From 06a5633cfc37465bd95b5ffb12acc660b75813c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Oct 2025 13:32:58 +0200 Subject: [PATCH 01/12] fix: decrease stack sampling size for python (#125) --- src/run/runner/wall_time/perf/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/run/runner/wall_time/perf/mod.rs b/src/run/runner/wall_time/perf/mod.rs index 7cefc097..c9386bc7 100644 --- a/src/run/runner/wall_time/perf/mod.rs +++ b/src/run/runner/wall_time/perf/mod.rs @@ -94,7 +94,9 @@ impl PerfRunner { || config.command.contains("uv") || config.command.contains("python") { - (UnwindingMode::Dwarf, Some(65528)) + // Max supported stack size is 64KiB, but this will increase the file size by a lot. In + // order to allow uploads and maintain accuracy, we limit this to 8KiB. + (UnwindingMode::Dwarf, Some(8 * 1024)) } else { // Default to dwarf unwinding since it works well with most binaries. debug!("No call graph mode detected, defaulting to dwarf"); From a5ddb7807b680b1589893b86488c0b5463619e48 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Mon, 6 Oct 2025 13:34:16 +0200 Subject: [PATCH 02/12] =?UTF-8?q?chore:=20Release=20runner=20version=204.1?= =?UTF-8?q?.1=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef5755a..74952331 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ +## [4.1.1] - 2025-10-06 + +### 🐛 Bug Fixes +- Decrease stack sampling size for python (#125) by @not-matthias in [#125](https://github.com/CodSpeedHQ/runner/pull/125) +- Break when parsing invalid command by @not-matthias in [#122](https://github.com/CodSpeedHQ/runner/pull/122) + + ## [4.1.0] - 2025-10-02 ### 🚀 Features @@ -489,6 +496,7 @@ - Add linting components to the toolchain by @art049 +[4.1.1]: https://github.com/CodSpeedHQ/runner/compare/v4.1.0..v4.1.1 [4.1.0]: https://github.com/CodSpeedHQ/runner/compare/v4.0.1..v4.1.0 [4.0.1]: https://github.com/CodSpeedHQ/runner/compare/v4.0.0..v4.0.1 [4.0.0]: https://github.com/CodSpeedHQ/runner/compare/v3.8.1..v4.0.0 diff --git a/Cargo.lock b/Cargo.lock index e592995f..69f42f34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,7 +284,7 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "codspeed-runner" -version = "4.1.0" +version = "4.1.1" dependencies = [ "anyhow", "async-compression", diff --git a/Cargo.toml b/Cargo.toml index 69e338bf..1783dccd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codspeed-runner" -version = "4.1.0" +version = "4.1.1" edition = "2024" repository = "https://github.com/CodSpeedHQ/runner" publish = false From e114915a519eb88b2c1eda4a6f7da0c5842c1b6a Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 13:03:17 -0600 Subject: [PATCH 03/12] Prototype Criterion log ingestion --- Cargo.toml | 4 + src/app.rs | 29 +++++++ src/run/ingest/criterion.rs | 167 ++++++++++++++++++++++++++++++++++++ src/run/ingest/mod.rs | 66 ++++++++++++++ src/run/mod.rs | 13 ++- src/run/runner/tests.rs | 2 + 6 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 src/run/ingest/criterion.rs create mode 100644 src/run/ingest/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 1783dccd..25d67087 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,10 @@ publish = false name = "codspeed" path = "src/main.rs" +[features] +# Enable slow executor integration tests (valgrind/walltime). Disabled by default. +executor_tests = [] + [dependencies] anyhow = "1.0.75" diff --git a/src/app.rs b/src/app.rs index 96ab23f2..c37e7e91 100644 --- a/src/app.rs +++ b/src/app.rs @@ -46,6 +46,8 @@ pub struct Cli { enum Commands { /// Run the bench command and upload the results to CodSpeed Run(run::RunArgs), + /// Ingest existing Criterion output and produce a CodSpeed profile folder + IngestCriterion(run::ingest::IngestArgs), /// Manage the CLI authentication state Auth(auth::AuthArgs), /// Pre-install the codspeed executors @@ -66,6 +68,33 @@ pub async fn run() -> Result<()> { match cli.command { Commands::Run(args) => run::run(args, &api_client, &codspeed_config).await?, + Commands::IngestCriterion(args) => { + // ingest and optionally upload the produced profile folder + let profile_folder = run::ingest::ingest_criterion(args.clone()).await?; + if args.upload { + // Reuse the existing `run` upload flow: construct RunArgs that skip running and point to the profile folder + let run_args = run::RunArgs { + upload_url: None, + token: None, + repository: None, + provider: None, + working_directory: None, + mode: run::RunnerMode::Walltime, + instruments: vec![], + mongo_uri_env_name: None, + profile_folder: Some(profile_folder), + message_format: None, + skip_upload: false, + skip_run: true, + skip_setup: true, + perf_run_args: run::PerfRunArgs::new(false, None), + command: vec![], + }; + + // Run the uploader path (this will call uploader::upload internally) + run::run(run_args, &api_client, &codspeed_config).await?; + } + } Commands::Auth(args) => auth::run(args, &api_client).await?, Commands::Setup => setup::setup().await?, } diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs new file mode 100644 index 00000000..f4d70ce9 --- /dev/null +++ b/src/run/ingest/criterion.rs @@ -0,0 +1,167 @@ +use crate::prelude::*; +use runner_shared::metadata::PerfMetadata; +use serde::Deserialize; +use std::fs; +use std::path::{Path, PathBuf}; + +/// Ingest Criterion results from `criterion_dir` and write CodSpeed-friendly artifacts into `profile_folder`. +pub fn ingest_criterion_results(criterion_dir: &Path, profile_folder: &Path) -> Result<()> { + // Walk criterion_dir recursively for per-benchmark folders. + if !criterion_dir.exists() { + bail!( + "Criterion directory does not exist: {}", + criterion_dir.display() + ); + } + + let mut collected_dirs: Vec = vec![]; + + fn collect_benchmark_dirs(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if !path.is_dir() { + continue; + } + + let estimates = path.join("new").join("estimates.json"); + let raw = path.join("new").join("raw.csv"); + if estimates.exists() || raw.exists() { + out.push(path.clone()); + continue; + } + + // Recurse into subdirectory + collect_benchmark_dirs(&path, out)?; + } + Ok(()) + } + + collect_benchmark_dirs(criterion_dir, &mut collected_dirs)?; + + // Now parse each collected benchmark dir + let mut benchmarks: Vec<(String, f64)> = Vec::new(); + for path in collected_dirs { + let estimates = path.join("new").join("estimates.json"); + if estimates.exists() { + if let Ok(time) = parse_estimates_json(&estimates) { + let name = path.file_name().unwrap().to_string_lossy().to_string(); + benchmarks.push((name, time)); + continue; + } + } + + let raw = path.join("new").join("raw.csv"); + if raw.exists() { + if let Ok(time) = parse_raw_csv_mean(&raw) { + let name = path.file_name().unwrap().to_string_lossy().to_string(); + benchmarks.push((name, time)); + continue; + } + } + } + + // Deduplicate by name (in case of nested duplicates) + let mut unique: Vec<(String, f64)> = Vec::new(); + for (n, t) in benchmarks.drain(..) { + if !unique.iter().any(|(un, _)| un == &n) { + unique.push((n, t)); + } + } + + benchmarks = unique; + + // Ensure profile folder exists + fs::create_dir_all(profile_folder)?; + + // Write a simple codspeed-benchmarks.json with name/time pairs + let bench_json_path = profile_folder.join("codspeed-benchmarks.json"); + let mut bench_json = serde_json::Map::new(); + let array = benchmarks + .iter() + .map(|(n, t)| { + let mut m = serde_json::Map::new(); + m.insert("name".into(), serde_json::Value::String(n.clone())); + m.insert( + "time".into(), + serde_json::Value::Number( + serde_json::Number::from_f64(*t).unwrap_or_else(|| serde_json::Number::from(0)), + ), + ); + serde_json::Value::Object(m) + }) + .collect::>(); + bench_json.insert("benchmarks".into(), serde_json::Value::Array(array)); + fs::write(bench_json_path, serde_json::to_string_pretty(&bench_json)?)?; + + // Write minimal perf.metadata so other tools can detect URIs + let metadata = PerfMetadata { + version: 1, + integration: ("criterion-ingest".into(), env!("CARGO_PKG_VERSION").into()), + uri_by_ts: benchmarks + .iter() + .map(|(n, _)| (current_time(), n.clone())) + .collect(), + ignored_modules: vec![], + markers: vec![], + }; + metadata.save_to(profile_folder)?; + + Ok(()) +} + +fn current_time() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[derive(Deserialize)] +struct EstimatesRoot { + mean: Option, + median: Option, +} + +#[derive(Deserialize)] +struct Estimate { + point_estimate: f64, +} + +fn parse_estimates_json(path: &Path) -> Result { + let data = fs::read_to_string(path)?; + let root: EstimatesRoot = + serde_json::from_str(&data).context("Failed to parse estimates.json")?; + if let Some(mean) = root.mean { + return Ok(mean.point_estimate); + } + if let Some(median) = root.median { + return Ok(median.point_estimate); + } + bail!("No mean/median estimate found in {}", path.display()); +} + +fn parse_raw_csv_mean(path: &Path) -> Result { + let data = fs::read_to_string(path)?; + let mut sum = 0f64; + let mut count = 0u64; + for line in data.lines() { + // skip header if present + if line.starts_with('#') || line.trim().is_empty() { + continue; + } + // CSV may have multiple columns, try last column as value + let cols: Vec<&str> = line.split(',').collect(); + if let Some(s) = cols.last() { + if let Ok(v) = s.trim().parse::() { + sum += v; + count += 1; + } + } + } + if count == 0 { + bail!("No numeric rows found in {}", path.display()); + } + Ok(sum / (count as f64)) +} diff --git a/src/run/ingest/mod.rs b/src/run/ingest/mod.rs new file mode 100644 index 00000000..1632b7a5 --- /dev/null +++ b/src/run/ingest/mod.rs @@ -0,0 +1,66 @@ +use crate::prelude::*; +use clap::Args; +use std::path::PathBuf; + +pub mod criterion; + +#[derive(Args, Debug, Clone)] +pub struct IngestArgs { + /// Path to the Criterion `target/criterion` directory (or parent of it) + #[arg(long)] + pub criterion_dir: PathBuf, + + /// Profile folder to write CodSpeed artifacts into. If omitted, a temporary folder will be used. + #[arg(long)] + pub profile_folder: Option, + + /// After ingestion, upload the produced profile folder to CodSpeed using the current config + #[arg(long, default_value_t = false)] + pub upload: bool, +} +pub async fn ingest_criterion(args: IngestArgs) -> Result { + let profile_folder = if let Some(p) = args.profile_folder { + p + } else { + // create a temporary directory + let tmp = tempfile::tempdir()?; + tmp.path().to_path_buf() + }; + + criterion::ingest_criterion_results(&args.criterion_dir, &profile_folder) + .context("Failed to ingest Criterion results")?; + + info!( + "Wrote CodSpeed profile artifacts to {}", + profile_folder.display() + ); + Ok(profile_folder) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use tempfile::tempdir; + + #[test] + fn test_ingest_samples() { + let samples_dir = PathBuf::from(format!( + "{}/src/run/ingest/samples/criterion_target", + env!("CARGO_MANIFEST_DIR") + )); + let tmp = tempdir().unwrap(); + let profile = tmp.path().to_path_buf(); + + criterion::ingest_criterion_results(&samples_dir, &profile).unwrap(); + + let bench_file = profile.join("codspeed-benchmarks.json"); + assert!(bench_file.exists()); + let content = std::fs::read_to_string(bench_file).unwrap(); + assert!(content.contains("bench1")); + assert!(content.contains("bench2")); + + let metadata = profile.join("perf.metadata"); + assert!(metadata.exists()); + } +} diff --git a/src/run/mod.rs b/src/run/mod.rs index b64b51f8..1c438d83 100644 --- a/src/run/mod.rs +++ b/src/run/mod.rs @@ -17,9 +17,10 @@ mod instruments; mod poll_results; pub mod run_environment; pub mod runner; -mod uploader; +pub mod uploader; pub mod config; +pub mod ingest; pub mod logger; fn show_banner() { @@ -60,6 +61,16 @@ pub struct PerfRunArgs { perf_unwinding_mode: Option, } +impl PerfRunArgs { + /// Public constructor to create PerfRunArgs programmatically + pub fn new(enable_perf: bool, perf_unwinding_mode: Option) -> Self { + Self { + enable_perf, + perf_unwinding_mode, + } + } +} + #[derive(Args, Debug)] pub struct RunArgs { /// The upload URL to use for uploading the results, useful for on-premises installations diff --git a/src/run/runner/tests.rs b/src/run/runner/tests.rs index 36316216..a9cb560d 100644 --- a/src/run/runner/tests.rs +++ b/src/run/runner/tests.rs @@ -129,6 +129,7 @@ async fn create_test_setup() -> (SystemInfo, RunData, TempDir) { (system_info, run_data, temp_dir) } +#[cfg(feature = "executor_tests")] mod valgrind { use super::*; @@ -185,6 +186,7 @@ mod valgrind { } } +#[cfg(feature = "executor_tests")] mod walltime { use super::*; From e17507bc9fa7c96fc02ffe78ba78e6d024ec2b86 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 14:44:54 -0600 Subject: [PATCH 04/12] Patch ` attempted to set a logger after the logging system was already initialized` error --- src/app.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/app.rs b/src/app.rs index c37e7e91..00615d49 100644 --- a/src/app.rs +++ b/src/app.rs @@ -59,16 +59,13 @@ pub async fn run() -> Result<()> { let codspeed_config = CodSpeedConfig::load_with_override(cli.oauth_token.as_deref())?; let api_client = CodSpeedAPIClient::try_from((&cli, &codspeed_config))?; - match cli.command { - Commands::Run(_) => {} // Run is responsible for its own logger initialization - _ => { - init_local_logger()?; - } - } - match cli.command { Commands::Run(args) => run::run(args, &api_client, &codspeed_config).await?, Commands::IngestCriterion(args) => { + if !args.upload { + init_local_logger()?; + } + // ingest and optionally upload the produced profile folder let profile_folder = run::ingest::ingest_criterion(args.clone()).await?; if args.upload { @@ -95,8 +92,14 @@ pub async fn run() -> Result<()> { run::run(run_args, &api_client, &codspeed_config).await?; } } - Commands::Auth(args) => auth::run(args, &api_client).await?, - Commands::Setup => setup::setup().await?, + Commands::Auth(args) => { + init_local_logger()?; + auth::run(args, &api_client).await?; + } + Commands::Setup => { + init_local_logger()?; + setup::setup().await?; + } } Ok(()) } From 4a4ae6013071789471f0e2d98d4ad8cf2d6c6643 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 14:48:11 -0600 Subject: [PATCH 05/12] Debug macOS support --- src/run/check_system.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/run/check_system.rs b/src/run/check_system.rs index efe7a29e..4f1f88f0 100644 --- a/src/run/check_system.rs +++ b/src/run/check_system.rs @@ -56,7 +56,11 @@ impl SystemInfo { pub fn new() -> Result { let os = System::distribution_id(); let os_version = System::os_version().ok_or(anyhow!("Failed to get OS version"))?; - let arch = System::cpu_arch(); + let arch_raw = System::cpu_arch(); + let arch = match arch_raw.as_str() { + "arm64" => "aarch64".to_string(), + other => other.to_string(), + }; let user = get_user()?; let host = System::host_name().ok_or(anyhow!("Failed to get host name"))?; From cdca1d557197ee2d9dcdd006e415ba43aa2736b5 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 15:48:56 -0600 Subject: [PATCH 06/12] Update Criterion parsing --- src/run/ingest/criterion.rs | 596 +++++++++++++++++++++++++++++------- 1 file changed, 488 insertions(+), 108 deletions(-) diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs index f4d70ce9..92626651 100644 --- a/src/run/ingest/criterion.rs +++ b/src/run/ingest/criterion.rs @@ -1,12 +1,15 @@ use crate::prelude::*; use runner_shared::metadata::PerfMetadata; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; +const NANOSECONDS_IN_SECOND: f64 = 1_000_000_000.0; +const IQR_OUTLIER_FACTOR: f64 = 1.5; +const STDEV_OUTLIER_FACTOR: f64 = 3.0; + /// Ingest Criterion results from `criterion_dir` and write CodSpeed-friendly artifacts into `profile_folder`. pub fn ingest_criterion_results(criterion_dir: &Path, profile_folder: &Path) -> Result<()> { - // Walk criterion_dir recursively for per-benchmark folders. if !criterion_dir.exists() { bail!( "Criterion directory does not exist: {}", @@ -14,154 +17,531 @@ pub fn ingest_criterion_results(criterion_dir: &Path, profile_folder: &Path) -> ); } - let mut collected_dirs: Vec = vec![]; - - fn collect_benchmark_dirs(dir: &Path, out: &mut Vec) -> Result<()> { - for entry in fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - if !path.is_dir() { - continue; - } - - let estimates = path.join("new").join("estimates.json"); - let raw = path.join("new").join("raw.csv"); - if estimates.exists() || raw.exists() { - out.push(path.clone()); - continue; - } + let mut benchmark_dirs = Vec::new(); + collect_benchmark_dirs(criterion_dir, &mut benchmark_dirs)?; - // Recurse into subdirectory - collect_benchmark_dirs(&path, out)?; - } - Ok(()) + if benchmark_dirs.is_empty() { + bail!( + "No Criterion benchmarks found under {}", + criterion_dir.display() + ); } - collect_benchmark_dirs(criterion_dir, &mut collected_dirs)?; - - // Now parse each collected benchmark dir - let mut benchmarks: Vec<(String, f64)> = Vec::new(); - for path in collected_dirs { - let estimates = path.join("new").join("estimates.json"); - if estimates.exists() { - if let Ok(time) = parse_estimates_json(&estimates) { - let name = path.file_name().unwrap().to_string_lossy().to_string(); - benchmarks.push((name, time)); - continue; - } - } - - let raw = path.join("new").join("raw.csv"); - if raw.exists() { - if let Ok(time) = parse_raw_csv_mean(&raw) { - let name = path.file_name().unwrap().to_string_lossy().to_string(); - benchmarks.push((name, time)); - continue; + let mut benchmarks = Vec::new(); + let mut skipped = 0usize; + for bench_dir in benchmark_dirs { + match build_walltime_benchmark(criterion_dir, &bench_dir) { + Ok(Some(bench)) => benchmarks.push(bench), + Ok(None) => skipped += 1, + Err(err) => { + skipped += 1; + debug!( + "Skipping benchmark at {} due to error: {err:?}", + bench_dir.display() + ); } } } - // Deduplicate by name (in case of nested duplicates) - let mut unique: Vec<(String, f64)> = Vec::new(); - for (n, t) in benchmarks.drain(..) { - if !unique.iter().any(|(un, _)| un == &n) { - unique.push((n, t)); - } + if benchmarks.is_empty() { + bail!( + "Failed to ingest any Criterion benchmarks from {} (skipped {skipped})", + criterion_dir.display() + ); } - benchmarks = unique; + // Stable ordering for deterministic outputs + benchmarks.sort_by(|a, b| a.name().cmp(b.name())); - // Ensure profile folder exists fs::create_dir_all(profile_folder)?; - // Write a simple codspeed-benchmarks.json with name/time pairs + let results = WalltimeResults::new(benchmarks.clone()); let bench_json_path = profile_folder.join("codspeed-benchmarks.json"); - let mut bench_json = serde_json::Map::new(); - let array = benchmarks + let bench_json_file = std::fs::File::create(&bench_json_path) + .context("Failed to create codspeed-benchmarks.json")?; + serde_json::to_writer_pretty(bench_json_file, &results) + .context("Failed to write codspeed-benchmarks.json")?; + + let base_ts = current_time_ns(); + let uri_by_ts = benchmarks .iter() - .map(|(n, t)| { - let mut m = serde_json::Map::new(); - m.insert("name".into(), serde_json::Value::String(n.clone())); - m.insert( - "time".into(), - serde_json::Value::Number( - serde_json::Number::from_f64(*t).unwrap_or_else(|| serde_json::Number::from(0)), - ), - ); - serde_json::Value::Object(m) - }) - .collect::>(); - bench_json.insert("benchmarks".into(), serde_json::Value::Array(array)); - fs::write(bench_json_path, serde_json::to_string_pretty(&bench_json)?)?; + .enumerate() + .map(|(idx, bench)| (base_ts + idx as u64, bench.uri().to_string())) + .collect(); - // Write minimal perf.metadata so other tools can detect URIs let metadata = PerfMetadata { version: 1, integration: ("criterion-ingest".into(), env!("CARGO_PKG_VERSION").into()), - uri_by_ts: benchmarks - .iter() - .map(|(n, _)| (current_time(), n.clone())) - .collect(), + uri_by_ts, ignored_modules: vec![], markers: vec![], }; - metadata.save_to(profile_folder)?; + metadata + .save_to(profile_folder) + .context("Failed to write perf.metadata")?; Ok(()) } -fn current_time() -> u64 { +fn collect_benchmark_dirs(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if !path.is_dir() { + continue; + } + + let new_dir = path.join("new"); + if new_dir.join("sample.json").exists() + || new_dir.join("estimates.json").exists() + || new_dir.join("raw.csv").exists() + { + out.push(path); + continue; + } + + collect_benchmark_dirs(&path, out)?; + } + Ok(()) +} + +fn build_walltime_benchmark( + criterion_root: &Path, + bench_dir: &Path, +) -> Result> { + let measurements = match load_benchmark_measurements(bench_dir) { + Some(data) => data, + None => return Ok(None), + }; + + let identity = determine_identity(criterion_root, bench_dir); + let stats = BenchmarkStats::from_measurements(&measurements); + + if stats.mean_ns < f64::EPSILON { + return Ok(None); + } + + let benchmark = WalltimeBenchmark { + metadata: BenchmarkMetadata { + name: identity.name, + uri: identity.uri, + }, + config: BenchmarkConfig { + warmup_time_ns: None, + min_round_time_ns: None, + max_time_ns: measurements.max_time_ns, + max_rounds: None, + }, + stats, + }; + + Ok(Some(benchmark)) +} + +fn load_benchmark_measurements(dir: &Path) -> Option { + let new_dir = dir.join("new"); + + if let Ok(sample) = fs::read_to_string(new_dir.join("sample.json")) { + if let Ok(sample) = serde_json::from_str::(&sample) { + if let Some(measurements) = BenchmarkMeasurements::from_sample(sample) { + return Some(measurements); + } + } + } + + if let Ok(estimates) = fs::read_to_string(new_dir.join("estimates.json")) { + if let Ok(estimates) = serde_json::from_str::(&estimates) { + if let Some(measurements) = BenchmarkMeasurements::from_estimates(estimates) { + return Some(measurements); + } + } + } + + None +} + +fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { + let new_dir = dir.join("new"); + if let Ok(benchmark_id) = fs::read_to_string(new_dir.join("benchmark.json")) { + if let Ok(id) = serde_json::from_str::(&benchmark_id) { + let mut name = id.group_id.clone(); + if let Some(function) = id.function_id { + if !function.is_empty() { + name.push_str("::"); + name.push_str(&function); + } + } + if let Some(parameter) = id.value_str { + if !parameter.is_empty() { + name.push_str(&format!("[{parameter}]")); + } + } + let uri = format!("criterion::{name}"); + return BenchmarkIdentity { name, uri }; + } + } + + let relative = dir + .strip_prefix(criterion_root) + .unwrap_or(dir) + .iter() + .map(|component| component.to_string_lossy()) + .collect::>() + .join("::"); + let name = if relative.is_empty() { + dir.file_name() + .map(|os| os.to_string_lossy().to_string()) + .unwrap_or_else(|| "benchmark".to_string()) + } else { + relative.clone() + }; + let uri_suffix = if relative.is_empty() { + name.clone() + } else { + relative + }; + let uri = format!("criterion::{uri_suffix}"); + + BenchmarkIdentity { name, uri } +} + +fn current_time_ns() -> u64 { use std::time::{SystemTime, UNIX_EPOCH}; SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) + .map(|d| d.as_nanos() as u64) .unwrap_or(0) } -#[derive(Deserialize)] -struct EstimatesRoot { - mean: Option, - median: Option, +#[derive(Clone, Debug, Serialize)] +struct WalltimeResults { + creator: Creator, + instrument: Instrument, + benchmarks: Vec, } -#[derive(Deserialize)] -struct Estimate { - point_estimate: f64, +impl WalltimeResults { + fn new(benchmarks: Vec) -> Self { + Self { + creator: Creator { + name: "codspeed-criterion-ingest".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + pid: std::process::id(), + }, + instrument: Instrument { + type_: "walltime".to_string(), + }, + benchmarks, + } + } } -fn parse_estimates_json(path: &Path) -> Result { - let data = fs::read_to_string(path)?; - let root: EstimatesRoot = - serde_json::from_str(&data).context("Failed to parse estimates.json")?; - if let Some(mean) = root.mean { - return Ok(mean.point_estimate); +#[derive(Clone, Debug, Serialize)] +struct Creator { + name: String, + version: String, + pid: u32, +} + +#[derive(Clone, Debug, Serialize)] +struct Instrument { + #[serde(rename = "type")] + type_: String, +} + +#[derive(Clone, Debug, Serialize)] +struct WalltimeBenchmark { + #[serde(flatten)] + metadata: BenchmarkMetadata, + config: BenchmarkConfig, + stats: BenchmarkStats, +} + +impl WalltimeBenchmark { + fn name(&self) -> &str { + &self.metadata.name } - if let Some(median) = root.median { - return Ok(median.point_estimate); + + fn uri(&self) -> &str { + &self.metadata.uri } - bail!("No mean/median estimate found in {}", path.display()); } -fn parse_raw_csv_mean(path: &Path) -> Result { - let data = fs::read_to_string(path)?; - let mut sum = 0f64; - let mut count = 0u64; - for line in data.lines() { - // skip header if present - if line.starts_with('#') || line.trim().is_empty() { - continue; +#[derive(Clone, Debug, Serialize)] +struct BenchmarkMetadata { + name: String, + uri: String, +} + +#[derive(Clone, Debug, Serialize)] +struct BenchmarkConfig { + warmup_time_ns: Option, + min_round_time_ns: Option, + max_time_ns: Option, + max_rounds: Option, +} + +#[derive(Clone, Debug, Serialize)] +struct BenchmarkStats { + min_ns: f64, + max_ns: f64, + mean_ns: f64, + stdev_ns: f64, + q1_ns: f64, + median_ns: f64, + q3_ns: f64, + rounds: u64, + total_time: f64, + iqr_outlier_rounds: u64, + stdev_outlier_rounds: u64, + iter_per_round: u64, + warmup_iters: u64, +} + +impl BenchmarkStats { + fn from_measurements(measurements: &BenchmarkMeasurements) -> Self { + let rounds = measurements.per_iter_ns.len() as u64; + + let min_ns = measurements + .per_iter_ns + .iter() + .copied() + .fold(f64::INFINITY, f64::min); + let max_ns = measurements + .per_iter_ns + .iter() + .copied() + .fold(f64::NEG_INFINITY, f64::max); + + let mean_ns = measurements.mean(); + let stdev_ns = measurements.stdev(); + + let (q1_ns, median_ns, q3_ns) = measurements.quantiles(); + let (iqr_outlier_rounds, stdev_outlier_rounds) = + measurements.outlier_counts(mean_ns, stdev_ns, q1_ns, q3_ns); + + Self { + min_ns: if !min_ns.is_finite() { mean_ns } else { min_ns }, + max_ns: if !max_ns.is_finite() { mean_ns } else { max_ns }, + mean_ns, + stdev_ns, + q1_ns, + median_ns, + q3_ns, + rounds, + total_time: measurements.total_time, + iqr_outlier_rounds, + stdev_outlier_rounds, + iter_per_round: measurements.iter_per_round, + warmup_iters: measurements.warmup_iters, + } + } +} + +#[derive(Clone, Debug)] +struct BenchmarkMeasurements { + per_iter_ns: Vec, + total_time: f64, + iter_per_round: u64, + warmup_iters: u64, + max_time_ns: Option, + stdev_override: Option, +} + +impl BenchmarkMeasurements { + fn from_sample(sample: SavedSample) -> Option { + if sample.iters.is_empty() + || sample.times.is_empty() + || sample.iters.len() != sample.times.len() + { + return None; } - // CSV may have multiple columns, try last column as value - let cols: Vec<&str> = line.split(',').collect(); - if let Some(s) = cols.last() { - if let Ok(v) = s.trim().parse::() { - sum += v; - count += 1; + + let mut per_iter_ns = Vec::with_capacity(sample.times.len()); + let mut total_time_ns = 0f64; + let mut iter_sum = 0f64; + + for (iters, time) in sample.iters.iter().zip(sample.times.iter()) { + if *iters <= f64::EPSILON { + return None; } + per_iter_ns.push(time / iters); + total_time_ns += *time; + iter_sum += *iters; + } + + if per_iter_ns.is_empty() { + return None; + } + + let iter_per_round = if iter_sum <= f64::EPSILON { + 1 + } else { + (iter_sum / per_iter_ns.len() as f64).round() as u64 + }; + + Some(Self { + per_iter_ns, + total_time: total_time_ns / NANOSECONDS_IN_SECOND, + iter_per_round: iter_per_round.max(1), + warmup_iters: 0, + max_time_ns: None, + stdev_override: None, + }) + } + + fn from_estimates(estimates: EstimatesRoot) -> Option { + let EstimatesRoot { + mean, + median, + std_dev, + } = estimates; + + let mean_estimate = mean.or(median).map(|estimate| estimate.point_estimate)?; + + if mean_estimate <= f64::EPSILON { + return None; + } + + let mean_ns = mean_estimate * NANOSECONDS_IN_SECOND; + let stdev_override = std_dev + .map(|std| std.point_estimate * NANOSECONDS_IN_SECOND) + .filter(|v| v.is_finite() && *v >= 0.0); + + Some(Self { + per_iter_ns: vec![mean_ns], + total_time: mean_estimate, + iter_per_round: 1, + warmup_iters: 0, + max_time_ns: None, + stdev_override, + }) + } + + fn mean(&self) -> f64 { + if self.per_iter_ns.is_empty() { + return 0.0; } + self.per_iter_ns.iter().sum::() / self.per_iter_ns.len() as f64 } - if count == 0 { - bail!("No numeric rows found in {}", path.display()); + + fn stdev(&self) -> f64 { + if let Some(override_value) = self.stdev_override { + return override_value; + } + + let n = self.per_iter_ns.len(); + if n < 2 { + return 0.0; + } + let mean = self.mean(); + let variance = self + .per_iter_ns + .iter() + .map(|value| { + let diff = value - mean; + diff * diff + }) + .sum::() + / (n as f64 - 1.0); + variance.sqrt() + } + + fn quantiles(&self) -> (f64, f64, f64) { + if self.per_iter_ns.is_empty() { + return (0.0, 0.0, 0.0); + } + + let mut sorted = self.per_iter_ns.clone(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let q1 = quantile(&sorted, 0.25); + let median = quantile(&sorted, 0.50); + let q3 = quantile(&sorted, 0.75); + (q1, median, q3) + } + + fn outlier_counts(&self, mean_ns: f64, stdev_ns: f64, q1_ns: f64, q3_ns: f64) -> (u64, u64) { + if self.per_iter_ns.is_empty() { + return (0, 0); + } + + let iqr = q3_ns - q1_ns; + let iqr_low = q1_ns - IQR_OUTLIER_FACTOR * iqr; + let iqr_high = q3_ns + IQR_OUTLIER_FACTOR * iqr; + let iqr_outlier_rounds = if iqr <= f64::EPSILON { + 0 + } else { + self.per_iter_ns + .iter() + .filter(|&&value| value < iqr_low || value > iqr_high) + .count() as u64 + }; + + let stdev_outlier_rounds = if stdev_ns <= f64::EPSILON { + 0 + } else { + let low = mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns; + let high = mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns; + self.per_iter_ns + .iter() + .filter(|&&value| value < low || value > high) + .count() as u64 + }; + + (iqr_outlier_rounds, stdev_outlier_rounds) + } +} + +fn quantile(sorted: &[f64], q: f64) -> f64 { + if sorted.is_empty() { + return 0.0; + } + if sorted.len() == 1 { + return sorted[0]; } - Ok(sum / (count as f64)) + + let clamped_q = q.clamp(0.0, 1.0); + let pos = clamped_q * (sorted.len() as f64 - 1.0); + let lower = pos.floor() as usize; + let upper = pos.ceil() as usize; + + if lower == upper { + sorted[lower] + } else { + let weight = pos - lower as f64; + sorted[lower] * (1.0 - weight) + sorted[upper] * weight + } +} + +#[derive(Debug, Deserialize)] +struct SavedSample { + #[serde(default)] + _sampling_mode: Option, + iters: Vec, + times: Vec, +} + +#[derive(Debug, Deserialize)] +struct EstimatesRoot { + mean: Option, + median: Option, + #[serde(rename = "std_dev")] + std_dev: Option, +} + +#[derive(Debug, Deserialize)] +struct Estimate { + point_estimate: f64, +} + +#[derive(Debug, Deserialize)] +struct BenchmarkIdRecord { + group_id: String, + function_id: Option, + value_str: Option, +} + +struct BenchmarkIdentity { + name: String, + uri: String, } From 19e53d13cb4d1cde26c7a253d262bf56149b0e94 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 16:35:27 -0600 Subject: [PATCH 07/12] Debug report upload --- src/run/ingest/criterion.rs | 38 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs index 92626651..2b3bd0fc 100644 --- a/src/run/ingest/criterion.rs +++ b/src/run/ingest/criterion.rs @@ -1,5 +1,5 @@ use crate::prelude::*; -use runner_shared::metadata::PerfMetadata; +use runner_shared::{fifo::MarkerType, metadata::PerfMetadata}; use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; @@ -62,19 +62,31 @@ pub fn ingest_criterion_results(criterion_dir: &Path, profile_folder: &Path) -> serde_json::to_writer_pretty(bench_json_file, &results) .context("Failed to write codspeed-benchmarks.json")?; - let base_ts = current_time_ns(); - let uri_by_ts = benchmarks - .iter() - .enumerate() - .map(|(idx, bench)| (base_ts + idx as u64, bench.uri().to_string())) - .collect(); + let mut uri_by_ts = Vec::with_capacity(benchmarks.len()); + let mut markers = Vec::with_capacity(benchmarks.len() * 2); + let mut timeline_cursor = 0u64; + + for bench in &benchmarks { + uri_by_ts.push((timeline_cursor, bench.uri().to_string())); + markers.push(MarkerType::SampleStart(timeline_cursor)); + + let raw_duration_ns = (bench.stats.total_time * NANOSECONDS_IN_SECOND).round(); + let duration_ns = raw_duration_ns + .is_finite() + .then_some(raw_duration_ns.max(1.0)) + .unwrap_or(1.0) as u64; + let end_ts = timeline_cursor + duration_ns; + + markers.push(MarkerType::SampleEnd(end_ts)); + timeline_cursor = end_ts.saturating_add(1); + } let metadata = PerfMetadata { version: 1, - integration: ("criterion-ingest".into(), env!("CARGO_PKG_VERSION").into()), + integration: ("codspeed-runner".into(), env!("CARGO_PKG_VERSION").into()), uri_by_ts, ignored_modules: vec![], - markers: vec![], + markers, }; metadata .save_to(profile_folder) @@ -205,14 +217,6 @@ fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { BenchmarkIdentity { name, uri } } -fn current_time_ns() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0) -} - #[derive(Clone, Debug, Serialize)] struct WalltimeResults { creator: Creator, From ba1c70a4cb46e9ef1009b0882cbace28f8216a68 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 17:08:17 -0600 Subject: [PATCH 08/12] Debug Criterion ingest --- src/run/ingest/criterion.rs | 11 ++++++++++- src/run/ingest/mod.rs | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs index 2b3bd0fc..12b52547 100644 --- a/src/run/ingest/criterion.rs +++ b/src/run/ingest/criterion.rs @@ -56,6 +56,15 @@ pub fn ingest_criterion_results(criterion_dir: &Path, profile_folder: &Path) -> fs::create_dir_all(profile_folder)?; let results = WalltimeResults::new(benchmarks.clone()); + + let results_dir = profile_folder.join("results"); + fs::create_dir_all(&results_dir).context("Failed to create results directory")?; + let results_json_path = results_dir.join(format!("{}.json", std::process::id())); + let results_json_file = + std::fs::File::create(&results_json_path).context("Failed to create results JSON file")?; + serde_json::to_writer_pretty(&results_json_file, &results) + .context("Failed to write results JSON file")?; + let bench_json_path = profile_folder.join("codspeed-benchmarks.json"); let bench_json_file = std::fs::File::create(&bench_json_path) .context("Failed to create codspeed-benchmarks.json")?; @@ -228,7 +237,7 @@ impl WalltimeResults { fn new(benchmarks: Vec) -> Self { Self { creator: Creator { - name: "codspeed-criterion-ingest".to_string(), + name: "codspeed-rust".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), pid: std::process::id(), }, diff --git a/src/run/ingest/mod.rs b/src/run/ingest/mod.rs index 1632b7a5..287ac11f 100644 --- a/src/run/ingest/mod.rs +++ b/src/run/ingest/mod.rs @@ -54,6 +54,13 @@ mod tests { criterion::ingest_criterion_results(&samples_dir, &profile).unwrap(); + let results_dir = profile.join("results"); + assert!(results_dir.is_dir()); + let results_file = results_dir.join(format!("{}.json", std::process::id())); + assert!(results_file.exists()); + let results_content = std::fs::read_to_string(&results_file).unwrap(); + assert!(results_content.contains("bench1")); + let bench_file = profile.join("codspeed-benchmarks.json"); assert!(bench_file.exists()); let content = std::fs::read_to_string(bench_file).unwrap(); From 4ad918feb97ddbf3ea6e2b471d71702bfc022830 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 17:41:19 -0600 Subject: [PATCH 09/12] Update bench identity resolution --- src/run/ingest/criterion.rs | 68 +++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs index 12b52547..e15f282f 100644 --- a/src/run/ingest/criterion.rs +++ b/src/run/ingest/criterion.rs @@ -144,7 +144,7 @@ fn build_walltime_benchmark( let benchmark = WalltimeBenchmark { metadata: BenchmarkMetadata { - name: identity.name, + name: identity.display_name, uri: identity.uri, }, config: BenchmarkConfig { @@ -183,47 +183,73 @@ fn load_benchmark_measurements(dir: &Path) -> Option { fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { let new_dir = dir.join("new"); + let relative_components = dir + .strip_prefix(criterion_root) + .unwrap_or(dir) + .iter() + .map(|component| component.to_string_lossy().to_string()) + .collect::>(); + if let Ok(benchmark_id) = fs::read_to_string(new_dir.join("benchmark.json")) { if let Ok(id) = serde_json::from_str::(&benchmark_id) { - let mut name = id.group_id.clone(); + let mut segments = Vec::new(); + if !id.group_id.is_empty() { + segments.push(id.group_id); + } + if let Some(function) = id.function_id { if !function.is_empty() { - name.push_str("::"); - name.push_str(&function); + segments.push(function); } } + if let Some(parameter) = id.value_str { if !parameter.is_empty() { - name.push_str(&format!("[{parameter}]")); + if let Some(last) = segments.last_mut() { + last.push_str(&format!("[{parameter}]")); + } else { + segments.push(format!("[{parameter}]")); + } } } - let uri = format!("criterion::{name}"); - return BenchmarkIdentity { name, uri }; + + let display_name = if segments.is_empty() { + relative_components.join("/") + } else { + segments.join("/") + }; + + let uri_suffix = if segments.is_empty() { + if relative_components.is_empty() { + display_name.clone() + } else { + relative_components.join("::") + } + } else { + segments.join("::") + }; + + let uri = format!("criterion::{uri_suffix}"); + return BenchmarkIdentity { display_name, uri }; } } - let relative = dir - .strip_prefix(criterion_root) - .unwrap_or(dir) - .iter() - .map(|component| component.to_string_lossy()) - .collect::>() - .join("::"); - let name = if relative.is_empty() { + let display_name = if relative_components.is_empty() { dir.file_name() .map(|os| os.to_string_lossy().to_string()) .unwrap_or_else(|| "benchmark".to_string()) } else { - relative.clone() + relative_components.join("/") }; - let uri_suffix = if relative.is_empty() { - name.clone() + + let uri_suffix = if relative_components.is_empty() { + display_name.replace('/', "::") } else { - relative + relative_components.join("::") }; let uri = format!("criterion::{uri_suffix}"); - BenchmarkIdentity { name, uri } + BenchmarkIdentity { display_name, uri } } #[derive(Clone, Debug, Serialize)] @@ -555,6 +581,6 @@ struct BenchmarkIdRecord { } struct BenchmarkIdentity { - name: String, + display_name: String, uri: String, } From 4e19a96066e84d308b81c4353abe5057ba2e4230 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 18:20:54 -0600 Subject: [PATCH 10/12] Prototype display segment update --- src/run/ingest/criterion.rs | 45 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs index e15f282f..f282d23a 100644 --- a/src/run/ingest/criterion.rs +++ b/src/run/ingest/criterion.rs @@ -213,21 +213,44 @@ fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { } } - let display_name = if segments.is_empty() { - relative_components.join("/") + let mut display_segments = Vec::new(); + if let Some(first) = relative_components.first() { + display_segments.push(first.clone()); + } + + if segments.is_empty() { + if relative_components.len() > 1 { + display_segments.extend(relative_components.iter().skip(1).cloned()); + } } else { - segments.join("/") - }; + for segment in &segments { + if display_segments + .last() + .map(|existing| existing == segment) + .unwrap_or(false) + { + continue; + } + display_segments.push(segment.clone()); + } + } - let uri_suffix = if segments.is_empty() { - if relative_components.is_empty() { - display_name.clone() + if display_segments.is_empty() { + if !segments.is_empty() { + display_segments = segments.clone(); + } else if !relative_components.is_empty() { + display_segments = relative_components.clone(); + } else if let Some(file_name) = + dir.file_name().map(|os| os.to_string_lossy().to_string()) + { + display_segments.push(file_name); } else { - relative_components.join("::") + display_segments.push("benchmark".to_string()); } - } else { - segments.join("::") - }; + } + + let display_name = display_segments.join("/"); + let uri_suffix = display_segments.join("::"); let uri = format!("criterion::{uri_suffix}"); return BenchmarkIdentity { display_name, uri }; From 3b795bc794ae3083434452096a1a8cb85a0d0ff4 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 18:37:54 -0600 Subject: [PATCH 11/12] Prototype display name resolution --- src/run/ingest/criterion.rs | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs index f282d23a..8b3c0b72 100644 --- a/src/run/ingest/criterion.rs +++ b/src/run/ingest/criterion.rs @@ -213,14 +213,14 @@ fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { } } - let mut display_segments = Vec::new(); - if let Some(first) = relative_components.first() { - display_segments.push(first.clone()); - } + let mut display_segments = relative_components.clone(); - if segments.is_empty() { - if relative_components.len() > 1 { - display_segments.extend(relative_components.iter().skip(1).cloned()); + if segments.is_empty() && display_segments.is_empty() { + if let Some(file_name) = dir.file_name().map(|os| os.to_string_lossy().to_string()) + { + display_segments.push(file_name); + } else { + display_segments.push("benchmark".to_string()); } } else { for segment in &segments { @@ -231,22 +231,17 @@ fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { { continue; } + + if display_segments.iter().any(|existing| existing == segment) { + continue; + } + display_segments.push(segment.clone()); } } if display_segments.is_empty() { - if !segments.is_empty() { - display_segments = segments.clone(); - } else if !relative_components.is_empty() { - display_segments = relative_components.clone(); - } else if let Some(file_name) = - dir.file_name().map(|os| os.to_string_lossy().to_string()) - { - display_segments.push(file_name); - } else { - display_segments.push("benchmark".to_string()); - } + display_segments.push("benchmark".to_string()); } let display_name = display_segments.join("/"); From 70e2e5f00faaf879a767647ec73a472069d1c287 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Mon, 6 Oct 2025 19:07:53 -0600 Subject: [PATCH 12/12] Continue prototype display name resolution --- src/run/ingest/criterion.rs | 99 +++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 25 deletions(-) diff --git a/src/run/ingest/criterion.rs b/src/run/ingest/criterion.rs index 8b3c0b72..820f3125 100644 --- a/src/run/ingest/criterion.rs +++ b/src/run/ingest/criterion.rs @@ -190,6 +190,8 @@ fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { .map(|component| component.to_string_lossy().to_string()) .collect::>(); + let mut uri_segments = relative_components.clone(); + if let Ok(benchmark_id) = fs::read_to_string(new_dir.join("benchmark.json")) { if let Ok(id) = serde_json::from_str::(&benchmark_id) { let mut segments = Vec::new(); @@ -213,18 +215,16 @@ fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { } } - let mut display_segments = relative_components.clone(); - - if segments.is_empty() && display_segments.is_empty() { + if segments.is_empty() && uri_segments.is_empty() { if let Some(file_name) = dir.file_name().map(|os| os.to_string_lossy().to_string()) { - display_segments.push(file_name); + uri_segments.push(file_name); } else { - display_segments.push("benchmark".to_string()); + uri_segments.push("benchmark".to_string()); } } else { for segment in &segments { - if display_segments + if uri_segments .last() .map(|existing| existing == segment) .unwrap_or(false) @@ -232,38 +232,40 @@ fn determine_identity(criterion_root: &Path, dir: &Path) -> BenchmarkIdentity { continue; } - if display_segments.iter().any(|existing| existing == segment) { + if uri_segments.iter().any(|existing| existing == segment) { continue; } - display_segments.push(segment.clone()); + uri_segments.push(segment.clone()); } } + } + } - if display_segments.is_empty() { - display_segments.push("benchmark".to_string()); - } - - let display_name = display_segments.join("/"); - let uri_suffix = display_segments.join("::"); - - let uri = format!("criterion::{uri_suffix}"); - return BenchmarkIdentity { display_name, uri }; + if uri_segments.is_empty() { + if let Some(file_name) = dir.file_name().map(|os| os.to_string_lossy().to_string()) { + uri_segments.push(file_name); + } else { + uri_segments.push("benchmark".to_string()); } } - let display_name = if relative_components.is_empty() { - dir.file_name() - .map(|os| os.to_string_lossy().to_string()) - .unwrap_or_else(|| "benchmark".to_string()) + let display_segments = if uri_segments.len() >= 2 { + uri_segments[uri_segments.len() - 2..].to_vec() } else { - relative_components.join("/") + uri_segments.clone() }; - let uri_suffix = if relative_components.is_empty() { - display_name.replace('/', "::") + let display_name = if display_segments.is_empty() { + "benchmark".to_string() } else { - relative_components.join("::") + display_segments.join("/") + }; + + let uri_suffix = if uri_segments.is_empty() { + "benchmark".to_string() + } else { + uri_segments.join("::") }; let uri = format!("criterion::{uri_suffix}"); @@ -602,3 +604,50 @@ struct BenchmarkIdentity { display_name: String, uri: String, } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn display_name_includes_parent_directory() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + let bench_dir = root + .join("column_store_fragmented_1M") + .join("sum_u64_fragmented_scan_only"); + std::fs::create_dir_all(bench_dir.join("new")).unwrap(); + + let identity = determine_identity(root, &bench_dir); + + assert_eq!( + identity.display_name, + "column_store_fragmented_1M/sum_u64_fragmented_scan_only" + ); + assert_eq!( + identity.uri, + "criterion::column_store_fragmented_1M::sum_u64_fragmented_scan_only" + ); + } + + #[test] + fn display_name_uses_last_two_uri_segments() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + let bench_dir = root + .join("very") + .join("deep") + .join("benchmark_group") + .join("inner_bench"); + std::fs::create_dir_all(bench_dir.join("new")).unwrap(); + + let identity = determine_identity(root, &bench_dir); + + assert_eq!(identity.display_name, "benchmark_group/inner_bench"); + assert_eq!( + identity.uri, + "criterion::very::deep::benchmark_group::inner_bench" + ); + } +}