diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e48e529..e03a3315 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: - uses: taiki-e/install-action@v2 with: - tool: cargo-udeps + tool: cargo-llvm-cov, cargo-udeps # smoelius: I expect this list to grow. - name: Install tools diff --git a/.gitignore b/.gitignore index ea8c4bf7..92ad3e07 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /target +/test-fuzz/coverage +/test-fuzz/lcov.info diff --git a/Cargo.lock b/Cargo.lock index 93d79a16..6af1446d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,7 @@ dependencies = [ "rlimit", "semver", "serde", + "snapbox", "strip-ansi-escapes", "strum_macros", "subprocess", @@ -255,7 +256,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror", + "thiserror 2.0.16", ] [[package]] @@ -338,7 +339,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror", + "thiserror 2.0.16", ] [[package]] @@ -525,7 +526,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -694,6 +695,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lcov" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ccfa6d5e585a884db65b37f38184e4364eaf74d884ac35d0a90fe9baf80b723" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "libc" version = "0.2.178" @@ -1049,7 +1059,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1168,6 +1178,28 @@ dependencies = [ "similar", ] +[[package]] +name = "snapbox" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96dcfc4581e3355d70ac2ee14cfdf81dce3d85c85f1ed9e2c1d3013f53b3436b" +dependencies = [ + "anstream", + "anstyle", + "normalize-line-endings", + "similar", + "snapbox-macros", +] + +[[package]] +name = "snapbox-macros" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16569f53ca23a41bb6f62e0a5084aa1661f4814a67fa33696a79073e03a664af" +dependencies = [ + "anstream", +] + [[package]] name = "spin" version = "0.9.8" @@ -1241,7 +1273,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1279,13 +1311,16 @@ dependencies = [ "cargo_metadata", "cast_checks", "ctor", + "lcov", "predicates", "regex", + "rustc-demangle", "semver", "serde", "serde_combinators", "serde_json", "similar-asserts", + "snapbox", "tempfile", "test-fuzz-internal", "test-fuzz-macro", @@ -1373,13 +1408,33 @@ dependencies = [ "test-fuzz-testing", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.16", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1546,7 +1601,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 38611823..9e28ef93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ serde_assert = "0.8" serde_json = "1.0" serde = { version = "1.0", features = ["derive", "rc"] } sha1 = "0.10" +snapbox = "0.6" similar-asserts = "1.7" strip-ansi-escapes = "0.2" strum_macros = "0.27" diff --git a/README.md b/README.md index 72d99fea..453dc4d8 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,13 @@ Options: --backtrace Display backtraces --consolidate Move one target's crashes, hangs, and work queue to its corpus; to consolidate all targets, use --consolidate-all + --coverage Generate coverage for corpus, crashes, hangs, or work queue. By + default, an uninstrumented fuzz target is used. To generate + coverage with instrumentation, append `-instrumented` to , + e.g., --coverage corpus-instrumented. [possible values: corpus, + corpus-instrumented, crashes, crashes-instrumented, generic-args, + hangs, hangs-instrumented, impl-generic-args, queue, + queue-instrumented] --cpus Fuzz using at most cpus; default is all but one --display Display corpus, crashes, generic args, `impl` generic args, hangs, or work queue. By default, an uninstrumented fuzz target is used. @@ -301,7 +308,8 @@ Options: --no-ui Disable user interface -p, --package Package containing fuzz target --persistent Enable persistent mode fuzzing - --pretty Pretty-print debug output when displaying/replaying + --pretty Pretty-print debug output when generating coverage, displaying, or + replaying --release Build in release mode --replay Replay corpus, crashes, hangs, or work queue. By default, an uninstrumented fuzz target is used. To replay with @@ -317,7 +325,8 @@ Options: --test Integration test containing fuzz target --timeout Number of seconds to consider a hang when fuzzing or replaying (equivalent to -- -t when fuzzing) - --verbose Show build output when displaying/replaying + --verbose Show build output when generating coverage, displaying, or + replaying -h, --help Print help -V, --version Print version diff --git a/cargo-test-fuzz/Cargo.toml b/cargo-test-fuzz/Cargo.toml index 08da0e40..a53926ce 100644 --- a/cargo-test-fuzz/Cargo.toml +++ b/cargo-test-fuzz/Cargo.toml @@ -47,6 +47,7 @@ runtime = { workspace = true } assert_cmd = { workspace = true } predicates = { workspace = true } rlimit = { workspace = true } +snapbox = { workspace = true } tempfile = { workspace = true } walkdir = { workspace = true } xshell = { workspace = true } diff --git a/cargo-test-fuzz/src/bin/cargo_test_fuzz/transition.rs b/cargo-test-fuzz/src/bin/cargo_test_fuzz/transition.rs index 24396bff..5173647f 100644 --- a/cargo-test-fuzz/src/bin/cargo_test_fuzz/transition.rs +++ b/cargo-test-fuzz/src/bin/cargo_test_fuzz/transition.rs @@ -34,6 +34,13 @@ struct TestFuzzWithDeprecations { consolidate: bool, #[arg(long, hide = true)] consolidate_all: bool, + #[arg( + long, + help = "Generate coverage for corpus, crashes, hangs, or work queue. By default, an \ + uninstrumented fuzz target is used. To generate coverage with instrumentation, \ + append `-instrumented` to , e.g., --coverage corpus-instrumented." + )] + coverage: Option, #[arg( long, value_name = "N", @@ -89,7 +96,7 @@ struct TestFuzzWithDeprecations { persistent: bool, #[arg( long, - help = "Pretty-print debug output when displaying/replaying", + help = "Pretty-print debug output when generating coverage, displaying, or replaying", alias = "pretty-print" )] pretty: bool, @@ -136,7 +143,10 @@ struct TestFuzzWithDeprecations { -t when fuzzing)" )] timeout: Option, - #[arg(long, help = "Show build output when displaying/replaying")] + #[arg( + long, + help = "Show build output when generating coverage, displaying, or replaying" + )] verbose: bool, #[arg( value_name = "TARGETNAME", @@ -153,6 +163,7 @@ impl From for super::TestFuzz { backtrace, consolidate, consolidate_all, + coverage, cpus, display, exact, @@ -191,6 +202,7 @@ impl From for super::TestFuzz { backtrace, consolidate, consolidate_all, + coverage, cpus, display, exact, diff --git a/cargo-test-fuzz/src/lib.rs b/cargo-test-fuzz/src/lib.rs index 4e6443fb..89c853b6 100644 --- a/cargo-test-fuzz/src/lib.rs +++ b/cargo-test-fuzz/src/lib.rs @@ -27,7 +27,7 @@ use internal::dirs::{ corpus_directory_from_target, crashes_directory_from_target, generic_args_directory_from_target, hangs_directory_from_target, impl_generic_args_directory_from_target, output_directory_from_target, - queue_directory_from_target, target_directory, + queue_directory_from_target, target_directory, workspace, }; use log::debug; use mio::{Events, Interest, Poll, Token, unix::pipe::Receiver}; @@ -47,6 +47,9 @@ use std::{ use strum_macros::Display; use subprocess::{CommunicateError, Exec, ExitStatus, NullFile, Redirection}; +mod var_guard; +use var_guard::VarGuard; + const AUTO_GENERATED_SUFFIX: &str = "_fuzz__::auto_generate"; const ENTRY_SUFFIX: &str = "_fuzz__::entry"; @@ -86,6 +89,7 @@ pub struct TestFuzz { pub backtrace: bool, pub consolidate: bool, pub consolidate_all: bool, + pub coverage: Option, pub cpus: Option, pub display: Option, pub exact: bool, @@ -114,6 +118,63 @@ pub struct TestFuzz { pub zzargs: Vec, } +static LLVM_PROFILE_FILE: OnceLock = OnceLock::new(); +static TARGET_DIR: OnceLock = OnceLock::new(); + +impl TestFuzz { + fn set_llvm_cov_env(&self, command: &mut Command) { + command.env("CARGO_LLVM_COV", "1"); + command.env("CARGO_LLVM_COV_SHOW_ENV", "1"); + command.env( + "CARGO_LLVM_COV_TARGET_DIR", + self.coverage_target_directory(), + ); + command.env("CARGO_LLVM_COV_BUILD_DIR", self.target_directory()); + } + + pub fn llvm_profile_file(&self) -> &PathBuf { + LLVM_PROFILE_FILE.get_or_init(|| { + let workspace = workspace(); + self.coverage_target_directory() + .join(format!("{workspace}-%p-%8m.profraw")) + }) + } + + #[must_use] + pub fn coverage_target_directory(&self) -> PathBuf { + self.target_directory().clone() // .join("test-fuzz-coverage") + } + + pub fn target_directory(&self) -> &PathBuf { + #[expect(clippy::disallowed_methods)] + TARGET_DIR.get_or_init(|| target_directory(self.use_instrumentation())) + } + + const fn use_instrumentation(&self) -> bool { + let no_instrumentation = self.list + || matches!( + self.coverage, + Some(Object::Corpus | Object::Crashes | Object::Hangs | Object::Queue) + ) + || matches!( + self.display, + Some( + Object::Corpus + | Object::Crashes + | Object::Hangs + | Object::ImplGenericArgs + | Object::GenericArgs + | Object::Queue + ) + ) + || matches!( + self.replay, + Some(Object::Corpus | Object::Crashes | Object::Hangs | Object::Queue) + ); + !no_instrumentation + } +} + #[derive(Clone, Deserialize, Serialize)] struct Executable { path: PathBuf, @@ -153,24 +214,22 @@ pub fn run(opts: TestFuzz) -> Result<()> { opts }; - let no_instrumentation = opts.list - || matches!( - opts.display, - Some( - Object::Corpus - | Object::Crashes - | Object::Hangs - | Object::ImplGenericArgs - | Object::GenericArgs - | Object::Queue - ) - ) - || matches!( - opts.replay, - Some(Object::Corpus | Object::Crashes | Object::Hangs | Object::Queue) - ); + // smoelius: `LLVM_PROFILE_FILE` must be set every time a binary with coverage instrumentation + // is run. This includes when listing a binary's fuzz targets. To avoid forgetting to set the + // environment variable, we set it here, once and for all. Note that we do so using `VarGuard`, + // which is from an old version of Clippy. `VarGuard` should restore `LLVM_PROFILE_FILE`'s value + // before `cargo-test-fuzz` terminates, and thus should allow one to generate coverage for + // `cargo-test-fuzz`. + // + // Note that we generate coverage for fuzz targets, not tests, which is contrary to what one + // might expect. + let _var_guard = if opts.coverage.is_some() { + Some(VarGuard::set("LLVM_PROFILE_FILE", opts.llvm_profile_file())) + } else { + None + }; - run_without_exit_code(&opts, !no_instrumentation).map_err(|error| { + run_without_exit_code(&opts).map_err(|error| { if opts.exit_code { eprintln!("{error:?}"); exit(2); @@ -179,8 +238,16 @@ pub fn run(opts: TestFuzz) -> Result<()> { }) } +#[allow(clippy::too_many_lines)] #[doc(hidden)] -pub fn run_without_exit_code(opts: &TestFuzz, use_instrumentation: bool) -> Result<()> { +pub fn run_without_exit_code(opts: &TestFuzz) -> Result<()> { + if let Some(object) = opts.coverage { + ensure!( + !matches!(object, Object::ImplGenericArgs | Object::GenericArgs), + "`--coverage {}` is invalid.", + object.to_string().to_kebab_case() + ); + } if let Some(object) = opts.replay { ensure!( !matches!(object, Object::ImplGenericArgs | Object::GenericArgs), @@ -192,11 +259,13 @@ pub fn run_without_exit_code(opts: &TestFuzz, use_instrumentation: bool) -> Resu // smoelius: Ensure `cargo-afl` is installed. let _ = cached_cargo_afl_version(); + let coverage = opts.coverage.is_some(); + let display = opts.display.is_some(); let replay = opts.replay.is_some(); - let executables = build(opts, use_instrumentation, display || replay)?; + let executables = build(opts, coverage || display || replay)?; let mut executable_targets = executable_targets(&executables)?; @@ -222,7 +291,7 @@ pub fn run_without_exit_code(opts: &TestFuzz, use_instrumentation: bool) -> Resu return reset(opts, &executable_targets); } - if opts.consolidate || opts.reset || display || replay { + if opts.consolidate || opts.reset || coverage || display || replay { let (executable, target) = executable_target(opts, &executable_targets)?; if opts.consolidate || opts.reset { @@ -232,7 +301,22 @@ pub fn run_without_exit_code(opts: &TestFuzz, use_instrumentation: bool) -> Resu return reset(opts, &executable_targets); } + if coverage { + let mut command = Command::new("cargo"); + command.args(["llvm-cov", "clean", "--profraw-only"]); + opts.set_llvm_cov_env(&mut command); + debug!("{command:?}"); + let status = command + .status() + .with_context(|| format!("Could not get status of `{command:?}`"))?; + ensure!(status.success(), "Command failed: {command:?}"); + } + let (flags, dir) = None + .or_else(|| { + opts.coverage + .map(|object| flags_and_dir(object, &executable.name, &target)) + }) .or_else(|| { opts.display .map(|object| flags_and_dir(object, &executable.name, &target)) @@ -243,10 +327,41 @@ pub fn run_without_exit_code(opts: &TestFuzz, use_instrumentation: bool) -> Resu }) .unwrap_or_else(|| (Flags::empty(), PathBuf::default())); - return for_each_entry(opts, &executable, &target, display, replay, flags, &dir); - } + for_each_entry( + opts, + &executable, + &target, + coverage, + display, + replay, + flags, + &dir, + )?; + + if coverage { + if Path::new("lcov.info") + .try_exists() + .with_context(|| "Could not determine whether lcov.info exists")? + { + eprintln!("Warning: Overwriting lcov.info"); + } + let mut command = Command::new("cargo"); + command.args(["llvm-cov", "report", "--lcov", "--output-path=lcov.info"]); + opts.set_llvm_cov_env(&mut command); + debug!("{command:?}"); + let status = command + .status() + .with_context(|| format!("Could not get status of `{command:?}`"))?; + ensure!(status.success(), "Command failed: {command:?}"); + eprintln!( + "\ +Wrote lcov to `lcov.info`. To view it as html, try running: + + genhtml lcov.info --output-directory coverage && open coverage/index.html +" + ); + } - if !use_instrumentation { return Ok(()); } @@ -256,12 +371,12 @@ pub fn run_without_exit_code(opts: &TestFuzz, use_instrumentation: bool) -> Resu } #[allow(clippy::too_many_lines)] -fn build(opts: &TestFuzz, use_instrumentation: bool, quiet: bool) -> Result> { +fn build(opts: &TestFuzz, quiet: bool) -> Result> { let metadata = metadata(opts)?; let silence_stderr = quiet && !opts.verbose; let mut args = vec![]; - if use_instrumentation { + if opts.use_instrumentation() { args.extend_from_slice(&["afl"]); } args.extend_from_slice(&["test", "--offline", "--no-run"]); @@ -274,9 +389,8 @@ fn build(opts: &TestFuzz, use_instrumentation: bool, quiet: bool) -> Result Result Result (Flags, PathBuf) } } -#[allow(clippy::too_many_lines)] +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] fn for_each_entry( opts: &TestFuzz, executable: &Executable, target: &str, + coverage: bool, display: bool, replay: bool, flags: Flags, @@ -749,6 +871,9 @@ fn for_each_entry( let mut envs = BASE_ENVS.to_vec(); envs.push(("AFL_QUIET", "1")); + if coverage { + envs.push(("TEST_FUZZ_COVERAGE", "1")); + } if display { envs.push(("TEST_FUZZ_DISPLAY", "1")); } @@ -861,12 +986,7 @@ fn for_each_entry( if !nonempty { eprintln!( "Nothing to {}.", - match (display, replay) { - (true, true) => "display/replay", - (true, false) => "display", - (false, true) => "replay", - (false, false) => unreachable!(), - } + present_participle(coverage, display, replay) ); return Ok(()); } @@ -876,9 +996,10 @@ fn for_each_entry( return Ok(()); } - if (failure || timeout) && !replay { + if (failure || timeout) && !coverage && !replay { eprintln!( - "Encountered a {} while not replaying. A buggy Debug implementation perhaps?", + "Encountered a {} while not generating coverage or replaying. A buggy Debug \ + implementation perhaps?", if failure { "failure" } else if timeout { @@ -893,6 +1014,26 @@ fn for_each_entry( Ok(()) } +fn present_participle(coverage: bool, display: bool, replay: bool) -> String { + let mut actions = String::new(); + if coverage { + actions.push_str("generate coverage for"); + } + if display { + if !actions.is_empty() { + actions.push('/'); + } + actions.push_str("display"); + } + if replay { + if !actions.is_empty() { + actions.push('/'); + } + actions.push_str("replay"); + } + actions +} + fn flatten_executable_targets( opts: &TestFuzz, executable_targets: Vec<(Executable, Vec)>, @@ -1035,6 +1176,12 @@ fn prefix_with_width(s: &str, width: usize) -> &str { #[allow(clippy::too_many_lines)] fn fuzz(opts: &TestFuzz, executable_targets: &[(Executable, String)]) -> Result<()> { + ensure!(opts.coverage.is_none(), { + // smoelius: I am not sure whether this `ensure!` is reachable. + debug_assert!(false); + "Fuzzing with coverage is currently disallowed." + }); + auto_generate_corpora(executable_targets)?; let mut config = Config { @@ -1367,7 +1514,7 @@ mod tests { set_current_dir("../fuzzable").unwrap(); - let executables = build(&TestFuzz::default(), false, false).unwrap(); + let executables = build(&TestFuzz::default(), false).unwrap(); let executable_targets = executable_targets(&executables).unwrap(); diff --git a/cargo-test-fuzz/src/var_guard.rs b/cargo-test-fuzz/src/var_guard.rs new file mode 100644 index 00000000..758e2b16 --- /dev/null +++ b/cargo-test-fuzz/src/var_guard.rs @@ -0,0 +1,36 @@ +// smoelius: `VarGuard` is based on: +// https://github.com/rust-lang/rust-clippy/blob/9cc8da222b3893bc13bc13c8827e93f8ea246854/tests/compile-test.rs +// smoelius: Clippy dropped `VarGuard` when it switched to `ui_test`: +// https://github.com/rust-lang/rust-clippy/commit/77d10ac63dae6ef0a691d9acd63d65de9b9bf88e + +use std::{ + env::{remove_var, set_var, var_os}, + ffi::{OsStr, OsString}, +}; + +/// Restores an env var on drop +#[derive(Debug)] +#[must_use] +pub struct VarGuard { + key: &'static str, + value: Option, +} + +impl VarGuard { + pub fn set(key: &'static str, val: impl AsRef) -> Self { + let value = var_os(key); + unsafe { + set_var(key, val); + } + Self { key, value } + } +} + +impl Drop for VarGuard { + fn drop(&mut self) { + match self.value.as_deref() { + None => unsafe { remove_var(self.key) }, + Some(value) => unsafe { set_var(self.key, value) }, + } + } +} diff --git a/cargo-test-fuzz/tests/integration/display.rs b/cargo-test-fuzz/tests/integration/display.rs index 04302204..ea1edda0 100644 --- a/cargo-test-fuzz/tests/integration/display.rs +++ b/cargo-test-fuzz/tests/integration/display.rs @@ -13,7 +13,8 @@ fn display_debug_crash() { "crash::target_fuzz__::auto_generate", "crash::target", "", - "Encountered a failure while not replaying. A buggy Debug implementation perhaps?", + "Encountered a failure while not generating coverage or replaying. A buggy Debug \ + implementation perhaps?", ); } @@ -24,7 +25,8 @@ fn display_debug_hang() { "hang::target_fuzz__::auto_generate", "hang::target", "", - "Encountered a timeout while not replaying. A buggy Debug implementation perhaps?", + "Encountered a timeout while not generating coverage or replaying. A buggy Debug \ + implementation perhaps?", ); } diff --git a/cargo-test-fuzz/tests/integration/llvm_cov_show_env.rs b/cargo-test-fuzz/tests/integration/llvm_cov_show_env.rs new file mode 100644 index 00000000..c905ee3b --- /dev/null +++ b/cargo-test-fuzz/tests/integration/llvm_cov_show_env.rs @@ -0,0 +1,43 @@ +use internal::dirs::workspace; +use snapbox::{Redactions, assert_data_eq}; +use std::{io::Write, path::Path, process::Command}; +use testing::CommandExt; + +// smoelius: We need the `WS` redaction because we cannot assume the `test-fuzz` repository was +// cloned into a directory named `test-fuzz`. +const EXPECTED_SHOW_ENV: &str = "\ +RUSTFLAGS='-C instrument-coverage --cfg=coverage --cfg=trybuild_no_target' +LLVM_PROFILE_FILE='[TARGET_DIR]/[WS]-%p-%[..]m.profraw' +CARGO_LLVM_COV=1 +CARGO_LLVM_COV_SHOW_ENV=1 +CARGO_LLVM_COV_TARGET_DIR=[TARGET_DIR] +CARGO_LLVM_COV_BUILD_DIR=[TARGET_DIR] +"; + +#[test] +fn llvm_cov_show_env() { + let mut command = Command::new("which"); + command.arg("cargo-llvm-cov"); + let output = command.output().unwrap(); + if !output.status.success() { + #[allow(clippy::explicit_write)] + writeln!( + std::io::stderr(), + "Skipping `llvm_cov_show_env` test as `cargo-llvm-cov` is unavailable" + ) + .unwrap(); + return; + } + let stdout = String::from_utf8(output.stdout).unwrap(); + let assert = Command::new(stdout.trim_end()) + .args(["llvm-cov", "show-env"]) + .logged_assert(); + let actual_show_env = std::str::from_utf8(&assert.get_output().stdout).unwrap(); + let mut redactions = Redactions::new(); + let parent = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + redactions + .insert("[TARGET_DIR]", parent.join("target")) + .unwrap(); + redactions.insert("[WS]", workspace()).unwrap(); + assert_data_eq!(redactions.redact(actual_show_env), EXPECTED_SHOW_ENV); +} diff --git a/cargo-test-fuzz/tests/integration/main.rs b/cargo-test-fuzz/tests/integration/main.rs index 67271a34..d532e1db 100644 --- a/cargo-test-fuzz/tests/integration/main.rs +++ b/cargo-test-fuzz/tests/integration/main.rs @@ -8,4 +8,5 @@ mod fuzz_generic; mod fuzz_parallel; mod fuzz_profile; mod generic_args; +mod llvm_cov_show_env; mod replay; diff --git a/clippy.toml b/clippy.toml index 4655f46b..1bee6091 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,4 +1,5 @@ disallowed-methods = [ { path = "assert_cmd::assert::OutputAssertExt::assert", reason = "use `test_fuzz_testing::CommandExt::logged_assert`" }, { path = "assert_cmd::cmd::Command::assert", reason = "use `test_fuzz_testing::CommandExt::logged_assert`" }, + { path = "test_fuzz_internal::dirs::target_directory", reason = "use `TestFuzz::target_directory`" }, ] diff --git a/fuzzable/Cargo.toml b/fuzzable/Cargo.toml index 98a1ef57..3ad4170a 100644 --- a/fuzzable/Cargo.toml +++ b/fuzzable/Cargo.toml @@ -4,10 +4,6 @@ version = "7.2.5" edition = "2024" publish = false -[[bin]] -name = "hello-world" -path = "src/main.rs" - [dependencies] serde = { workspace = true } test-fuzz = { workspace = true } diff --git a/fuzzable/src/main.rs b/fuzzable/src/bin/hello-world.rs similarity index 56% rename from fuzzable/src/main.rs rename to fuzzable/src/bin/hello-world.rs index 737dad3c..0fb75652 100644 --- a/fuzzable/src/main.rs +++ b/fuzzable/src/bin/hello-world.rs @@ -1,8 +1,12 @@ fn main() { - target("Hello, world!"); + intermediary("Hello, world!"); } #[test_fuzz::test_fuzz(enable_in_production)] +fn intermediary(s: &str) { + target(s); +} + fn target(s: &str) { println!("{s}"); } diff --git a/internal/src/dirs.rs b/internal/src/dirs.rs index 324be782..674e8798 100644 --- a/internal/src/dirs.rs +++ b/internal/src/dirs.rs @@ -60,21 +60,25 @@ pub fn output_directory_from_target(krate: &str, target: &str) -> PathBuf { #[must_use] fn impl_generic_args_directory() -> PathBuf { + #[expect(clippy::disallowed_methods)] target_directory(false).join(path_segment("impl_generic_args")) } #[must_use] fn generic_args_directory() -> PathBuf { + #[expect(clippy::disallowed_methods)] target_directory(false).join(path_segment("generic_args")) } #[must_use] fn corpus_directory() -> PathBuf { + #[expect(clippy::disallowed_methods)] target_directory(false).join(path_segment("corpus")) } #[must_use] fn output_directory() -> PathBuf { + #[expect(clippy::disallowed_methods)] target_directory(true).join(path_segment("output")) } @@ -115,6 +119,19 @@ pub fn target_directory(instrumented: bool) -> PathBuf { target_dir.into() } +#[must_use] +pub fn workspace() -> String { + let root = workspace_root(); + let file_name = root.file_name().unwrap(); + file_name.to_str().map(ToOwned::to_owned).unwrap() +} + +fn workspace_root() -> PathBuf { + let mut command = MetadataCommand::new(); + let workspace_root = command.no_deps().exec().unwrap().workspace_root; + workspace_root.into() +} + #[must_use] fn path_from_args_type() -> String { let type_name = type_name::(); diff --git a/macro/src/lib.rs b/macro/src/lib.rs index de36705d..9be3e06c 100644 --- a/macro/src/lib.rs +++ b/macro/src/lib.rs @@ -700,16 +700,21 @@ fn map_method_or_fn( // smoelius: Do not set the panic hook when replaying. Leave cargo test's // panic hook in place. if test_fuzz::runtime::test_fuzz_enabled() { - if test_fuzz::runtime::display_enabled() + if test_fuzz::runtime::coverage_enabled() + || test_fuzz::runtime::display_enabled() || test_fuzz::runtime::replay_enabled() { #input_args if test_fuzz::runtime::display_enabled() { #output_args } - if test_fuzz::runtime::replay_enabled() { + if test_fuzz::runtime::coverage_enabled() + || test_fuzz::runtime::replay_enabled() + { #call_in_environment_with_deserialized_arguments - #output_ret + if test_fuzz::runtime::replay_enabled() { + #output_ret + } } } else { std::panic::set_hook(std::boxed::Box::new(|_| std::process::abort())); diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index e2a3748b..dd6ba2ef 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -137,6 +137,11 @@ pub fn test_fuzz_enabled() -> bool { enabled("") } +#[must_use] +pub fn coverage_enabled() -> bool { + enabled("COVERAGE") +} + #[must_use] pub fn display_enabled() -> bool { enabled("DISPLAY") diff --git a/test-fuzz/Cargo.toml b/test-fuzz/Cargo.toml index 415e9b6e..bf4188fe 100644 --- a/test-fuzz/Cargo.toml +++ b/test-fuzz/Cargo.toml @@ -28,12 +28,17 @@ regex = { workspace = true } semver = { workspace = true } serde_json = { workspace = true } similar-asserts = { workspace = true } +snapbox = { workspace = true } tempfile = { workspace = true } toml_edit = { workspace = true } walkdir = { workspace = true } testing = { workspace = true } +[target.'cfg(target_os = "linux")'.dev-dependencies] +lcov = "0.8" +rustc-demangle = "0.1" + # smoelius: A list of formats we might support can be found here: # https://github.com/djkoloski/rust_serialization_benchmark diff --git a/test-fuzz/tests/integration/in_production.rs b/test-fuzz/tests/integration/in_production.rs index a0f6daee..e0f564c8 100644 --- a/test-fuzz/tests/integration/in_production.rs +++ b/test-fuzz/tests/integration/in_production.rs @@ -26,7 +26,7 @@ static MUTEX: Mutex<()> = Mutex::new(()); fn test(write: bool, n: usize) { let _lock = MUTEX.lock().unwrap(); - let thread_specific_corpus = corpus_directory_from_target("hello-world", "target"); + let thread_specific_corpus = corpus_directory_from_target("hello-world", "intermediary"); // smoelius: HACK. `hello-world` writes to `target/corpus`, not, e.g., // `target/corpus_ThreadId_3_`. For now, just replace `corpus_ThreadId_3_` with `corpus`. @@ -69,4 +69,133 @@ fn test(write: bool, n: usize) { .success(); assert_eq!(read_dir(corpus).map(Iterator::count).unwrap_or_default(), n); + + #[cfg(target_os = "linux")] + if write { + linux::generate_and_check_coverage(); + } +} + +#[cfg(target_os = "linux")] +mod linux { + use super::CommandExt; + use lcov::{Reader, Record}; + use regex::Regex; + use rustc_demangle::demangle; + use std::{ + collections::{BTreeMap, BTreeSet}, + fs::read_to_string, + os::unix::ffi::OsStrExt, + path::Path, + sync::LazyLock, + }; + use testing::fuzzable::test_fuzz_all; + + type FunctionCoverage = BTreeMap; + + type LineCoverage = BTreeMap>; + + static EXPECTED_FUNCTION_COVERAGE: LazyLock = LazyLock::new(|| { + [ + (String::from("hello_world::main"), 0), + (String::from("hello_world::target"), 1), + ] + .into_iter() + .collect() + }); + + static EXPECTED_LINE_COVERAGE: LazyLock = LazyLock::new(|| { + [( + String::from("fuzzable/src/bin/hello-world.rs"), + [10, 11, 12].into_iter().collect(), + )] + .into_iter() + .collect() + }); + + pub fn generate_and_check_coverage() { + #[cfg_attr(dylint_lib = "general", allow(abs_home_path))] + let parent = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + + let mut command = test_fuzz_all().unwrap(); + // smoelius: As mentioned above, `hello-world` does not write to, e.g., + // `target/corpus_ThreadId_3_`. So do not set `TEST_FUZZ_ID`. + command.env_remove("TEST_FUZZ_ID"); + command.args(["--coverage=corpus", "intermediary", "--exact"]); + command.logged_assert().success(); + + let lcov_info = read_to_string("lcov.info").unwrap(); + let (function_coverage, line_coverage) = lcov_demangle(&lcov_info); + + let hello_world_function_coverage = function_coverage + .into_iter() + .filter(|(name, _)| name.starts_with("hello_world::")) + .collect::(); + assert_eq!(*EXPECTED_FUNCTION_COVERAGE, hello_world_function_coverage); + + let hello_world_line_coverage = line_coverage + .into_iter() + .filter_map(|(path, count)| { + let stripped = Path::new(&path).strip_prefix(parent).ok()?; + if stripped.starts_with("fuzzable") { + Some((stripped.to_string_lossy().to_string(), count)) + } else { + None + } + }) + .collect::(); + assert_eq!(*EXPECTED_LINE_COVERAGE, hello_world_line_coverage); + } + + // smoelius: `lcov_demangle` is loosely based on `lcov_read` from `cargo-line-test`: + // https://github.com/trailofbits/cargo-line-test/blob/ebe2fb110eaeb2255d919c838edcc078a0147467/src/db/read.rs#L97 + fn lcov_demangle(lcov: &str) -> (FunctionCoverage, LineCoverage) { + let mut function_coverage = FunctionCoverage::default(); + let mut line_coverage = LineCoverage::default(); + let mut source_file = None; + for result in Reader::new(lcov.as_bytes()) { + match result.unwrap() { + Record::SourceFile { path } => { + if let Some(source_file) = &source_file { + panic!("source file already given: {source_file}"); + } + let path_utf8 = std::str::from_utf8(path.as_os_str().as_bytes()).unwrap(); + source_file = Some(path_utf8.to_owned()); + } + Record::LineData { + line, + count, + checksum: _, + } if count != 0 => { + let Some(source_file) = &source_file else { + panic!("source file not given"); + }; + line_coverage + .entry(source_file.clone()) + .or_default() + .insert(line); + } + Record::FunctionData { name, count } => { + let demangled_name = demangle(&name).to_string(); + let stripped_demangled_name = strip_disambiguators(&demangled_name); + let existing_count = function_coverage + .entry(stripped_demangled_name) + .or_default(); + *existing_count += count; + } + Record::EndOfRecord => { + source_file = None; + } + _ => {} + } + } + (function_coverage, line_coverage) + } + + static DISAMBIGUATOR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\[[[:xdigit:]]*\]").unwrap()); + + fn strip_disambiguators(name: &str) -> String { + DISAMBIGUATOR_RE.replace_all(name, "").to_string() + } } diff --git a/test-fuzz/tests/integration/link.rs b/test-fuzz/tests/integration/link.rs index 0b5815c3..05ce894c 100644 --- a/test-fuzz/tests/integration/link.rs +++ b/test-fuzz/tests/integration/link.rs @@ -24,6 +24,7 @@ fn link() { let pred = pred.not(); // smoelius: https://stackoverflow.com/questions/7219845/difference-between-nm-and-objdump + #[expect(clippy::disallowed_methods)] Command::new("nm") .args([target_directory(false) .join("debug/hello-world") diff --git a/testing/src/fuzzable.rs b/testing/src/fuzzable.rs index 95af4960..2b4ab8b5 100644 --- a/testing/src/fuzzable.rs +++ b/testing/src/fuzzable.rs @@ -32,6 +32,10 @@ pub fn test(krate: &str, test: &str) -> Result { ]; args.extend_from_slice(&["--no-run", "--message-format=json"]); + // smoelius: As mentioned in cargo-test-fuzz/src/lib.rs, `LLVM_PROFILE_FILE` must be set every + // time a binary with coverage instrumentation is run. However, there is no easy way to tell + // whether a test binary was compiled with coverage instrumentation. So it is unclear whether + // `LLVM_PROFILE_FILE` should be set here. let exec = Exec::cmd("cargo").args(&args).stdout(Redirection::Pipe); debug!("{exec:?}"); let mut popen = exec.clone().popen()?;