From d58beebe5ed058cc764563c4420b0307d7dc5b4c Mon Sep 17 00:00:00 2001 From: Alexander Kiselev Date: Tue, 20 Jan 2026 20:16:35 -0800 Subject: [PATCH] Add command coverage scenarios per language --- src/testing/config.rs | 3 + src/testing/runner.rs | 70 +++++++++- tests/scenarios/commands_c.yml | 207 +++++++++++++++++++++++++++ tests/scenarios/commands_go.yml | 208 ++++++++++++++++++++++++++++ tests/scenarios/commands_python.yml | 205 +++++++++++++++++++++++++++ tests/scenarios/commands_rust.yml | 207 +++++++++++++++++++++++++++ 6 files changed, 899 insertions(+), 1 deletion(-) create mode 100644 tests/scenarios/commands_c.yml create mode 100644 tests/scenarios/commands_go.yml create mode 100644 tests/scenarios/commands_python.yml create mode 100644 tests/scenarios/commands_rust.yml diff --git a/src/testing/config.rs b/src/testing/config.rs index 4cd1b2a..3608752 100644 --- a/src/testing/config.rs +++ b/src/testing/config.rs @@ -92,6 +92,9 @@ pub struct CommandExpectation { pub success: Option, /// Substring that should be in the output pub output_contains: Option, + /// Allow failures without failing the test + #[serde(default)] + pub allow_failure: bool, } /// Expectations for a stop event diff --git a/src/testing/runner.rs b/src/testing/runner.rs index be5f6fa..c111b46 100644 --- a/src/testing/runner.rs +++ b/src/testing/runner.rs @@ -230,6 +230,7 @@ async fn execute_command_step( let cmd = parse_command(command_str)?; let result = client.send_command(cmd).await; + let allow_failure = expect.map(|exp| exp.allow_failure).unwrap_or(false); // Check expectations if let Some(exp) = expect { @@ -255,7 +256,29 @@ async fn execute_command_step( return Ok(()); } - result?; + if allow_failure && result.is_err() { + println!( + " {} Step {}: {} (allowed failure)", + "✓".green(), + step_num, + command_str.dimmed() + ); + return Ok(()); + } + + let value = result?; + + if let Some(exp) = expect { + if let Some(expected_substr) = &exp.output_contains { + let output = serde_json::to_string(&value).unwrap_or_default(); + if !output.contains(expected_substr) { + return Err(Error::TestAssertion(format!( + "Command '{}' output missing '{}'", + command_str, expected_substr + ))); + } + } + } println!( " {} Step {}: {}", @@ -736,6 +759,51 @@ fn parse_command(s: &str) -> Result { }) } + "context" | "where" => { + let lines = if let Some(value) = args.first() { + value.parse().map_err(|_| { + Error::Config(format!("Invalid context line count: {}", value)) + })? + } else { + 5 + }; + Ok(Command::Context { lines }) + } + + "output" => { + let mut tail = None; + let mut clear = false; + let mut idx = 0; + + while idx < args.len() { + match args[idx] { + "--tail" | "-t" => { + if idx + 1 >= args.len() { + return Err(Error::Config( + "output --tail requires a number".to_string(), + )); + } + tail = Some(args[idx + 1].parse().map_err(|_| { + Error::Config(format!("Invalid tail value: {}", args[idx + 1])) + })?); + idx += 2; + } + "--clear" => { + clear = true; + idx += 1; + } + other => { + return Err(Error::Config(format!( + "Unknown output option: {}", + other + ))); + } + } + } + + Ok(Command::GetOutput { tail, clear }) + } + "stop" => Ok(Command::Stop), "detach" => Ok(Command::Detach), "restart" => Ok(Command::Restart), diff --git a/tests/scenarios/commands_c.yml b/tests/scenarios/commands_c.yml new file mode 100644 index 0000000..eb1aa7f --- /dev/null +++ b/tests/scenarios/commands_c.yml @@ -0,0 +1,207 @@ +# C Command Coverage Test +# Exercises debugger commands and subcommands against a C fixture. + +name: "C Command Coverage Test" +description: "C coverage for breakpoints, stepping, stack, threads, context, output, and session commands" + +setup: + - shell: "mkdir -p tests/fixtures/bin && gcc -g -O0 tests/fixtures/simple.c -o tests/fixtures/bin/simple_c" + +target: + program: "tests/fixtures/bin/simple_c" + args: [] + stop_on_entry: true + +steps: + - action: command + command: "b tests/fixtures/simple.c:24" + expect: + success: true + + - action: command + command: "breakpoint add tests/fixtures/simple.c:6" + expect: + success: true + + - action: command + command: "breakpoint list" + expect: + output_contains: "breakpoints" + + - action: command + command: "breakpoint disable 2" + expect: + output_contains: "disabled" + + - action: command + command: "breakpoint enable 2" + expect: + output_contains: "enabled" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "breakpoint" + file: "simple.c" + line: 24 + + - action: command + command: "pause" + expect: + allow_failure: true + + - action: command + command: "locals" + expect: + output_contains: "variables" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "print x + y" + expect: + output_contains: "30" + + - action: command + command: "eval x + y" + expect: + output_contains: "30" + + - action: command + command: "step" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "backtrace" + expect: + output_contains: "frames" + + - action: inspect_stack + asserts: + - index: 0 + function: "add" + file: "simple.c" + - index: 1 + function: "main" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "up" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "down" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "a" + value_contains: "10" + - name: "b" + value_contains: "20" + + - action: command + command: "next" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "finish" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + + - action: command + command: "context 5" + expect: + output_contains: "source_lines" + + - action: command + command: "threads" + expect: + output_contains: "threads" + + - action: command + command: "thread 1" + expect: + output_contains: "selected" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "breakpoint remove 2" + expect: + output_contains: "removed" + + - action: command + command: "breakpoint remove --all" + expect: + output_contains: "removed" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "exited" + + - action: command + command: "output" + expect: + output_contains: "Sum" + + - action: check_output + contains: "Factorial" + + - action: command + command: "restart" + expect: + allow_failure: true + + - action: command + command: "stop" + expect: + allow_failure: true + + - action: command + command: "detach" + expect: + allow_failure: true diff --git a/tests/scenarios/commands_go.yml b/tests/scenarios/commands_go.yml new file mode 100644 index 0000000..93dd7b6 --- /dev/null +++ b/tests/scenarios/commands_go.yml @@ -0,0 +1,208 @@ +# Go Command Coverage Test +# Exercises debugger commands and subcommands against a Go fixture. + +name: "Go Command Coverage Test" +description: "Go coverage for breakpoints, stepping, stack, threads, context, output, and session commands" + +setup: + - shell: "mkdir -p tests/fixtures/bin && go build -gcflags='all=-N -l' -o tests/fixtures/bin/simple_go tests/fixtures/simple.go" + +target: + program: "tests/fixtures/bin/simple_go" + args: [] + adapter: "go" + stop_on_entry: true + +steps: + - action: command + command: "b tests/fixtures/simple.go:26" + expect: + success: true + + - action: command + command: "breakpoint add tests/fixtures/simple.go:8" + expect: + success: true + + - action: command + command: "breakpoint list" + expect: + output_contains: "breakpoints" + + - action: command + command: "breakpoint disable 2" + expect: + output_contains: "disabled" + + - action: command + command: "breakpoint enable 2" + expect: + output_contains: "enabled" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "breakpoint" + file: "simple.go" + line: 26 + + - action: command + command: "pause" + expect: + allow_failure: true + + - action: command + command: "locals" + expect: + output_contains: "variables" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "print x + y" + expect: + output_contains: "30" + + - action: command + command: "eval x + y" + expect: + output_contains: "30" + + - action: command + command: "step" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "backtrace" + expect: + output_contains: "frames" + + - action: inspect_stack + asserts: + - index: 0 + function: "add" + file: "simple.go" + - index: 1 + function: "main" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "up" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "down" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "a" + value_contains: "10" + - name: "b" + value_contains: "20" + + - action: command + command: "next" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "finish" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + + - action: command + command: "context 5" + expect: + output_contains: "source_lines" + + - action: command + command: "threads" + expect: + output_contains: "threads" + + - action: command + command: "thread 1" + expect: + output_contains: "selected" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "breakpoint remove 2" + expect: + output_contains: "removed" + + - action: command + command: "breakpoint remove --all" + expect: + output_contains: "removed" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "exited" + + - action: command + command: "output" + expect: + output_contains: "Sum" + + - action: check_output + contains: "Factorial" + + - action: command + command: "restart" + expect: + allow_failure: true + + - action: command + command: "stop" + expect: + allow_failure: true + + - action: command + command: "detach" + expect: + allow_failure: true diff --git a/tests/scenarios/commands_python.yml b/tests/scenarios/commands_python.yml new file mode 100644 index 0000000..7e156ff --- /dev/null +++ b/tests/scenarios/commands_python.yml @@ -0,0 +1,205 @@ +# Python Command Coverage Test +# Exercises debugger commands and subcommands against a Python fixture. + +name: "Python Command Coverage Test" +description: "Python coverage for breakpoints, stepping, stack, threads, context, output, and session commands" + +target: + program: "tests/fixtures/simple.py" + args: [] + adapter: "debugpy" + stop_on_entry: true + +steps: + - action: command + command: "b tests/fixtures/simple.py:23" + expect: + success: true + + - action: command + command: "breakpoint add tests/fixtures/simple.py:8" + expect: + success: true + + - action: command + command: "breakpoint list" + expect: + output_contains: "breakpoints" + + - action: command + command: "breakpoint disable 2" + expect: + output_contains: "disabled" + + - action: command + command: "breakpoint enable 2" + expect: + output_contains: "enabled" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "breakpoint" + file: "simple.py" + line: 23 + + - action: command + command: "pause" + expect: + allow_failure: true + + - action: command + command: "locals" + expect: + output_contains: "variables" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "print x + y" + expect: + output_contains: "30" + + - action: command + command: "eval x + y" + expect: + output_contains: "30" + + - action: command + command: "step" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "backtrace" + expect: + output_contains: "frames" + + - action: inspect_stack + asserts: + - index: 0 + function: "add" + file: "simple.py" + - index: 1 + function: "main" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "up" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "down" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "a" + value_contains: "10" + - name: "b" + value_contains: "20" + + - action: command + command: "next" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "finish" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + + - action: command + command: "context 5" + expect: + output_contains: "source_lines" + + - action: command + command: "threads" + expect: + output_contains: "threads" + + - action: command + command: "thread 1" + expect: + output_contains: "selected" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "breakpoint remove 2" + expect: + output_contains: "removed" + + - action: command + command: "breakpoint remove --all" + expect: + output_contains: "removed" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "exited" + + - action: command + command: "output" + expect: + output_contains: "Sum" + + - action: check_output + contains: "Factorial" + + - action: command + command: "restart" + expect: + allow_failure: true + + - action: command + command: "stop" + expect: + allow_failure: true + + - action: command + command: "detach" + expect: + allow_failure: true diff --git a/tests/scenarios/commands_rust.yml b/tests/scenarios/commands_rust.yml new file mode 100644 index 0000000..b3f4e8e --- /dev/null +++ b/tests/scenarios/commands_rust.yml @@ -0,0 +1,207 @@ +# Rust Command Coverage Test +# Exercises debugger commands and subcommands against a Rust fixture. + +name: "Rust Command Coverage Test" +description: "Rust coverage for breakpoints, stepping, stack, threads, context, output, and session commands" + +setup: + - shell: "mkdir -p tests/fixtures/bin && rustc -g tests/fixtures/simple.rs -o tests/fixtures/bin/simple_rs" + +target: + program: "tests/fixtures/bin/simple_rs" + args: [] + stop_on_entry: true + +steps: + - action: command + command: "b tests/fixtures/simple.rs:23" + expect: + success: true + + - action: command + command: "breakpoint add tests/fixtures/simple.rs:5" + expect: + success: true + + - action: command + command: "breakpoint list" + expect: + output_contains: "breakpoints" + + - action: command + command: "breakpoint disable 2" + expect: + output_contains: "disabled" + + - action: command + command: "breakpoint enable 2" + expect: + output_contains: "enabled" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "breakpoint" + file: "simple.rs" + line: 23 + + - action: command + command: "pause" + expect: + allow_failure: true + + - action: command + command: "locals" + expect: + output_contains: "variables" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "print x + y" + expect: + output_contains: "30" + + - action: command + command: "eval x + y" + expect: + output_contains: "30" + + - action: command + command: "step" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "backtrace" + expect: + output_contains: "frames" + + - action: inspect_stack + asserts: + - index: 0 + function: "add" + file: "simple.rs" + - index: 1 + function: "main" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "up" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "x" + value_contains: "10" + - name: "y" + value_contains: "20" + + - action: command + command: "down" + expect: + output_contains: "selected" + + - action: inspect_locals + asserts: + - name: "a" + value_contains: "10" + - name: "b" + value_contains: "20" + + - action: command + command: "next" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + expect: + reason: "step" + + - action: command + command: "finish" + expect: + output_contains: "stepping" + + - action: await + timeout: 10 + + - action: command + command: "context 5" + expect: + output_contains: "source_lines" + + - action: command + command: "threads" + expect: + output_contains: "threads" + + - action: command + command: "thread 1" + expect: + output_contains: "selected" + + - action: command + command: "frame 0" + expect: + output_contains: "selected" + + - action: command + command: "breakpoint remove 2" + expect: + output_contains: "removed" + + - action: command + command: "breakpoint remove --all" + expect: + output_contains: "removed" + + - action: command + command: "continue" + + - action: await + timeout: 15 + expect: + reason: "exited" + + - action: command + command: "output" + expect: + output_contains: "Sum" + + - action: check_output + contains: "Factorial" + + - action: command + command: "restart" + expect: + allow_failure: true + + - action: command + command: "stop" + expect: + allow_failure: true + + - action: command + command: "detach" + expect: + allow_failure: true