-
Notifications
You must be signed in to change notification settings - Fork 1
Add command-coverage scenarios and extend scenario runner for command output/assertions #7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -230,6 +230,7 @@ async fn execute_command_step( | |
| let cmd = parse_command(command_str)?; | ||
|
|
||
| let result = client.send_command(cmd).await; | ||
| let allow_failure = expect.map(|exp| exp.allow_failure).unwrap_or(false); | ||
|
|
||
| // Check expectations | ||
| if let Some(exp) = expect { | ||
|
|
@@ -255,7 +256,29 @@ async fn execute_command_step( | |
| return Ok(()); | ||
| } | ||
|
|
||
| result?; | ||
| if allow_failure && result.is_err() { | ||
| println!( | ||
| " {} Step {}: {} (allowed failure)", | ||
| "✓".green(), | ||
| step_num, | ||
| command_str.dimmed() | ||
| ); | ||
| return Ok(()); | ||
| } | ||
|
|
||
| let value = result?; | ||
|
|
||
| if let Some(exp) = expect { | ||
| if let Some(expected_substr) = &exp.output_contains { | ||
| let output = serde_json::to_string(&value).unwrap_or_default(); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The use of let output = serde_json::to_string(&value).map_err(|e| {
Error::TestAssertion(format!(
"Failed to serialize command result for output check: {}",
e
))
})?; |
||
| if !output.contains(expected_substr) { | ||
| return Err(Error::TestAssertion(format!( | ||
| "Command '{}' output missing '{}'", | ||
| command_str, expected_substr | ||
| ))); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| println!( | ||
| " {} Step {}: {}", | ||
|
|
@@ -736,6 +759,51 @@ fn parse_command(s: &str) -> Result<Command> { | |
| }) | ||
| } | ||
|
|
||
| "context" | "where" => { | ||
| let lines = if let Some(value) = args.first() { | ||
| value.parse().map_err(|_| { | ||
| Error::Config(format!("Invalid context line count: {}", value)) | ||
| })? | ||
|
Comment on lines
+762
to
+766
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The scenario parser only accepts a positional number for Useful? React with 👍 / 👎. |
||
| } else { | ||
| 5 | ||
| }; | ||
| Ok(Command::Context { lines }) | ||
| } | ||
|
Comment on lines
+762
to
+771
|
||
|
|
||
| "output" => { | ||
| let mut tail = None; | ||
| let mut clear = false; | ||
| let mut idx = 0; | ||
|
|
||
| while idx < args.len() { | ||
| match args[idx] { | ||
| "--tail" | "-t" => { | ||
| if idx + 1 >= args.len() { | ||
| return Err(Error::Config( | ||
| "output --tail requires a number".to_string(), | ||
| )); | ||
| } | ||
| tail = Some(args[idx + 1].parse().map_err(|_| { | ||
| Error::Config(format!("Invalid tail value: {}", args[idx + 1])) | ||
| })?); | ||
| idx += 2; | ||
| } | ||
| "--clear" => { | ||
| clear = true; | ||
| idx += 1; | ||
| } | ||
| other => { | ||
| return Err(Error::Config(format!( | ||
| "Unknown output option: {}", | ||
| other | ||
| ))); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| Ok(Command::GetOutput { tail, clear }) | ||
| } | ||
|
Comment on lines
+762
to
+805
|
||
|
|
||
| "stop" => Ok(Command::Stop), | ||
| "detach" => Ok(Command::Detach), | ||
| "restart" => Ok(Command::Restart), | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,207 @@ | ||
| # C Command Coverage Test | ||
| # Exercises debugger commands and subcommands against a C fixture. | ||
|
|
||
| name: "C Command Coverage Test" | ||
| description: "C coverage for breakpoints, stepping, stack, threads, context, output, and session commands" | ||
|
|
||
| setup: | ||
| - shell: "mkdir -p tests/fixtures/bin && gcc -g -O0 tests/fixtures/simple.c -o tests/fixtures/bin/simple_c" | ||
|
|
||
| target: | ||
| program: "tests/fixtures/bin/simple_c" | ||
| args: [] | ||
| stop_on_entry: true | ||
|
|
||
| steps: | ||
| - action: command | ||
| command: "b tests/fixtures/simple.c:24" | ||
| expect: | ||
| success: true | ||
|
|
||
| - action: command | ||
| command: "breakpoint add tests/fixtures/simple.c:6" | ||
| expect: | ||
| success: true | ||
|
|
||
| - action: command | ||
| command: "breakpoint list" | ||
| expect: | ||
| output_contains: "breakpoints" | ||
|
|
||
| - action: command | ||
| command: "breakpoint disable 2" | ||
| expect: | ||
| output_contains: "disabled" | ||
|
|
||
| - action: command | ||
| command: "breakpoint enable 2" | ||
| expect: | ||
| output_contains: "enabled" | ||
|
|
||
| - action: command | ||
| command: "continue" | ||
|
|
||
| - action: await | ||
| timeout: 15 | ||
| expect: | ||
| reason: "breakpoint" | ||
| file: "simple.c" | ||
| line: 24 | ||
|
|
||
| - action: command | ||
| command: "pause" | ||
| expect: | ||
| allow_failure: true | ||
|
|
||
| - action: command | ||
| command: "locals" | ||
| expect: | ||
| output_contains: "variables" | ||
|
|
||
| - action: inspect_locals | ||
| asserts: | ||
| - name: "x" | ||
| value_contains: "10" | ||
| - name: "y" | ||
| value_contains: "20" | ||
|
|
||
| - action: command | ||
| command: "print x + y" | ||
| expect: | ||
| output_contains: "30" | ||
|
|
||
| - action: command | ||
| command: "eval x + y" | ||
| expect: | ||
| output_contains: "30" | ||
|
|
||
| - action: command | ||
| command: "step" | ||
| expect: | ||
| output_contains: "stepping" | ||
|
|
||
| - action: await | ||
| timeout: 10 | ||
| expect: | ||
| reason: "step" | ||
|
|
||
| - action: command | ||
| command: "backtrace" | ||
| expect: | ||
| output_contains: "frames" | ||
|
|
||
| - action: inspect_stack | ||
| asserts: | ||
| - index: 0 | ||
| function: "add" | ||
| file: "simple.c" | ||
| - index: 1 | ||
| function: "main" | ||
|
|
||
| - action: command | ||
| command: "frame 0" | ||
| expect: | ||
| output_contains: "selected" | ||
|
|
||
| - action: command | ||
| command: "up" | ||
| expect: | ||
| output_contains: "selected" | ||
|
|
||
| - action: inspect_locals | ||
| asserts: | ||
| - name: "x" | ||
| value_contains: "10" | ||
| - name: "y" | ||
| value_contains: "20" | ||
|
|
||
| - action: command | ||
| command: "down" | ||
| expect: | ||
| output_contains: "selected" | ||
|
|
||
| - action: inspect_locals | ||
| asserts: | ||
| - name: "a" | ||
| value_contains: "10" | ||
| - name: "b" | ||
| value_contains: "20" | ||
|
|
||
| - action: command | ||
| command: "next" | ||
| expect: | ||
| output_contains: "stepping" | ||
|
|
||
| - action: await | ||
| timeout: 10 | ||
| expect: | ||
| reason: "step" | ||
|
|
||
| - action: command | ||
| command: "finish" | ||
| expect: | ||
| output_contains: "stepping" | ||
|
|
||
| - action: await | ||
| timeout: 10 | ||
|
|
||
| - action: command | ||
| command: "context 5" | ||
| expect: | ||
| output_contains: "source_lines" | ||
|
|
||
| - action: command | ||
| command: "threads" | ||
| expect: | ||
| output_contains: "threads" | ||
|
|
||
| - action: command | ||
| command: "thread 1" | ||
| expect: | ||
| output_contains: "selected" | ||
|
|
||
| - action: command | ||
| command: "frame 0" | ||
| expect: | ||
| output_contains: "selected" | ||
|
|
||
| - action: command | ||
| command: "breakpoint remove 2" | ||
| expect: | ||
| output_contains: "removed" | ||
|
|
||
| - action: command | ||
| command: "breakpoint remove --all" | ||
| expect: | ||
| output_contains: "removed" | ||
|
|
||
| - action: command | ||
| command: "continue" | ||
|
|
||
| - action: await | ||
| timeout: 15 | ||
| expect: | ||
| reason: "exited" | ||
|
|
||
| - action: command | ||
| command: "output" | ||
| expect: | ||
| output_contains: "Sum" | ||
|
|
||
| - action: check_output | ||
| contains: "Factorial" | ||
|
|
||
| - action: command | ||
| command: "restart" | ||
| expect: | ||
| allow_failure: true | ||
|
|
||
| - action: command | ||
| command: "stop" | ||
| expect: | ||
| allow_failure: true | ||
|
|
||
| - action: command | ||
| command: "detach" | ||
| expect: | ||
| allow_failure: true |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The documentation comment for the
allow_failurefield is incomplete. It should explain when this flag should be used versussuccess: false, and clarify the behavior when a command succeeds despiteallow_failure: truebeing set (does it pass or fail?). Consider expanding this to: "Allow the command to fail without failing the test. Unlikesuccess: falsewhich expects failure, this permits either success or failure."