From 5116b58d9d741090a7154858d236f2393cee8581 Mon Sep 17 00:00:00 2001 From: msmps <7691252+msmps@users.noreply.github.com> Date: Thu, 29 Jan 2026 15:56:58 +0000 Subject: [PATCH 1/3] feat(snapshot): add await-change and settle flags for reliable screen change detection Adds --await-change and --settle flags to the snapshot command, eliminating the need for manual sleep() calls in TUI automation scripts. Features: - --await-change : Block until content_hash differs from baseline - --settle : Wait for screen to be stable for N ms (handles progressive renders) - --timeout : Maximum wait time (default: 30s) prevents infinite loops Implementation: - Two-phase polling in handle_snapshot: await change, then settle - Hash tracking across phases (settle starts from await_change exit point) - 50ms poll interval (SNAPSHOT_POLL_INTERVAL_MS constant) - Timeout errors use configured timeout_ms value (not elapsed time) - Timeout errors include helpful suggestions and context for debugging - Client-side hash tracking keeps server stateless Usage: HASH=$(pilotty snapshot | jq '.content_hash') pilotty key Enter pilotty snapshot --await-change $HASH --settle 50 Addresses Pain Point #4: Manual sleep required between actions. Includes: - Protocol changes with serde(default) for backward compatibility - CLI args with comprehensive help text and examples - Test: test_snapshot_await_change_detects_update - Documentation updates to README, SKILL.md, and npm README Reviewed by Oracle: Fixed hash tracking across phases, improved error messages. --- README.md | 58 ++++- crates/pilotty-cli/src/args.rs | 20 +- crates/pilotty-cli/src/daemon/server.rs | 300 +++++++++++++++++++++++- crates/pilotty-cli/src/main.rs | 3 + crates/pilotty-core/src/protocol.rs | 17 ++ npm/README.md | 5 + skills/pilotty/SKILL.md | 77 +++++- 7 files changed, 449 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 1f18cad..268767c 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,10 @@ pilotty examples # Show end-to-end workflow example pilotty snapshot # Full JSON with text pilotty snapshot --format compact # JSON without text field pilotty snapshot --format text # Plain text with cursor indicator + +# Wait for screen to change before returning (no more manual sleep!) +pilotty snapshot --await-change $HASH # Block until hash differs +pilotty snapshot --await-change $HASH --settle 100 # Then wait for stability ``` ### Input @@ -155,6 +159,11 @@ pilotty resize 120 40 # Resize terminal to 120x40 pilotty wait-for "Ready" # Wait for text to appear pilotty wait-for "Error" --regex # Wait for regex pattern pilotty wait-for "Done" -t 5000 # Wait with 5s timeout + +# Wait for screen changes (preferred over sleep) +HASH=$(pilotty snapshot | jq '.content_hash') +pilotty key Enter +pilotty snapshot --await-change $HASH --settle 50 # Wait for change + 50ms stability ``` ## Snapshot Output @@ -204,9 +213,40 @@ pilotty automatically detects interactive UI elements in terminal applications. | `focused` | Whether element has focus (only present if true) | | `checked` | Toggle state (only present for toggles) | -### Change Detection +### Wait for Screen Changes -The `content_hash` field enables screen change detection between snapshots: +The `--await-change` flag solves the fundamental problem of TUI automation: **"How long should I wait after an action?"** + +Instead of guessing sleep durations (too short = race condition, too long = slow), wait for the screen to actually change: + +```bash +# Capture baseline hash +HASH=$(pilotty snapshot | jq '.content_hash') + +# Perform action +pilotty key Enter + +# Wait for screen to change (blocks until hash differs) +pilotty snapshot --await-change $HASH + +# Or wait for screen to stabilize (useful for apps that render progressively) +pilotty snapshot --await-change $HASH --settle 100 # Wait 100ms after last change +``` + +**Flags:** +- `--await-change `: Block until `content_hash` differs from this value +- `--settle `: After change detected, wait for screen to be stable for this many ms +- `--timeout `: Maximum wait time (default: 30000) + +**Why this matters:** +- No more flaky automation due to race conditions +- No more slow scripts due to conservative sleep values +- Works regardless of how fast/slow the target app is +- The `--settle` flag handles apps that render progressively + +### Manual Change Detection + +For manual polling, use `content_hash` directly: ```bash # Get initial snapshot @@ -389,18 +429,20 @@ pilotty spawn vim myfile.txt # 2. Wait for it to be ready pilotty wait-for "myfile.txt" -# 3. Take a snapshot to understand the screen -pilotty snapshot +# 3. Take a snapshot to understand the screen and capture hash +HASH=$(pilotty snapshot | jq '.content_hash') # 4. Navigate using keyboard commands pilotty key i # Enter insert mode pilotty type "Hello, World!" pilotty key Escape -pilotty type ":wq" -pilotty key Enter -# 5. Re-snapshot after screen changes -pilotty snapshot +# 5. Wait for screen to update, then save (no manual sleep needed!) +pilotty snapshot --await-change $HASH --settle 50 +pilotty key "Escape : w q Enter" # vim :wq sequence + +# 6. Verify vim exited +pilotty list-sessions ``` ## Key Combinations diff --git a/crates/pilotty-cli/src/args.rs b/crates/pilotty-cli/src/args.rs index 8365014..1e0c8c2 100644 --- a/crates/pilotty-cli/src/args.rs +++ b/crates/pilotty-cli/src/args.rs @@ -37,7 +37,13 @@ Examples: pilotty snapshot # Snapshot default session (full JSON) pilotty snapshot --format compact # JSON without text field pilotty snapshot --format text # Plain text with cursor indicator - pilotty snapshot -s editor # Snapshot a specific session")] + pilotty snapshot -s editor # Snapshot a specific session + +Wait for change: + HASH=$(pilotty snapshot | jq -r '.content_hash') + pilotty key Enter + pilotty snapshot --await-change $HASH # Block until screen changes + pilotty snapshot --await-change $HASH --settle 100 # Wait for 100ms stability")] Snapshot(SnapshotArgs), /// Type text at the current cursor position @@ -147,6 +153,18 @@ pub struct SnapshotArgs { #[arg(short, long, help = SESSION_HELP)] pub session: Option, + + /// Block until content_hash differs from this value + #[arg(long, value_name = "HASH")] + pub await_change: Option, + + /// Wait for screen to stabilize for this many ms before returning + #[arg(long, default_value_t = 0, value_name = "MS")] + pub settle: u64, + + /// Timeout in milliseconds for await-change/settle (default: 30s) + #[arg(short, long, default_value_t = 30000)] + pub timeout: u64, } #[derive(Debug, Clone, Copy, ValueEnum)] diff --git a/crates/pilotty-cli/src/daemon/server.rs b/crates/pilotty-cli/src/daemon/server.rs index 64929ed..f59e060 100644 --- a/crates/pilotty-cli/src/daemon/server.rs +++ b/crates/pilotty-cli/src/daemon/server.rs @@ -511,8 +511,23 @@ async fn handle_request( cwd, } => handle_spawn(&request.id, &sessions, command, session_name, cwd).await, - Command::Snapshot { session, format } => { - handle_snapshot(&request.id, &sessions, session, format).await + Command::Snapshot { + session, + format, + await_change, + settle_ms, + timeout_ms, + } => { + handle_snapshot( + &request.id, + &sessions, + session, + format, + await_change, + settle_ms, + timeout_ms, + ) + .await } Command::ListSessions => handle_list_sessions(&request.id, &sessions).await, @@ -613,25 +628,123 @@ async fn handle_spawn( } } +/// Poll interval for await_change/settle operations. +const SNAPSHOT_POLL_INTERVAL_MS: u64 = 50; + /// Handle snapshot command. +/// +/// Supports optional wait-for-change semantics: +/// - `await_change`: Block until content_hash differs from this value +/// - `settle_ms`: After change detected, wait for screen to be stable this long +/// - `timeout_ms`: Maximum time to wait for change/settle async fn handle_snapshot( request_id: &str, sessions: &SessionManager, session: Option, format: Option, + await_change: Option, + settle_ms: u64, + timeout_ms: u64, ) -> Response { - // Resolve session + use std::time::{Duration, Instant}; + + // Resolve session first let session_id = match sessions.resolve_session(session.as_deref()).await { Ok(id) => id, Err(e) => return Response::error(request_id, e), }; let format = format.unwrap_or(SnapshotFormat::Full); - - // Full format includes UI element detection let with_elements = matches!(format, SnapshotFormat::Full); + let timeout = Duration::from_millis(timeout_ms); + let settle = Duration::from_millis(settle_ms); + let poll_interval = Duration::from_millis(SNAPSHOT_POLL_INTERVAL_MS); + let start = Instant::now(); + + // Track hash across phases (set during await_change, used by settle) + let mut current_hash: Option = None; + + // Phase 1: If await_change is set, wait until content_hash differs + if let Some(baseline_hash) = await_change { + loop { + if start.elapsed() >= timeout { + return Response::error( + request_id, + ApiError::command_failed_with_suggestion( + format!( + "Timeout after {}ms waiting for screen to change from hash {}", + timeout_ms, + baseline_hash + ), + "Screen content did not change. The application may be idle or waiting for input.", + ), + ); + } - // Get snapshot data (drains PTY output first) + let snapshot = match sessions.get_snapshot_data(&session_id, false).await { + Ok(data) => data, + Err(e) => return Response::error(request_id, e), + }; + + current_hash = snapshot.content_hash; + if current_hash != Some(baseline_hash) { + debug!( + "Screen changed from hash {} to {:?} after {}ms", + baseline_hash, + current_hash, + start.elapsed().as_millis() + ); + break; + } + + tokio::time::sleep(poll_interval).await; + } + } + + // Phase 2: If settle_ms > 0, wait for screen stability + if settle_ms > 0 { + // Start from where await_change left off (or None if no await_change) + let mut last_hash = current_hash; + let mut stable_since = Instant::now(); + + loop { + if start.elapsed() >= timeout { + return Response::error( + request_id, + ApiError::command_failed_with_suggestion( + format!( + "Timeout after {}ms waiting for screen to stabilize for {}ms (last hash: {:?})", + timeout_ms, + settle_ms, + last_hash + ), + "Screen kept changing. Try increasing --timeout or --settle.", + ), + ); + } + + let snapshot = match sessions.get_snapshot_data(&session_id, false).await { + Ok(data) => data, + Err(e) => return Response::error(request_id, e), + }; + + if snapshot.content_hash != last_hash { + last_hash = snapshot.content_hash; + stable_since = Instant::now(); + } else if stable_since.elapsed() >= settle { + debug!( + "Screen stabilized for {}ms after {}ms total", + settle_ms, + start.elapsed().as_millis() + ); + break; + } + + tokio::time::sleep(poll_interval).await; + } + } + + // Phase 3: Take final snapshot with requested format let snapshot = match sessions.get_snapshot_data(&session_id, with_elements).await { Ok(data) => data, Err(e) => return Response::error(request_id, e), @@ -640,7 +753,6 @@ async fn handle_snapshot( match format { SnapshotFormat::Text => { - // Format as plain text with cursor indicator let output = format_text_snapshot(&snapshot.text, cursor_row, cursor_col, snapshot.size); Response::success( @@ -652,9 +764,7 @@ async fn handle_snapshot( ) } SnapshotFormat::Full => { - // Full: text + elements + metadata + content_hash let snapshot_id = sessions.next_snapshot_id(); - let screen_state = ScreenState { snapshot_id, size: TerminalSize { @@ -673,7 +783,6 @@ async fn handle_snapshot( Response::success(request_id, ResponseData::ScreenState(screen_state)) } SnapshotFormat::Compact => { - // Compact: metadata only, no text, elements, or hash let snapshot_id = sessions.next_snapshot_id(); let screen_state = ScreenState { snapshot_id, @@ -1415,6 +1524,9 @@ mod tests { command: Command::Snapshot { session: Some("test-snap".to_string()), format: Some(SnapshotFormat::Text), + await_change: None, + settle_ms: 0, + timeout_ms: 30000, }, }; let snap_json = serde_json::to_string(&snap_request).unwrap(); @@ -1512,6 +1624,9 @@ mod tests { command: Command::Snapshot { session: Some("full-test".to_string()), format: Some(SnapshotFormat::Full), + await_change: None, + settle_ms: 0, + timeout_ms: 30000, }, }; let snap_json = serde_json::to_string(&snap_request).unwrap(); @@ -1656,6 +1771,9 @@ mod tests { command: Command::Snapshot { session: Some("type-test".to_string()), format: Some(SnapshotFormat::Text), + await_change: None, + settle_ms: 0, + timeout_ms: 30000, }, }; let snap_json = serde_json::to_string(&snap_request).unwrap(); @@ -2404,6 +2522,165 @@ mod tests { let _ = std::fs::remove_file(&socket_path); } + #[tokio::test] + async fn test_snapshot_await_change_detects_update() { + let temp_dir = std::env::temp_dir(); + let socket_path = + temp_dir.join(format!("pilotty-await-change-{}.sock", std::process::id())); + let pid_path = socket_path.with_extension("pid"); + + let server = DaemonServer::bind_to(socket_path.clone(), pid_path.clone()) + .await + .expect("Failed to bind server"); + + let server_handle = tokio::spawn(async move { + let _ = timeout(Duration::from_secs(10), server.run()).await; + }); + + tokio::time::sleep(Duration::from_millis(50)).await; + + let stream = UnixStream::connect(&socket_path) + .await + .expect("Failed to connect"); + let (reader, mut writer) = stream.into_split(); + let mut reader = BufReader::new(reader); + + // Spawn cat (echoes input) + let spawn_request = Request { + id: "spawn-1".to_string(), + command: Command::Spawn { + command: vec!["cat".to_string()], + session_name: Some("await-test".to_string()), + cwd: None, + }, + }; + let request_json = serde_json::to_string(&spawn_request).unwrap(); + writer + .write_all(request_json.as_bytes()) + .await + .expect("write"); + writer.write_all(b"\n").await.expect("newline"); + writer.flush().await.expect("flush"); + + let mut response_line = String::new(); + timeout(Duration::from_secs(2), reader.read_line(&mut response_line)) + .await + .expect("timeout") + .expect("read"); + + tokio::time::sleep(Duration::from_millis(100)).await; + + // Take baseline snapshot to get initial hash + let snap_request = Request { + id: "snap-baseline".to_string(), + command: Command::Snapshot { + session: Some("await-test".to_string()), + format: Some(SnapshotFormat::Full), + await_change: None, + settle_ms: 0, + timeout_ms: 30000, + }, + }; + let snap_json = serde_json::to_string(&snap_request).unwrap(); + writer.write_all(snap_json.as_bytes()).await.expect("write"); + writer.write_all(b"\n").await.expect("newline"); + writer.flush().await.expect("flush"); + + response_line.clear(); + timeout(Duration::from_secs(2), reader.read_line(&mut response_line)) + .await + .expect("timeout") + .expect("read"); + + let baseline_response: Response = + serde_json::from_str(&response_line).expect("parse baseline response"); + assert!( + baseline_response.success, + "Baseline snapshot should succeed" + ); + + let baseline_hash = match baseline_response.data { + Some(ResponseData::ScreenState(state)) => state.content_hash.expect("should have hash"), + _ => panic!("Expected ScreenState"), + }; + + // Type something to change the screen + let type_request = Request { + id: "type-1".to_string(), + command: Command::Type { + text: "hello".to_string(), + session: Some("await-test".to_string()), + }, + }; + let type_json = serde_json::to_string(&type_request).unwrap(); + writer.write_all(type_json.as_bytes()).await.expect("write"); + writer.write_all(b"\n").await.expect("newline"); + writer.flush().await.expect("flush"); + + response_line.clear(); + timeout(Duration::from_secs(2), reader.read_line(&mut response_line)) + .await + .expect("timeout") + .expect("read"); + + // Request snapshot with await_change - should detect the change + let start = std::time::Instant::now(); + let await_request = Request { + id: "snap-await".to_string(), + command: Command::Snapshot { + session: Some("await-test".to_string()), + format: Some(SnapshotFormat::Full), + await_change: Some(baseline_hash), + settle_ms: 50, + timeout_ms: 5000, + }, + }; + let await_json = serde_json::to_string(&await_request).unwrap(); + writer + .write_all(await_json.as_bytes()) + .await + .expect("write"); + writer.write_all(b"\n").await.expect("newline"); + writer.flush().await.expect("flush"); + + response_line.clear(); + timeout(Duration::from_secs(5), reader.read_line(&mut response_line)) + .await + .expect("timeout") + .expect("read"); + let elapsed = start.elapsed(); + + let await_response: Response = + serde_json::from_str(&response_line).expect("parse await response"); + assert!(await_response.success, "Await snapshot should succeed"); + + // Verify it detected change quickly (not timing out) + assert!( + elapsed < Duration::from_secs(3), + "Should detect change quickly, took {:?}", + elapsed + ); + + // Verify hash actually changed + if let Some(ResponseData::ScreenState(state)) = await_response.data { + assert_ne!( + state.content_hash, + Some(baseline_hash), + "Hash should have changed" + ); + // Verify content contains what we typed + assert!( + state.text.as_ref().is_some_and(|t| t.contains("hello")), + "Text should contain 'hello'" + ); + } else { + panic!("Expected ScreenState response"); + } + + server_handle.abort(); + let _ = std::fs::remove_file(&socket_path); + } + #[tokio::test] async fn test_spawn_with_invalid_cwd_fails() { let temp_dir = std::env::temp_dir(); @@ -2534,6 +2811,9 @@ mod tests { command: Command::Snapshot { session: Some("elem-test".to_string()), format: Some(SnapshotFormat::Full), + await_change: None, + settle_ms: 0, + timeout_ms: 30000, }, }; let snap_json = serde_json::to_string(&snap_request).unwrap(); diff --git a/crates/pilotty-cli/src/main.rs b/crates/pilotty-cli/src/main.rs index c05dbc5..bd3d365 100644 --- a/crates/pilotty-cli/src/main.rs +++ b/crates/pilotty-cli/src/main.rs @@ -56,6 +56,9 @@ fn cli_to_command(cli: &Cli) -> Option { crate::args::SnapshotFormat::Compact => SnapshotFormat::Compact, crate::args::SnapshotFormat::Text => SnapshotFormat::Text, }), + await_change: args.await_change, + settle_ms: args.settle, + timeout_ms: args.timeout, }), Commands::Type(args) => Some(Command::Type { text: args.text.clone(), diff --git a/crates/pilotty-core/src/protocol.rs b/crates/pilotty-core/src/protocol.rs index 687e2c9..5f8c02a 100644 --- a/crates/pilotty-core/src/protocol.rs +++ b/crates/pilotty-core/src/protocol.rs @@ -5,6 +5,11 @@ use serde::{Deserialize, Serialize}; use crate::error::ApiError; use crate::snapshot::ScreenState; +/// Default timeout for snapshot await_change/settle operations (30 seconds). +fn default_snapshot_timeout() -> u64 { + 30000 +} + /// A request from CLI to daemon. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Request { @@ -30,9 +35,21 @@ pub enum Command { /// Kill a session. Kill { session: Option }, /// Get a snapshot of the terminal screen. + /// + /// Optionally block until the screen changes from a baseline hash and/or + /// stabilizes for a specified duration. Snapshot { session: Option, format: Option, + /// If set, block until content_hash differs from this value. + #[serde(default)] + await_change: Option, + /// Wait for screen to be stable for this many ms before returning. + #[serde(default)] + settle_ms: u64, + /// Timeout in ms for await_change/settle operations. + #[serde(default = "default_snapshot_timeout")] + timeout_ms: u64, }, /// Type text at cursor. Type { diff --git a/npm/README.md b/npm/README.md index 775b05c..68cde68 100644 --- a/npm/README.md +++ b/npm/README.md @@ -51,6 +51,11 @@ pilotty key Ctrl+C pilotty key "Ctrl+X m" # Emacs chord pilotty key "Escape : w q Enter" # vim :wq +# Wait for screen to change (no more guessing sleep durations!) +HASH=$(pilotty snapshot | jq '.content_hash') +pilotty key Enter +pilotty snapshot --await-change $HASH --settle 50 + # Click at specific coordinates (row, col) pilotty click 10 5 diff --git a/skills/pilotty/SKILL.md b/skills/pilotty/SKILL.md index 712a1dc..162337e 100644 --- a/skills/pilotty/SKILL.md +++ b/skills/pilotty/SKILL.md @@ -68,6 +68,12 @@ pilotty snapshot # Full JSON with text content and elements pilotty snapshot --format compact # JSON without text field pilotty snapshot --format text # Plain text with cursor indicator pilotty snapshot -s myapp # Snapshot specific session + +# Wait for screen to change (eliminates need for sleep!) +HASH=$(pilotty snapshot | jq '.content_hash') +pilotty key Enter +pilotty snapshot --await-change $HASH # Block until screen changes +pilotty snapshot --await-change $HASH --settle 50 # Wait for 50ms stability ``` ### Input @@ -120,10 +126,12 @@ pilotty wait-for "~" -s editor # Wait in specific session |--------|-------------| | `-s, --session ` | Target specific session (default: "default") | | `--format ` | Snapshot format: full, compact, text | -| `-t, --timeout ` | Timeout for wait-for (default: 30000) | +| `-t, --timeout ` | Timeout for wait-for and await-change (default: 30000) | | `-r, --regex` | Treat wait-for pattern as regex | | `--name ` | Session name for spawn command | | `--delay ` | Delay between keys in a sequence (default: 0, max: 10000) | +| `--await-change ` | Block snapshot until content_hash differs | +| `--settle ` | Wait for screen to be stable for this many ms (default: 0) | ### Environment variables @@ -197,9 +205,39 @@ pilotty automatically detects interactive UI elements in terminal applications. | **0.8** | Medium confidence: Bracket patterns `[OK]`, `` | | **0.6** | Lower confidence: Underscore input fields `____` | -### Change Detection +### Wait for Screen Changes (Recommended) + +**Stop guessing sleep durations!** Use `--await-change` to wait for the screen to actually update: + +```bash +# Capture baseline hash +HASH=$(pilotty snapshot | jq '.content_hash') + +# Perform action +pilotty key Enter + +# Wait for screen to change (blocks until hash differs) +pilotty snapshot --await-change $HASH + +# Or wait for screen to stabilize (for apps that render progressively) +pilotty snapshot --await-change $HASH --settle 100 +``` + +**Flags:** +| Flag | Description | +|------|-------------| +| `--await-change ` | Block until `content_hash` differs from this value | +| `--settle ` | After change detected, wait for screen to be stable for MS | +| `--timeout ` | Maximum wait time (default: 30000) | + +**Why this is better than sleep:** +- `sleep 1` is a guess - too short causes race conditions, too long slows automation +- `--await-change` waits exactly as long as needed - no more, no less +- `--settle` handles apps that render progressively (show partial, then complete) -The `content_hash` field enables efficient screen change detection: +### Manual Change Detection + +For manual polling (not recommended), use `content_hash` directly: ```bash # Get initial state @@ -273,8 +311,9 @@ pilotty click 5 10 # Click at row 5, col 10 # 1. Spawn vim pilotty spawn --name editor vim /tmp/hello.txt -# 2. Wait for vim to load +# 2. Wait for vim to load and capture baseline hash pilotty wait-for -s editor "hello.txt" +HASH=$(pilotty snapshot -s editor | jq '.content_hash') # 3. Enter insert mode pilotty key -s editor i @@ -282,7 +321,8 @@ pilotty key -s editor i # 4. Type content pilotty type -s editor "Hello from pilotty!" -# 5. Exit insert mode and save (using key sequence) +# 5. Wait for screen to update, then exit (no sleep needed!) +pilotty snapshot -s editor --await-change $HASH --settle 50 pilotty key -s editor "Escape : w q Enter" # 6. Verify session ended @@ -306,19 +346,20 @@ pilotty spawn --name opts dialog --checklist "Select features:" 12 50 4 \ "autosave" "Auto-save documents" on \ "telemetry" "Usage analytics" off -# 2. Wait for dialog to render -sleep 0.5 +# 2. Wait for dialog to render (use await-change, not sleep!) +pilotty snapshot -s opts --settle 200 # Wait for initial render to stabilize -# 3. Get snapshot and examine elements -pilotty snapshot -s opts | jq '.elements[] | select(.kind == "toggle")' -# Shows toggle elements with checked state and positions +# 3. Get snapshot and examine elements, capture hash +SNAP=$(pilotty snapshot -s opts) +echo "$SNAP" | jq '.elements[] | select(.kind == "toggle")' +HASH=$(echo "$SNAP" | jq '.content_hash') # 4. Navigate to "darkmode" and toggle it pilotty key -s opts Down # Move to second option pilotty key -s opts Space # Toggle it on -# 5. Verify the change -pilotty snapshot -s opts | jq '.elements[] | select(.kind == "toggle") | {text, checked}' +# 5. Wait for change and verify +pilotty snapshot -s opts --await-change $HASH | jq '.elements[] | select(.kind == "toggle") | {text, checked}' # 6. Confirm selection pilotty key -s opts Enter @@ -440,6 +481,18 @@ Errors include actionable suggestions: ## Common Patterns +### Reliable action + wait (recommended) + +```bash +# The pattern: capture hash, act, await change +HASH=$(pilotty snapshot | jq '.content_hash') +pilotty key Enter +pilotty snapshot --await-change $HASH --settle 50 + +# This replaces fragile patterns like: +# pilotty key Enter && sleep 1 && pilotty snapshot # BAD: guessing +``` + ### Wait then act ```bash From 2fdf7a38921ef4c822ecb3bf468a0de83d3a8a6f Mon Sep 17 00:00:00 2001 From: msmps <7691252+msmps@users.noreply.github.com> Date: Thu, 29 Jan 2026 16:24:25 +0000 Subject: [PATCH 2/3] docs: readme + skill --- README.md | 46 +++++++++++++++++++++++++++- skills/pilotty/SKILL.md | 66 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 268767c..307e9f8 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,7 @@ pilotty snapshot --await-change $HASH --settle 100 # Wait 100ms after last chan **Flags:** - `--await-change `: Block until `content_hash` differs from this value - `--settle `: After change detected, wait for screen to be stable for this many ms -- `--timeout `: Maximum wait time (default: 30000) +- `-t, --timeout `: Maximum wait time (default: 30000) **Why this matters:** - No more flaky automation due to race conditions @@ -244,6 +244,23 @@ pilotty snapshot --await-change $HASH --settle 100 # Wait 100ms after last chan - Works regardless of how fast/slow the target app is - The `--settle` flag handles apps that render progressively +### Streaming AI Responses + +For AI-powered TUIs that stream responses (opencode, etc.), use longer settle times: + +```bash +HASH=$(pilotty snapshot -s ai | jq -r '.content_hash') +pilotty type -s ai "explain this code" +pilotty key -s ai Enter + +# Wait for streaming to complete: 3s settle, 60s timeout +pilotty snapshot -s ai --await-change "$HASH" --settle 3000 -t 60000 +``` + +- Use `--settle 2000-3000` because AI responses pause between chunks +- Extend timeout with `-t 60000` for longer generations +- Long responses may scroll; use `pilotty scroll up` to see the full output + ### Manual Change Detection For manual polling, use `content_hash` directly: @@ -445,6 +462,33 @@ pilotty key "Escape : w q Enter" # vim :wq sequence pilotty list-sessions ``` +### Example: AI TUI Interaction + +For AI-powered terminal apps that stream responses: + +```bash +# 1. Spawn the AI app +pilotty spawn --name ai opencode + +# 2. Wait for prompt +pilotty wait-for -s ai "Ask anything" -t 15000 + +# 3. Capture baseline hash, type prompt, submit +HASH=$(pilotty snapshot -s ai | jq -r '.content_hash') +pilotty type -s ai "write a haiku about rust" +pilotty key -s ai Enter + +# 4. Wait for streaming response (3s settle, 60s timeout) +pilotty snapshot -s ai --await-change "$HASH" --settle 3000 -t 60000 --format text + +# 5. Scroll up if response is long +pilotty scroll -s ai up 10 +pilotty snapshot -s ai --format text + +# 6. Clean up +pilotty kill -s ai +``` + ## Key Combinations Supported key formats: diff --git a/skills/pilotty/SKILL.md b/skills/pilotty/SKILL.md index 162337e..50bc909 100644 --- a/skills/pilotty/SKILL.md +++ b/skills/pilotty/SKILL.md @@ -228,13 +228,40 @@ pilotty snapshot --await-change $HASH --settle 100 |------|-------------| | `--await-change ` | Block until `content_hash` differs from this value | | `--settle ` | After change detected, wait for screen to be stable for MS | -| `--timeout ` | Maximum wait time (default: 30000) | +| `-t, --timeout ` | Maximum wait time (default: 30000) | **Why this is better than sleep:** - `sleep 1` is a guess - too short causes race conditions, too long slows automation - `--await-change` waits exactly as long as needed - no more, no less - `--settle` handles apps that render progressively (show partial, then complete) +### Waiting for Streaming AI Responses + +When interacting with AI-powered TUIs (like opencode, etc.) that stream responses, you need a longer `--settle` time since the screen keeps updating as tokens arrive: + +```bash +# 1. Capture hash before sending prompt +HASH=$(pilotty snapshot -s myapp | jq -r '.content_hash') + +# 2. Type prompt and submit +pilotty type -s myapp "write me a poem about ai agents" +pilotty key -s myapp Enter + +# 3. Wait for streaming response to complete +# - Use longer settle (2-3s) since AI apps pause between chunks +# - Extend timeout for long responses (60s+) +pilotty snapshot -s myapp --await-change "$HASH" --settle 3000 -t 60000 + +# 4. Response may be scrolled - scroll up if needed to see full output +pilotty scroll -s myapp up 10 +pilotty snapshot -s myapp --format text +``` + +**Key parameters for streaming:** +- `--settle 2000-3000`: AI responses have pauses between chunks; 2-3 seconds ensures streaming is truly done +- `-t 60000`: Extend timeout beyond the 30s default for longer generations +- The settle timer resets on each screen change, so it naturally waits until streaming stops + ### Manual Change Detection For manual polling (not recommended), use `content_hash` directly: @@ -417,6 +444,43 @@ pilotty key -s monitor q # Quit pilotty kill -s monitor ``` +## Example: Interact with AI TUI (opencode, etc.) + +AI-powered TUIs stream responses, requiring special handling: + +```bash +# 1. Spawn the AI app +pilotty spawn --name ai opencode + +# 2. Wait for the prompt to be ready +pilotty wait-for -s ai "Ask anything" -t 15000 + +# 3. Capture baseline hash +HASH=$(pilotty snapshot -s ai | jq -r '.content_hash') + +# 4. Type prompt and submit +pilotty type -s ai "explain the architecture of this codebase" +pilotty key -s ai Enter + +# 5. Wait for streaming response to complete +# - settle=3000: Wait 3s of no changes to ensure streaming is done +# - timeout=60000: Allow up to 60s for long responses +pilotty snapshot -s ai --await-change "$HASH" --settle 3000 -t 60000 --format text + +# 6. If response is long and scrolled, scroll up to see full output +pilotty scroll -s ai up 20 +pilotty snapshot -s ai --format text + +# 7. Clean up +pilotty kill -s ai +``` + +**Gotchas with AI apps:** +- Use `--settle 2000-3000` because AI responses pause between chunks +- Extend timeout with `-t 60000` for complex prompts +- Long responses may scroll the terminal; use `scroll up` to see the beginning +- The settle timer resets on each screen update, so it waits for true completion + --- ## Sessions From fdbe1fd6cee01e74201c024097dd355032344bab Mon Sep 17 00:00:00 2001 From: msmps <7691252+msmps@users.noreply.github.com> Date: Thu, 29 Jan 2026 16:56:15 +0000 Subject: [PATCH 3/3] fix: tweaks --- crates/pilotty-cli/src/args.rs | 2 +- crates/pilotty-cli/src/daemon/server.rs | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/pilotty-cli/src/args.rs b/crates/pilotty-cli/src/args.rs index 1e0c8c2..916db5d 100644 --- a/crates/pilotty-cli/src/args.rs +++ b/crates/pilotty-cli/src/args.rs @@ -162,7 +162,7 @@ pub struct SnapshotArgs { #[arg(long, default_value_t = 0, value_name = "MS")] pub settle: u64, - /// Timeout in milliseconds for await-change/settle (default: 30s) + /// Total timeout in milliseconds for await-change and settle combined (default: 30s) #[arg(short, long, default_value_t = 30000)] pub timeout: u64, } diff --git a/crates/pilotty-cli/src/daemon/server.rs b/crates/pilotty-cli/src/daemon/server.rs index f59e060..bd55d25 100644 --- a/crates/pilotty-cli/src/daemon/server.rs +++ b/crates/pilotty-cli/src/daemon/server.rs @@ -657,13 +657,15 @@ async fn handle_snapshot( let format = format.unwrap_or(SnapshotFormat::Full); let with_elements = matches!(format, SnapshotFormat::Full); let timeout = Duration::from_millis(timeout_ms); - let settle = Duration::from_millis(settle_ms); + // Settle must be at least one poll interval to be meaningful + let settle = Duration::from_millis(if settle_ms > 0 { + settle_ms.max(SNAPSHOT_POLL_INTERVAL_MS) + } else { + 0 + }); let poll_interval = Duration::from_millis(SNAPSHOT_POLL_INTERVAL_MS); let start = Instant::now(); - // Track hash across phases (set during await_change, used by settle) - let mut current_hash: Option = None; - // Phase 1: If await_change is set, wait until content_hash differs if let Some(baseline_hash) = await_change { loop { @@ -686,12 +688,11 @@ async fn handle_snapshot( Err(e) => return Response::error(request_id, e), }; - current_hash = snapshot.content_hash; - if current_hash != Some(baseline_hash) { + if snapshot.content_hash != Some(baseline_hash) { debug!( "Screen changed from hash {} to {:?} after {}ms", baseline_hash, - current_hash, + snapshot.content_hash, start.elapsed().as_millis() ); break; @@ -703,8 +704,12 @@ async fn handle_snapshot( // Phase 2: If settle_ms > 0, wait for screen stability if settle_ms > 0 { - // Start from where await_change left off (or None if no await_change) - let mut last_hash = current_hash; + // Get fresh snapshot - don't rely on potentially stale hash from Phase 1 + let snapshot = match sessions.get_snapshot_data(&session_id, false).await { + Ok(data) => data, + Err(e) => return Response::error(request_id, e), + }; + let mut last_hash = snapshot.content_hash; let mut stable_since = Instant::now(); loop {