From c7ba815528716b6aa1ac94e85dbce5b8f74136ef Mon Sep 17 00:00:00 2001 From: Paulius Date: Tue, 1 Apr 2025 14:11:25 +0200 Subject: [PATCH 1/3] For unknown reasons keydown + keyup is more reliable than hotkey --- src/generalagents/macos/computer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/generalagents/macos/computer.py b/src/generalagents/macos/computer.py index cdd6e82..5940957 100644 --- a/src/generalagents/macos/computer.py +++ b/src/generalagents/macos/computer.py @@ -52,7 +52,10 @@ def _scaled(self, coord: Coordinate) -> tuple[int, int]: def _execute_action(self, action: Action) -> None: match action: case ActionKeyPress(kind="key_press", keys=keys) if keys: - pyautogui.hotkey(*keys) + for key in keys: + pyautogui.keyDown(key) + for key in reversed(keys): + pyautogui.keyUp(key) case ActionType(kind="type", text=text) if text: pyautogui.write(text) From b6bdefeee22bd917794c996aecd9080cbd394f81 Mon Sep 17 00:00:00 2001 From: Paulius Date: Tue, 1 Apr 2025 14:21:18 +0200 Subject: [PATCH 2/3] Reduce additional built-in wait by pyautogui --- src/generalagents/macos/computer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generalagents/macos/computer.py b/src/generalagents/macos/computer.py index 5940957..343e976 100644 --- a/src/generalagents/macos/computer.py +++ b/src/generalagents/macos/computer.py @@ -23,6 +23,7 @@ ) pyautogui.FAILSAFE = True # Move mouse to corner to abort +pyautogui.PAUSE = 0.001 # We are waiting manually in the code class Computer: From 5fb939ac31ff1e5c86d107e509162f8e76eb98b5 Mon Sep 17 00:00:00 2001 From: Paulius Date: Tue, 1 Apr 2025 14:27:06 +0200 Subject: [PATCH 3/3] Make mouse movements visible --- src/generalagents/macos/computer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/generalagents/macos/computer.py b/src/generalagents/macos/computer.py index 343e976..7eebed0 100644 --- a/src/generalagents/macos/computer.py +++ b/src/generalagents/macos/computer.py @@ -5,6 +5,7 @@ import pyautogui from PIL import Image +from pytweening import easeInOutQuad from generalagents.action import ( Action, @@ -24,6 +25,7 @@ pyautogui.FAILSAFE = True # Move mouse to corner to abort pyautogui.PAUSE = 0.001 # We are waiting manually in the code +MOUSE_SETTINGS = {"duration": 0.101, "tween": easeInOutQuad} # duration <= 0.1 is treated as 0 by pyautogui class Computer: @@ -62,19 +64,19 @@ def _execute_action(self, action: Action) -> None: pyautogui.write(text) case ActionLeftClick(kind="left_click", coordinate=coord): - pyautogui.click(*self._scaled(coord), button="left") + pyautogui.click(*self._scaled(coord), button="left", **MOUSE_SETTINGS) case ActionRightClick(kind="right_click", coordinate=coord): - pyautogui.click(*self._scaled(coord), button="right") + pyautogui.click(*self._scaled(coord), button="right", **MOUSE_SETTINGS) case ActionDoubleClick(kind="double_click", coordinate=coord): - pyautogui.doubleClick(*self._scaled(coord)) + pyautogui.doubleClick(*self._scaled(coord), **MOUSE_SETTINGS) case ActionTripleClick(kind="triple_click", coordinate=coord): - pyautogui.tripleClick(*self._scaled(coord)) + pyautogui.tripleClick(*self._scaled(coord), **MOUSE_SETTINGS) case ActionMouseMove(kind="mouse_move", coordinate=coord): - pyautogui.moveTo(*self._scaled(coord)) + pyautogui.moveTo(*self._scaled(coord), **MOUSE_SETTINGS) case ActionDrag(kind="drag", drag_start=start, drag_end=end): pyautogui.moveTo(*self._scaled(start))