feat: merge AT-SPI targeting and delay post-action screenshots

test/model.test.ts

@@ -30,4 +30,6 @@ test("tool inputs reject invalid commands, bounds, and action names", () => {
     assert.throws(() => desktopArguments.parse({ actions: [{ type: "click", x: -1, y: 0 }] }));
     assert.throws(() => desktopArguments.parse({ actions: [{ type: "execute" }] }));
     assert.deepEqual(desktopArguments.parse({ actions: [] }), { actions: [] });
+    assert.throws(() => desktopArguments.parse({ actions: [{ type: "click_target", target: "1" }] }));
+    assert.equal(desktopArguments.parse({ actions: [{ type: "click_target", target: "1", observationId: "frame" }] }).actions[0]?.type, "click_target");
 });

test/test_accessibility.py

@@ -0,0 +1,59 @@
+import sys
+from pathlib import Path
+import unittest
+from unittest.mock import patch
+import subprocess
+
+sys.path.insert(0, str(Path(__file__).parents[1] / "vm"))
+import accessibility
+import desktop
+
+
+class AccessibilityTests(unittest.TestCase):
+    def setUp(self):
+        self.target = {"id": "1", "path": [0, 0, 2], "app": "editor", "name": "Input", "role": "text", "bounds": [10, 20, 100, 40]}
+        self.saved = {"observationId": "abc", "size": [800, 600], "targets": [self.target]}
+        self.current = {"targets": [self.target]}
+
+    def test_badges_stay_in_their_menu_rows(self):
+        for y in (31, 59, 87, 223):
+            left, top = accessibility.badge_position([0, y, 166, 27], 22, 18, 1280, 800)
+            self.assertGreaterEqual(left, 0)
+            self.assertGreaterEqual(top, y)
+            self.assertLessEqual(top + 18, y + 27)
+
+    def test_badge_clamped_to_screen(self):
+        left, top = accessibility.badge_position([1275, 795, 5, 5], 22, 18, 1280, 800)
+        self.assertLessEqual(left + 22, 1280)
+        self.assertLessEqual(top + 18, 800)
+
+    def test_center(self):
+        self.assertEqual(accessibility.resolve_target(self.saved, self.current, "abc", "1", 800, 600), (60, 40))
+
+    def test_stale_and_missing(self):
+        for observation, target, width in [("old", "1", 800), ("abc", "2", 800), ("abc", "1", 900)]:
+            with self.assertRaises(ValueError):
+                accessibility.resolve_target(self.saved, self.current, observation, target, width, 600)
+        with self.assertRaises(ValueError):
+            accessibility.resolve_target(self.saved, {"targets": []}, "abc", "1", 800, 600)
+
+    def test_moved_target(self):
+        moved = {**self.target, "bounds": [100, 100, 100, 40]}
+        with self.assertRaises(ValueError):
+            accessibility.resolve_target(self.saved, {"targets": [moved]}, "abc", "1", 800, 600)
+
+    def test_timeout_fallback(self):
+        with patch.object(subprocess, "run", side_effect=subprocess.TimeoutExpired("scan", 3)):
+            result = accessibility.collect(800, 600)
+        self.assertEqual(result["targets"], [])
+        self.assertIn("unavailable", result["warning"])
+
+    def test_target_must_be_first(self):
+        target = {"type": "click_target", "target": "1", "observationId": "abc"}
+        desktop.validate({"actions": [target]}, 800, 600)
+        with self.assertRaises(ValueError):
+            desktop.validate({"actions": [{"type": "keys", "keys": ["Return"]}, target]}, 800, 600)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_desktop.py

@@ -1,9 +1,14 @@
 import importlib.util
 from pathlib import Path
 import unittest
+import sys
+sys.path.insert(0, str(Path(__file__).parents[1] / "vm"))
 import tempfile
 import json
-from unittest.mock import patch
+from unittest.mock import patch, Mock
+from types import SimpleNamespace
+from contextlib import ExitStack
+import io
 
 spec = importlib.util.spec_from_file_location("desktop", Path(__file__).parents[1] / "vm/desktop.py")
 desktop = importlib.util.module_from_spec(spec)
@@ -11,6 +16,32 @@ spec.loader.exec_module(desktop)
 
 
 class ValidationTests(unittest.TestCase):
+    def test_settling_delay_precedes_final_capture_only_after_actions(self):
+        for actions in ([], [{"type": "click", "x": 500, "y": 500}]):
+            events = []
+            image = Mock(size=(1280, 800), width=1280, height=800)
+            def grab(**kwargs):
+                events.append("capture")
+                return image
+            with tempfile.TemporaryDirectory() as directory, ExitStack() as stack:
+                state = Path(directory)
+                (state / "session.json").write_text('{"DISPLAY": ":0"}')
+                stack.enter_context(patch.object(desktop, "STATE", state))
+                stack.enter_context(patch.object(sys, "argv", ["desktop.py"]))
+                stack.enter_context(patch.object(sys, "stdin", io.StringIO(json.dumps({"actions": actions, "coordinateSpace": "normalized_1000"}))))
+                stack.enter_context(patch.object(sys, "stdout", io.StringIO()))
+                stack.enter_context(patch.dict(sys.modules, {"PIL": SimpleNamespace(ImageGrab=SimpleNamespace(grab=grab))}))
+                stack.enter_context(patch("time.sleep", side_effect=lambda seconds: events.append(seconds)))
+                command = stack.enter_context(patch.object(desktop, "run", side_effect=lambda *args: events.append("action")))
+                stack.enter_context(patch.object(desktop.accessibility, "collect", return_value={"targets": [], "warning": None}))
+                stack.enter_context(patch.object(desktop.accessibility, "annotate", return_value=image))
+                desktop.main()
+                if actions:
+                    self.assertEqual(events, ["capture", "action", 0.5, "capture"])
+                    command.assert_called_once_with("xdotool", "mousemove", "--sync", "640", "400", "click", "1")
+                else:
+                    self.assertEqual(events, ["capture", "capture"])
+
     def test_session_save(self):
         with tempfile.TemporaryDirectory() as directory:
             state = Path(directory)