feat: AT-SPI interactions

vm/desktop.py

@@ -9,6 +9,7 @@ import sys
 import uuid
 import fcntl
 import tempfile
+import accessibility
 
 STATE = Path.home() / ".local/state/desktop-harness"
 
@@ -39,6 +40,13 @@ def validate(request, width, height):
             integer(action.get("y"), 0, height - 1)
             if action.get("button", "left") not in ("left", "middle", "right"):
                 raise ValueError("Unknown mouse button")
+        elif kind == "click_target":
+            if not isinstance(action.get("target"), str) or not action["target"].isdigit():
+                raise ValueError("Target must be a numbered label string")
+            if not isinstance(action.get("observationId"), str):
+                raise ValueError("click_target requires observationId")
+            if action is not actions[0] or sum(a.get("type") == "click_target" for a in actions) > 1:
+                raise ValueError("click_target must be the first and only target click; capture again before another")
         elif kind == "scroll":
             if action.get("direction") not in ("up", "down"):
                 raise ValueError("Unknown scroll direction")
@@ -129,6 +137,11 @@ def main():
                 if kind == "click":
                     button = {"left": "1", "middle": "2", "right": "3"}[action.get("button", "left")]
                     run("xdotool", "mousemove", "--sync", str(action["x"]), str(action["y"]), "click", button)
+                elif kind == "click_target":
+                    saved = json.loads((STATE / "targets.json").read_text())
+                    current = accessibility.collect(width, height)
+                    x, y = accessibility.resolve_target(saved, current, action["observationId"], action["target"], width, height)
+                    run("xdotool", "mousemove", "--sync", str(x), str(y), "click", "1")
                 elif kind == "scroll":
                     run("xdotool", "click", "--repeat", str(action["steps"]), "--delay", "50", "4" if action["direction"] == "up" else "5")
                 elif kind == "keys":
@@ -143,10 +156,20 @@ def main():
             except Exception as error:
                 raise RuntimeError(f"Action {completed} failed after {completed} completed actions; partial effects possible: {error}") from error
         screenshot = ImageGrab.grab(xdisplay=os.environ["DISPLAY"])
+        accessible = accessibility.collect(screenshot.width, screenshot.height)
+        observation_id = str(uuid.uuid4())
+        (STATE / "targets.json").write_text(json.dumps({
+            "observationId": observation_id, "size": list(screenshot.size), "targets": accessible["targets"],
+        }))
+        raw_buffer = io.BytesIO()
+        screenshot.save(raw_buffer, format="PNG")
         buffer = io.BytesIO()
-        screenshot.save(buffer, format="PNG")
+        accessibility.annotate(screenshot, accessible["targets"]).save(buffer, format="PNG")
         print(json.dumps({
-            "observationId": str(uuid.uuid4()),
+            "observationId": observation_id,
+            "targets": accessible["targets"],
+            "accessibilityWarning": accessible["warning"],
+            "rawImage": base64.b64encode(raw_buffer.getvalue()).decode(),
             "capturedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
             "width": screenshot.width,
             "height": screenshot.height,