feat: merge AT-SPI targeting and delay post-action screenshots

vm/desktop.py

@@ -9,6 +9,7 @@ import sys
 import uuid
 import fcntl
 import tempfile
+import accessibility
 
 STATE = Path.home() / ".local/state/desktop-harness"
 
@@ -49,6 +50,13 @@ def validate(request, width, height):
             click_pixels(action, width, height, space)
             if action.get("button", "left") not in ("left", "middle", "right"):
                 raise ValueError("Unknown mouse button")
+        elif kind == "click_target":
+            if not isinstance(action.get("target"), str) or not action["target"].isdigit():
+                raise ValueError("Target must be a numbered label string")
+            if not isinstance(action.get("observationId"), str):
+                raise ValueError("click_target requires observationId")
+            if action is not actions[0] or sum(a.get("type") == "click_target" for a in actions) > 1:
+                raise ValueError("click_target must be the first and only target click; capture again before another")
         elif kind == "scroll":
             if action.get("direction") not in ("up", "down"):
                 raise ValueError("Unknown scroll direction")
@@ -143,6 +151,11 @@ def main():
                     x, y = click_pixels(action, width, height, space)
                     run("xdotool", "mousemove", "--sync", str(x), str(y), "click", button)
                     clicks.append({"supplied": [action["x"], action["y"]], "pixels": [x, y]})
+                elif kind == "click_target":
+                    saved = json.loads((STATE / "targets.json").read_text())
+                    current = accessibility.collect(width, height)
+                    x, y = accessibility.resolve_target(saved, current, action["observationId"], action["target"], width, height)
+                    run("xdotool", "mousemove", "--sync", str(x), str(y), "click", "1")
                 elif kind == "scroll":
                     run("xdotool", "click", "--repeat", str(action["steps"]), "--delay", "50", "4" if action["direction"] == "up" else "5")
                 elif kind == "keys":
@@ -156,11 +169,24 @@ def main():
                 completed += 1
             except Exception as error:
                 raise RuntimeError(f"Action {completed} failed after {completed} completed actions; partial effects possible: {error}") from error
+        if actions:
+            # Input delivery can finish before the application has repainted.
+            time.sleep(0.5)
         screenshot = ImageGrab.grab(xdisplay=os.environ["DISPLAY"])
+        accessible = accessibility.collect(screenshot.width, screenshot.height)
+        observation_id = str(uuid.uuid4())
+        (STATE / "targets.json").write_text(json.dumps({
+            "observationId": observation_id, "size": list(screenshot.size), "targets": accessible["targets"],
+        }))
+        raw_buffer = io.BytesIO()
+        screenshot.save(raw_buffer, format="PNG")
         buffer = io.BytesIO()
-        screenshot.save(buffer, format="PNG")
+        accessibility.annotate(screenshot, accessible["targets"]).save(buffer, format="PNG")
         print(json.dumps({
-            "observationId": str(uuid.uuid4()),
+            "observationId": observation_id,
+            "targets": accessible["targets"],
+            "accessibilityWarning": accessible["warning"],
+            "rawImage": base64.b64encode(raw_buffer.getvalue()).decode(),
             "capturedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
             "width": screenshot.width,
             "height": screenshot.height,