zoeissleeping/token
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support normalizing coordinate spaces

Browse Source
This commit is contained in:
Zoe
2026-09-19 18:23:51 -05:00
parent c49f228847
commit 5389d4541c
9 changed files with 85 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vm/desktop.py
+19 -3
View File
@@ -23,9 +23,20 @@ def integer(value, low, high):
return value
def click_pixels(action, width, height, space):
if space == "normalized_1000":
x = integer(action.get("x"), 0, 1000)
y = integer(action.get("y"), 0, 1000)
return (x * (width - 1) + 500) // 1000, (y * (height - 1) + 500) // 1000
return integer(action.get("x"), 0, width - 1), integer(action.get("y"), 0, height - 1)
def validate(request, width, height):
if not isinstance(request, dict) or not isinstance(request.get("actions"), list):
raise ValueError("Request must contain an actions array")
space = request.get("coordinateSpace", "pixels")
if space not in ("pixels", "normalized_1000"):
raise ValueError("Unknown coordinate space")
actions = request["actions"]
if len(actions) > 20:
raise ValueError("At most 20 actions per request")
@@ -35,8 +46,7 @@ def validate(request, width, height):
raise ValueError("Actions must be objects")
kind = action.get("type")
if kind == "click":
integer(action.get("x"), 0, width - 1)
integer(action.get("y"), 0, height - 1)
click_pixels(action, width, height, space)
if action.get("button", "left") not in ("left", "middle", "right"):
raise ValueError("Unknown mouse button")
elif kind == "scroll":
@@ -122,13 +132,17 @@ def main():
expected = request.get("expectedSize")
if expected is not None and expected != [width, height]:
raise ValueError("Display geometry changed; capture again before acting")
space = request.get("coordinateSpace", "pixels")
clicks = []
completed = 0
for action in actions:
kind = action["type"]
try:
if kind == "click":
button = {"left": "1", "middle": "2", "right": "3"}[action.get("button", "left")]
run("xdotool", "mousemove", "--sync", str(action["x"]), str(action["y"]), "click", button)
x, y = click_pixels(action, width, height, space)
run("xdotool", "mousemove", "--sync", str(x), str(y), "click", button)
clicks.append({"supplied": [action["x"], action["y"]], "pixels": [x, y]})
elif kind == "scroll":
run("xdotool", "click", "--repeat", str(action["steps"]), "--delay", "50", "4" if action["direction"] == "up" else "5")
elif kind == "keys":
@@ -151,6 +165,8 @@ def main():
"width": screenshot.width,
"height": screenshot.height,
"completedActions": completed,
"coordinateSpace": space,
"clicks": clicks,
"image": base64.b64encode(buffer.getvalue()).decode(),
}))