zoeissleeping/token
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: test fix to change clicks to box drawing

Browse Source
This commit is contained in:
Zoe
2026-09-21 00:04:35 -05:00
parent d4b5ea0254
commit 4b08ce05a4
5 changed files with 37 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -77,14 +77,14 @@ Other action examples:
{
"expectedSize": [1280, 800],
"actions": [
{ "type": "click", "x": 300, "y": 200, "button": "left" },
{ "type": "click", "ymin": 190, "xmin": 280, "ymax": 210, "xmax": 320, "button": "left" },
{ "type": "keys", "keys": ["ctrl", "a"] },
{ "type": "scroll", "direction": "down", "steps": 2 }
]
}
```
Use coordinates from your own screenshot, not these example coordinates. `expectedSize` is optional and rejects input if the screen size changed. Supported actions are click, click_target, scroll, keys, text, and wait. Drag is deferred. Actions are serialized by a VM-side lock; avoid manual interaction while a request runs. The full request is validated before any actions execute, but runtime failures can still leave partial effects.
Use coordinates from your own screenshot, not these example coordinates. Coordinate clicks draw a bounding box with `ymin`, `xmin`, `ymax`, and `xmax`; the click is delivered to the center of the box. `expectedSize` is optional and rejects input if the screen size changed. Supported actions are click, click_target, scroll, keys, text, and wait. Drag is deferred. Actions are serialized by a VM-side lock; avoid manual interaction while a request runs. The full request is validated before any actions execute, but runtime failures can still leave partial effects.
After a non-empty action batch, the helper waits **500 ms** before taking the resulting screenshot and scanning accessibility targets. Capture-only requests have no added delay. This allows ordinary UI updates to paint; slow loads still need an explicit wait or another observation.
src/tools.ts
+3 -3
View File
@@ -5,7 +5,7 @@ import { quoteShell, remote } from "./ssh.js";
const coordinate = z.number().int().nonnegative();
const action = z.discriminatedUnion("type", [
z.object({ type: z.literal("click_target"), target: z.string().regex(/^\d+$/), observationId: z.string().min(1) }),
z.object({ type: z.literal("click"), x: coordinate, y: coordinate, button: z.enum(["left", "middle", "right"]).default("left") }),
z.object({ type: z.literal("click"), ymin: coordinate, xmin: coordinate, ymax: coordinate, xmax: coordinate, button: z.enum(["left", "middle", "right"]).default("left") }),
z.object({ type: z.literal("scroll"), direction: z.enum(["up", "down"]), steps: z.number().int().min(1).max(30) }),
z.object({ type: z.literal("keys"), keys: z.array(z.string().regex(/^[a-zA-Z0-9_]+$/)).min(1).max(8) }),
z.object({ type: z.literal("text"), text: z.string().max(20000) }),
@@ -14,10 +14,10 @@ const action = z.discriminatedUnion("type", [
export const desktopArguments = z.object({ actions: z.array(action).max(20) });
export const shellArguments = z.object({ command: z.string().min(1).max(20000), timeoutSeconds: z.number().int().min(1).max(120).default(30) });
const target = z.object({ id: z.string(), name: z.string(), role: z.string(), app: z.string(), bounds: z.tuple([z.number(), z.number(), z.number(), z.number()]), states: z.array(z.string()).default([]), value: z.string().optional() });
const observation = z.object({ image: z.string().min(1), rawImage: z.string().optional(), targets: z.array(target).default([]), accessibilityWarning: z.string().nullable().optional(), observationId: z.string(), width: z.number().int().positive(), height: z.number().int().positive(), completedActions: z.number(), coordinateSpace: z.enum(["pixels", "normalized_1000"]), clicks: z.array(z.object({ supplied: z.tuple([z.number(), z.number()]), pixels: z.tuple([z.number(), z.number()]) })) });
const observation = z.object({ image: z.string().min(1), rawImage: z.string().optional(), targets: z.array(target).default([]), accessibilityWarning: z.string().nullable().optional(), observationId: z.string(), width: z.number().int().positive(), height: z.number().int().positive(), completedActions: z.number(), coordinateSpace: z.enum(["pixels", "normalized_1000"]), clicks: z.array(z.object({ supplied: z.array(z.number()), pixels: z.tuple([z.number(), z.number()]) })) });
export const toolDefinitions = [
{ type: "function", function: { name: "desktop", description: "Operate the XFCE desktop and receive a fresh screenshot. Empty actions captures only. Prefer click_target using a numbered accessibility label and its observationId from the latest screenshot. A target click must be the first action, with at most one per request. Labels are overlay annotations, not actual UI. If no suitable label exists, use coordinate clicks. Coordinates must be normalized integers in the range [0, 1000], where (0, 0) is top-left and (1000, 1000) is bottom-right. Text replaces the clipboard and pastes with Ctrl+V; not appropriate for terminals. Use X11 key names such as ctrl, Return, Escape. A 500ms settling delay precedes post-action screenshots. Observe after uncertain transitions; slower applications may need an explicit wait. Total waits must not exceed 5000ms.", parameters: z.toJSONSchema(desktopArguments, { io: "input" }) } },
{ type: "function", function: { name: "desktop", description: "Operate the XFCE desktop and receive a fresh screenshot. Empty actions captures only. Prefer click_target using a numbered accessibility label and its observationId from the latest screenshot. A target click must be the first action, with at most one per request. Labels are overlay annotations, not actual UI. If no suitable label exists, use coordinate clicks to draw a box with ymin, xmin, ymax, and xmax; the click occurs at the center of the box. Coordinates must be normalized integers in the range [0, 1000], where (0, 0) is top-left and (1000, 1000) is bottom-right. Text replaces the clipboard and pastes with Ctrl+V; not appropriate for terminals. Use X11 key names such as ctrl, Return, Escape. A 500ms settling delay precedes post-action screenshots. Observe after uncertain transitions; slower applications may need an explicit wait. Total waits must not exceed 5000ms.", parameters: z.toJSONSchema(desktopArguments, { io: "input" }) } },
{ type: "function", function: { name: "shell", description: "Run a bounded shell command inside home with the desktop user's permissions. Not for managed background jobs. Output is truncated to 16000 characters per stream.", parameters: z.toJSONSchema(shellArguments, { io: "input" }) } },
];
test/model.test.ts
+4 -1
View File
@@ -22,6 +22,7 @@ test("adapter accepts tool calls and rejects truncated responses", async (t) =>
test("tool descriptions advertise coordinate conventions", () => {
const normalized = toolsForCoordinates("normalized_1000")[0]?.function.description ?? "";
assert.match(normalized, /Coordinates must be normalized integers in the range \[0, 1000\], where \(0, 0\) is top-left and \(1000, 1000\) is bottom-right\./);
assert.match(normalized, /ymin, xmin, ymax, and xmax/);
const pixels = toolsForCoordinates("pixels")[0]?.function.description ?? "";
assert.match(pixels, /Click coordinates are screenshot pixels, not normalized coordinates\./);
@@ -31,9 +32,11 @@ test("tool descriptions advertise coordinate conventions", () => {
test("tool inputs reject invalid commands, bounds, and action names", () => {
assert.throws(() => shellArguments.parse({ command: "", timeoutSeconds: 1 }));
assert.throws(() => shellArguments.parse({ command: "true", timeoutSeconds: 999 }));
assert.throws(() => desktopArguments.parse({ actions: [{ type: "click", x: -1, y: 0 }] }));
assert.throws(() => desktopArguments.parse({ actions: [{ type: "click", ymin: -1, xmin: 0, ymax: 0, xmax: 0 }] }));
assert.throws(() => desktopArguments.parse({ actions: [{ type: "click", x: 10, y: 20 }] }));
assert.throws(() => desktopArguments.parse({ actions: [{ type: "execute" }] }));
assert.deepEqual(desktopArguments.parse({ actions: [] }), { actions: [] });
assert.throws(() => desktopArguments.parse({ actions: [{ type: "click_target", target: "1" }] }));
assert.equal(desktopArguments.parse({ actions: [{ type: "click_target", target: "1", observationId: "frame" }] }).actions[0]?.type, "click_target");
assert.equal(desktopArguments.parse({ actions: [{ type: "click", ymin: 10, xmin: 20, ymax: 30, xmax: 40 }] }).actions[0]?.type, "click");
});
test/test_desktop.py
+9 -1
View File
@@ -17,7 +17,7 @@ spec.loader.exec_module(desktop)
class ValidationTests(unittest.TestCase):
def test_settling_delay_precedes_final_capture_only_after_actions(self):
for actions in ([], [{"type": "click", "x": 500, "y": 500}]):
for actions in ([], [{"type": "click", "ymin": 400, "xmin": 400, "ymax": 600, "xmax": 600}]):
events = []
image = Mock(size=(1280, 800), width=1280, height=800)
def grab(**kwargs):
@@ -66,11 +66,17 @@ class ValidationTests(unittest.TestCase):
def test_normalized_coordinates(self):
for value, expected in [(0, (0, 0)), (500, (640, 400)), (1000, (1279, 799))]:
self.assertEqual(desktop.click_pixels({"x": value, "y": value}, 1280, 800, "normalized_1000"), expected)
self.assertEqual(desktop.click_pixels({"ymin": value, "xmin": value, "ymax": value, "xmax": value}, 1280, 800, "normalized_1000"), expected)
self.assertEqual(desktop.click_pixels({"ymin": 300, "xmin": 400, "ymax": 500, "xmax": 600}, 1280, 800, "normalized_1000"), (640, 320))
self.assertEqual(desktop.click_pixels({"x": 1000, "y": 1000}, 1, 1, "normalized_1000"), (0, 0))
self.assertEqual(desktop.click_pixels({"ymin": 0, "xmin": 0, "ymax": 1000, "xmax": 1000}, 1, 1, "normalized_1000"), (0, 0))
self.assertEqual(desktop.click_pixels({"x": 480, "y": 425}, 1280, 800, "pixels"), (480, 425))
self.assertEqual(desktop.click_pixels({"ymin": 400, "xmin": 460, "ymax": 450, "xmax": 500}, 1280, 800, "pixels"), (480, 425))
for value in (-1, 1001, 0.5, True):
with self.assertRaises(ValueError):
desktop.click_pixels({"x": value, "y": 0}, 1280, 800, "normalized_1000")
with self.assertRaises(ValueError):
desktop.click_pixels({"ymin": value, "xmin": 0, "ymax": 0, "xmax": 0}, 1280, 800, "normalized_1000")
def test_invalid_coordinate_space(self):
with self.assertRaises(ValueError):
@@ -83,6 +89,8 @@ class ValidationTests(unittest.TestCase):
for x in (-1, 100, True, "2"):
with self.assertRaises(ValueError):
desktop.validate({"actions": [{"type": "click", "x": x, "y": 0}]}, 100, 100)
with self.assertRaises(ValueError):
desktop.validate({"actions": [{"type": "click", "ymin": 0, "xmin": x, "ymax": 0, "xmax": 0}]}, 100, 100)
def test_invalid_keys(self):
with self.assertRaises(ValueError):
vm/desktop.py
+19 -1
View File
@@ -25,6 +25,20 @@ def integer(value, low, high):
def click_pixels(action, width, height, space):
if "ymin" in action or "xmin" in action or "ymax" in action or "xmax" in action:
if space == "normalized_1000":
ymin = integer(action.get("ymin"), 0, 1000)
xmin = integer(action.get("xmin"), 0, 1000)
ymax = integer(action.get("ymax"), 0, 1000)
xmax = integer(action.get("xmax"), 0, 1000)
x = (xmin + xmax) // 2
y = (ymin + ymax) // 2
return (x * (width - 1) + 500) // 1000, (y * (height - 1) + 500) // 1000
ymin = integer(action.get("ymin"), 0, height - 1)
xmin = integer(action.get("xmin"), 0, width - 1)
ymax = integer(action.get("ymax"), 0, height - 1)
xmax = integer(action.get("xmax"), 0, width - 1)
return (xmin + xmax) // 2, (ymin + ymax) // 2
if space == "normalized_1000":
x = integer(action.get("x"), 0, 1000)
y = integer(action.get("y"), 0, 1000)
@@ -150,7 +164,11 @@ def main():
button = {"left": "1", "middle": "2", "right": "3"}[action.get("button", "left")]
x, y = click_pixels(action, width, height, space)
run("xdotool", "mousemove", "--sync", str(x), str(y), "click", button)
clicks.append({"supplied": [action["x"], action["y"]], "pixels": [x, y]})
if "ymin" in action or "xmin" in action or "ymax" in action or "xmax" in action:
supplied = [action["ymin"], action["xmin"], action["ymax"], action["xmax"]]
else:
supplied = [action["x"], action["y"]]
clicks.append({"supplied": supplied, "pixels": [x, y]})
elif kind == "click_target":
saved = json.loads((STATE / "targets.json").read_text())
current = accessibility.collect(width, height)