From d4b5ea02542148dc4b9bd7dd96f022b7104fcb6e Mon Sep 17 00:00:00 2001 From: Zoe Date: Sun, 20 Sep 2026 23:58:02 -0500 Subject: [PATCH] fix: test fix to normalized coordinate space. --- src/tools.ts | 25 +++++++++++++++++++++---- test/model.test.ts | 14 +++++++------- test/test_desktop.py | 16 ++++------------ vm/desktop.py | 6 +++--- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/tools.ts b/src/tools.ts index 83425fc..ab3e835 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -5,7 +5,7 @@ import { quoteShell, remote } from "./ssh.js"; const coordinate = z.number().int().nonnegative(); const action = z.discriminatedUnion("type", [ z.object({ type: z.literal("click_target"), target: z.string().regex(/^\d+$/), observationId: z.string().min(1) }), - z.object({ type: z.literal("click"), x: coordinate.describe("X coordinate where to click"), y: coordinate.describe("Y coordinate where to click"), button: z.enum(["left", "middle", "right"]).default("left") }), + z.object({ type: z.literal("click"), x: coordinate, y: coordinate, button: z.enum(["left", "middle", "right"]).default("left") }), z.object({ type: z.literal("scroll"), direction: z.enum(["up", "down"]), steps: z.number().int().min(1).max(30) }), z.object({ type: z.literal("keys"), keys: z.array(z.string().regex(/^[a-zA-Z0-9_]+$/)).min(1).max(8) }), z.object({ type: z.literal("text"), text: z.string().max(20000) }), @@ -17,12 +17,29 @@ const target = z.object({ id: z.string(), name: z.string(), role: z.string(), ap const observation = z.object({ image: z.string().min(1), rawImage: z.string().optional(), targets: z.array(target).default([]), accessibilityWarning: z.string().nullable().optional(), observationId: z.string(), width: z.number().int().positive(), height: z.number().int().positive(), completedActions: z.number(), coordinateSpace: z.enum(["pixels", "normalized_1000"]), clicks: z.array(z.object({ supplied: z.tuple([z.number(), z.number()]), pixels: z.tuple([z.number(), z.number()]) })) }); export const toolDefinitions = [ - { type: "function", function: { name: "desktop", description: "Operate the XFCE desktop and receive a fresh screenshot. Empty actions captures only. Prefer click_target using a numbered accessibility label and its observationId from the latest screenshot. A target click must be the first action, with at most one per request. Labels are overlay annotations, not actual UI. If no suitable label exists, use coordinate clicks where x and y are where to click. Text replaces the clipboard and pastes with Ctrl+V; not appropriate for terminals. Use X11 key names such as ctrl, Return, Escape. A 500ms settling delay precedes post-action screenshots. Observe after uncertain transitions; slower applications may need an explicit wait. Total waits must not exceed 5000ms.", parameters: z.toJSONSchema(desktopArguments, { io: "input" }) } }, + { type: "function", function: { name: "desktop", description: "Operate the XFCE desktop and receive a fresh screenshot. Empty actions captures only. Prefer click_target using a numbered accessibility label and its observationId from the latest screenshot. A target click must be the first action, with at most one per request. Labels are overlay annotations, not actual UI. If no suitable label exists, use coordinate clicks. Coordinates must be normalized integers in the range [0, 1000], where (0, 0) is top-left and (1000, 1000) is bottom-right. Text replaces the clipboard and pastes with Ctrl+V; not appropriate for terminals. Use X11 key names such as ctrl, Return, Escape. A 500ms settling delay precedes post-action screenshots. Observe after uncertain transitions; slower applications may need an explicit wait. Total waits must not exceed 5000ms.", parameters: z.toJSONSchema(desktopArguments, { io: "input" }) } }, { type: "function", function: { name: "shell", description: "Run a bounded shell command inside home with the desktop user's permissions. Not for managed background jobs. Output is truncated to 16000 characters per stream.", parameters: z.toJSONSchema(shellArguments, { io: "input" }) } }, ]; -export function toolsForCoordinates(_space?: CoordinateSpace) { - return toolDefinitions; +export function toolsForCoordinates(space: CoordinateSpace = "normalized_1000") { + const convention = space === "normalized_1000" + ? "Coordinates must be normalized integers in the range [0, 1000], where (0, 0) is top-left and (1000, 1000) is bottom-right." + : "Click coordinates are screenshot pixels, not normalized coordinates."; + return toolDefinitions.map((tool) => { + if (tool.function.name !== "desktop") { + return tool; + } + return { + ...tool, + function: { + ...tool.function, + description: tool.function.description.replace( + "Coordinates must be normalized integers in the range [0, 1000], where (0, 0) is top-left and (1000, 1000) is bottom-right.", + convention, + ), + }, + }; + }); } export async function captureDesktop(destination: string, args: unknown, coordinateSpace: CoordinateSpace = "pixels") { diff --git a/test/model.test.ts b/test/model.test.ts index cb64b0c..2404a74 100644 --- a/test/model.test.ts +++ b/test/model.test.ts @@ -19,13 +19,13 @@ test("adapter accepts tool calls and rejects truncated responses", async (t) => await assert.rejects(complete(config, [], []), /incomplete/); }); -test("tool descriptions do not inject coordinate normalization or pixel prompting", () => { - for (const space of ["pixels", "normalized_1000"] as const) { - const description = toolsForCoordinates(space)[0]?.function.description ?? ""; - assert.doesNotMatch(description, /normalized/i); - assert.doesNotMatch(description, /pixel/i); - assert.doesNotMatch(description, /0\.\.1000/); - } +test("tool descriptions advertise coordinate conventions", () => { + const normalized = toolsForCoordinates("normalized_1000")[0]?.function.description ?? ""; + assert.match(normalized, /Coordinates must be normalized integers in the range \[0, 1000\], where \(0, 0\) is top-left and \(1000, 1000\) is bottom-right\./); + + const pixels = toolsForCoordinates("pixels")[0]?.function.description ?? ""; + assert.match(pixels, /Click coordinates are screenshot pixels, not normalized coordinates\./); + assert.doesNotMatch(pixels, /normalized integers in the range/); }); test("tool inputs reject invalid commands, bounds, and action names", () => { diff --git a/test/test_desktop.py b/test/test_desktop.py index c41aa3f..307e335 100644 --- a/test/test_desktop.py +++ b/test/test_desktop.py @@ -38,7 +38,7 @@ class ValidationTests(unittest.TestCase): desktop.main() if actions: self.assertEqual(events, ["capture", "action", 0.5, "capture"]) - command.assert_called_once_with("xdotool", "mousemove", "--sync", "640", "500", "click", "1") + command.assert_called_once_with("xdotool", "mousemove", "--sync", "640", "400", "click", "1") else: self.assertEqual(events, ["capture", "capture"]) @@ -64,17 +64,9 @@ class ValidationTests(unittest.TestCase): self.assertIn("OnlyShowIn=XFCE;", entries[0].read_text()) def test_normalized_coordinates(self): - # 1280x800: width > 1000 (normalized 0..1000), height <= 1000 (pixels 0..799) - self.assertEqual(desktop.click_pixels({"x": 0, "y": 0}, 1280, 800, "normalized_1000"), (0, 0)) - self.assertEqual(desktop.click_pixels({"x": 500, "y": 400}, 1280, 800, "normalized_1000"), (640, 400)) - self.assertEqual(desktop.click_pixels({"x": 1000, "y": 799}, 1280, 800, "normalized_1000"), (1279, 799)) - # 1920x1080: both > 1000 (both normalized 0..1000) - self.assertEqual(desktop.click_pixels({"x": 500, "y": 500}, 1920, 1080, "normalized_1000"), (960, 540)) - # 800x600: both <= 1000 (both pixels) - self.assertEqual(desktop.click_pixels({"x": 400, "y": 300}, 800, 600, "normalized_1000"), (400, 300)) - # 1x1: both <= 1000 - self.assertEqual(desktop.click_pixels({"x": 0, "y": 0}, 1, 1, "normalized_1000"), (0, 0)) - # pixels mode + for value, expected in [(0, (0, 0)), (500, (640, 400)), (1000, (1279, 799))]: + self.assertEqual(desktop.click_pixels({"x": value, "y": value}, 1280, 800, "normalized_1000"), expected) + self.assertEqual(desktop.click_pixels({"x": 1000, "y": 1000}, 1, 1, "normalized_1000"), (0, 0)) self.assertEqual(desktop.click_pixels({"x": 480, "y": 425}, 1280, 800, "pixels"), (480, 425)) for value in (-1, 1001, 0.5, True): with self.assertRaises(ValueError): diff --git a/vm/desktop.py b/vm/desktop.py index f1e1054..61ca94f 100644 --- a/vm/desktop.py +++ b/vm/desktop.py @@ -26,9 +26,9 @@ def integer(value, low, high): def click_pixels(action, width, height, space): if space == "normalized_1000": - x = (integer(action.get("x"), 0, 1000) * (width - 1) + 500) // 1000 if width > 1000 else integer(action.get("x"), 0, width - 1) - y = (integer(action.get("y"), 0, 1000) * (height - 1) + 500) // 1000 if height > 1000 else integer(action.get("y"), 0, height - 1) - return x, y + x = integer(action.get("x"), 0, 1000) + y = integer(action.get("y"), 0, 1000) + return (x * (width - 1) + 500) // 1000, (y * (height - 1) + 500) // 1000 return integer(action.get("x"), 0, width - 1), integer(action.get("y"), 0, height - 1)