commit 75b3a01aee15945b8eccad7dc7fd25cbc9cf1eec Author: Zoe Date: Sat Sep 19 14:53:40 2026 -0500 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..152109f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +dist/ +artifacts/ +__pycache__/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..bf3de91 --- /dev/null +++ b/README.md @@ -0,0 +1,104 @@ +# Desktop harness — first slice + +Manual SSH and X11 controls. There is **no model loop, persistent conversation, scheduler, or Jev integration yet**. The package name is a placeholder. + +## On berlin (or your development computer) + +Requires Node.js 22+ and an SSH client. Configure an SSH alias `home` for the VM's desktop user, with key authentication. Connect manually first to verify and save its host key: + +```sh +ssh home +``` + +The CLI requires known host keys and noninteractive authentication. It does not bypass SSH verification. + +```sh +npm install +npm run build +npm run home -- install home +``` + +## Inside home + +Install dependencies: + +```sh +sudo pacman -S --needed python python-pillow xdotool xclip +``` + +From a terminal **inside the logged-in XFCE desktop**, run: + +```sh +python3 ~/.local/lib/desktop-harness/desktop.py --register-session +``` + +This records the current graphical session environment for SSH requests. Installation also creates `~/.config/autostart/desktop-harness-session.desktop`, so subsequent XFCE logins refresh registration automatically. Re-running `install` updates both the helper and its autostart entry. No periodic refresh is needed within the same session. Keep the desktop logged in and unlocked for these initial checks. + +To test automatic registration, log out and back into XFCE, then capture from berlin without running the registration command manually. The helper assumes one graphical session for this account; it does not log in, unlock the screen, or choose between concurrent sessions. To disable automatic registration, remove the autostart file. + +## Try capturing + +From berlin: + +```sh +npm run home -- capture home artifacts/first.png +``` + +Open the PNG. Check dimensions and that it shows the actual desktop rather than a blank or different display. + +## Try input + +Open Mousepad or another ordinary GUI text editor manually, focus a blank document, and create `actions.json` on berlin: + +```json +{ + "actions": [ + { "type": "text", "text": "Hello from berlin!\nUnicode: café — こんにちは" }, + { "type": "wait", "milliseconds": 300 } + ] +} +``` + +```sh +npm run home -- act home actions.json artifacts/typed.png +``` + +Verify the text and screenshot. Text insertion **replaces the clipboard** and pastes with Ctrl+V; this is for ordinary GUI text fields, not terminals (which often need Ctrl+Shift+V). For shell work use the shell command below. + +Other action examples: + +```json +{ + "expectedSize": [1280, 800], + "actions": [ + { "type": "click", "x": 300, "y": 200, "button": "left" }, + { "type": "keys", "keys": ["ctrl", "a"] }, + { "type": "scroll", "direction": "down", "steps": 2 } + ] +} +``` + +Use coordinates from your own screenshot, not these example coordinates. `expectedSize` is optional and rejects input if the screen size changed. Supported actions are click, scroll, keys, text, and wait. Drag is deferred. Actions are serialized by a VM-side lock; avoid manual interaction while a request runs. The full request is validated before any actions execute, but runtime failures can still leave partial effects. + +## Try shell execution + +```sh +npm run home -- shell home 'printf "hello\n"; uname -a' +npm run home -- shell home 'sleep 10' 2 +``` + +The second command should time out. GNU `timeout` runs inside the VM and sends TERM, then KILL after two seconds. This is not a sandbox: commands have the desktop user's permissions, and processes that deliberately detach can escape the timeout. Output is limited to 32 MiB across stdout/stderr; exceeding it disconnects SSH and reports an unknown outcome. Commands must be one quoted local argument. Default timeout is 30 seconds, maximum 300. + +## Local checks + +```sh +npm run check +npm test +python3 -m unittest discover -s test -p 'test_*.py' +``` + +Desktop dependencies are imported only when operating the display, so validation tests can run without X11 or Pillow. + +## Next milestone + +Once capture, Unicode insertion, key combinations, and shell timeout work on the real VM, add the llama.cpp adapter, durable tool-call records, and a single model/tool loop. Wake/sleep and restart recovery follow. No SSH connection or real graphical session is available in the development sandbox, so those checks must be run on your setup. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..8391e78 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,46 @@ +{ + "name": "desktop-harness", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "desktop-harness", + "devDependencies": { + "@types/node": "^25.0.0", + "typescript": "^5.9.0" + } + }, + "node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..aad2d25 --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "desktop-harness", + "private": true, + "type": "module", + "scripts": { + "build": "tsc", + "check": "tsc --noEmit", + "test": "npm run build && node --test dist/test/*.test.js", + "home": "node dist/src/main.js" + }, + "devDependencies": { + "@types/node": "^25.0.0", + "typescript": "^5.9.0" + } +} diff --git a/src/main.ts b/src/main.ts new file mode 100644 index 0000000..b6c5121 --- /dev/null +++ b/src/main.ts @@ -0,0 +1,65 @@ +import { readFile, mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { quoteShell, remote } from "./ssh.js"; + +async function main(): Promise { + const [operation, destination, argument, extra] = process.argv.slice(2); + if (!operation || !destination) { + throw new Error("Usage: home [output.png|actions.json|command] [seconds]"); + } + const helper = '"$HOME/.local/lib/desktop-harness/desktop.py"'; + if (operation === "install") { + const source = await readFile(new URL("../../vm/desktop.py", import.meta.url), "utf8"); + const result = await remote(destination, `mkdir -p "$HOME/.local/lib/desktop-harness" && cat > ${helper} && python3 ${helper} --install-autostart`, source); + if (result.code !== 0) { + throw new Error(result.stderr || "Installation failed."); + } + console.log("Installed desktop helper and XFCE autostart. Future logins register automatically; for the current session, run registration once in an XFCE terminal (see README.md)."); + return; + } + if (operation === "shell") { + if (!argument) { + throw new Error("Supply a shell command as one quoted argument."); + } + const seconds = Number(extra ?? 30); + if (!Number.isInteger(seconds) || seconds < 1 || seconds > 300) { + throw new Error("Timeout must be 1–300 seconds."); + } + const result = await remote(destination, + `timeout --signal=TERM --kill-after=2s ${seconds}s bash -lc ${quoteShell(argument)}`, + "", (seconds + 20) * 1000); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + if (result.code === 255 || result.code === null) { + throw new Error("SSH disconnected; command outcome is unknown."); + } + console.error(`\nExit status: ${result.code}${result.code === 124 || result.code === 137 ? " (timeout/termination)" : ""}`); + process.exitCode = result.code; + return; + } + if (operation !== "capture" && operation !== "act") { + throw new Error(`Unknown operation: ${operation}`); + } + const request = operation === "capture" + ? { actions: [] } + : JSON.parse(await readFile(argument ?? "actions.json", "utf8")) as unknown; + const result = await remote(destination, `python3 ${helper}`, JSON.stringify(request)); + if (result.code !== 0) { + throw new Error(result.stderr || "Desktop request failed; some actions may already have executed."); + } + const response: unknown = JSON.parse(result.stdout.toString("utf8")); + if (typeof response !== "object" || response === null || !("image" in response) + || typeof response.image !== "string") { + throw new Error("Invalid desktop response."); + } + const output = resolve(operation === "capture" ? argument ?? "artifacts/desktop.png" : extra ?? "artifacts/desktop.png"); + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, Buffer.from(response.image, "base64")); + const { image, ...metadata } = response; + console.log(JSON.stringify({ ...metadata, screenshot: output }, null, 2)); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/src/ssh.ts b/src/ssh.ts new file mode 100644 index 0000000..84082ac --- /dev/null +++ b/src/ssh.ts @@ -0,0 +1,59 @@ +import { spawn } from "node:child_process"; + +export function quoteShell(value: string): string { + return "'" + value.replaceAll("'", "'\\''") + "'"; +} + +export interface RemoteResult { + code: number | null; + stdout: Buffer; + stderr: string; +} + +export function remote(destination: string, command: string, input = "", timeoutMs = 30_000): Promise { + if (!destination || destination.startsWith("-") || /\s/.test(destination)) { + throw new Error("Use an SSH config alias or user@hostname as the destination."); + } + return new Promise((resolve, reject) => { + const child = spawn("ssh", [ + "-T", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", + "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=10", + "-o", "ServerAliveCountMax=2", destination, command, + ], { stdio: "pipe" }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let size = 0; + let failure: Error | undefined; + const timer = setTimeout(() => { + failure = new Error("SSH deadline exceeded; remote outcome is unknown."); + child.kill("SIGKILL"); + }, timeoutMs); + const collect = (chunks: Buffer[], chunk: Buffer): void => { + size += chunk.length; + if (size > 32 * 1024 * 1024) { + failure = new Error("SSH output exceeded 32 MiB; remote outcome is unknown."); + child.kill("SIGKILL"); + return; + } + chunks.push(chunk); + }; + child.stdout.on("data", (chunk: Buffer) => collect(stdout, chunk)); + child.stderr.on("data", (chunk: Buffer) => collect(stderr, chunk)); + child.stdin.on("error", () => { + // SSH's exit status and stderr explain a closed input pipe. + }); + child.on("error", (error) => { + clearTimeout(timer); + reject(error); + }); + child.on("close", (code) => { + clearTimeout(timer); + if (failure) { + reject(failure); + } else { + resolve({ code, stdout: Buffer.concat(stdout), stderr: Buffer.concat(stderr).toString("utf8") }); + } + }); + child.stdin.end(input); + }); +} diff --git a/test/ssh.test.ts b/test/ssh.test.ts new file mode 100644 index 0000000..61e36d4 --- /dev/null +++ b/test/ssh.test.ts @@ -0,0 +1,15 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { quoteShell, remote } from "../src/ssh.js"; + +test("shell quoting preserves literal input", () => { + for (const value of ["", "hello", "a'b", "$(echo unsafe);\n*", "こんにちは"]) { + assert.equal(execFileSync("sh", ["-c", `printf %s ${quoteShell(value)}`], { encoding: "utf8" }), value); + } +}); + +test("rejects option-like SSH destinations", () => { + assert.throws(() => remote("-oProxyCommand=bad", "true")); + assert.throws(() => remote("home extra", "true")); +}); diff --git a/test/test_desktop.py b/test/test_desktop.py new file mode 100644 index 0000000..5184126 --- /dev/null +++ b/test/test_desktop.py @@ -0,0 +1,53 @@ +import importlib.util +from pathlib import Path +import unittest +import tempfile +import json +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("desktop", Path(__file__).parents[1] / "vm/desktop.py") +desktop = importlib.util.module_from_spec(spec) +spec.loader.exec_module(desktop) + + +class ValidationTests(unittest.TestCase): + def test_session_save(self): + with tempfile.TemporaryDirectory() as directory: + state = Path(directory) + with patch.object(desktop, "STATE", state): + desktop.save_session({"DISPLAY": ":0"}) + desktop.save_session({"DISPLAY": ":1"}) + path = state / "session.json" + self.assertEqual(json.loads(path.read_text()), {"DISPLAY": ":1"}) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(len(list(state.iterdir())), 1) + + def test_autostart(self): + with tempfile.TemporaryDirectory() as directory: + with patch.object(Path, "home", return_value=Path(directory)): + desktop.install_autostart() + desktop.install_autostart() + entries = list((Path(directory) / ".config/autostart").iterdir()) + self.assertEqual(len(entries), 1) + self.assertIn("--register-session\n", entries[0].read_text()) + self.assertIn("OnlyShowIn=XFCE;", entries[0].read_text()) + + def test_capture(self): + self.assertEqual(desktop.validate({"actions": []}, 100, 100), []) + + def test_bounds(self): + for x in (-1, 100, True, "2"): + with self.assertRaises(ValueError): + desktop.validate({"actions": [{"type": "click", "x": x, "y": 0}]}, 100, 100) + + def test_invalid_keys(self): + with self.assertRaises(ValueError): + desktop.validate({"actions": [{"type": "keys", "keys": ["--repeat"]}]}, 100, 100) + + def test_wait_budget(self): + with self.assertRaises(ValueError): + desktop.validate({"actions": [{"type": "wait", "milliseconds": 3000}] * 2}, 100, 100) + + +if __name__ == "__main__": + unittest.main() diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8460e50 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "outDir": "dist", + "rootDir": ".", + "noUncheckedIndexedAccess": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/vm/desktop.py b/vm/desktop.py new file mode 100644 index 0000000..c726e24 --- /dev/null +++ b/vm/desktop.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +import base64 +import datetime +import json +import os +from pathlib import Path +import subprocess +import sys +import uuid +import fcntl +import tempfile + +STATE = Path.home() / ".local/state/desktop-harness" + + +def run(*args, input=None): + return subprocess.run(args, input=input, check=True, capture_output=True, timeout=10).stdout + + +def integer(value, low, high): + if type(value) is not int or not low <= value <= high: + raise ValueError(f"Expected integer between {low} and {high}") + return value + + +def validate(request, width, height): + if not isinstance(request, dict) or not isinstance(request.get("actions"), list): + raise ValueError("Request must contain an actions array") + actions = request["actions"] + if len(actions) > 20: + raise ValueError("At most 20 actions per request") + total_wait = 0 + for action in actions: + if not isinstance(action, dict): + raise ValueError("Actions must be objects") + kind = action.get("type") + if kind == "click": + integer(action.get("x"), 0, width - 1) + integer(action.get("y"), 0, height - 1) + if action.get("button", "left") not in ("left", "middle", "right"): + raise ValueError("Unknown mouse button") + elif kind == "scroll": + if action.get("direction") not in ("up", "down"): + raise ValueError("Unknown scroll direction") + integer(action.get("steps"), 1, 30) + elif kind == "keys": + keys = action.get("keys") + if not isinstance(keys, list) or not 1 <= len(keys) <= 8: + raise ValueError("Supply 1–8 key names") + if any(not isinstance(key, str) or not key or not all(c.isalnum() or c == "_" for c in key) for key in keys): + raise ValueError("Invalid X11 key name") + elif kind == "text": + if not isinstance(action.get("text"), str) or len(action["text"]) > 20000 or "\0" in action["text"]: + raise ValueError("Text must contain at most 20000 characters and no NUL") + elif kind == "wait": + total_wait += integer(action.get("milliseconds"), 0, 5000) + else: + raise ValueError(f"Unsupported action: {kind}") + if total_wait > 5000: + raise ValueError("Total requested wait exceeds 5 seconds") + return actions + + +def install_autostart(): + directory = Path.home() / ".config/autostart" + directory.mkdir(parents=True, exist_ok=True) + # Desktop Exec quoting is not shell quoting; percent signs are field codes. + helper = str(Path(__file__).resolve()).replace("%", "%%") + for character in ("\\", '"', "`", "$"): + helper = helper.replace(character, "\\" + character) + (directory / "desktop-harness-session.desktop").write_text( + "[Desktop Entry]\n" + "Type=Application\n" + "Name=Desktop harness session registration\n" + f'Exec=python3 "{helper}" --register-session\n' + "Terminal=false\n" + "OnlyShowIn=XFCE;\n" + ) + + +def save_session(environment): + # Readers must see either the previous complete environment or the new one. + with tempfile.NamedTemporaryFile(mode="w", dir=STATE, delete=False) as file: + temporary = Path(file.name) + try: + json.dump(environment, file) + file.close() + temporary.replace(STATE / "session.json") + finally: + temporary.unlink(missing_ok=True) + + +def main(): + STATE.mkdir(parents=True, exist_ok=True) + if sys.argv[1:] == ["--install-autostart"]: + install_autostart() + print("Installed XFCE session registration autostart") + return + if sys.argv[1:] == ["--register-session"]: + if not os.environ.get("DISPLAY") or os.environ.get("XDG_SESSION_TYPE") == "wayland": + raise ValueError("Run registration from a terminal inside the XFCE/X11 desktop") + environment = {key: os.environ[key] for key in ("DISPLAY", "XAUTHORITY", "DBUS_SESSION_BUS_ADDRESS") if key in os.environ} + save_session(environment) + print("Registered graphical session") + return + if not (STATE / "session.json").exists(): + raise ValueError("No desktop session registered. Log into XFCE or run desktop.py --register-session in an XFCE terminal.") + environment = json.loads((STATE / "session.json").read_text()) + os.environ.update(environment) + with (STATE / "desktop.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + from PIL import ImageGrab + import io + import time + screenshot = ImageGrab.grab(xdisplay=os.environ["DISPLAY"]) + width, height = screenshot.size + raw = sys.stdin.read(256001) + if len(raw) > 256000: + raise ValueError("Request too large") + request = json.loads(raw) + actions = validate(request, width, height) + expected = request.get("expectedSize") + if expected is not None and expected != [width, height]: + raise ValueError("Display geometry changed; capture again before acting") + completed = 0 + for action in actions: + kind = action["type"] + try: + if kind == "click": + button = {"left": "1", "middle": "2", "right": "3"}[action.get("button", "left")] + run("xdotool", "mousemove", "--sync", str(action["x"]), str(action["y"]), "click", button) + elif kind == "scroll": + run("xdotool", "click", "--repeat", str(action["steps"]), "--delay", "50", "4" if action["direction"] == "up" else "5") + elif kind == "keys": + run("xdotool", "key", "--clearmodifiers", "+".join(action["keys"])) + elif kind == "text": + # xclip owns the selection in the background. Replacing the clipboard is intentional. + subprocess.run(["xclip", "-selection", "clipboard"], input=action["text"].encode(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=5) + run("xdotool", "key", "--clearmodifiers", "ctrl+v") + elif kind == "wait": + time.sleep(action["milliseconds"] / 1000) + completed += 1 + except Exception as error: + raise RuntimeError(f"Action {completed} failed after {completed} completed actions; partial effects possible: {error}") from error + screenshot = ImageGrab.grab(xdisplay=os.environ["DISPLAY"]) + buffer = io.BytesIO() + screenshot.save(buffer, format="PNG") + print(json.dumps({ + "observationId": str(uuid.uuid4()), + "capturedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "width": screenshot.width, + "height": screenshot.height, + "completedActions": completed, + "image": base64.b64encode(buffer.getvalue()).decode(), + })) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print(str(error), file=sys.stderr) + sys.exit(1)