From be20af6b84c7e3e22f9292a69f5df38c55e511c0 Mon Sep 17 00:00:00 2001 From: Daniel Ehrhardt Date: Sun, 11 Oct 2026 14:21:39 +0200 Subject: [PATCH] Android phones: review fixes - Saved logins on phones only for runs that may reveal secrets: what is typed on a phone can be read through its shell. Secrets go to adb on stdin (not in process arguments here), 2FA codes are masked too, the UI dump gets a fresh name each time. - android_assign only gives phones to other agents, never to the caller or its own chat. - adb stderr is capped like stdout. - Live view: video and still pictures side by side (viewers without WebCodecs watch along), stills only go to video viewers when the phone can't stream, restarts are spaced out per device (a congested viewer can't restart everyone's stream), no screenrecord left running after a stop. - Input on letterboxed video maps to the right pixel; edge pixels clamp. - pull_file refuses folders and cleans up after itself; open_app quoting; a refresh that must start adb isn't swallowed by one in flight. - Emulators nobody named, described or assigned are forgotten after a day out of sight; the page groups phones that aren't connected. --- .../src/components/android/device-card.tsx | 11 +++ .../src/components/android/phone-screen.tsx | 11 ++- .../components/settings/android-section.tsx | 4 +- .../src/pages/android/android-page.tsx | 46 +++++++---- packages/core/src/android/adb.ts | 9 ++- packages/core/src/android/screen.ts | 32 ++++++-- packages/core/src/android/service.ts | 39 +++++++-- packages/core/src/android/stream.ts | 48 ++++++++--- packages/core/src/android/tools.ts | 50 +++++++++--- packages/core/src/mcp/tools.ts | 18 ++--- packages/core/src/runner/prompt.ts | 2 +- packages/core/src/runner/runner.ts | 2 +- packages/core/src/server/routes/android.ts | 10 +-- packages/core/src/server/ws.ts | 8 +- packages/core/test/android.test.ts | 81 ++++++++++++++++--- packages/core/test/fixtures/fake-adb.ts | 9 ++- packages/shared/src/events.ts | 13 ++- packages/shared/src/models.ts | 6 +- 18 files changed, 307 insertions(+), 92 deletions(-) diff --git a/apps/desktop/src/components/android/device-card.tsx b/apps/desktop/src/components/android/device-card.tsx index e026325d..777adfd0 100644 --- a/apps/desktop/src/components/android/device-card.tsx +++ b/apps/desktop/src/components/android/device-card.tsx @@ -55,6 +55,7 @@ export function DeviceCard({ device, onEdit, onForget }: { device: AndroidDevice return (
+ {connected && ( )} + )}
+ {!connected && ( + + + + )}

{device.name}

diff --git a/apps/desktop/src/components/android/phone-screen.tsx b/apps/desktop/src/components/android/phone-screen.tsx index fdaedbd9..6339ed39 100644 --- a/apps/desktop/src/components/android/phone-screen.tsx +++ b/apps/desktop/src/components/android/phone-screen.tsx @@ -63,6 +63,7 @@ export function PhoneScreen({ const canvasRef = useRef(null); const viewRef = useRef(null); const size = useRef({ width: 0, height: 0 }); + const screenSize = useRef({ width: 0, height: 0 }); const [frameSize, setFrameSize] = useState<{ width: number; height: number } | null>(null); const [hasPicture, setHasPicture] = useState(false); const [error, setError] = useState(null); @@ -81,9 +82,15 @@ export function PhoneScreen({ const canvas = canvasRef.current; const { width, height } = size.current; if (!canvas || !width || !height) return null; + // The screen sits letterboxed inside the video (its edges are rounded to multiples of 16), the video inside the view. + const sw = screenSize.current.width || width; + const sh = screenSize.current.height || height; + const inVideo = Math.min(width / sw, height / sh); + const vx = (width - sw * inVideo) / 2 + rx * sw * inVideo; + const vy = (height - sh * inVideo) / 2 + ry * sh * inVideo; const rect = canvas.getBoundingClientRect(); const scale = Math.min(rect.width / width, rect.height / height); - return { x: (rect.width - width * scale) / 2 + rx * width * scale, y: (rect.height - height * scale) / 2 + ry * height * scale }; + return { x: (rect.width - width * scale) / 2 + vx * scale, y: (rect.height - height * scale) / 2 + vy * scale }; }, []); /* ------------------------------ the stream ------------------------------ */ @@ -141,6 +148,7 @@ export function PhoneScreen({ } if (e.kind === "frame") { size.current = { width: e.width, height: e.height }; + screenSize.current = e.screen; setFrameSize({ width: e.width, height: e.height }); const img = new Image(); img.onload = () => draw(img, e.width, e.height); @@ -151,6 +159,7 @@ export function PhoneScreen({ if (e.kind === "config") { decoder?.close(); size.current = { width: e.width, height: e.height }; + screenSize.current = e.screen; setFrameSize({ width: e.width, height: e.height }); waitingKey = true; try { diff --git a/apps/desktop/src/components/settings/android-section.tsx b/apps/desktop/src/components/settings/android-section.tsx index 513dd074..7c718b50 100644 --- a/apps/desktop/src/components/settings/android-section.tsx +++ b/apps/desktop/src/components/settings/android-section.tsx @@ -109,12 +109,12 @@ export function AndroidSection({ settings }: { settings: Settings }) { label="Type logins and 2FA codes into phones" htmlFor="android-vault-fill" disabled={!a.enabled} - description="Godmode types saved logins into password fields on the phone without the AI seeing them, but it can't tell which app a field belongs to. An agent that can use the phone's shell could read what is typed. Turning this on asks for your vault passphrase." + description="Godmode types saved logins into password fields on the phone. Whatever is typed on a phone can be read through its shell, so only agents you allow to see passwords get this; the others ask you to sign in on the phone. Turning this on asks for your vault passphrase." > void setVaultFill(on)} />
    - }>Typed by Godmode, not the AI + }>Only agents that may see passwords }>Passwords only into password fields }>Every fill audited } caution> diff --git a/apps/desktop/src/pages/android/android-page.tsx b/apps/desktop/src/pages/android/android-page.tsx index e501831e..955e2828 100644 --- a/apps/desktop/src/pages/android/android-page.tsx +++ b/apps/desktop/src/pages/android/android-page.tsx @@ -36,6 +36,28 @@ function SetupSteps() { ); } +function DeviceGrid({ devices, onEdit, onForget }: { devices: AndroidDevice[]; onEdit: (d: AndroidDevice) => void; onForget: (d: AndroidDevice) => void }) { + return ( +
    + + {devices.map((device, i) => ( + + onEdit(device)} onForget={() => onForget(device)} /> + + ))} + +
    + ); +} + /** Android phones connected over adb: watch them, take control, give them to agents. */ export default function AndroidPage() { const qc = useQueryClient(); @@ -178,23 +200,13 @@ export default function AndroidPage() { {connected} of {devices.length} connected · adb {adb?.version ?? ""} {adb?.managed ? " (Godmode's copy)" : ""}

    -
    - - {devices.map((device, i) => ( - - setEditing(device)} onForget={() => setForgetting(device)} /> - - ))} - -
    + d.state !== "disconnected")} onEdit={setEditing} onForget={setForgetting} /> + {devices.some((d) => d.state === "disconnected") && ( +
    +

    Not connected

    + d.state === "disconnected")} onEdit={setEditing} onForget={setForgetting} /> +
    + )} )}
diff --git a/packages/core/src/android/adb.ts b/packages/core/src/android/adb.ts index b51e5912..ab8c8faa 100644 --- a/packages/core/src/android/adb.ts +++ b/packages/core/src/android/adb.ts @@ -24,6 +24,7 @@ const DOWNLOADS: Partial { + const decoder = new TextDecoder(); try { - errText = await new Response(proc.stderr as ReadableStream).text(); + for await (const chunk of proc.stderr as ReadableStream) { + errText += decoder.decode(chunk, { stream: true }); + if (errText.length > 2 * MAX_STDERR) errText = errText.slice(-MAX_STDERR); + } } catch { /* closed */ } + if (errText.length > MAX_STDERR) errText = errText.slice(-MAX_STDERR); }; const outputs = Promise.all([readOut(), readErr()]); if (opts.stdin !== undefined) { diff --git a/packages/core/src/android/screen.ts b/packages/core/src/android/screen.ts index e80f1700..811e03f6 100644 --- a/packages/core/src/android/screen.ts +++ b/packages/core/src/android/screen.ts @@ -3,6 +3,7 @@ * here), touches and swipes (`input`), keys, typed text and the tree of what is on screen (`uiautomator dump`). * Coordinates are always the phone's own pixels in its current orientation; tools and the live view scale them. */ +import { randomBytes } from "node:crypto"; import { gunzipSync } from "node:zlib"; import type { AndroidKey } from "@godmode/shared"; import { fitSize, MAX_MODEL_PIXELS } from "../computer/geometry"; @@ -40,6 +41,18 @@ export function parseScreenSize(out: string): Size | null { return m ? { width: Number(m[1]), height: Number(m[2]) } : null; } +/** + * A point on a picture of the screen → the phone's pixel. The picture may be letterboxed (screenrecord keeps the + * screen's proportions inside a video rounded to multiples of 16); points on the bars land on the nearest edge. + */ +export function frameToScreen(screen: Size, frame: Size, x: number, y: number): { x: number; y: number } { + const scale = Math.min(frame.width / screen.width, frame.height / screen.height); + const offX = (frame.width - screen.width * scale) / 2; + const offY = (frame.height - screen.height * scale) / 2; + const clamp = (v: number, max: number) => Math.min(max - 1, Math.max(0, v)); + return { x: clamp((x - offX) / scale, screen.width), y: clamp((y - offY) / scale, screen.height) }; +} + export function rememberSize(serial: string, size: Size): void { sizes.set(serial, { size, at: Date.now() }); } @@ -181,8 +194,7 @@ export function keyCommand(key: AndroidKey | string): string | null { if (key === "notifications") return "cmd statusbar expand-notifications"; if (key === "quick_settings") return "cmd statusbar expand-settings"; if (key === "select_all") return "input keycombination 113 29"; - const code = KEYCODES[key as keyof typeof KEYCODES]; - if (code !== undefined) return `input keyevent ${code}`; + if (Object.hasOwn(KEYCODES, key)) return `input keyevent ${KEYCODES[key as keyof typeof KEYCODES]}`; if (/^KEYCODE_[A-Z0-9_]{1,40}$/.test(key)) return `input keyevent ${key}`; if (/^\d{1,3}$/.test(key)) return `input keyevent ${key}`; return null; @@ -227,6 +239,16 @@ export function typeCommands(text: string, chunk = 400): string[] { return out; } +/** + * Type a saved secret: the commands go to the phone's shell on stdin, so the value isn't in any process's arguments on + * this computer (on the phone, `input` still gets it as an argument — the shell user can see that, see tools.ts). + */ +export async function typeSecretText(serial: string, text: string, signal?: AbortSignal): Promise { + if (untypable(text).length || /[\n\t]/.test(text)) throw new AdbError("The value has characters adb can't type"); + const res = await adb(["shell"], { serial, stdin: `${typeCommands(text).join(" && ")} && exit 0 || exit 1\n`, timeoutMs: 30_000, signal }); + if (res.code !== 0) throw deviceError(res, "Typing"); +} + export async function typeText(serial: string, text: string, signal?: AbortSignal): Promise { const bad = untypable(text); if (bad.length) { @@ -306,10 +328,10 @@ export function parseUiXml(xml: string): UiNode[] { return nodes; } -const DUMP = "/data/local/tmp/godmode-ui.xml"; - export async function uiNodes(serial: string, signal?: AbortSignal): Promise { - const res = await adb(["shell", `uiautomator dump ${DUMP} >/dev/null 2>&1; cat ${DUMP} 2>/dev/null; rm -f ${DUMP}`], { serial, timeoutMs: 30_000, signal }); + // A fresh name each time: nothing else on the phone can prepare the file that is read back. + const dump = `/data/local/tmp/godmode-ui-${randomBytes(6).toString("hex")}.xml`; + const res = await adb(["shell", `uiautomator dump ${dump} >/dev/null 2>&1; cat ${dump} 2>/dev/null; rm -f ${dump}`], { serial, timeoutMs: 30_000, signal }); const xml = res.stdout.toString("utf8"); if (!xml.includes(" { const CONNECTION_RANK: Record = { usb: 0, emulator: 1, wifi: 2 }; -let refreshing: Promise | null = null; +let refreshing: { done: Promise; startServer: boolean } | null = null; let lastRefresh = 0; /** Look at what adb sees now. `startServer`: also when no adb server runs yet (adb starts one). */ export function refreshDevices(opts: { startServer?: boolean } = {}): Promise { - if (opts.startServer) lastInterest = Date.now(); - refreshing ??= doRefresh(!!opts.startServer).finally(() => { - refreshing = null; - lastRefresh = Date.now(); - }); - return refreshing; + const startServer = !!opts.startServer; + if (startServer) lastInterest = Date.now(); + // A look in progress that wouldn't start the server doesn't answer one that must. + if (refreshing && startServer && !refreshing.startServer) return refreshing.done.then(() => refreshDevices(opts)); + if (!refreshing) { + const done = doRefresh(startServer).finally(() => { + refreshing = null; + lastRefresh = Date.now(); + }); + refreshing = { done, startServer }; + } + return refreshing.done; } function snapshot(): string { @@ -265,6 +271,7 @@ async function doRefresh(startServer: boolean): Promise { live.clear(); for (const [id, l] of seen) live.set(id, l); pending = nextPending; + pruneEmulators(); await refreshPhoneStates(); } catch (err) { lastError = err instanceof AdbError ? err.message : err instanceof Error ? err.message : String(err); @@ -274,6 +281,24 @@ async function doRefresh(startServer: boolean): Promise { } } +let lastPrune = 0; + +/** + * Emulators come and go (a test run can start dozens): one nobody named, described or gave to an agent or chat is + * forgotten after a day out of sight. Real phones are always kept. + */ +function pruneEmulators(): void { + if (Date.now() - lastPrune < 3600_000) return; + lastPrune = Date.now(); + const cutoff = new Date(Date.now() - 24 * 3600_000).toISOString(); + const stale = all("SELECT * FROM android_devices WHERE kind = 'emulator' AND description = '' AND COALESCE(last_seen_at, created_at) < ?", cutoff); + for (const r of stale) { + if (live.has(r.id) || androidAssignments(r.id).length || r.updated_at !== r.created_at) continue; + run("DELETE FROM android_devices WHERE id = ?", r.id); + log.info("forgot an emulator not seen for a day", { device: r.id, name: r.name }); + } +} + /** Battery and screen of connected phones, at most every 30 s; the screen size once per connection. */ async function refreshPhoneStates(): Promise { await Promise.all( diff --git a/packages/core/src/android/stream.ts b/packages/core/src/android/stream.ts index 23e8ca66..a0bf4b2a 100644 --- a/packages/core/src/android/stream.ts +++ b/packages/core/src/android/stream.ts @@ -172,7 +172,10 @@ class PhoneStream { constructor(readonly deviceId: string) {} start() { - this.loop ??= this.run().catch((err) => log.warn(`live view of ${this.deviceId} stopped`, err)); + // Video and pictures side by side: a viewer that can't decode H.264 watches along with one that can. + this.loop ??= Promise.all([this.videoLoop(), this.frameLoop()]) + .then(() => undefined) + .catch((err) => log.warn(`live view of ${this.deviceId} stopped`, err)); } restart() { @@ -180,7 +183,12 @@ class PhoneStream { this.kill(); } - private kill() { + /** Nobody watches the video anymore: don't keep screenrecord waiting for the next change on screen. */ + videoGone() { + if (!this.wantsVideo()) this.kill(); + } + + kill() { try { this.proc?.kill("SIGKILL"); } catch { @@ -200,17 +208,27 @@ class PhoneStream { emit({ type: "android.video", deviceId: this.deviceId, kind: "error", message }); } - private async run() { + private async videoLoop() { + await this.loopWhile(() => this.wantsVideo(), () => this.streamVideo()); + } + + private async frameLoop() { + await this.loopWhile(() => this.wantsFrames(), () => this.sendFrame()); + } + + private async loopWhile(wanted: () => boolean, step: () => Promise) { while (!this.stopped) { + if (!wanted()) { + await sleep(300); + continue; + } if (!isConnected(this.deviceId)) { this.error(`${safeName(this.deviceId)} isn't connected.`); await sleep(2000); continue; } try { - if (this.wantsVideo()) await this.streamVideo(); - else if (this.wantsFrames()) await this.sendFrame(); - else await sleep(500); + await step(); } catch (err) { if (this.stopped) break; this.error(err instanceof Error ? err.message : String(err)); @@ -235,6 +253,7 @@ class PhoneStream { width: size.width, height: size.height, screen: { width: raw.width, height: raw.height }, + ...(this.noVideo ? { all: true } : {}), }); await sleep(Math.max(100, 400 - (Date.now() - started))); } @@ -244,12 +263,15 @@ class PhoneStream { const bin = resolveAdb(); if (!bin) throw new AdbError("adb is not installed", "not_installed"); const screen = await screenSize(serial, 0); + if (this.stopped || !this.wantsVideo()) return; const size = videoSize(screen); const proc = Bun.spawn( [bin.path, "-s", serial, "exec-out", `screenrecord --output-format=h264 --size ${size.width}x${size.height} --bit-rate ${BIT_RATE} -`], { stdin: "ignore", stdout: "pipe", stderr: "pipe", env: childEnv() as Record }, ); this.proc = proc; + // Stopped while it was starting: stop() found no process to kill. + if (this.stopped || !this.wantsVideo()) this.kill(); this.restartWanted = false; let frames = 0; const startedAt = Date.now(); @@ -354,6 +376,9 @@ function safeName(id: string): string { const streams = new Map(); const restartTimers = new Map>(); +const lastRestart = new Map(); +/** A new viewer gets its keyframe soon; a congested one waits longer, so one slow viewer can't keep restarting everyone's. */ +const RESTART_GAP_MS = { join: 1500, congested: 6000 }; /** Viewers of a phone changed (from the WebSocket hub): start, restart for a keyframe, or stop its stream. */ export function setViewers(deviceId: string, video: number, frames: number, joined: boolean): void { @@ -375,18 +400,21 @@ export function setViewers(deviceId: string, video: number, frames: number, join } stream.videoViewers = video; stream.frameViewers = frames; - if (joined && video > 0) requestKeyframe(deviceId); + if (video === 0) stream.videoGone(); + if (joined && video > 0) requestKeyframe(deviceId, "join"); } -/** A viewer needs a keyframe (joined, or dropped frames): restart the stream, at most every 1.5 s. */ -export function requestKeyframe(deviceId: string): void { +/** A viewer needs a keyframe (it joined, or dropped frames): restart the stream, spaced out per device. */ +export function requestKeyframe(deviceId: string, reason: keyof typeof RESTART_GAP_MS = "congested"): void { if (restartTimers.has(deviceId)) return; + const wait = Math.max(300, (lastRestart.get(deviceId) ?? 0) + RESTART_GAP_MS[reason] - Date.now()); restartTimers.set( deviceId, setTimeout(() => { restartTimers.delete(deviceId); + lastRestart.set(deviceId, Date.now()); streams.get(deviceId)?.restart(); - }, 300), + }, wait), ); } diff --git a/packages/core/src/android/tools.ts b/packages/core/src/android/tools.ts index 11804d14..baacb09f 100644 --- a/packages/core/src/android/tools.ts +++ b/packages/core/src/android/tools.ts @@ -10,6 +10,7 @@ import { randomBytes } from "node:crypto"; import { z } from "zod"; import { ANDROID_KEYS } from "@godmode/shared"; import { getAgent } from "../agents/service"; +import { chatFillOnly } from "../services/conversations"; import { bus } from "../events/bus"; import { logger } from "../log"; import { audit } from "../services/audit"; @@ -23,7 +24,7 @@ import { localPath } from "../ssh/tools"; import { AdbError, adb, adbErrorText, shq } from "./adb"; import { runAndroidDeviceIds } from "./assignments"; import { androidRun, ensureConnected, noticeFailure, promptDevices, type PromptAndroidDevice } from "./service"; -import { focusedField, longPress, modelSize, pressKey, screenSize, screenshot, swipe, tap, typeText, uiNodes, untypable, type Size, type UiNode } from "./screen"; +import { focusedField, longPress, modelSize, pressKey, screenSize, screenshot, swipe, tap, typeSecretText, typeText, uiNodes, untypable, type Size, type UiNode } from "./screen"; const log = logger("android"); @@ -41,6 +42,18 @@ export const ANDROID_INSTRUCTIONS = const VAULT_FILL_OFF = 'Typing saved logins and 2FA codes into phones is turned off. Ask the human to turn on "Logins and 2FA codes" in Settings → Android phones, then try again.'; +const FILL_NEEDS_REVEAL = + "Godmode only types saved logins into a phone for agents that may see passwords: the phone's shell can read what is typed. Ask the human to sign in on the phone (they can do it in the live view)."; + +/** + * Typing a secret on a phone is as good as showing it: `input` gets it as an argument, which the phone's shell user — + * the one the `shell` tool runs as — can read. So only runs that may reveal secrets get the fills. + */ +function mayFill(ctx: { agentId: string; conversationId: string | null }): boolean { + if (!getSettings().android.vaultFill) return false; + const agent = getAgent(ctx.agentId); + return agent.permissions.secretAccess === "reveal" && !(ctx.conversationId && chatFillOnly(ctx.conversationId)); +} function text(t: string, isError = false): AndroidToolResult { return { content: [{ type: "text", text: t }], ...(isError ? { isError: true } : {}) }; @@ -75,6 +88,12 @@ function mask(runId: string, s: string): string { return out; } +function remember(runId: string, secret: string) { + const set = typedSecrets.get(runId) ?? new Set(); + set.add(secret); + typedSecrets.set(runId, set); +} + export function forgetAndroidRun(runId: string): void { typedSecrets.delete(runId); for (const key of [...shots.keys()]) if (key.startsWith(`${runId}:`)) shots.delete(key); @@ -89,6 +108,7 @@ interface ToolEnv { serial: string; runId: string; agentId: string; + conversationId: string | null; folders: string[]; signal?: AbortSignal; } @@ -129,12 +149,13 @@ async function after(env: ToolEnv, note: string, want: boolean | undefined): Pro async function toScreen(env: ToolEnv, c: [number, number], what: string): Promise<{ x: number; y: number; screen: Size }> { const screen = await screenSize(env.serial); let shot = shots.get(shotKey(env.runId, env.device.id)); - if (shot && (shot.screen.width !== screen.width || shot.screen.height !== screen.height)) { + const portrait = (s: Size) => s.height >= s.width; + if (shot && portrait(shot.screen) !== portrait(screen)) { throw new AdbError(`The phone turned since your last screenshot (now ${screen.width}×${screen.height}). Take a new screenshot and use its coordinates.`); } shot ??= { ...modelSize(screen, maxEdge()), screen }; const [x, y] = c; - if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x > shot.width || y > shot.height) { + if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x >= shot.width || y >= shot.height) { throw new AdbError(`${what} [${x}, ${y}] is outside the screenshot (${shot.width}×${shot.height}).`); } return { x: (x * screen.width) / shot.width, y: (y * screen.height) / shot.height, screen }; @@ -447,6 +468,14 @@ const TOOLS: AndroidTool[] = [ local_path: z.string().max(4096).optional(), }), run: async (args, env) => { + const kind = await adb(["shell", `if [ -d ${shq(args.remote_path)} ]; then echo dir; elif [ -e ${shq(args.remote_path)} ]; then echo file; else echo none; fi`], { + serial: env.serial, + timeoutMs: 15_000, + signal: env.signal, + }); + const what = kind.stdout.toString("utf8").trim(); + if (what === "dir") return text(`${args.remote_path} is a folder. Copy files one at a time (pack a folder with tar on the phone first).`, true); + if (what === "none") return text(`No such file on ${env.device.name}: ${args.remote_path}`, true); const name = posix.basename(args.remote_path.replace(/\/+$/, "")) || "download"; const target = args.local_path?.trim() || join(getAgent(env.agentId).repoPath, "workspace", "downloads", name); let local = localPath(target, env.folders, "write"); @@ -458,7 +487,7 @@ const TOOLS: AndroidTool[] = [ if (res.code !== 0) return text(`Copying failed: ${adbErrorText(res)}`, true); renameSync(partial, local); } finally { - rmSync(partial, { force: true }); + rmSync(partial, { force: true, recursive: true }); } return text(`Copied ${args.remote_path} from ${env.device.name} to ${local}.`); }, @@ -477,6 +506,7 @@ const TOOLS: AndroidTool[] = [ }), run: async (args, env) => { if (!getSettings().android.vaultFill) return text(VAULT_FILL_OFF, true); + if (!mayFill(env)) return text(FILL_NEEDS_REVEAL, true); const agent = getAgent(env.agentId); const secret = revealForAgent(agent, args.credentialId); const value = args.field === "username" ? secret.username : secret.password; @@ -499,10 +529,10 @@ const TOOLS: AndroidTool[] = [ record(false, { error: "not_password", pkg: field.pkg }); return text("The focused field isn't a password field, so Godmode didn't type the password there. Tap the password field and try again.", true); } - (typedSecrets.get(env.runId) ?? typedSecrets.set(env.runId, new Set()).get(env.runId)!).add(value); + remember(env.runId, value); ripple(env, "type"); try { - await typeText(env.serial, value, env.signal); + await typeSecretText(env.serial, value, env.signal); if (args.field === "password") { // Focus may have moved while typing: take the password back out of whatever field got it. const now = focusedField(await uiNodes(env.serial, env.signal).catch(() => [])); @@ -535,6 +565,7 @@ const TOOLS: AndroidTool[] = [ }), run: async (args, env) => { if (!getSettings().android.vaultFill) return text(VAULT_FILL_OFF, true); + if (!mayFill(env)) return text(FILL_NEEDS_REVEAL, true); const agent = getAgent(env.agentId); let id = args.totpId ?? null; if (args.credentialId) { @@ -557,8 +588,9 @@ const TOOLS: AndroidTool[] = [ code = codeForAgent(agent, id); } const meta = { field: "totp", runId: env.runId, deviceId: env.device.id, credentialId: args.credentialId ?? null }; + remember(env.runId, code.code); try { - await typeText(env.serial, code.code, env.signal); + await typeSecretText(env.serial, code.code, env.signal); if (args.submit) await pressKey(env.serial, "enter", { signal: env.signal }); } catch (err) { audit(`agent:${agent.id}`, "totp.fill", id, { ...meta, ok: false, error: err instanceof Error ? err.message : String(err) }); @@ -592,7 +624,7 @@ function devicesOf(ctx: RunContext): PromptAndroidDevice[] { export function listAndroidTools(ctx: RunContext): { name: string; description: string; inputSchema: Record }[] { if (!androidRun(ctx.runId)) return []; - const fill = getSettings().android.vaultFill; + const fill = mayFill(ctx); return [LIST_TOOL, ...TOOLS.filter((t) => fill || !t.name.startsWith("fill_"))].map((t) => ({ name: t.name, description: t.description, inputSchema: jsonSchema(t.name, t.schema) })); } @@ -649,7 +681,7 @@ export async function callAndroidTool(ctx: RunContext, name: string, args: unkno setTimeout(() => audited.delete(key), 6 * 3600_000).unref?.(); audit(`agent:${ctx.agentId}`, "android.use", device.id, { runId: ctx.runId }); } - const result = await tool.run(parsed as never, { device, serial, runId: ctx.runId, agentId: ctx.agentId, folders: run.folders, signal: run.signal }); + const result = await tool.run(parsed as never, { device, serial, runId: ctx.runId, agentId: ctx.agentId, conversationId: ctx.conversationId, folders: run.folders, signal: run.signal }); return { ...result, content: result.content.map((c) => (c.type === "text" ? { ...c, text: mask(ctx.runId, c.text) } : c)) }; } catch (err) { if (err instanceof z.ZodError) return text(`Invalid arguments: ${err.issues.map((i) => `${i.path.join(".") || "input"}: ${i.message}`).join("; ")}`, true); diff --git a/packages/core/src/mcp/tools.ts b/packages/core/src/mcp/tools.ts index a9242e0a..80758700 100644 --- a/packages/core/src/mcp/tools.ts +++ b/packages/core/src/mcp/tools.ts @@ -2695,22 +2695,18 @@ const TOOLS: ToolDef[] = [ defineTool({ name: "android_assign", description: - "Let an agent (every run) or this chat see and control an Android phone with the android tools, from its next run. The phone is real: give it only to agents whose job needs it. Only the human can take it away again.", + "Let another agent see and control an Android phone with the android tools in every run, from its next run. The phone is real: give it only to agents whose job needs it. You can't give a phone to yourself or your own chat (the human picks phones for a chat in its message box), and only the human can take one away again.", schema: z.object({ deviceId: z.string(), - target: z.enum(["agent", "this_chat"]), - id: z.string().optional().describe("Agent id (not needed for this_chat)"), + agentId: z.string().describe("The agent that gets the phone"), }), when: managesSetup, - run: async ({ deviceId, target, id }, { agent, ctx }) => { + run: async ({ deviceId, agentId }, { agent, ctx }) => { if (lockedVm(ctx)) return fail("This task runs in a virtual machine and is kept off the human's computer, where the phones are plugged in."); - if (target === "agent" && !id) return fail("Pass the agent's id."); - if (target === "this_chat" && !ctx.conversationId) return fail("There is no chat here: assign the phone to an agent."); - if (target === "agent") { - const refusal = revealTargetRefusal(agent, ctx, getAgent(id!), "give it a phone"); - if (refusal) return fail(refusal); - } - const device = await assignDevice(deviceId, { kind: target === "agent" ? "agent" : "conversation", id: target === "agent" ? id! : ctx.conversationId, assigned: true }, `agent:${agent.id}`); + if (agentId === agent.id) return fail("You can't give yourself a phone. Ask the human: they pick phones for a chat in its message box, or for an agent in its settings."); + const refusal = revealTargetRefusal(agent, ctx, getAgent(agentId), "give it a phone"); + if (refusal) return fail(refusal); + const device = await assignDevice(deviceId, { kind: "agent", id: agentId, assigned: true }, `agent:${agent.id}`); return json({ id: device.id, name: device.name, state: device.state, usedBy: device.assignments.map((a) => `${a.kind} ${a.name}`), note: "Applies from the next run." }); }, }), diff --git a/packages/core/src/runner/prompt.ts b/packages/core/src/runner/prompt.ts index 429c447a..6093913f 100644 --- a/packages/core/src/runner/prompt.ts +++ b/packages/core/src/runner/prompt.ts @@ -506,7 +506,7 @@ ${devices.map(androidLine).join("\n")} - Look, act, look again: take a screenshot (or \`ui_elements\`) before you tap, and check the result after every step. Popups, permission dialogs and keyboards change the screen. - ${human} watches the phone live and may take over. If it is locked with a PIN, pattern or fingerprint, ask ${human} to unlock it — never try to guess or bypass a lock screen. - It is a real phone with ${human}'s accounts and data: don't buy, post, send messages, delete, factory-reset, or change accounts, security or developer settings unless ${human} asked for exactly that. -- ${vaultFill ? "Signing in inside an app: find the login with `vault_list_logins`, tap the field, then call `fill_login({ credentialId, field })` or `fill_totp({ credentialId })` — Godmode types the value and you never see it; passwords only go into password fields. Only fill a login in its own app or site." : `Godmode doesn't type saved logins or 2FA codes into phones: ${human} hasn't allowed it. If a task needs a sign-in on the phone, ask ${human} to sign in, or to turn on "Logins and 2FA codes" in Settings → Android phones.`} Never type a password or 2FA code with \`type_text\`.`; +- ${vaultFill ? "Signing in inside an app: find the login with `vault_list_logins`, tap the field, then call `fill_login({ credentialId, field })` or `fill_totp({ credentialId })` — Godmode types the value and you never see it; passwords only go into password fields. Only fill a login in its own app or site." : `Godmode doesn't type saved logins or 2FA codes into phones for you. If a task needs a sign-in on the phone, ask ${human} to sign in there (they can do it in the live view); apps stay signed in.`} Never type a password or 2FA code with \`type_text\`.`; } function computerSection(target: ComputerTarget, human: string, canReveal: boolean): string { diff --git a/packages/core/src/runner/runner.ts b/packages/core/src/runner/runner.ts index 80b1cbac..182ab6d1 100644 --- a/packages/core/src/runner/runner.ts +++ b/packages/core/src/runner/runner.ts @@ -2010,7 +2010,7 @@ async function runClaude(job: Job, agent: Agent, res: Resources): Promise { - const p = { x: x * sx, y: y * sy }; - if (p.x < 0 || p.y < 0 || p.x > screen.width || p.y > screen.height) throw badRequest("That point is outside the screen."); - return p; - }; + const at = (x: number, y: number) => frameToScreen(screen, frame?.width && frame.height ? frame : screen, x, y); if (event.type === "tap") { const p = at(event.x, event.y); return await tap(serial, p.x, p.y); diff --git a/packages/core/src/server/ws.ts b/packages/core/src/server/ws.ts index e39e3357..e1c5849a 100644 --- a/packages/core/src/server/ws.ts +++ b/packages/core/src/server/ws.ts @@ -130,7 +130,7 @@ let onComputerSubscribe: ((view: string, subscribed: boolean) => void) | null = /** Hook invoked when the viewers of an Android phone change (see android/stream.ts). */ let onAndroidViewers: ((deviceId: string, video: number, frames: number, joined: boolean) => void) | null = null; -let onAndroidKeyframe: ((deviceId: string) => void) | null = null; +let onAndroidKeyframe: ((deviceId: string, reason: "join" | "congested") => void) | null = null; export function setAndroidViewHandlers(handlers: { viewers: typeof onAndroidViewers; keyframe: typeof onAndroidKeyframe }) { onAndroidViewers = handlers.viewers; @@ -154,7 +154,7 @@ function changeAndroidSubscription(ws: HubSocket, deviceId: string, mode: "video if (current === mode) { if (mode === "video") { (ws.data.androidNeedsKey ??= new Set()).add(deviceId); - onAndroidKeyframe?.(deviceId); + onAndroidKeyframe?.(deviceId, "join"); } return; } @@ -182,10 +182,10 @@ function sendAndroid(event: Extract) { writeFileSync(join(dir, "state.json"), JSON.stringify({ devices, phones })); } -function calls(): { serial: string | null; args: string[] }[] { +function calls(): { serial: string | null; args: string[]; stdin?: string }[] { try { return readFileSync(join(dir, "calls.jsonl"), "utf8") .split("\n") @@ -62,7 +62,7 @@ function clearCalls() { writeFileSync(join(dir, "calls.jsonl"), ""); } -const shellCalls = () => calls().filter((c) => c.args[0] === "shell").map((c) => c.args.slice(1).join(" ")); +const shellCalls = () => calls().filter((c) => c.args[0] === "shell").map((c) => c.stdin ?? c.args.slice(1).join(" ")); const PIXEL = { serialno: "R58M12ABCDE", manufacturer: "Google", model: "Pixel 8", release: "15", sdk: 35, width: 1080, height: 2400, ui: UI.replace('focused="FOCUSED"', 'focused="true"').replace('password="PASSWORD"', 'password="true"') }; @@ -162,6 +162,17 @@ describe("parsing", () => { ]); expect(videoSize({ width: 1080, height: 2400 })).toEqual({ width: 576, height: 1280 }); }); + + test("points on a letterboxed video map to the phone's pixels", () => { + // 1080×2340 in a 576×1280 video: 576/1080 scales the height to 1248, so 16 px bars top and bottom. + const screen = { width: 1080, height: 2340 }; + expect(videoSize(screen)).toEqual({ width: 576, height: 1280 }); + const video = { width: 576, height: 1280 }; + expect(frameToScreen(screen, video, 288, 16)).toEqual({ x: 540, y: 0 }); + expect(frameToScreen(screen, video, 288, 640)).toEqual({ x: 540, y: 1170 }); + expect(frameToScreen(screen, video, 576, 1280)).toEqual({ x: 1079, y: 2339 }); + expect(frameToScreen(screen, screen, 10, 20)).toEqual({ x: 10, y: 20 }); + }); }); describe("phones", () => { @@ -242,11 +253,11 @@ describe("android MCP tools", () => { return { text: r.result.content.find((c) => c.type === "text")?.text ?? "", images: r.result.content.filter((c) => c.type === "image").length, isError: !!r.result.isError }; } - function runWith(ids: string[]) { - const conv = createConversation({ agentId: agent.id, androidDeviceIds: ids }); + function runWith(ids: string[], who: Agent = agent) { + const conv = createConversation({ agentId: who.id, androidDeviceIds: ids }); const runId = `run_and_${Math.random().toString(36).slice(2)}`; attachAndroid(runId, ids, [dir]); - const token = issueRunToken({ runId, agentId: agent.id, conversationId: conv.id, workspaceId: null, depth: 0 }); + const token = issueRunToken({ runId, agentId: who.id, conversationId: conv.id, workspaceId: null, depth: 0 }); return { token, runId, done: () => (revokeRunToken(token), detachAndroid(runId), forgetAndroidRun(runId)) }; } @@ -332,17 +343,25 @@ describe("android MCP tools", () => { } }); - test("saved logins go into password fields only, and never come back", async () => { + test("saved logins: only for agents that may see them, into password fields only, never in the output", async () => { const login = createCredential({ name: "Shop", url: "https://shop.example.com", username: "owner@example.com", password: "s3cret-Pass!" }); - const run = runWith([pixelId]); + const revealer = await makeAgent({ name: "Revealer", permissions: { secretAccess: "reveal" } }); + const run = runWith([pixelId], revealer); + const fillOnly = runWith([pixelId]); expect((await call(run.token, "fill_login", { credentialId: login.id, field: "password" })).text).toContain("turned off"); updateSettings({ android: { vaultFill: true } }); try { + // The phone's shell can read what is typed: an agent that may only fill doesn't get the tools. + expect((await rpc(fillOnly.token, "tools/list")).result.tools!.map((t) => t.name)).not.toContain("fill_login"); + expect((await call(fillOnly.token, "fill_login", { credentialId: login.id, field: "password" })).text).toContain("only types saved logins into a phone for agents that may see passwords"); + expect((await rpc(run.token, "tools/list")).result.tools!.map((t) => t.name)).toContain("fill_login"); clearCalls(); const filled = await call(run.token, "fill_login", { credentialId: login.id, field: "password" }); expect(filled.isError).toBe(false); expect(filled.text).toContain('Typed the password of "Shop" into the password field of com.example.shop'); - expect(shellCalls().some((c) => c.includes("input text 's3cret-Pass!'"))).toBe(true); + // Typed through stdin: the password isn't in adb's arguments on this computer. + expect(calls().some((c) => c.stdin?.startsWith("input text 's3cret-Pass!'"))).toBe(true); + expect(calls().some((c) => c.args.join(" ").includes("s3cret"))).toBe(false); // Whatever a command prints afterwards, the password is masked. expect((await call(run.token, "shell", { command: "echo s3cret-Pass!" })).text).toBe("Exit code: 0\n••••••••"); // Not a password field: refused. @@ -354,6 +373,48 @@ describe("android MCP tools", () => { updateSettings({ android: { vaultFill: false } }); writeState("R58M12ABCDE device\n", { R58M12ABCDE: PIXEL }); run.done(); + fillOnly.done(); + } + }); + + test("files from the phone: one at a time, into the run's folders", async () => { + const run = runWith([pixelId]); + try { + expect((await call(run.token, "pull_file", { remote_path: "/sdcard/DCIM" })).text).toContain("is a folder"); + expect((await call(run.token, "pull_file", { remote_path: "/sdcard/missing.jpg" })).text).toContain("No such file"); + const pulled = await call(run.token, "pull_file", { remote_path: "/sdcard/Download/a.txt", local_path: join(dir, "a.txt") }); + expect(pulled.text).toBe(`Copied /sdcard/Download/a.txt from Shop phone to ${realpathSync(dir)}/a.txt.`); + expect(readFileSync(join(dir, "a.txt"), "utf8")).toStartWith("pulled /sdcard/Download/a.txt"); + expect((await call(run.token, "pull_file", { remote_path: "/sdcard/Download/a.txt", local_path: "/etc/a.txt" })).text).toContain("outside the folders"); + } finally { + run.done(); + } + }); + + test("a manager gives phones to other agents, never to itself", async () => { + const boss = await makeAgent({ name: "Boss", permissions: { canManageAgents: true } }); + const conv = createConversation({ agentId: boss.id }); + const token = issueRunToken({ runId: "run_and_boss", agentId: boss.id, conversationId: conv.id, workspaceId: null, depth: 0 }); + const gateway = async (name: string, args: Record) => { + const res = await fetch(`${env.baseUrl}/mcp`, { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", Accept: "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }), + }); + const r = (await res.json()) as { result: { content: { text: string }[]; isError?: boolean } }; + return { text: r.result.content[0]!.text, isError: !!r.result.isError }; + }; + try { + const self = await gateway("android_assign", { deviceId: pixelId, agentId: boss.id }); + expect(self.isError).toBe(true); + expect(self.text).toContain("can't give yourself a phone"); + expect(getAgent(boss.id).androidDeviceIds).toEqual([]); + const worker = await makeAgent({ name: "Phone Worker" }); + expect((await gateway("android_assign", { deviceId: pixelId, agentId: worker.id })).isError).toBe(false); + expect(getAgent(worker.id).androidDeviceIds).toEqual([pixelId]); + expect((await gateway("android_devices_list", {})).text).toContain('"name": "Shop phone"'); + } finally { + revokeRunToken(token); } }); diff --git a/packages/core/test/fixtures/fake-adb.ts b/packages/core/test/fixtures/fake-adb.ts index 87718521..5b6963ef 100755 --- a/packages/core/test/fixtures/fake-adb.ts +++ b/packages/core/test/fixtures/fake-adb.ts @@ -26,7 +26,10 @@ if (args[0] === "-s") { serial = args[1]!; args = args.slice(2); } -appendFileSync(join(dir, "calls.jsonl"), `${JSON.stringify({ serial, args })}\n`); +// `adb shell` without a command reads its script from stdin (logged apart from the arguments). +const stdin = args.length === 1 && args[0] === "shell" ? (await Bun.stdin.text()).trim() : undefined; +appendFileSync(join(dir, "calls.jsonl"), `${JSON.stringify({ serial, args, ...(stdin !== undefined ? { stdin } : {}) })}\n`); +if (stdin !== undefined) args = ["shell", stdin]; const phone = serial ? state.phones[serial] : undefined; const out = (s: string) => process.stdout.write(s); @@ -67,6 +70,8 @@ if (args[0] === "version") { out(" mWakefulness=Awake\n status: 2\n level: 81\n USB powered: true\n"); } else if (cmd.includes("uiautomator dump")) { out(phone!.ui ?? ""); + } else if (cmd.startsWith("if [ -d ")) { + out(cmd.includes("/DCIM'") ? "dir\n" : cmd.includes("/missing") ? "none\n" : "file\n"); } else if (cmd.startsWith("cmd package resolve-activity")) { out(cmd.endsWith("com.example.shop") ? "priority=0 preferredOrder=0 match=0x108000\ncom.example.shop/.MainActivity$Launcher\n" : "No activity found\n"); } else if (cmd.startsWith("am start")) { @@ -81,6 +86,8 @@ if (args[0] === "version") { } else if (cmd.startsWith("exit ")) { process.exit(Number(cmd.slice(5))); } +} else if (args[0] === "pull") { + Bun.write(args[2]!, `pulled ${args[1]}`); } else if (args[0] === "connect") { out(`connected to ${args[1]}\n`); } diff --git a/packages/shared/src/events.ts b/packages/shared/src/events.ts index 184f2d02..f242606a 100644 --- a/packages/shared/src/events.ts +++ b/packages/shared/src/events.ts @@ -133,7 +133,18 @@ export type ServerEvent = screen: { width: number; height: number }; } | { type: "android.video"; deviceId: ID; kind: "chunk"; data: string; key: boolean; timestamp: number } - | { type: "android.video"; deviceId: ID; kind: "frame"; data: string; mime: "image/png" | "image/jpeg"; width: number; height: number; screen: { width: number; height: number } } + | { + type: "android.video"; + deviceId: ID; + kind: "frame"; + data: string; + mime: "image/png" | "image/jpeg"; + width: number; + height: number; + screen: { width: number; height: number }; + /** For video viewers too: this phone can't stream video. */ + all?: boolean; + } | { type: "android.video"; deviceId: ID; kind: "error"; message: string } /** An agent touched a phone's screen (drawn as a ripple in the live view). x/y 0–1 of the screen. */ | { type: "android.action"; deviceId: ID; runId: ID; action: string; x?: number; y?: number; toX?: number; toY?: number } diff --git a/packages/shared/src/models.ts b/packages/shared/src/models.ts index fd865ed2..5b9e0dcb 100644 --- a/packages/shared/src/models.ts +++ b/packages/shared/src/models.ts @@ -1363,9 +1363,9 @@ export interface AndroidSettings { /** Godmode looks for Android phones connected over adb, and agents may control the phones assigned to them. */ enabled: boolean; /** - * Agents may type saved logins and 2FA codes into a phone (android tools `fill_login` / `fill_totp`). Godmode types - * the values, so the model never sees them, and passwords only go into password fields — but the fill can't be bound - * to the login's app or website. Turning it on needs a vault grant. + * Godmode may type saved logins and 2FA codes into a phone (android tools `fill_login` / `fill_totp`), passwords only + * into password fields. What is typed on a phone can be read through its shell, so only runs that may reveal secrets + * get the tools. Turning it on needs a vault grant. */ vaultFill: boolean; /** Custom adb binary; empty = one from the Android SDK, PATH or Homebrew, else Godmode's own copy (installed on demand). */