diff --git a/apps/desktop/src/components/android/device-card.tsx b/apps/desktop/src/components/android/device-card.tsx index e026325d..777adfd0 100644 --- a/apps/desktop/src/components/android/device-card.tsx +++ b/apps/desktop/src/components/android/device-card.tsx @@ -55,6 +55,7 @@ export function DeviceCard({ device, onEdit, onForget }: { device: AndroidDevice return (
+ {connected && ( )} + )}
+ {!connected && ( + + + + )}

{device.name}

diff --git a/apps/desktop/src/components/android/phone-screen.tsx b/apps/desktop/src/components/android/phone-screen.tsx index fdaedbd9..6339ed39 100644 --- a/apps/desktop/src/components/android/phone-screen.tsx +++ b/apps/desktop/src/components/android/phone-screen.tsx @@ -63,6 +63,7 @@ export function PhoneScreen({ const canvasRef = useRef(null); const viewRef = useRef(null); const size = useRef({ width: 0, height: 0 }); + const screenSize = useRef({ width: 0, height: 0 }); const [frameSize, setFrameSize] = useState<{ width: number; height: number } | null>(null); const [hasPicture, setHasPicture] = useState(false); const [error, setError] = useState(null); @@ -81,9 +82,15 @@ export function PhoneScreen({ const canvas = canvasRef.current; const { width, height } = size.current; if (!canvas || !width || !height) return null; + // The screen sits letterboxed inside the video (its edges are rounded to multiples of 16), the video inside the view. + const sw = screenSize.current.width || width; + const sh = screenSize.current.height || height; + const inVideo = Math.min(width / sw, height / sh); + const vx = (width - sw * inVideo) / 2 + rx * sw * inVideo; + const vy = (height - sh * inVideo) / 2 + ry * sh * inVideo; const rect = canvas.getBoundingClientRect(); const scale = Math.min(rect.width / width, rect.height / height); - return { x: (rect.width - width * scale) / 2 + rx * width * scale, y: (rect.height - height * scale) / 2 + ry * height * scale }; + return { x: (rect.width - width * scale) / 2 + vx * scale, y: (rect.height - height * scale) / 2 + vy * scale }; }, []); /* ------------------------------ the stream ------------------------------ */ @@ -141,6 +148,7 @@ export function PhoneScreen({ } if (e.kind === "frame") { size.current = { width: e.width, height: e.height }; + screenSize.current = e.screen; setFrameSize({ width: e.width, height: e.height }); const img = new Image(); img.onload = () => draw(img, e.width, e.height); @@ -151,6 +159,7 @@ export function PhoneScreen({ if (e.kind === "config") { decoder?.close(); size.current = { width: e.width, height: e.height }; + screenSize.current = e.screen; setFrameSize({ width: e.width, height: e.height }); waitingKey = true; try { diff --git a/apps/desktop/src/components/settings/android-section.tsx b/apps/desktop/src/components/settings/android-section.tsx index 513dd074..7c718b50 100644 --- a/apps/desktop/src/components/settings/android-section.tsx +++ b/apps/desktop/src/components/settings/android-section.tsx @@ -109,12 +109,12 @@ export function AndroidSection({ settings }: { settings: Settings }) { label="Type logins and 2FA codes into phones" htmlFor="android-vault-fill" disabled={!a.enabled} - description="Godmode types saved logins into password fields on the phone without the AI seeing them, but it can't tell which app a field belongs to. An agent that can use the phone's shell could read what is typed. Turning this on asks for your vault passphrase." + description="Godmode types saved logins into password fields on the phone. Whatever is typed on a phone can be read through its shell, so only agents you allow to see passwords get this; the others ask you to sign in on the phone. Turning this on asks for your vault passphrase." > void setVaultFill(on)} />
    - }>Typed by Godmode, not the AI + }>Only agents that may see passwords }>Passwords only into password fields }>Every fill audited } caution> diff --git a/apps/desktop/src/pages/android/android-page.tsx b/apps/desktop/src/pages/android/android-page.tsx index e501831e..955e2828 100644 --- a/apps/desktop/src/pages/android/android-page.tsx +++ b/apps/desktop/src/pages/android/android-page.tsx @@ -36,6 +36,28 @@ function SetupSteps() { ); } +function DeviceGrid({ devices, onEdit, onForget }: { devices: AndroidDevice[]; onEdit: (d: AndroidDevice) => void; onForget: (d: AndroidDevice) => void }) { + return ( +
    + + {devices.map((device, i) => ( + + onEdit(device)} onForget={() => onForget(device)} /> + + ))} + +
    + ); +} + /** Android phones connected over adb: watch them, take control, give them to agents. */ export default function AndroidPage() { const qc = useQueryClient(); @@ -178,23 +200,13 @@ export default function AndroidPage() { {connected} of {devices.length} connected · adb {adb?.version ?? ""} {adb?.managed ? " (Godmode's copy)" : ""}

    -
    - - {devices.map((device, i) => ( - - setEditing(device)} onForget={() => setForgetting(device)} /> - - ))} - -
    + d.state !== "disconnected")} onEdit={setEditing} onForget={setForgetting} /> + {devices.some((d) => d.state === "disconnected") && ( +
    +

    Not connected

    + d.state === "disconnected")} onEdit={setEditing} onForget={setForgetting} /> +
    + )} )}
diff --git a/packages/core/src/android/adb.ts b/packages/core/src/android/adb.ts index b51e5912..ab8c8faa 100644 --- a/packages/core/src/android/adb.ts +++ b/packages/core/src/android/adb.ts @@ -24,6 +24,7 @@ const DOWNLOADS: Partial { + const decoder = new TextDecoder(); try { - errText = await new Response(proc.stderr as ReadableStream).text(); + for await (const chunk of proc.stderr as ReadableStream) { + errText += decoder.decode(chunk, { stream: true }); + if (errText.length > 2 * MAX_STDERR) errText = errText.slice(-MAX_STDERR); + } } catch { /* closed */ } + if (errText.length > MAX_STDERR) errText = errText.slice(-MAX_STDERR); }; const outputs = Promise.all([readOut(), readErr()]); if (opts.stdin !== undefined) { diff --git a/packages/core/src/android/screen.ts b/packages/core/src/android/screen.ts index e80f1700..811e03f6 100644 --- a/packages/core/src/android/screen.ts +++ b/packages/core/src/android/screen.ts @@ -3,6 +3,7 @@ * here), touches and swipes (`input`), keys, typed text and the tree of what is on screen (`uiautomator dump`). * Coordinates are always the phone's own pixels in its current orientation; tools and the live view scale them. */ +import { randomBytes } from "node:crypto"; import { gunzipSync } from "node:zlib"; import type { AndroidKey } from "@godmode/shared"; import { fitSize, MAX_MODEL_PIXELS } from "../computer/geometry"; @@ -40,6 +41,18 @@ export function parseScreenSize(out: string): Size | null { return m ? { width: Number(m[1]), height: Number(m[2]) } : null; } +/** + * A point on a picture of the screen → the phone's pixel. The picture may be letterboxed (screenrecord keeps the + * screen's proportions inside a video rounded to multiples of 16); points on the bars land on the nearest edge. + */ +export function frameToScreen(screen: Size, frame: Size, x: number, y: number): { x: number; y: number } { + const scale = Math.min(frame.width / screen.width, frame.height / screen.height); + const offX = (frame.width - screen.width * scale) / 2; + const offY = (frame.height - screen.height * scale) / 2; + const clamp = (v: number, max: number) => Math.min(max - 1, Math.max(0, v)); + return { x: clamp((x - offX) / scale, screen.width), y: clamp((y - offY) / scale, screen.height) }; +} + export function rememberSize(serial: string, size: Size): void { sizes.set(serial, { size, at: Date.now() }); } @@ -181,8 +194,7 @@ export function keyCommand(key: AndroidKey | string): string | null { if (key === "notifications") return "cmd statusbar expand-notifications"; if (key === "quick_settings") return "cmd statusbar expand-settings"; if (key === "select_all") return "input keycombination 113 29"; - const code = KEYCODES[key as keyof typeof KEYCODES]; - if (code !== undefined) return `input keyevent ${code}`; + if (Object.hasOwn(KEYCODES, key)) return `input keyevent ${KEYCODES[key as keyof typeof KEYCODES]}`; if (/^KEYCODE_[A-Z0-9_]{1,40}$/.test(key)) return `input keyevent ${key}`; if (/^\d{1,3}$/.test(key)) return `input keyevent ${key}`; return null; @@ -227,6 +239,16 @@ export function typeCommands(text: string, chunk = 400): string[] { return out; } +/** + * Type a saved secret: the commands go to the phone's shell on stdin, so the value isn't in any process's arguments on + * this computer (on the phone, `input` still gets it as an argument — the shell user can see that, see tools.ts). + */ +export async function typeSecretText(serial: string, text: string, signal?: AbortSignal): Promise { + if (untypable(text).length || /[\n\t]/.test(text)) throw new AdbError("The value has characters adb can't type"); + const res = await adb(["shell"], { serial, stdin: `${typeCommands(text).join(" && ")} && exit 0 || exit 1\n`, timeoutMs: 30_000, signal }); + if (res.code !== 0) throw deviceError(res, "Typing"); +} + export async function typeText(serial: string, text: string, signal?: AbortSignal): Promise { const bad = untypable(text); if (bad.length) { @@ -306,10 +328,10 @@ export function parseUiXml(xml: string): UiNode[] { return nodes; } -const DUMP = "/data/local/tmp/godmode-ui.xml"; - export async function uiNodes(serial: string, signal?: AbortSignal): Promise { - const res = await adb(["shell", `uiautomator dump ${DUMP} >/dev/null 2>&1; cat ${DUMP} 2>/dev/null; rm -f ${DUMP}`], { serial, timeoutMs: 30_000, signal }); + // A fresh name each time: nothing else on the phone can prepare the file that is read back. + const dump = `/data/local/tmp/godmode-ui-${randomBytes(6).toString("hex")}.xml`; + const res = await adb(["shell", `uiautomator dump ${dump} >/dev/null 2>&1; cat ${dump} 2>/dev/null; rm -f ${dump}`], { serial, timeoutMs: 30_000, signal }); const xml = res.stdout.toString("utf8"); if (!xml.includes(" { const CONNECTION_RANK: Record = { usb: 0, emulator: 1, wifi: 2 }; -let refreshing: Promise | null = null; +let refreshing: { done: Promise; startServer: boolean } | null = null; let lastRefresh = 0; /** Look at what adb sees now. `startServer`: also when no adb server runs yet (adb starts one). */ export function refreshDevices(opts: { startServer?: boolean } = {}): Promise { - if (opts.startServer) lastInterest = Date.now(); - refreshing ??= doRefresh(!!opts.startServer).finally(() => { - refreshing = null; - lastRefresh = Date.now(); - }); - return refreshing; + const startServer = !!opts.startServer; + if (startServer) lastInterest = Date.now(); + // A look in progress that wouldn't start the server doesn't answer one that must. + if (refreshing && startServer && !refreshing.startServer) return refreshing.done.then(() => refreshDevices(opts)); + if (!refreshing) { + const done = doRefresh(startServer).finally(() => { + refreshing = null; + lastRefresh = Date.now(); + }); + refreshing = { done, startServer }; + } + return refreshing.done; } function snapshot(): string { @@ -265,6 +271,7 @@ async function doRefresh(startServer: boolean): Promise { live.clear(); for (const [id, l] of seen) live.set(id, l); pending = nextPending; + pruneEmulators(); await refreshPhoneStates(); } catch (err) { lastError = err instanceof AdbError ? err.message : err instanceof Error ? err.message : String(err); @@ -274,6 +281,24 @@ async function doRefresh(startServer: boolean): Promise { } } +let lastPrune = 0; + +/** + * Emulators come and go (a test run can start dozens): one nobody named, described or gave to an agent or chat is + * forgotten after a day out of sight. Real phones are always kept. + */ +function pruneEmulators(): void { + if (Date.now() - lastPrune < 3600_000) return; + lastPrune = Date.now(); + const cutoff = new Date(Date.now() - 24 * 3600_000).toISOString(); + const stale = all("SELECT * FROM android_devices WHERE kind = 'emulator' AND description = '' AND COALESCE(last_seen_at, created_at) < ?", cutoff); + for (const r of stale) { + if (live.has(r.id) || androidAssignments(r.id).length || r.updated_at !== r.created_at) continue; + run("DELETE FROM android_devices WHERE id = ?", r.id); + log.info("forgot an emulator not seen for a day", { device: r.id, name: r.name }); + } +} + /** Battery and screen of connected phones, at most every 30 s; the screen size once per connection. */ async function refreshPhoneStates(): Promise { await Promise.all( diff --git a/packages/core/src/android/stream.ts b/packages/core/src/android/stream.ts index 23e8ca66..a0bf4b2a 100644 --- a/packages/core/src/android/stream.ts +++ b/packages/core/src/android/stream.ts @@ -172,7 +172,10 @@ class PhoneStream { constructor(readonly deviceId: string) {} start() { - this.loop ??= this.run().catch((err) => log.warn(`live view of ${this.deviceId} stopped`, err)); + // Video and pictures side by side: a viewer that can't decode H.264 watches along with one that can. + this.loop ??= Promise.all([this.videoLoop(), this.frameLoop()]) + .then(() => undefined) + .catch((err) => log.warn(`live view of ${this.deviceId} stopped`, err)); } restart() { @@ -180,7 +183,12 @@ class PhoneStream { this.kill(); } - private kill() { + /** Nobody watches the video anymore: don't keep screenrecord waiting for the next change on screen. */ + videoGone() { + if (!this.wantsVideo()) this.kill(); + } + + kill() { try { this.proc?.kill("SIGKILL"); } catch { @@ -200,17 +208,27 @@ class PhoneStream { emit({ type: "android.video", deviceId: this.deviceId, kind: "error", message }); } - private async run() { + private async videoLoop() { + await this.loopWhile(() => this.wantsVideo(), () => this.streamVideo()); + } + + private async frameLoop() { + await this.loopWhile(() => this.wantsFrames(), () => this.sendFrame()); + } + + private async loopWhile(wanted: () => boolean, step: () => Promise) { while (!this.stopped) { + if (!wanted()) { + await sleep(300); + continue; + } if (!isConnected(this.deviceId)) { this.error(`${safeName(this.deviceId)} isn't connected.`); await sleep(2000); continue; } try { - if (this.wantsVideo()) await this.streamVideo(); - else if (this.wantsFrames()) await this.sendFrame(); - else await sleep(500); + await step(); } catch (err) { if (this.stopped) break; this.error(err instanceof Error ? err.message : String(err)); @@ -235,6 +253,7 @@ class PhoneStream { width: size.width, height: size.height, screen: { width: raw.width, height: raw.height }, + ...(this.noVideo ? { all: true } : {}), }); await sleep(Math.max(100, 400 - (Date.now() - started))); } @@ -244,12 +263,15 @@ class PhoneStream { const bin = resolveAdb(); if (!bin) throw new AdbError("adb is not installed", "not_installed"); const screen = await screenSize(serial, 0); + if (this.stopped || !this.wantsVideo()) return; const size = videoSize(screen); const proc = Bun.spawn( [bin.path, "-s", serial, "exec-out", `screenrecord --output-format=h264 --size ${size.width}x${size.height} --bit-rate ${BIT_RATE} -`], { stdin: "ignore", stdout: "pipe", stderr: "pipe", env: childEnv() as Record }, ); this.proc = proc; + // Stopped while it was starting: stop() found no process to kill. + if (this.stopped || !this.wantsVideo()) this.kill(); this.restartWanted = false; let frames = 0; const startedAt = Date.now(); @@ -354,6 +376,9 @@ function safeName(id: string): string { const streams = new Map(); const restartTimers = new Map>(); +const lastRestart = new Map(); +/** A new viewer gets its keyframe soon; a congested one waits longer, so one slow viewer can't keep restarting everyone's. */ +const RESTART_GAP_MS = { join: 1500, congested: 6000 }; /** Viewers of a phone changed (from the WebSocket hub): start, restart for a keyframe, or stop its stream. */ export function setViewers(deviceId: string, video: number, frames: number, joined: boolean): void { @@ -375,18 +400,21 @@ export function setViewers(deviceId: string, video: number, frames: number, join } stream.videoViewers = video; stream.frameViewers = frames; - if (joined && video > 0) requestKeyframe(deviceId); + if (video === 0) stream.videoGone(); + if (joined && video > 0) requestKeyframe(deviceId, "join"); } -/** A viewer needs a keyframe (joined, or dropped frames): restart the stream, at most every 1.5 s. */ -export function requestKeyframe(deviceId: string): void { +/** A viewer needs a keyframe (it joined, or dropped frames): restart the stream, spaced out per device. */ +export function requestKeyframe(deviceId: string, reason: keyof typeof RESTART_GAP_MS = "congested"): void { if (restartTimers.has(deviceId)) return; + const wait = Math.max(300, (lastRestart.get(deviceId) ?? 0) + RESTART_GAP_MS[reason] - Date.now()); restartTimers.set( deviceId, setTimeout(() => { restartTimers.delete(deviceId); + lastRestart.set(deviceId, Date.now()); streams.get(deviceId)?.restart(); - }, 300), + }, wait), ); } diff --git a/packages/core/src/android/tools.ts b/packages/core/src/android/tools.ts index 11804d14..baacb09f 100644 --- a/packages/core/src/android/tools.ts +++ b/packages/core/src/android/tools.ts @@ -10,6 +10,7 @@ import { randomBytes } from "node:crypto"; import { z } from "zod"; import { ANDROID_KEYS } from "@godmode/shared"; import { getAgent } from "../agents/service"; +import { chatFillOnly } from "../services/conversations"; import { bus } from "../events/bus"; import { logger } from "../log"; import { audit } from "../services/audit"; @@ -23,7 +24,7 @@ import { localPath } from "../ssh/tools"; import { AdbError, adb, adbErrorText, shq } from "./adb"; import { runAndroidDeviceIds } from "./assignments"; import { androidRun, ensureConnected, noticeFailure, promptDevices, type PromptAndroidDevice } from "./service"; -import { focusedField, longPress, modelSize, pressKey, screenSize, screenshot, swipe, tap, typeText, uiNodes, untypable, type Size, type UiNode } from "./screen"; +import { focusedField, longPress, modelSize, pressKey, screenSize, screenshot, swipe, tap, typeSecretText, typeText, uiNodes, untypable, type Size, type UiNode } from "./screen"; const log = logger("android"); @@ -41,6 +42,18 @@ export const ANDROID_INSTRUCTIONS = const VAULT_FILL_OFF = 'Typing saved logins and 2FA codes into phones is turned off. Ask the human to turn on "Logins and 2FA codes" in Settings → Android phones, then try again.'; +const FILL_NEEDS_REVEAL = + "Godmode only types saved logins into a phone for agents that may see passwords: the phone's shell can read what is typed. Ask the human to sign in on the phone (they can do it in the live view)."; + +/** + * Typing a secret on a phone is as good as showing it: `input` gets it as an argument, which the phone's shell user — + * the one the `shell` tool runs as — can read. So only runs that may reveal secrets get the fills. + */ +function mayFill(ctx: { agentId: string; conversationId: string | null }): boolean { + if (!getSettings().android.vaultFill) return false; + const agent = getAgent(ctx.agentId); + return agent.permissions.secretAccess === "reveal" && !(ctx.conversationId && chatFillOnly(ctx.conversationId)); +} function text(t: string, isError = false): AndroidToolResult { return { content: [{ type: "text", text: t }], ...(isError ? { isError: true } : {}) }; @@ -75,6 +88,12 @@ function mask(runId: string, s: string): string { return out; } +function remember(runId: string, secret: string) { + const set = typedSecrets.get(runId) ?? new Set(); + set.add(secret); + typedSecrets.set(runId, set); +} + export function forgetAndroidRun(runId: string): void { typedSecrets.delete(runId); for (const key of [...shots.keys()]) if (key.startsWith(`${runId}:`)) shots.delete(key); @@ -89,6 +108,7 @@ interface ToolEnv { serial: string; runId: string; agentId: string; + conversationId: string | null; folders: string[]; signal?: AbortSignal; } @@ -129,12 +149,13 @@ async function after(env: ToolEnv, note: string, want: boolean | undefined): Pro async function toScreen(env: ToolEnv, c: [number, number], what: string): Promise<{ x: number; y: number; screen: Size }> { const screen = await screenSize(env.serial); let shot = shots.get(shotKey(env.runId, env.device.id)); - if (shot && (shot.screen.width !== screen.width || shot.screen.height !== screen.height)) { + const portrait = (s: Size) => s.height >= s.width; + if (shot && portrait(shot.screen) !== portrait(screen)) { throw new AdbError(`The phone turned since your last screenshot (now ${screen.width}×${screen.height}). Take a new screenshot and use its coordinates.`); } shot ??= { ...modelSize(screen, maxEdge()), screen }; const [x, y] = c; - if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x > shot.width || y > shot.height) { + if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x >= shot.width || y >= shot.height) { throw new AdbError(`${what} [${x}, ${y}] is outside the screenshot (${shot.width}×${shot.height}).`); } return { x: (x * screen.width) / shot.width, y: (y * screen.height) / shot.height, screen }; @@ -447,6 +468,14 @@ const TOOLS: AndroidTool[] = [ local_path: z.string().max(4096).optional(), }), run: async (args, env) => { + const kind = await adb(["shell", `if [ -d ${shq(args.remote_path)} ]; then echo dir; elif [ -e ${shq(args.remote_path)} ]; then echo file; else echo none; fi`], { + serial: env.serial, + timeoutMs: 15_000, + signal: env.signal, + }); + const what = kind.stdout.toString("utf8").trim(); + if (what === "dir") return text(`${args.remote_path} is a folder. Copy files one at a time (pack a folder with tar on the phone first).`, true); + if (what === "none") return text(`No such file on ${env.device.name}: ${args.remote_path}`, true); const name = posix.basename(args.remote_path.replace(/\/+$/, "")) || "download"; const target = args.local_path?.trim() || join(getAgent(env.agentId).repoPath, "workspace", "downloads", name); let local = localPath(target, env.folders, "write"); @@ -458,7 +487,7 @@ const TOOLS: AndroidTool[] = [ if (res.code !== 0) return text(`Copying failed: ${adbErrorText(res)}`, true); renameSync(partial, local); } finally { - rmSync(partial, { force: true }); + rmSync(partial, { force: true, recursive: true }); } return text(`Copied ${args.remote_path} from ${env.device.name} to ${local}.`); }, @@ -477,6 +506,7 @@ const TOOLS: AndroidTool[] = [ }), run: async (args, env) => { if (!getSettings().android.vaultFill) return text(VAULT_FILL_OFF, true); + if (!mayFill(env)) return text(FILL_NEEDS_REVEAL, true); const agent = getAgent(env.agentId); const secret = revealForAgent(agent, args.credentialId); const value = args.field === "username" ? secret.username : secret.password; @@ -499,10 +529,10 @@ const TOOLS: AndroidTool[] = [ record(false, { error: "not_password", pkg: field.pkg }); return text("The focused field isn't a password field, so Godmode didn't type the password there. Tap the password field and try again.", true); } - (typedSecrets.get(env.runId) ?? typedSecrets.set(env.runId, new Set()).get(env.runId)!).add(value); + remember(env.runId, value); ripple(env, "type"); try { - await typeText(env.serial, value, env.signal); + await typeSecretText(env.serial, value, env.signal); if (args.field === "password") { // Focus may have moved while typing: take the password back out of whatever field got it. const now = focusedField(await uiNodes(env.serial, env.signal).catch(() => [])); @@ -535,6 +565,7 @@ const TOOLS: AndroidTool[] = [ }), run: async (args, env) => { if (!getSettings().android.vaultFill) return text(VAULT_FILL_OFF, true); + if (!mayFill(env)) return text(FILL_NEEDS_REVEAL, true); const agent = getAgent(env.agentId); let id = args.totpId ?? null; if (args.credentialId) { @@ -557,8 +588,9 @@ const TOOLS: AndroidTool[] = [ code = codeForAgent(agent, id); } const meta = { field: "totp", runId: env.runId, deviceId: env.device.id, credentialId: args.credentialId ?? null }; + remember(env.runId, code.code); try { - await typeText(env.serial, code.code, env.signal); + await typeSecretText(env.serial, code.code, env.signal); if (args.submit) await pressKey(env.serial, "enter", { signal: env.signal }); } catch (err) { audit(`agent:${agent.id}`, "totp.fill", id, { ...meta, ok: false, error: err instanceof Error ? err.message : String(err) }); @@ -592,7 +624,7 @@ function devicesOf(ctx: RunContext): PromptAndroidDevice[] { export function listAndroidTools(ctx: RunContext): { name: string; description: string; inputSchema: Record }[] { if (!androidRun(ctx.runId)) return []; - const fill = getSettings().android.vaultFill; + const fill = mayFill(ctx); return [LIST_TOOL, ...TOOLS.filter((t) => fill || !t.name.startsWith("fill_"))].map((t) => ({ name: t.name, description: t.description, inputSchema: jsonSchema(t.name, t.schema) })); } @@ -649,7 +681,7 @@ export async function callAndroidTool(ctx: RunContext, name: string, args: unkno setTimeout(() => audited.delete(key), 6 * 3600_000).unref?.(); audit(`agent:${ctx.agentId}`, "android.use", device.id, { runId: ctx.runId }); } - const result = await tool.run(parsed as never, { device, serial, runId: ctx.runId, agentId: ctx.agentId, folders: run.folders, signal: run.signal }); + const result = await tool.run(parsed as never, { device, serial, runId: ctx.runId, agentId: ctx.agentId, conversationId: ctx.conversationId, folders: run.folders, signal: run.signal }); return { ...result, content: result.content.map((c) => (c.type === "text" ? { ...c, text: mask(ctx.runId, c.text) } : c)) }; } catch (err) { if (err instanceof z.ZodError) return text(`Invalid arguments: ${err.issues.map((i) => `${i.path.join(".") || "input"}: ${i.message}`).join("; ")}`, true); diff --git a/packages/core/src/mcp/tools.ts b/packages/core/src/mcp/tools.ts index a9242e0a..80758700 100644 --- a/packages/core/src/mcp/tools.ts +++ b/packages/core/src/mcp/tools.ts @@ -2695,22 +2695,18 @@ const TOOLS: ToolDef[] = [ defineTool({ name: "android_assign", description: - "Let an agent (every run) or this chat see and control an Android phone with the android tools, from its next run. The phone is real: give it only to agents whose job needs it. Only the human can take it away again.", + "Let another agent see and control an Android phone with the android tools in every run, from its next run. The phone is real: give it only to agents whose job needs it. You can't give a phone to yourself or your own chat (the human picks phones for a chat in its message box), and only the human can take one away again.", schema: z.object({ deviceId: z.string(), - target: z.enum(["agent", "this_chat"]), - id: z.string().optional().describe("Agent id (not needed for this_chat)"), + agentId: z.string().describe("The agent that gets the phone"), }), when: managesSetup, - run: async ({ deviceId, target, id }, { agent, ctx }) => { + run: async ({ deviceId, agentId }, { agent, ctx }) => { if (lockedVm(ctx)) return fail("This task runs in a virtual machine and is kept off the human's computer, where the phones are plugged in."); - if (target === "agent" && !id) return fail("Pass the agent's id."); - if (target === "this_chat" && !ctx.conversationId) return fail("There is no chat here: assign the phone to an agent."); - if (target === "agent") { - const refusal = revealTargetRefusal(agent, ctx, getAgent(id!), "give it a phone"); - if (refusal) return fail(refusal); - } - const device = await assignDevice(deviceId, { kind: target === "agent" ? "agent" : "conversation", id: target === "agent" ? id! : ctx.conversationId, assigned: true }, `agent:${agent.id}`); + if (agentId === agent.id) return fail("You can't give yourself a phone. Ask the human: they pick phones for a chat in its message box, or for an agent in its settings."); + const refusal = revealTargetRefusal(agent, ctx, getAgent(agentId), "give it a phone"); + if (refusal) return fail(refusal); + const device = await assignDevice(deviceId, { kind: "agent", id: agentId, assigned: true }, `agent:${agent.id}`); return json({ id: device.id, name: device.name, state: device.state, usedBy: device.assignments.map((a) => `${a.kind} ${a.name}`), note: "Applies from the next run." }); }, }), diff --git a/packages/core/src/runner/prompt.ts b/packages/core/src/runner/prompt.ts index 429c447a..6093913f 100644 --- a/packages/core/src/runner/prompt.ts +++ b/packages/core/src/runner/prompt.ts @@ -506,7 +506,7 @@ ${devices.map(androidLine).join("\n")} - Look, act, look again: take a screenshot (or \`ui_elements\`) before you tap, and check the result after every step. Popups, permission dialogs and keyboards change the screen. - ${human} watches the phone live and may take over. If it is locked with a PIN, pattern or fingerprint, ask ${human} to unlock it — never try to guess or bypass a lock screen. - It is a real phone with ${human}'s accounts and data: don't buy, post, send messages, delete, factory-reset, or change accounts, security or developer settings unless ${human} asked for exactly that. -- ${vaultFill ? "Signing in inside an app: find the login with `vault_list_logins`, tap the field, then call `fill_login({ credentialId, field })` or `fill_totp({ credentialId })` — Godmode types the value and you never see it; passwords only go into password fields. Only fill a login in its own app or site." : `Godmode doesn't type saved logins or 2FA codes into phones: ${human} hasn't allowed it. If a task needs a sign-in on the phone, ask ${human} to sign in, or to turn on "Logins and 2FA codes" in Settings → Android phones.`} Never type a password or 2FA code with \`type_text\`.`; +- ${vaultFill ? "Signing in inside an app: find the login with `vault_list_logins`, tap the field, then call `fill_login({ credentialId, field })` or `fill_totp({ credentialId })` — Godmode types the value and you never see it; passwords only go into password fields. Only fill a login in its own app or site." : `Godmode doesn't type saved logins or 2FA codes into phones for you. If a task needs a sign-in on the phone, ask ${human} to sign in there (they can do it in the live view); apps stay signed in.`} Never type a password or 2FA code with \`type_text\`.`; } function computerSection(target: ComputerTarget, human: string, canReveal: boolean): string { diff --git a/packages/core/src/runner/runner.ts b/packages/core/src/runner/runner.ts index 80b1cbac..182ab6d1 100644 --- a/packages/core/src/runner/runner.ts +++ b/packages/core/src/runner/runner.ts @@ -2010,7 +2010,7 @@ async function runClaude(job: Job, agent: Agent, res: Resources): Promise { - const p = { x: x * sx, y: y * sy }; - if (p.x < 0 || p.y < 0 || p.x > screen.width || p.y > screen.height) throw badRequest("That point is outside the screen."); - return p; - }; + const at = (x: number, y: number) => frameToScreen(screen, frame?.width && frame.height ? frame : screen, x, y); if (event.type === "tap") { const p = at(event.x, event.y); return await tap(serial, p.x, p.y); diff --git a/packages/core/src/server/ws.ts b/packages/core/src/server/ws.ts index e39e3357..e1c5849a 100644 --- a/packages/core/src/server/ws.ts +++ b/packages/core/src/server/ws.ts @@ -130,7 +130,7 @@ let onComputerSubscribe: ((view: string, subscribed: boolean) => void) | null = /** Hook invoked when the viewers of an Android phone change (see android/stream.ts). */ let onAndroidViewers: ((deviceId: string, video: number, frames: number, joined: boolean) => void) | null = null; -let onAndroidKeyframe: ((deviceId: string) => void) | null = null; +let onAndroidKeyframe: ((deviceId: string, reason: "join" | "congested") => void) | null = null; export function setAndroidViewHandlers(handlers: { viewers: typeof onAndroidViewers; keyframe: typeof onAndroidKeyframe }) { onAndroidViewers = handlers.viewers; @@ -154,7 +154,7 @@ function changeAndroidSubscription(ws: HubSocket, deviceId: string, mode: "video if (current === mode) { if (mode === "video") { (ws.data.androidNeedsKey ??= new Set()).add(deviceId); - onAndroidKeyframe?.(deviceId); + onAndroidKeyframe?.(deviceId, "join"); } return; } @@ -182,10 +182,10 @@ function sendAndroid(event: Extract) { writeFileSync(join(dir, "state.json"), JSON.stringify({ devices, phones })); } -function calls(): { serial: string | null; args: string[] }[] { +function calls(): { serial: string | null; args: string[]; stdin?: string }[] { try { return readFileSync(join(dir, "calls.jsonl"), "utf8") .split("\n") @@ -62,7 +62,7 @@ function clearCalls() { writeFileSync(join(dir, "calls.jsonl"), ""); } -const shellCalls = () => calls().filter((c) => c.args[0] === "shell").map((c) => c.args.slice(1).join(" ")); +const shellCalls = () => calls().filter((c) => c.args[0] === "shell").map((c) => c.stdin ?? c.args.slice(1).join(" ")); const PIXEL = { serialno: "R58M12ABCDE", manufacturer: "Google", model: "Pixel 8", release: "15", sdk: 35, width: 1080, height: 2400, ui: UI.replace('focused="FOCUSED"', 'focused="true"').replace('password="PASSWORD"', 'password="true"') }; @@ -162,6 +162,17 @@ describe("parsing", () => { ]); expect(videoSize({ width: 1080, height: 2400 })).toEqual({ width: 576, height: 1280 }); }); + + test("points on a letterboxed video map to the phone's pixels", () => { + // 1080×2340 in a 576×1280 video: 576/1080 scales the height to 1248, so 16 px bars top and bottom. + const screen = { width: 1080, height: 2340 }; + expect(videoSize(screen)).toEqual({ width: 576, height: 1280 }); + const video = { width: 576, height: 1280 }; + expect(frameToScreen(screen, video, 288, 16)).toEqual({ x: 540, y: 0 }); + expect(frameToScreen(screen, video, 288, 640)).toEqual({ x: 540, y: 1170 }); + expect(frameToScreen(screen, video, 576, 1280)).toEqual({ x: 1079, y: 2339 }); + expect(frameToScreen(screen, screen, 10, 20)).toEqual({ x: 10, y: 20 }); + }); }); describe("phones", () => { @@ -242,11 +253,11 @@ describe("android MCP tools", () => { return { text: r.result.content.find((c) => c.type === "text")?.text ?? "", images: r.result.content.filter((c) => c.type === "image").length, isError: !!r.result.isError }; } - function runWith(ids: string[]) { - const conv = createConversation({ agentId: agent.id, androidDeviceIds: ids }); + function runWith(ids: string[], who: Agent = agent) { + const conv = createConversation({ agentId: who.id, androidDeviceIds: ids }); const runId = `run_and_${Math.random().toString(36).slice(2)}`; attachAndroid(runId, ids, [dir]); - const token = issueRunToken({ runId, agentId: agent.id, conversationId: conv.id, workspaceId: null, depth: 0 }); + const token = issueRunToken({ runId, agentId: who.id, conversationId: conv.id, workspaceId: null, depth: 0 }); return { token, runId, done: () => (revokeRunToken(token), detachAndroid(runId), forgetAndroidRun(runId)) }; } @@ -332,17 +343,25 @@ describe("android MCP tools", () => { } }); - test("saved logins go into password fields only, and never come back", async () => { + test("saved logins: only for agents that may see them, into password fields only, never in the output", async () => { const login = createCredential({ name: "Shop", url: "https://shop.example.com", username: "owner@example.com", password: "s3cret-Pass!" }); - const run = runWith([pixelId]); + const revealer = await makeAgent({ name: "Revealer", permissions: { secretAccess: "reveal" } }); + const run = runWith([pixelId], revealer); + const fillOnly = runWith([pixelId]); expect((await call(run.token, "fill_login", { credentialId: login.id, field: "password" })).text).toContain("turned off"); updateSettings({ android: { vaultFill: true } }); try { + // The phone's shell can read what is typed: an agent that may only fill doesn't get the tools. + expect((await rpc(fillOnly.token, "tools/list")).result.tools!.map((t) => t.name)).not.toContain("fill_login"); + expect((await call(fillOnly.token, "fill_login", { credentialId: login.id, field: "password" })).text).toContain("only types saved logins into a phone for agents that may see passwords"); + expect((await rpc(run.token, "tools/list")).result.tools!.map((t) => t.name)).toContain("fill_login"); clearCalls(); const filled = await call(run.token, "fill_login", { credentialId: login.id, field: "password" }); expect(filled.isError).toBe(false); expect(filled.text).toContain('Typed the password of "Shop" into the password field of com.example.shop'); - expect(shellCalls().some((c) => c.includes("input text 's3cret-Pass!'"))).toBe(true); + // Typed through stdin: the password isn't in adb's arguments on this computer. + expect(calls().some((c) => c.stdin?.startsWith("input text 's3cret-Pass!'"))).toBe(true); + expect(calls().some((c) => c.args.join(" ").includes("s3cret"))).toBe(false); // Whatever a command prints afterwards, the password is masked. expect((await call(run.token, "shell", { command: "echo s3cret-Pass!" })).text).toBe("Exit code: 0\n••••••••"); // Not a password field: refused. @@ -354,6 +373,48 @@ describe("android MCP tools", () => { updateSettings({ android: { vaultFill: false } }); writeState("R58M12ABCDE device\n", { R58M12ABCDE: PIXEL }); run.done(); + fillOnly.done(); + } + }); + + test("files from the phone: one at a time, into the run's folders", async () => { + const run = runWith([pixelId]); + try { + expect((await call(run.token, "pull_file", { remote_path: "/sdcard/DCIM" })).text).toContain("is a folder"); + expect((await call(run.token, "pull_file", { remote_path: "/sdcard/missing.jpg" })).text).toContain("No such file"); + const pulled = await call(run.token, "pull_file", { remote_path: "/sdcard/Download/a.txt", local_path: join(dir, "a.txt") }); + expect(pulled.text).toBe(`Copied /sdcard/Download/a.txt from Shop phone to ${realpathSync(dir)}/a.txt.`); + expect(readFileSync(join(dir, "a.txt"), "utf8")).toStartWith("pulled /sdcard/Download/a.txt"); + expect((await call(run.token, "pull_file", { remote_path: "/sdcard/Download/a.txt", local_path: "/etc/a.txt" })).text).toContain("outside the folders"); + } finally { + run.done(); + } + }); + + test("a manager gives phones to other agents, never to itself", async () => { + const boss = await makeAgent({ name: "Boss", permissions: { canManageAgents: true } }); + const conv = createConversation({ agentId: boss.id }); + const token = issueRunToken({ runId: "run_and_boss", agentId: boss.id, conversationId: conv.id, workspaceId: null, depth: 0 }); + const gateway = async (name: string, args: Record) => { + const res = await fetch(`${env.baseUrl}/mcp`, { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", Accept: "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }), + }); + const r = (await res.json()) as { result: { content: { text: string }[]; isError?: boolean } }; + return { text: r.result.content[0]!.text, isError: !!r.result.isError }; + }; + try { + const self = await gateway("android_assign", { deviceId: pixelId, agentId: boss.id }); + expect(self.isError).toBe(true); + expect(self.text).toContain("can't give yourself a phone"); + expect(getAgent(boss.id).androidDeviceIds).toEqual([]); + const worker = await makeAgent({ name: "Phone Worker" }); + expect((await gateway("android_assign", { deviceId: pixelId, agentId: worker.id })).isError).toBe(false); + expect(getAgent(worker.id).androidDeviceIds).toEqual([pixelId]); + expect((await gateway("android_devices_list", {})).text).toContain('"name": "Shop phone"'); + } finally { + revokeRunToken(token); } }); diff --git a/packages/core/test/fixtures/fake-adb.ts b/packages/core/test/fixtures/fake-adb.ts index 87718521..5b6963ef 100755 --- a/packages/core/test/fixtures/fake-adb.ts +++ b/packages/core/test/fixtures/fake-adb.ts @@ -26,7 +26,10 @@ if (args[0] === "-s") { serial = args[1]!; args = args.slice(2); } -appendFileSync(join(dir, "calls.jsonl"), `${JSON.stringify({ serial, args })}\n`); +// `adb shell` without a command reads its script from stdin (logged apart from the arguments). +const stdin = args.length === 1 && args[0] === "shell" ? (await Bun.stdin.text()).trim() : undefined; +appendFileSync(join(dir, "calls.jsonl"), `${JSON.stringify({ serial, args, ...(stdin !== undefined ? { stdin } : {}) })}\n`); +if (stdin !== undefined) args = ["shell", stdin]; const phone = serial ? state.phones[serial] : undefined; const out = (s: string) => process.stdout.write(s); @@ -67,6 +70,8 @@ if (args[0] === "version") { out(" mWakefulness=Awake\n status: 2\n level: 81\n USB powered: true\n"); } else if (cmd.includes("uiautomator dump")) { out(phone!.ui ?? ""); + } else if (cmd.startsWith("if [ -d ")) { + out(cmd.includes("/DCIM'") ? "dir\n" : cmd.includes("/missing") ? "none\n" : "file\n"); } else if (cmd.startsWith("cmd package resolve-activity")) { out(cmd.endsWith("com.example.shop") ? "priority=0 preferredOrder=0 match=0x108000\ncom.example.shop/.MainActivity$Launcher\n" : "No activity found\n"); } else if (cmd.startsWith("am start")) { @@ -81,6 +86,8 @@ if (args[0] === "version") { } else if (cmd.startsWith("exit ")) { process.exit(Number(cmd.slice(5))); } +} else if (args[0] === "pull") { + Bun.write(args[2]!, `pulled ${args[1]}`); } else if (args[0] === "connect") { out(`connected to ${args[1]}\n`); } diff --git a/packages/shared/src/events.ts b/packages/shared/src/events.ts index 184f2d02..f242606a 100644 --- a/packages/shared/src/events.ts +++ b/packages/shared/src/events.ts @@ -133,7 +133,18 @@ export type ServerEvent = screen: { width: number; height: number }; } | { type: "android.video"; deviceId: ID; kind: "chunk"; data: string; key: boolean; timestamp: number } - | { type: "android.video"; deviceId: ID; kind: "frame"; data: string; mime: "image/png" | "image/jpeg"; width: number; height: number; screen: { width: number; height: number } } + | { + type: "android.video"; + deviceId: ID; + kind: "frame"; + data: string; + mime: "image/png" | "image/jpeg"; + width: number; + height: number; + screen: { width: number; height: number }; + /** For video viewers too: this phone can't stream video. */ + all?: boolean; + } | { type: "android.video"; deviceId: ID; kind: "error"; message: string } /** An agent touched a phone's screen (drawn as a ripple in the live view). x/y 0–1 of the screen. */ | { type: "android.action"; deviceId: ID; runId: ID; action: string; x?: number; y?: number; toX?: number; toY?: number } diff --git a/packages/shared/src/models.ts b/packages/shared/src/models.ts index fd865ed2..5b9e0dcb 100644 --- a/packages/shared/src/models.ts +++ b/packages/shared/src/models.ts @@ -1363,9 +1363,9 @@ export interface AndroidSettings { /** Godmode looks for Android phones connected over adb, and agents may control the phones assigned to them. */ enabled: boolean; /** - * Agents may type saved logins and 2FA codes into a phone (android tools `fill_login` / `fill_totp`). Godmode types - * the values, so the model never sees them, and passwords only go into password fields — but the fill can't be bound - * to the login's app or website. Turning it on needs a vault grant. + * Godmode may type saved logins and 2FA codes into a phone (android tools `fill_login` / `fill_totp`), passwords only + * into password fields. What is typed on a phone can be read through its shell, so only runs that may reveal secrets + * get the tools. Turning it on needs a vault grant. */ vaultFill: boolean; /** Custom adb binary; empty = one from the Android SDK, PATH or Homebrew, else Godmode's own copy (installed on demand). */