Android phones: review fixes that missed #143

apps/desktop/src/components/android/device-card.tsx

@@ -55,6 +55,7 @@ export function DeviceCard({ device, onEdit, onForget }: { device: AndroidDevice
 
   return (
     <article aria-label={device.name} className={cn("flex min-w-0 flex-col rounded-xl border bg-card shadow-card transition hover:border-foreground/15", device.busy && "glow-border")}>
+      {connected && (
       <Link
         to={`/android/${device.id}`}
         aria-label={`Open ${device.name}`}
@@ -69,8 +70,18 @@ export function DeviceCard({ device, onEdit, onForget }: { device: AndroidDevice
           </span>
         )}
       </Link>
+      )}
 
       <div className="flex items-start gap-3 p-4">
+        {!connected && (
+          <Link
+            to={`/android/${device.id}`}
+            aria-label={`Open ${device.name}`}
+            className="grid size-10 shrink-0 place-items-center rounded-lg border bg-paper-2 text-muted-foreground outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
+          >
+            <Smartphone className="size-5" />
+          </Link>
+        )}
         <div className="min-w-0 flex-1">
           <div className="flex flex-wrap items-center gap-x-2 gap-y-1">
             <h3 className="min-w-0 truncate text-[15px] leading-snug font-medium tracking-[-0.01em]">{device.name}</h3>

apps/desktop/src/components/android/phone-screen.tsx

@@ -63,6 +63,7 @@ export function PhoneScreen({
   const canvasRef = useRef<HTMLCanvasElement>(null);
   const viewRef = useRef<HTMLDivElement>(null);
   const size = useRef({ width: 0, height: 0 });
+  const screenSize = useRef({ width: 0, height: 0 });
   const [frameSize, setFrameSize] = useState<{ width: number; height: number } | null>(null);
   const [hasPicture, setHasPicture] = useState(false);
   const [error, setError] = useState<string | null>(null);
@@ -81,9 +82,15 @@ export function PhoneScreen({
     const canvas = canvasRef.current;
     const { width, height } = size.current;
     if (!canvas || !width || !height) return null;
+    // The screen sits letterboxed inside the video (its edges are rounded to multiples of 16), the video inside the view.
+    const sw = screenSize.current.width || width;
+    const sh = screenSize.current.height || height;
+    const inVideo = Math.min(width / sw, height / sh);
+    const vx = (width - sw * inVideo) / 2 + rx * sw * inVideo;
+    const vy = (height - sh * inVideo) / 2 + ry * sh * inVideo;
     const rect = canvas.getBoundingClientRect();
     const scale = Math.min(rect.width / width, rect.height / height);
-    return { x: (rect.width - width * scale) / 2 + rx * width * scale, y: (rect.height - height * scale) / 2 + ry * height * scale };
+    return { x: (rect.width - width * scale) / 2 + vx * scale, y: (rect.height - height * scale) / 2 + vy * scale };
   }, []);
 
   /* ------------------------------ the stream ------------------------------ */
@@ -141,6 +148,7 @@ export function PhoneScreen({
       }
       if (e.kind === "frame") {
         size.current = { width: e.width, height: e.height };
+        screenSize.current = e.screen;
         setFrameSize({ width: e.width, height: e.height });
         const img = new Image();
         img.onload = () => draw(img, e.width, e.height);
@@ -151,6 +159,7 @@ export function PhoneScreen({
       if (e.kind === "config") {
         decoder?.close();
         size.current = { width: e.width, height: e.height };
+        screenSize.current = e.screen;
         setFrameSize({ width: e.width, height: e.height });
         waitingKey = true;
         try {

apps/desktop/src/components/settings/android-section.tsx

@@ -109,12 +109,12 @@ export function AndroidSection({ settings }: { settings: Settings }) {
           label="Type logins and 2FA codes into phones"
           htmlFor="android-vault-fill"
           disabled={!a.enabled}
-          description="Godmode types saved logins into password fields on the phone without the AI seeing them, but it can't tell which app a field belongs to. An agent that can use the phone's shell could read what is typed. Turning this on asks for your vault passphrase."
+          description="Godmode types saved logins into password fields on the phone. Whatever is typed on a phone can be read through its shell, so only agents you allow to see passwords get this; the others ask you to sign in on the phone. Turning this on asks for your vault passphrase."
         >
           <Switch id="android-vault-fill" checked={a.vaultFill} disabled={!a.enabled} onCheckedChange={(on) => void setVaultFill(on)} />
         </SettingRow>
         <ul aria-label="How it works" className="flex flex-wrap gap-2 py-3.5">
-          <Fact icon={<EyeOff />}>Typed by Godmode, not the AI</Fact>
+          <Fact icon={<EyeOff />}>Only agents that may see passwords</Fact>
           <Fact icon={<RectangleEllipsis />}>Passwords only into password fields</Fact>
           <Fact icon={<ScrollText />}>Every fill audited</Fact>
           <Fact icon={<AppWindow />} caution>

apps/desktop/src/pages/android/android-page.tsx

@@ -36,6 +36,28 @@ function SetupSteps() {
   );
 }
 
+function DeviceGrid({ devices, onEdit, onForget }: { devices: AndroidDevice[]; onEdit: (d: AndroidDevice) => void; onForget: (d: AndroidDevice) => void }) {
+  return (
+    <div className="grid grid-cols-1 items-start gap-4 @3xl:grid-cols-2 @6xl:grid-cols-3">
+      <AnimatePresence initial={false}>
+        {devices.map((device, i) => (
+          <motion.div
+            key={device.id}
+            layout="position"
+            initial={{ opacity: 0, y: 8 }}
+            animate={{ opacity: 1, y: 0 }}
+            exit={{ opacity: 0, scale: 0.97 }}
+            transition={{ delay: Math.min(i, 8) * 0.03 }}
+            className="min-w-0"
+          >
+            <DeviceCard device={device} onEdit={() => onEdit(device)} onForget={() => onForget(device)} />
+          </motion.div>
+        ))}
+      </AnimatePresence>
+    </div>
+  );
+}
+
 /** Android phones connected over adb: watch them, take control, give them to agents. */
 export default function AndroidPage() {
   const qc = useQueryClient();
@@ -178,23 +200,13 @@ export default function AndroidPage() {
                   {connected} of {devices.length} connected · adb {adb?.version ?? ""}
                   {adb?.managed ? " (Godmode's copy)" : ""}
                 </p>
-                <div className="grid grid-cols-1 items-start gap-4 @3xl:grid-cols-2 @6xl:grid-cols-3">
-                  <AnimatePresence initial={false}>
-                    {devices.map((device, i) => (
-                      <motion.div
-                        key={device.id}
-                        layout="position"
-                        initial={{ opacity: 0, y: 8 }}
-                        animate={{ opacity: 1, y: 0 }}
-                        exit={{ opacity: 0, scale: 0.97 }}
-                        transition={{ delay: Math.min(i, 8) * 0.03 }}
-                        className="min-w-0"
-                      >
-                        <DeviceCard device={device} onEdit={() => setEditing(device)} onForget={() => setForgetting(device)} />
-                      </motion.div>
-                    ))}
-                  </AnimatePresence>
-                </div>
+                <DeviceGrid devices={devices.filter((d) => d.state !== "disconnected")} onEdit={setEditing} onForget={setForgetting} />
+                {devices.some((d) => d.state === "disconnected") && (
+                  <section aria-label="Not connected" className="mt-4 grid gap-3">
+                    <h2 className="eyebrow text-[10.5px]">Not connected</h2>
+                    <DeviceGrid devices={devices.filter((d) => d.state === "disconnected")} onEdit={setEditing} onForget={setForgetting} />
+                  </section>
+                )}
               </>
             )}
           </div>

packages/core/src/android/adb.ts

@@ -24,6 +24,7 @@ const DOWNLOADS: Partial<Record<NodeJS.Platform, { file: string; sha256: string
   win32: { file: "platform-tools_r37.0.1-win.zip", sha256: "45f4d63113e895ebde0c90f194099a4676b6ac653bd28d54314a9e022bbc1a99" },
 };
 const DEFAULT_TIMEOUT_MS = 30_000;
+const MAX_STDERR = 1_000_000;
 const EXE = process.platform === "win32" ? "adb.exe" : "adb";
 
 export class AdbError extends Error {
@@ -146,12 +147,18 @@ export async function adb(args: string[], opts: AdbOptions = {}): Promise<AdbRes
       /* closed */
     }
   };
+  // A command can write without end to stderr too: keep its tail only.
   const readErr = async () => {
+    const decoder = new TextDecoder();
     try {
-      errText = await new Response(proc.stderr as ReadableStream<Uint8Array>).text();
+      for await (const chunk of proc.stderr as ReadableStream<Uint8Array>) {
+        errText += decoder.decode(chunk, { stream: true });
+        if (errText.length > 2 * MAX_STDERR) errText = errText.slice(-MAX_STDERR);
+      }
     } catch {
       /* closed */
     }
+    if (errText.length > MAX_STDERR) errText = errText.slice(-MAX_STDERR);
   };
   const outputs = Promise.all([readOut(), readErr()]);
   if (opts.stdin !== undefined) {

packages/core/src/android/screen.ts

@@ -3,6 +3,7 @@
  * here), touches and swipes (`input`), keys, typed text and the tree of what is on screen (`uiautomator dump`).
  * Coordinates are always the phone's own pixels in its current orientation; tools and the live view scale them.
  */
+import { randomBytes } from "node:crypto";
 import { gunzipSync } from "node:zlib";
 import type { AndroidKey } from "@godmode/shared";
 import { fitSize, MAX_MODEL_PIXELS } from "../computer/geometry";
@@ -40,6 +41,18 @@ export function parseScreenSize(out: string): Size | null {
   return m ? { width: Number(m[1]), height: Number(m[2]) } : null;
 }
 
+/**
+ * A point on a picture of the screen → the phone's pixel. The picture may be letterboxed (screenrecord keeps the
+ * screen's proportions inside a video rounded to multiples of 16); points on the bars land on the nearest edge.
+ */
+export function frameToScreen(screen: Size, frame: Size, x: number, y: number): { x: number; y: number } {
+  const scale = Math.min(frame.width / screen.width, frame.height / screen.height);
+  const offX = (frame.width - screen.width * scale) / 2;
+  const offY = (frame.height - screen.height * scale) / 2;
+  const clamp = (v: number, max: number) => Math.min(max - 1, Math.max(0, v));
+  return { x: clamp((x - offX) / scale, screen.width), y: clamp((y - offY) / scale, screen.height) };
+}
+
 export function rememberSize(serial: string, size: Size): void {
   sizes.set(serial, { size, at: Date.now() });
 }
@@ -181,8 +194,7 @@ export function keyCommand(key: AndroidKey | string): string | null {
   if (key === "notifications") return "cmd statusbar expand-notifications";
   if (key === "quick_settings") return "cmd statusbar expand-settings";
   if (key === "select_all") return "input keycombination 113 29";
-  const code = KEYCODES[key as keyof typeof KEYCODES];
-  if (code !== undefined) return `input keyevent ${code}`;
+  if (Object.hasOwn(KEYCODES, key)) return `input keyevent ${KEYCODES[key as keyof typeof KEYCODES]}`;
   if (/^KEYCODE_[A-Z0-9_]{1,40}$/.test(key)) return `input keyevent ${key}`;
   if (/^\d{1,3}$/.test(key)) return `input keyevent ${key}`;
   return null;
@@ -227,6 +239,16 @@ export function typeCommands(text: string, chunk = 400): string[] {
   return out;
 }
 
+/**
+ * Type a saved secret: the commands go to the phone's shell on stdin, so the value isn't in any process's arguments on
+ * this computer (on the phone, `input` still gets it as an argument — the shell user can see that, see tools.ts).
+ */
+export async function typeSecretText(serial: string, text: string, signal?: AbortSignal): Promise<void> {
+  if (untypable(text).length || /[\n\t]/.test(text)) throw new AdbError("The value has characters adb can't type");
+  const res = await adb(["shell"], { serial, stdin: `${typeCommands(text).join(" && ")} && exit 0 || exit 1\n`, timeoutMs: 30_000, signal });
+  if (res.code !== 0) throw deviceError(res, "Typing");
+}
+
 export async function typeText(serial: string, text: string, signal?: AbortSignal): Promise<void> {
   const bad = untypable(text);
   if (bad.length) {
@@ -306,10 +328,10 @@ export function parseUiXml(xml: string): UiNode[] {
   return nodes;
 }
 
-const DUMP = "/data/local/tmp/godmode-ui.xml";
-
 export async function uiNodes(serial: string, signal?: AbortSignal): Promise<UiNode[]> {
-  const res = await adb(["shell", `uiautomator dump ${DUMP} >/dev/null 2>&1; cat ${DUMP} 2>/dev/null; rm -f ${DUMP}`], { serial, timeoutMs: 30_000, signal });
+  // A fresh name each time: nothing else on the phone can prepare the file that is read back.
+  const dump = `/data/local/tmp/godmode-ui-${randomBytes(6).toString("hex")}.xml`;
+  const res = await adb(["shell", `uiautomator dump ${dump} >/dev/null 2>&1; cat ${dump} 2>/dev/null; rm -f ${dump}`], { serial, timeoutMs: 30_000, signal });
   const xml = res.stdout.toString("utf8");
   if (!xml.includes("<hierarchy")) {
     if (res.code !== 0 && !xml) throw deviceError(res, "Reading the screen");

packages/core/src/android/service.ts

@@ -191,17 +191,23 @@ async function identify(d: AdbDevice): Promise<string> {
 
 const CONNECTION_RANK: Record<AndroidConnection, number> = { usb: 0, emulator: 1, wifi: 2 };
 
-let refreshing: Promise<void> | null = null;
+let refreshing: { done: Promise<void>; startServer: boolean } | null = null;
 let lastRefresh = 0;
 
 /** Look at what adb sees now. `startServer`: also when no adb server runs yet (adb starts one). */
 export function refreshDevices(opts: { startServer?: boolean } = {}): Promise<void> {
-  if (opts.startServer) lastInterest = Date.now();
-  refreshing ??= doRefresh(!!opts.startServer).finally(() => {
-    refreshing = null;
-    lastRefresh = Date.now();
-  });
-  return refreshing;
+  const startServer = !!opts.startServer;
+  if (startServer) lastInterest = Date.now();
+  // A look in progress that wouldn't start the server doesn't answer one that must.
+  if (refreshing && startServer && !refreshing.startServer) return refreshing.done.then(() => refreshDevices(opts));
+  if (!refreshing) {
+    const done = doRefresh(startServer).finally(() => {
+      refreshing = null;
+      lastRefresh = Date.now();
+    });
+    refreshing = { done, startServer };
+  }
+  return refreshing.done;
 }
 
 function snapshot(): string {
@@ -265,6 +271,7 @@ async function doRefresh(startServer: boolean): Promise<void> {
     live.clear();
     for (const [id, l] of seen) live.set(id, l);
     pending = nextPending;
+    pruneEmulators();
     await refreshPhoneStates();
   } catch (err) {
     lastError = err instanceof AdbError ? err.message : err instanceof Error ? err.message : String(err);
@@ -274,6 +281,24 @@ async function doRefresh(startServer: boolean): Promise<void> {
   }
 }
 
+let lastPrune = 0;
+
+/**
+ * Emulators come and go (a test run can start dozens): one nobody named, described or gave to an agent or chat is
+ * forgotten after a day out of sight. Real phones are always kept.
+ */
+function pruneEmulators(): void {
+  if (Date.now() - lastPrune < 3600_000) return;
+  lastPrune = Date.now();
+  const cutoff = new Date(Date.now() - 24 * 3600_000).toISOString();
+  const stale = all<DeviceRow>("SELECT * FROM android_devices WHERE kind = 'emulator' AND description = '' AND COALESCE(last_seen_at, created_at) < ?", cutoff);
+  for (const r of stale) {
+    if (live.has(r.id) || androidAssignments(r.id).length || r.updated_at !== r.created_at) continue;
+    run("DELETE FROM android_devices WHERE id = ?", r.id);
+    log.info("forgot an emulator not seen for a day", { device: r.id, name: r.name });
+  }
+}
+
 /** Battery and screen of connected phones, at most every 30 s; the screen size once per connection. */
 async function refreshPhoneStates(): Promise<void> {
   await Promise.all(