From 429473687582ea9623dcf2648e75a0b59d40ca04 Mon Sep 17 00:00:00 2001 From: Lana Begunova <70295997+lana-20@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:36:14 -0700 Subject: [PATCH] app_speech: listening, in every client (mobiumdev/mobium#255) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two scenarios: listen start before an action and listen stop after it, in all five clients — Java and .NET as listenStart and listenStop. Captured on the Android 15 emulator around a tap on the Screen Reader Demo's Add aloud, whose count is a live region: the stop heard "Live cart: 1" as a change on screen. TalkBack's log output level was set to VERBOSE by hand before and back to ERROR after, and TalkBack is off again. check_examples: 118 scenarios, 0 problems in every client, .NET included. --- data/runs.json | 16 ++++++++++++++++ examples/canonical.json | 14 ++++++++++++++ examples/dotnet.json | 6 ++++++ examples/go.json | 6 ++++++ examples/java.json | 6 ++++++ examples/javascript.json | 6 ++++++ examples/python.json | 6 ++++++ scripts/capture.py | 5 +++++ 8 files changed, 65 insertions(+) diff --git a/data/runs.json b/data/runs.json index 7f146dc..296ce52 100644 --- a/data/runs.json +++ b/data/runs.json @@ -802,5 +802,21 @@ "verbatim": true, "exit": 0, "output": "TalkBack says:\n 1. \"not checked. Team. Radio button\" — Team (radio)\n 2. \"OFF. Dark mode. Switch\" — Dark mode (switch)\n 3. \"Dark mode, for the whole app\" — Dark mode, for the whole app, not in map\n 4. \"Read-only\" — Read-only, not in map\n 5. \"This field cannot be edited. Edit box. disabled\"\n 6. \"notify=false terms=false plan=free theme=light\" — notify=false terms=false plan=free theme=light, not in map\n 7. (nothing)\n 8. \"Back. Button\" — Back (button)\n 9. \"Form\" — Form, not in map\n 10. \"not checked. Email me. Check box\" — Email me (checkbox)\n 11. \"not checked. Accept terms. Check box\" — Accept terms (checkbox)\n 12. \"checked. Free. Radio button\" — Free (radio)\n 13. \"not checked. Pro. Radio button\" — Pro (radio)\nended: the cursor came back round to the first element" + }, + "app_speech[3]": { + "tool": "app_speech", + "index": 3, + "command": "mobium speech --listen start", + "verbatim": true, + "exit": 0, + "output": "listening to TalkBack — act, then stop it with listen stop" + }, + "app_speech[4]": { + "tool": "app_speech", + "index": 4, + "command": "mobium speech --listen stop", + "verbatim": true, + "exit": 0, + "output": "TalkBack said, since listening started: \"Live cart: 1\" — Live cart: 1, not in map [a change on screen]" } } diff --git a/examples/canonical.json b/examples/canonical.json index 72ff483..825f68c 100644 --- a/examples/canonical.json +++ b/examples/canonical.json @@ -937,6 +937,20 @@ "args": { "target": "label=Pro" } + }, + { + "title": "Start listening to the screen reader, before an action", + "cli": "mobium speech --listen start", + "args": { + "listen": "start" + } + }, + { + "title": "Stop listening, and hear what the action made it say", + "cli": "mobium speech --listen stop", + "args": { + "listen": "stop" + } } ], "app_hit_test": [ diff --git a/examples/dotnet.json b/examples/dotnet.json index 1a7bc3a..43c2b4d 100644 --- a/examples/dotnet.json +++ b/examples/dotnet.json @@ -468,6 +468,12 @@ }, { "code": "var said = device.Speech(\"label=Pro\");\nConsole.WriteLine(said[\"steps\"]);" + }, + { + "code": "device.ListenStart();\ndevice.Tap(\"testid=addLive\");" + }, + { + "code": "var heard = device.ListenStop();\nConsole.WriteLine(heard[\"steps\"]);" } ], "app_hit_test": [ diff --git a/examples/go.json b/examples/go.json index be6527d..4187aa2 100644 --- a/examples/go.json +++ b/examples/go.json @@ -468,6 +468,12 @@ }, { "code": "said, err := dev.Speech(ctx, mobium.SpeechOptions{Target: \"label=Pro\"})\nif err != nil {\n\treturn err\n}\nfmt.Println(said.Steps[0].Said)" + }, + { + "code": "if _, err := dev.Speech(ctx, mobium.SpeechOptions{Listen: \"start\"}); err != nil {\n\treturn err\n}\nif err := dev.Tap(ctx, \"testid=addLive\"); err != nil {\n\treturn err\n}" + }, + { + "code": "heard, err := dev.Speech(ctx, mobium.SpeechOptions{Listen: \"stop\"})\nif err != nil {\n\treturn err\n}\nfor _, step := range heard.Steps {\n\tfmt.Println(step.Said, step.Cause)\n}" } ], "app_hit_test": [ diff --git a/examples/java.json b/examples/java.json index 0952461..6822f1f 100644 --- a/examples/java.json +++ b/examples/java.json @@ -468,6 +468,12 @@ }, { "code": "Map said = device.speech(\"label=Pro\", false, 0);\nSystem.out.println(said.get(\"steps\"));" + }, + { + "code": "device.listenStart();\ndevice.tap(\"testid=addLive\");" + }, + { + "code": "Map heard = device.listenStop();\nSystem.out.println(heard.get(\"steps\"));" } ], "app_hit_test": [ diff --git a/examples/javascript.json b/examples/javascript.json index 5ac165f..a4c09f7 100644 --- a/examples/javascript.json +++ b/examples/javascript.json @@ -468,6 +468,12 @@ }, { "code": "const said = await device.speech({ target: 'label=Pro' })\nconsole.log(said.steps)" + }, + { + "code": "await device.speech({ listen: 'start' })\nawait device.tap('testid=addLive')" + }, + { + "code": "const heard = await device.speech({ listen: 'stop' })\nconsole.log(heard.steps)" } ], "app_hit_test": [ diff --git a/examples/python.json b/examples/python.json index 64c42df..7e88cff 100644 --- a/examples/python.json +++ b/examples/python.json @@ -468,6 +468,12 @@ }, { "code": "said = device.speech(\"label=Pro\")\nprint(said[\"steps\"])" + }, + { + "code": "device.speech(listen=\"start\")\ndevice.tap(\"testid=addLive\")" + }, + { + "code": "heard = device.speech(listen=\"stop\")\nfor step in heard[\"steps\"]:\n print(step[\"said\"], step.get(\"cause\"))" } ], "app_hit_test": [ diff --git a/scripts/capture.py b/scripts/capture.py index 129f437..14e6dfa 100644 --- a/scripts/capture.py +++ b/scripts/capture.py @@ -166,6 +166,11 @@ def app_screen(name): "$sleep 3"], after=True, pause=1.0), dict(tool="app_speech", i=0), dict(tool="app_speech", i=1, restore=["accessibility screen_reader off"]), + # Listening around a tap on the Screen Reader Demo's Add aloud, whose + # count is a live region: TalkBack says the new count. + dict(tool="app_speech", i=3, setup=app_screen("Screen Reader Demo") + ["wait testid=addLive", + "accessibility screen_reader on", "$sleep 4"]), + dict(tool="app_speech", i=4, setup=["tap testid=addLive"], restore=["accessibility screen_reader off"]), dict(tool="app_list_apps", i=0), dict(tool="app_contexts", i=0, setup=app_screen("WebViews") + ["tap 'text=Plain page'", "wait text=Back"], after=True),