app_speech: listening, in every client

data/runs.json

@@ -802,5 +802,21 @@
   "verbatim": true,
   "exit": 0,
   "output": "TalkBack says:\n  1. \"not checked. Team. Radio button\" — Team (radio)\n  2. \"OFF. Dark mode. Switch\" — Dark mode (switch)\n  3. \"Dark mode, for the whole app\" — Dark mode, for the whole app, not in map\n  4. \"Read-only\" — Read-only, not in map\n  5. \"This field cannot be edited. Edit box. disabled\"\n  6. \"notify=false terms=false plan=free theme=light\" — notify=false terms=false plan=free theme=light, not in map\n  7. (nothing)\n  8. \"Back. Button\" — Back (button)\n  9. \"Form\" — Form, not in map\n 10. \"not checked. Email me. Check box\" — Email me (checkbox)\n 11. \"not checked. Accept terms. Check box\" — Accept terms (checkbox)\n 12. \"checked. Free. Radio button\" — Free (radio)\n 13. \"not checked. Pro. Radio button\" — Pro (radio)\nended: the cursor came back round to the first element"
+ },
+ "app_speech[3]": {
+  "tool": "app_speech",
+  "index": 3,
+  "command": "mobium speech --listen start",
+  "verbatim": true,
+  "exit": 0,
+  "output": "listening to TalkBack — act, then stop it with listen stop"
+ },
+ "app_speech[4]": {
+  "tool": "app_speech",
+  "index": 4,
+  "command": "mobium speech --listen stop",
+  "verbatim": true,
+  "exit": 0,
+  "output": "TalkBack said, since listening started: \"Live cart: 1\" — Live cart: 1, not in map [a change on screen]"
  }
 }

examples/canonical.json

@@ -937,6 +937,20 @@
       "args": {
         "target": "label=Pro"
       }
+    },
+    {
+      "title": "Start listening to the screen reader, before an action",
+      "cli": "mobium speech --listen start",
+      "args": {
+        "listen": "start"
+      }
+    },
+    {
+      "title": "Stop listening, and hear what the action made it say",
+      "cli": "mobium speech --listen stop",
+      "args": {
+        "listen": "stop"
+      }
     }
   ],
   "app_hit_test": [

examples/dotnet.json

@@ -468,6 +468,12 @@
       },
       {
         "code": "var said = device.Speech(\"label=Pro\");\nConsole.WriteLine(said[\"steps\"]);"
+      },
+      {
+        "code": "device.ListenStart();\ndevice.Tap(\"testid=addLive\");"
+      },
+      {
+        "code": "var heard = device.ListenStop();\nConsole.WriteLine(heard[\"steps\"]);"
       }
     ],
     "app_hit_test": [

examples/go.json

@@ -468,6 +468,12 @@
       },
       {
         "code": "said, err := dev.Speech(ctx, mobium.SpeechOptions{Target: \"label=Pro\"})\nif err != nil {\n\treturn err\n}\nfmt.Println(said.Steps[0].Said)"
+      },
+      {
+        "code": "if _, err := dev.Speech(ctx, mobium.SpeechOptions{Listen: \"start\"}); err != nil {\n\treturn err\n}\nif err := dev.Tap(ctx, \"testid=addLive\"); err != nil {\n\treturn err\n}"
+      },
+      {
+        "code": "heard, err := dev.Speech(ctx, mobium.SpeechOptions{Listen: \"stop\"})\nif err != nil {\n\treturn err\n}\nfor _, step := range heard.Steps {\n\tfmt.Println(step.Said, step.Cause)\n}"
       }
     ],
     "app_hit_test": [

examples/java.json

@@ -468,6 +468,12 @@
       },
       {
         "code": "Map<String, Object> said = device.speech(\"label=Pro\", false, 0);\nSystem.out.println(said.get(\"steps\"));"
+      },
+      {
+        "code": "device.listenStart();\ndevice.tap(\"testid=addLive\");"
+      },
+      {
+        "code": "Map<String, Object> heard = device.listenStop();\nSystem.out.println(heard.get(\"steps\"));"
       }
     ],
     "app_hit_test": [

examples/javascript.json

@@ -468,6 +468,12 @@
       },
       {
         "code": "const said = await device.speech({ target: 'label=Pro' })\nconsole.log(said.steps)"
+      },
+      {
+        "code": "await device.speech({ listen: 'start' })\nawait device.tap('testid=addLive')"
+      },
+      {
+        "code": "const heard = await device.speech({ listen: 'stop' })\nconsole.log(heard.steps)"
       }
     ],
     "app_hit_test": [

examples/python.json

@@ -468,6 +468,12 @@
       },
       {
         "code": "said = device.speech(\"label=Pro\")\nprint(said[\"steps\"])"
+      },
+      {
+        "code": "device.speech(listen=\"start\")\ndevice.tap(\"testid=addLive\")"
+      },
+      {
+        "code": "heard = device.speech(listen=\"stop\")\nfor step in heard[\"steps\"]:\n    print(step[\"said\"], step.get(\"cause\"))"
       }
     ],
     "app_hit_test": [

scripts/capture.py

@@ -166,6 +166,11 @@ def app_screen(name):
                                                                   "$sleep 3"], after=True, pause=1.0),
     dict(tool="app_speech", i=0),
     dict(tool="app_speech", i=1, restore=["accessibility screen_reader off"]),
+    # Listening around a tap on the Screen Reader Demo's Add aloud, whose
+    # count is a live region: TalkBack says the new count.
+    dict(tool="app_speech", i=3, setup=app_screen("Screen Reader Demo") + ["wait testid=addLive",
+                                                                           "accessibility screen_reader on", "$sleep 4"]),
+    dict(tool="app_speech", i=4, setup=["tap testid=addLive"], restore=["accessibility screen_reader off"]),
     dict(tool="app_list_apps", i=0),
     dict(tool="app_contexts", i=0, setup=app_screen("WebViews") + ["tap 'text=Plain page'", "wait text=Back"],
          after=True),