From c3e03e9c2de78dc8875a4ac1d6d793f3bbdafe32 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:17:11 +0800 Subject: [PATCH 01/44] Record that the client-roots opt-in is confirmed --- Progress.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Progress.md b/Progress.md index dd14e3088..4d3854d28 100644 --- a/Progress.md +++ b/Progress.md @@ -245,7 +245,7 @@ U-20261009-06 與 -04 的 macOS 部分是在 Windows 上對假的 pyobjc／Quart 有時才是路徑的參數（`ac_open_path`／`ac_plan_open`／`ac_file_association`／`ac_act_in_view` 的 `target`，`ac_launch_process`／`ac_shell` 的 `argv`／`command`）、 `ac_handle_file_dialog` 的 `path`、自由格式物件裡的路徑（`ac_run_suite` 的 `spec`、`ac_run_dag` 的 `definition`、`ac_assert_all` 的 `specs`）、沒標註的外掛工具。 檔案 symlink 的跳脫在這台機器上建不出來（只跑了目錄 junction）；POSIX 的 `:` 分隔與 `~` 沒在 Linux／macOS 跑。 - `roots/list` 來的根目錄要另外開 `JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT` 才算數——這是實作時定的，請維護者確認。 + `roots/list` 來的根目錄要另外開 `JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT` 才算數（維護者 2026-10-09 確認）。 - **USB passthrough**：client 仍在 `reply_timeout_s`（10 秒）放棄，即使呼叫端要的 `timeout_ms` 更長（host 接受到 60 秒）； 對新 host 逾時已經無害，對舊 host 正常使用就會進入「需重連」。通道層的 ERROR（passthrough 關閉、bad frame、host 沒有 session）在 session 之前送出、不帶請求編號， 呼叫仍是等到逾時。沒對真的舊版 host、真的 WebRTC 通道或真的 USB 裝置跑過。 From 6546c0eb7c8f37825678812ea4593daaf495ed93 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:24:55 +0800 Subject: [PATCH 02/44] Split the WebRTC panel into one module per interaction group so each fits the file-length limit --- .../remote_desktop/webrtc_host_connection.py | 217 ++ .../gui/remote_desktop/webrtc_host_media.py | 276 ++ .../remote_desktop/webrtc_host_sessions.py | 227 ++ .../gui/remote_desktop/webrtc_host_trust.py | 131 + .../gui/remote_desktop/webrtc_host_ui.py | 266 ++ .../gui/remote_desktop/webrtc_panel.py | 2217 +---------------- .../gui/remote_desktop/webrtc_panel_common.py | 113 + .../webrtc_viewer_address_book.py | 239 ++ .../webrtc_viewer_connection.py | 252 ++ .../gui/remote_desktop/webrtc_viewer_files.py | 214 ++ .../gui/remote_desktop/webrtc_viewer_media.py | 236 ++ .../gui/remote_desktop/webrtc_viewer_ui.py | 328 +++ 12 files changed, 2530 insertions(+), 2186 deletions(-) create mode 100644 je_auto_control/gui/remote_desktop/webrtc_host_connection.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_host_media.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_host_sessions.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_host_trust.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_host_ui.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_panel_common.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_viewer_files.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_viewer_media.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_connection.py b/je_auto_control/gui/remote_desktop/webrtc_host_connection.py new file mode 100644 index 000000000..9f6c6cd3f --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_host_connection.py @@ -0,0 +1,217 @@ +"""Publishing, manual SDP exchange and shutdown for the WebRTC host panel. + +One interaction group of ``webrtc_panel._WebRTCHostPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QMessageBox, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _t, +) +from je_auto_control.gui.remote_desktop.webrtc_workers import ( + HostPublishLoopWorker, generate_host_id, retire_worker, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop import ( + MultiViewerHost, +) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _read_webrtc_config, +) + + +class _HostConnectionMixin: + """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" + + def _on_regen_id(self) -> None: + self._host_id_edit.setText(generate_host_id()) + + def _on_publish_via_server(self) -> None: + if not self._validate_required_fields(needs_server=True): + return + self._stop_host_if_any() + try: + self._multi_host = self._build_multi_host( + self._token_edit.text().strip(), + ) + except (ValueError, RuntimeError, OSError) as error: + self._show_error(error) + return + self._set_hosting(True) + self._publish_loop = HostPublishLoopWorker( + multi_host=self._multi_host, + server_url=self._server_edit.text().strip(), + host_id=self._host_id_edit.text().strip(), + secret=self._secret_edit.text() or None, + ) + self._publish_loop.offer_published.connect(self._on_loop_offer_published) + self._publish_loop.session_connected.connect(self._on_loop_session_connected) + self._publish_loop.failed.connect(self._on_signaling_failed) + self._status_label.setText(_t("rd_webrtc_publishing_offer")) + self._publish_loop.start() + self._start_lan_advertise() + + def _start_lan_advertise(self) -> None: + try: + from je_auto_control.utils.remote_desktop.lan_discovery import ( + HostAdvertiser, is_discovery_available, + ) + except ImportError: + return + if not is_discovery_available(): + return + try: + if self._lan_advertiser is not None: + self._lan_advertiser.stop() + self._lan_advertiser = HostAdvertiser( + host_id=self._host_id_edit.text().strip(), + signaling_url=self._server_edit.text().strip(), + ) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("lan advertise: %r", error) + + def _stop_lan_advertise(self) -> None: + if self._lan_advertiser is not None: + try: + self._lan_advertiser.stop() + except (RuntimeError, OSError): + pass + self._lan_advertiser = None + + def _on_loop_offer_published(self, session_id: str) -> None: + autocontrol_logger.debug("publish loop: offer for %s", session_id) + + # Optional: surface the session in the UI; for now just log. + + def _on_loop_session_connected(self, session_id: str) -> None: + if self._multi_host is not None: + self._signals.session_count.emit(self._multi_host.session_count()) + + def _on_signaling_failed(self, message: str) -> None: + QMessageBox.warning(self, "WebRTC", message) + self._status_label.setText(_t("rd_webrtc_status_idle")) + + def _on_generate_offer(self) -> None: + token = self._token_edit.text().strip() + if not token: + QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_token_required")) + return + try: + if self._multi_host is None: + self._multi_host = self._build_multi_host(token) + except (ValueError, RuntimeError, OSError) as error: + self._show_error(error) + return + self._set_hosting(True) + self._status_label.setText(_t("rd_webrtc_generating_offer")) + self._offer_view.setPlainText("") + QTimer.singleShot(0, self, self._produce_offer) + + def _require_multi_host(self) -> MultiViewerHost: + """Return the running host, or say the session is not up yet.""" + host = self._multi_host + if host is None: + raise RuntimeError(_t("rd_webrtc_not_started")) + return host + + def _produce_offer(self) -> None: + try: + session_id, offer = self._require_multi_host().create_session_offer() + except (RuntimeError, OSError) as error: # PermissionError is an OSError + self._show_error(error) + return + self._manual_session_id = session_id + self._offer_view.setPlainText(offer) + self._status_label.setText(_t("rd_webrtc_offer_ready")) + + def _on_apply_answer(self) -> None: + if self._multi_host is None or not self._manual_session_id: + QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_no_offer_yet")) + return + answer = self._answer_input.toPlainText().strip() + if not answer: + QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_no_answer")) + return + try: + self._multi_host.accept_session_answer(self._manual_session_id, answer) + self._status_label.setText(_t("rd_webrtc_answer_applied")) + except (ValueError, RuntimeError, OSError, KeyError) as error: + self._show_error(error) + return + self._manual_session_id = None # consumed; next Generate creates new session + + def _on_stop(self) -> None: + self._stop_host_if_any() + self._status_label.setText(_t("rd_webrtc_status_idle")) + self._signals.session_count.emit(0) + + def _validate_required_fields(self, *, needs_server: bool) -> bool: + token = self._token_edit.text().strip() + if not token: + QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_token_required")) + return False + if needs_server: + if not self._server_edit.text().strip(): + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_server_required"), + ) + return False + if not self._host_id_edit.text().strip(): + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_host_id_required"), + ) + return False + return True + + def _build_multi_host(self, token: str) -> MultiViewerHost: + whitelist_text = self._ip_whitelist_edit.toPlainText().strip() + whitelist = [line.strip() for line in whitelist_text.splitlines() + if line.strip() and not line.strip().startswith("#")] + host = MultiViewerHost( + token=token, + config=_read_webrtc_config(self), + trust_list=self._trust_list, + read_only=self._readonly_check.isChecked(), + ip_whitelist=whitelist, + on_annotation=self._signals.annotation.emit, + on_session_state=lambda _sid, state: self._signals.state.emit(state), + on_session_authenticated=self._on_session_authed, + on_pending_viewer=self._signals.pending_viewer.emit, + ) + return host + + def _stop_host_if_any(self) -> None: + self._stop_adaptive() + self._stop_lan_advertise() + if self._annotation_overlay is not None: + self._annotation_overlay.clear() + self._annotation_overlay.hide() + # Snapshot before clear(), as in _refresh_session_pollers. + for poller in list(self._session_pollers.values()): # NOSONAR python:S7504 + poller.stop() + self._session_pollers.clear() + self._session_cache.reset() + retire_worker(self._publish_loop) + self._publish_loop = None + if self._viewer_screen_window is not None: + self._viewer_screen_window.set_image(None) + self._viewer_screen_window.hide() + if self._multi_host is None: + return + try: + self._multi_host.stop_all() + except (RuntimeError, OSError): + pass + finally: + self._multi_host = None + self._manual_session_id = None + self._set_hosting(False) + + +__all__ = ["_HostConnectionMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_media.py b/je_auto_control/gui/remote_desktop/webrtc_host_media.py new file mode 100644 index 000000000..937fc7576 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_host_media.py @@ -0,0 +1,276 @@ +"""Monitor selection, bandwidth adaptation and viewer media for the WebRTC host panel. + +One interaction group of ``webrtc_panel._WebRTCHostPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtGui import QImage +from PySide6.QtWidgets import ( + QMessageBox, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _t, +) +from je_auto_control.gui.remote_desktop.blanking_overlay import BlankingOverlay +from je_auto_control.gui.remote_desktop.annotation_overlay import ( + HostAnnotationOverlay, +) +from je_auto_control.gui.remote_desktop.viewer_screen_window import ( + ViewerScreenWindow, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop import ( + install_hardware_codec, uninstall_hardware_codec, +) +from je_auto_control.utils.remote_desktop.adaptive_bitrate import ( + AdaptiveBitrateController, +) +from je_auto_control.utils.remote_desktop.webrtc_stats import ( + StatsPoller, StatsSnapshot, +) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _DEFAULT_MONITOR, _QUALITY_DOT_STYLE, _av_frame_to_qimage, +) + + +class _HostMediaMixin: + """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" + + def _on_hw_codec_changed(self) -> None: + codec = self._hw_codec_combo.currentData() or "" + if not codec: + uninstall_hardware_codec() + self._status_label.setText(_t("rd_webrtc_hw_codec_off_status")) + return + if install_hardware_codec(codec): + self._status_label.setText( + _t("rd_webrtc_hw_codec_active").format(codec=codec), + ) + else: + self._status_label.setText( + _t("rd_webrtc_hw_codec_failed").format(codec=codec), + ) + + def _on_toggle_accept_viewer_video(self, value: bool) -> None: + if self._multi_host is None: + return + with self._multi_host._lock: + sessions = list(self._multi_host._sessions.values()) + for host in sessions: + try: + if value: + host.set_viewer_video_callback( + self._on_viewer_video_av_frame, + ) + host.enable_accept_viewer_video() + else: + host.disable_accept_viewer_video() + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("toggle accept viewer video: %r", error) + if not value and self._viewer_screen_window is not None: + self._viewer_screen_window.set_image(None) + self._viewer_screen_window.hide() + + def _on_toggle_accept_opus_audio(self, value: bool) -> None: + if self._multi_host is None: + return + with self._multi_host._lock: + sessions = list(self._multi_host._sessions.values()) + for host in sessions: + try: + if value: + host.enable_accept_viewer_audio_opus() + else: + host.disable_accept_viewer_audio_opus() + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("toggle accept opus: %r", error) + + def _on_toggle_mic_receive(self, value: bool) -> None: + if self._multi_host is None: + return + # Apply to every active session. + with self._multi_host._lock: + sessions = list(self._multi_host._sessions.values()) + for host in sessions: + try: + if value: + host.enable_mic_receive() + else: + host.disable_mic_receive() + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("mic receive toggle: %r", error) + + def _on_toggle_adaptive(self, value: bool) -> None: + if value: + self._maybe_start_adaptive() + else: + self._stop_adaptive() + + def _populate_monitor_combo(self) -> None: + try: + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + with mss_grabber() as sct: + monitors = sct.monitors + for idx, mon in enumerate(monitors): + if idx == 0: + label = _t("rd_webrtc_monitor_all") + else: + label = f"#{idx}: {mon['width']}x{mon['height']} @"\ + f" ({mon['left']},{mon['top']})" + self._monitor_combo.addItem(label, idx) + except (ImportError, RuntimeError, OSError): + for idx in range(4): + self._monitor_combo.addItem(f"#{idx}", idx) + # Default to monitor #1 (the first real screen for mss) + idx_default = self._monitor_combo.findData(_DEFAULT_MONITOR) + if idx_default >= 0: + self._monitor_combo.setCurrentIndex(idx_default) + + def _on_pick_region(self) -> None: + try: + from je_auto_control.gui.selector import open_region_selector + region = open_region_selector(self) + except (ImportError, RuntimeError, OSError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + return + if region is None: + return + x, y, w, h = region + self._region_edit.setText(f"{x},{y},{w},{h}") + + def _on_monitor_changed(self, _i: int) -> None: + idx = self._monitor_combo.currentData() + if idx is None or self._multi_host is None: + return + track = self._multi_host.screen_track() + if track is None: + return + try: + track.set_target_monitor(int(idx)) + autocontrol_logger.info("monitor switched to #%d live", int(idx)) + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("set_target_monitor: %r", error) + + def _on_toggle_readonly(self, value: bool) -> None: + if self._multi_host is not None: + self._multi_host.set_read_only(value) + + def _on_toggle_blanking(self, checked: bool) -> None: + if checked: + if self._blanking is None: + self._blanking = BlankingOverlay() + self._blanking.show() + elif self._blanking is not None: + self._blanking.hide() + + def _maybe_start_adaptive(self) -> None: + # Always start a stats poller when sessions are active so the host + # quality dot updates; the adaptive controller is an optional + # consumer enabled via the checkbox. + if self._adaptive_poller is not None or self._multi_host is None: + return + track = self._multi_host.screen_track() + pc = self._multi_host.first_session_pc() + if pc is None: + return + if track is not None and self._adaptive_check.isChecked(): + max_fps = int(self._fps_spin.value()) + self._adaptive_controller = AdaptiveBitrateController( + track, max_fps=max_fps, + max_bitrate_kbps=int(self._max_bitrate_spin.value()), + ) + else: + self._adaptive_controller = None + self._adaptive_poller = StatsPoller( + pc, self._on_host_stats, interval_s=1.0, + ) + self._adaptive_poller.start() + autocontrol_logger.info( + "host stats poller active (adaptive=%s)", + self._adaptive_controller is not None, + ) + + def _on_host_stats(self, snapshot: StatsSnapshot) -> None: + # Fan-out: feed adaptive controller (if enabled) + update quality dot + if self._adaptive_controller is not None: + try: + self._adaptive_controller.on_stats(snapshot) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("adaptive on_stats: %r", error) + self._signals.stats.emit(snapshot) + + def _update_host_quality_dot(self, snapshot: StatsSnapshot) -> None: + rtt = snapshot.rtt_ms + loss = snapshot.packet_loss_pct or 0.0 + if rtt is None: + color = "#555" + tip_key = "rd_webrtc_quality_unknown" + elif rtt < 80 and loss < 1.0: + color = "#3a9c3a" + tip_key = "rd_webrtc_quality_good" + elif rtt < 200 and loss < 5.0: + color = "#c9a23a" + tip_key = "rd_webrtc_quality_fair" + else: + color = "#cc4444" + tip_key = "rd_webrtc_quality_poor" + self._host_quality_dot.setStyleSheet( + f"background-color: {color}; border-radius: 7px;", + ) + self._host_quality_dot.setToolTip(_t(tip_key)) + + def _reset_host_quality_dot(self) -> None: + self._host_quality_dot.setStyleSheet( + _QUALITY_DOT_STYLE, + ) + self._host_quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) + + def _stop_adaptive(self) -> None: + if self._adaptive_poller is not None: + self._adaptive_poller.stop() + self._adaptive_poller = None + self._adaptive_controller = None + + def _on_annotation_event(self, data) -> None: + if not isinstance(data, dict): + return + if self._annotation_overlay is None: + self._annotation_overlay = HostAnnotationOverlay(parent=self) + self._annotation_overlay.apply(data) + + def _on_session_authed(self, session_id: str) -> None: + self._signals.auth.emit(True) + if (self._multi_host is None + or not self._accept_viewer_video_check.isChecked()): + return + # Wire viewer-video callback on this freshly-authed session + with self._multi_host._lock: + host = self._multi_host._sessions.get(session_id) + if host is None: + return + host.set_viewer_video_callback(self._on_viewer_video_av_frame) + + def _on_viewer_video_av_frame(self, frame) -> None: + image = _av_frame_to_qimage(frame) + if image is not None: + self._signals.viewer_video_frame.emit(image) + + def _on_viewer_video_image(self, image: QImage) -> None: + if self._viewer_screen_window is None: + self._viewer_screen_window = ViewerScreenWindow(parent=self) + self._viewer_screen_window.closed.connect( + self._on_viewer_screen_closed, + ) + if not self._viewer_screen_window.isVisible(): + self._viewer_screen_window.show() + self._viewer_screen_window.set_image(image) + + def _on_viewer_screen_closed(self) -> None: + if self._viewer_screen_window is not None: + self._viewer_screen_window.set_image(None) + + +__all__ = ["_HostMediaMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py b/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py new file mode 100644 index 000000000..ccacd54f8 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py @@ -0,0 +1,227 @@ +"""Session table, per-session quality and disconnect for the WebRTC host panel. + +One interaction group of ``webrtc_panel._WebRTCHostPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + +from typing import Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QTableWidgetItem, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _t, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.webrtc_inspector import ( + default_webrtc_inspector, +) +from je_auto_control.utils.remote_desktop.webrtc_stats import ( + StatsPoller, StatsSnapshot, +) + + +class _HostSessionsMixin: + """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" + + def _on_session_count(self, count: int) -> None: + self._sessions_label.setText( + _t("rd_webrtc_sessions_count").format(n=count), + ) + if self._tray is not None: + self._tray.set_state(sessions=count) + # Color the badge by load: gray=0, green=1-3, yellow=4-10, red=>10 + if count == 0: + bg, fg = "#3a3a3a", "#888" + elif count <= 3: + bg, fg = "#1f4d1f", "#a6e3a6" + elif count <= 10: + bg, fg = "#5a4710", "#f5d99a" + else: + bg, fg = "#5a1010", "#ffaaaa" + self._sessions_label.setStyleSheet( + f"background: {bg}; color: {fg}; padding: 2px 8px;" + "border-radius: 8px; font-weight: bold;", + ) + self._sync_session_pollers() + self._refresh_sessions_table() + if count > 0: + self._maybe_start_adaptive() + else: + self._stop_adaptive() + self._reset_host_quality_dot() + + def _sync_session_pollers(self) -> None: + """Spawn StatsPoller for new sessions; stop pollers for gone ones.""" + if self._multi_host is None: + # Snapshot before clear() so a slow stop() doesn't race with the + # clear that follows. + for poller in list(self._session_pollers.values()): # NOSONAR python:S7504 + poller.stop() + self._session_pollers.clear() + self._session_cache.reset() + return + active_sids = {s["session_id"] for s in self._multi_host.list_sessions()} + # Stop pollers whose session is gone + # The loop deletes from self._session_pollers, so list() is required + # to avoid a RuntimeError. + for sid in list(self._session_pollers.keys()): # NOSONAR python:S7504 + if sid not in active_sids: + self._session_pollers[sid].stop() + del self._session_pollers[sid] + self._session_cache.drop(sid) + # Spawn pollers for new sessions + for sid in active_sids: + if sid in self._session_pollers: + continue + pc = self._multi_host.session_pc(sid) + if pc is None: + continue + poller = StatsPoller(pc, self._make_session_stats_handler(sid), + interval_s=1.0) + poller.start() + self._session_pollers[sid] = poller + + def _make_session_stats_handler(self, session_id: str): + """Closure capturing session_id for the per-session poller.""" + def _handle(snapshot: StatsSnapshot) -> None: + default_webrtc_inspector().record(snapshot) + color = self._quality_color(snapshot) + self._session_cache.set( + session_id, color=color, snapshot=snapshot, + ) + # Re-paint just the dot cell for this session_id (avoid full reflow). + # One read: the GUI thread can clear it between a check and a call. + host = self._multi_host + self._signals.session_count.emit(host.session_count() if host else 0) + return _handle + + @staticmethod + def _format_quality_tooltip(snapshot: Optional[StatsSnapshot]) -> str: + if snapshot is None: + return _t("rd_webrtc_quality_unknown") + parts = [] + if snapshot.rtt_ms is not None: + parts.append(f"RTT {snapshot.rtt_ms:.0f}ms") + if snapshot.packet_loss_pct is not None: + parts.append(f"loss {snapshot.packet_loss_pct:.1f}%") + if snapshot.fps is not None: + parts.append(f"FPS {snapshot.fps:.1f}") + if snapshot.bitrate_kbps is not None: + parts.append(f"{snapshot.bitrate_kbps:.0f}kbps") + return " | ".join(parts) if parts else _t("rd_webrtc_quality_unknown") + + @staticmethod + def _quality_color(snapshot: StatsSnapshot) -> str: + rtt = snapshot.rtt_ms + loss = snapshot.packet_loss_pct or 0.0 + if rtt is None: + return "#555" + if rtt < 80 and loss < 1.0: + return "#3a9c3a" + if rtt < 200 and loss < 5.0: + return "#c9a23a" + return "#cc4444" + + def _refresh_sessions_table(self) -> None: + from datetime import datetime + from PySide6.QtGui import QColor + if self._multi_host is None: + self._sessions_table.setRowCount(0) + return + sessions = self._multi_host.list_sessions() + self._sessions_table.setRowCount(len(sessions)) + for row, info in enumerate(sessions): + sid = info.get("session_id", "") + vid = info.get("pending_viewer_id") or "" + state = info.get("state", "") + connected = info.get("connected_at") or "" + if connected: + try: + dt = datetime.fromisoformat(connected) + connected = dt.astimezone().strftime("%H:%M:%S") + except (TypeError, ValueError): + pass + color = self._session_cache.get_color(sid) + dot_item = QTableWidgetItem("●") + dot_item.setForeground(QColor(color)) + dot_item.setTextAlignment(Qt.AlignmentFlag.AlignCenter) + dot_item.setToolTip(self._format_quality_tooltip( + self._session_cache.get_snapshot(sid), + )) + self._sessions_table.setItem(row, 0, dot_item) + id_item = QTableWidgetItem(sid[:8] if sid else "") + id_item.setData(Qt.ItemDataRole.UserRole, sid) + self._sessions_table.setItem(row, 1, id_item) + self._sessions_table.setItem( + row, 2, QTableWidgetItem(vid[:12] if vid else ""), + ) + self._sessions_table.setItem(row, 3, QTableWidgetItem(state)) + self._sessions_table.setItem(row, 4, QTableWidgetItem(connected)) + + def _on_sessions_context_menu(self, position) -> None: + from PySide6.QtWidgets import QMenu + if self._multi_host is None: + return + row = self._sessions_table.rowAt(position.y()) + if row < 0: + return + self._sessions_table.selectRow(row) + sid_item = self._sessions_table.item(row, 1) + viewer_item = self._sessions_table.item(row, 2) + if sid_item is None: + return + sid = sid_item.data(Qt.ItemDataRole.UserRole) or "" + viewer_id = viewer_item.text() if viewer_item is not None else "" + menu = QMenu(self._sessions_table) + actions = { + "disconnect": menu.addAction(_t("rd_webrtc_disconnect_selected")), + "trust": menu.addAction(_t("rd_webrtc_sess_trust_viewer")), + "copy": menu.addAction(_t("rd_webrtc_sess_copy_id")), + } + actions["trust"].setEnabled(bool(viewer_id)) + chosen = menu.exec( + self._sessions_table.viewport().mapToGlobal(position), + ) + self._dispatch_session_menu(chosen, actions, sid, viewer_id) + + def _dispatch_session_menu(self, chosen, actions: dict, + sid: str, viewer_id: str) -> None: + """Run the action chosen from the sessions context menu.""" + if chosen is actions["disconnect"]: + self._on_disconnect_selected() + elif chosen is actions["trust"] and viewer_id: + self._trust_session_viewer(sid) + elif chosen is actions["copy"] and sid: + self._copy_session_id_to_clipboard(sid) + + @staticmethod + def _copy_session_id_to_clipboard(sid: str) -> None: + from PySide6.QtWidgets import QApplication + clip = QApplication.clipboard() + if clip is not None: + clip.setText(sid) + + def _on_disconnect_selected(self) -> None: + if self._multi_host is None: + return + row = self._sessions_table.currentRow() + if row < 0: + return + item = self._sessions_table.item(row, 1) + if item is None: + return + sid = item.data(Qt.ItemDataRole.UserRole) + if not isinstance(sid, str) or not sid: + return + try: + self._multi_host.stop_session(sid) + except (KeyError, RuntimeError, OSError) as error: + autocontrol_logger.warning("disconnect session: %r", error) + self._signals.session_count.emit(self._multi_host.session_count()) + + +__all__ = ["_HostSessionsMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_trust.py b/je_auto_control/gui/remote_desktop/webrtc_host_trust.py new file mode 100644 index 000000000..29352e1d8 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_host_trust.py @@ -0,0 +1,131 @@ +"""Trust list, audit log and file push for the WebRTC host panel. + +One interaction group of ``webrtc_panel._WebRTCHostPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QFileDialog, QMessageBox, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _read_import_entries, _t, +) +from je_auto_control.gui.remote_desktop.webrtc_dialogs import ( + AuditLogDialog, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _JSON_FILE_FILTER, +) + + +class _HostTrustMixin: + """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" + + def _on_view_audit(self) -> None: + from je_auto_control.utils.remote_desktop.audit_log import ( + default_audit_log, + ) + AuditLogDialog(default_audit_log(), parent=self).exec() + + def _on_push_file(self) -> None: + if self._multi_host is None or self._multi_host.session_count() == 0: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_no_viewers"), + ) + return + path, _filter = QFileDialog.getOpenFileName( + self, _t("rd_webrtc_push_file"), "", + ) + if not path: + return + try: + sent = self._multi_host.broadcast_file(path) + QMessageBox.information( + self, "WebRTC", + _t("rd_webrtc_push_done").format(n=sent, name=path), + ) + except (RuntimeError, OSError, ValueError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_export_trust(self) -> None: + import json as _json + path, _filter = QFileDialog.getSaveFileName( + self, _t("rd_webrtc_trust_export"), "trusted_viewers.json", + _JSON_FILE_FILTER, + ) + if not path: + return + try: + with open(path, "w", encoding="utf-8") as fh: + _json.dump({"viewers": self._trust_list.list_entries()}, + fh, indent=2, ensure_ascii=False) + except OSError as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_import_trust(self) -> None: + path, _filter = QFileDialog.getOpenFileName( + self, _t("rd_webrtc_trust_import"), "", _JSON_FILE_FILTER, + ) + if not path: + return + try: + viewers = _read_import_entries(path, "viewers") + except (OSError, ValueError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + return + added = 0 + for entry in viewers: + vid, label = entry.get("viewer_id"), entry.get("label") + if isinstance(vid, str) and vid: + self._trust_list.add(vid, label=label if isinstance(label, str) else "") + added += 1 + QMessageBox.information( + self, "WebRTC", + _t("rd_webrtc_trust_import_done").format(n=added), + ) + self._refresh_trusted_list() + + def _refresh_trusted_list(self) -> None: + self._trusted_list.populate(self._trust_list.list_entries()) + + def _trust_session_viewer(self, sid: str) -> None: + try: + multi_host = self._require_multi_host() + with multi_host._lock: + host = multi_host._sessions.get(sid) + full_vid = host.pending_viewer_id if host is not None else None + if full_vid: + self._trust_list.add(full_vid, label=f"sess {sid[:6]}") + self._refresh_trusted_list() + except (RuntimeError, OSError, ValueError) as error: + autocontrol_logger.warning("trust viewer: %r", error) + + def _on_remove_trust(self, viewer_id: str) -> None: + self._trust_list.remove(viewer_id) + self._refresh_trusted_list() + + def _on_remove_trust_button(self) -> None: + item = self._trusted_list.currentItem() + if item is None: + return + viewer_id = item.data(Qt.ItemDataRole.UserRole) + if isinstance(viewer_id, str): + self._on_remove_trust(viewer_id) + + def _on_clear_trust(self) -> None: + result = QMessageBox.question( + self, "WebRTC", _t("rd_webrtc_clear_trust_confirm"), + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + ) + if result != QMessageBox.StandardButton.Yes: + return + self._trust_list.clear() + self._refresh_trusted_list() + + +__all__ = ["_HostTrustMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_ui.py b/je_auto_control/gui/remote_desktop/webrtc_host_ui.py new file mode 100644 index 000000000..88e098062 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_host_ui.py @@ -0,0 +1,266 @@ +"""Widget construction for the WebRTC host panel. + +One interaction group of ``webrtc_panel._WebRTCHostPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QAbstractItemView, QCheckBox, QComboBox, QGridLayout, + QGroupBox, QHBoxLayout, QHeaderView, QLabel, QLineEdit, + QPushButton, QSpinBox, QTableWidget, QTextEdit, QVBoxLayout, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _t, +) +from je_auto_control.gui.remote_desktop.advanced_group import ( + build_advanced_group, +) +from je_auto_control.gui.remote_desktop.trusted_group import ( + build_trusted_group, +) +from je_auto_control.gui.remote_desktop.webrtc_workers import ( + generate_host_id, +) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _DEFAULT_FPS, _DEFAULT_SIGNALING_URL, _QUALITY_DOT_STYLE, +) + + +class _HostUiMixin: + """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" + + def _build_ui(self) -> None: + layout = QVBoxLayout(self) + layout.addWidget(self._build_signaling_group()) + layout.addWidget(self._build_config_group()) + layout.addWidget(self._build_manual_group()) + layout.addWidget(build_advanced_group(self, include_hw_codec=True)) + layout.addWidget(build_trusted_group(self)) + self._status_label = QLabel(_t("rd_webrtc_status_idle")) + layout.addWidget(self._status_label) + sessions_row = QHBoxLayout() + self._host_quality_dot = QLabel() + self._host_quality_dot.setFixedSize(14, 14) + self._host_quality_dot.setStyleSheet( + _QUALITY_DOT_STYLE, + ) + self._host_quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) + sessions_row.addWidget(self._host_quality_dot) + self._sessions_label = QLabel(_t("rd_webrtc_sessions_count").format(n=0)) + sessions_row.addWidget(self._sessions_label, stretch=1) + layout.addLayout(sessions_row) + self._sessions_table = QTableWidget(0, 5) + self._sessions_table.setHorizontalHeaderLabels([ + "", # quality dot column + _t("rd_webrtc_sess_col_id"), + _t("rd_webrtc_sess_col_viewer"), + _t("rd_webrtc_sess_col_state"), + _t("rd_webrtc_sess_col_connected"), + ]) + self._sessions_table.setColumnWidth(0, 18) + self._sessions_table.horizontalHeader().setSectionResizeMode( + 2, QHeaderView.ResizeMode.Stretch, + ) + self._sessions_table.setEditTriggers( + QAbstractItemView.EditTrigger.NoEditTriggers, + ) + # Hint at a comfortable starting height, but let the table + # grow with the window instead of pinning it at 140 px even + # when the operator has a 4K monitor's worth of space. + self._sessions_table.setMinimumHeight(140) + self._sessions_table.setSelectionBehavior( + QAbstractItemView.SelectionBehavior.SelectRows, + ) + self._sessions_table.setSelectionMode( + QAbstractItemView.SelectionMode.SingleSelection, + ) + self._sessions_table.setContextMenuPolicy( + Qt.ContextMenuPolicy.CustomContextMenu, + ) + self._sessions_table.customContextMenuRequested.connect( + self._on_sessions_context_menu, + ) + layout.addWidget(self._sessions_table) + sessions_btn_row = QHBoxLayout() + self._disconnect_btn = self._tr( + QPushButton(), "rd_webrtc_disconnect_selected", + ) + self._disconnect_btn.clicked.connect(self._on_disconnect_selected) + sessions_btn_row.addWidget(self._disconnect_btn) + sessions_btn_row.addStretch() + layout.addLayout(sessions_btn_row) + push_row = QHBoxLayout() + self._push_file_btn = self._tr(QPushButton(), "rd_webrtc_push_file") + self._push_file_btn.clicked.connect(self._on_push_file) + push_row.addWidget(self._push_file_btn) + audit_btn = self._tr(QPushButton(), "rd_webrtc_view_audit") + audit_btn.clicked.connect(self._on_view_audit) + push_row.addWidget(audit_btn) + push_row.addStretch() + layout.addLayout(push_row) + + def _build_signaling_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_signaling_group") + grid = QGridLayout() + grid.addWidget(self._tr(QLabel(), "rd_webrtc_server_label"), 0, 0) + self._server_edit = QLineEdit(_DEFAULT_SIGNALING_URL) + grid.addWidget(self._server_edit, 0, 1, 1, 3) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_host_id_label"), 1, 0) + self._host_id_edit = QLineEdit(generate_host_id()) + grid.addWidget(self._host_id_edit, 1, 1, 1, 2) + regen = self._tr(QPushButton(), "rd_webrtc_regen_id") + regen.clicked.connect(self._on_regen_id) + grid.addWidget(regen, 1, 3) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_secret_label"), 2, 0) + self._secret_edit = QLineEdit() + self._secret_edit.setEchoMode(QLineEdit.EchoMode.Password) + grid.addWidget(self._secret_edit, 2, 1, 1, 3) + self._publish_btn = self._tr( + QPushButton(), "rd_webrtc_publish_via_server", + ) + self._publish_btn.clicked.connect(self._on_publish_via_server) + grid.addWidget(self._publish_btn, 3, 0, 1, 4) + # Read-only fingerprint label + copy button + from je_auto_control.utils.remote_desktop.fingerprint import ( + fingerprint_for_display, load_or_create_host_fingerprint, + ) + try: + fp = load_or_create_host_fingerprint() + except OSError: + fp = "" + grid.addWidget(self._tr(QLabel(), "rd_webrtc_my_fingerprint"), 4, 0) + self._fingerprint_label = QLabel( + fingerprint_for_display(fp) if fp else "", + ) + self._fingerprint_label.setStyleSheet( + "color: #888; font-family: 'Consolas', monospace; font-size: 10pt;", + ) + self._fingerprint_label.setTextInteractionFlags( + Qt.TextInteractionFlag.TextSelectableByMouse, + ) + grid.addWidget(self._fingerprint_label, 4, 1, 1, 2) + copy_fp_btn = self._tr(QPushButton(), "rd_webrtc_copy_fingerprint") + copy_fp_btn.clicked.connect(lambda: self._on_copy_fingerprint(fp)) + grid.addWidget(copy_fp_btn, 4, 3) + group.setLayout(grid) + return group + + def _on_copy_fingerprint(self, fp: str) -> None: + from PySide6.QtWidgets import QApplication + clipboard = QApplication.clipboard() + if clipboard is not None: + clipboard.setText(fp) + + def _build_config_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_config_group") + grid = QGridLayout() + grid.addWidget(self._tr(QLabel(), "rd_token_label"), 0, 0) + self._token_edit = self._tr(QLineEdit(), "rd_token_placeholder") + grid.addWidget(self._token_edit, 0, 1) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_monitor_label"), 1, 0) + self._monitor_combo = QComboBox() + self._populate_monitor_combo() + self._monitor_combo.currentIndexChanged.connect( + self._on_monitor_changed, + ) + grid.addWidget(self._monitor_combo, 1, 1) + grid.addWidget(self._tr(QLabel(), "rd_fps_label"), 2, 0) + self._fps_spin = QSpinBox() + self._fps_spin.setRange(1, 60) + self._fps_spin.setValue(_DEFAULT_FPS) + grid.addWidget(self._fps_spin, 2, 1) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_region_label"), 11, 0) + self._region_edit = QLineEdit() + self._tr(self._region_edit, "rd_webrtc_region_placeholder", + "setPlaceholderText") + grid.addWidget(self._region_edit, 11, 1) + pick_region_btn = self._tr(QPushButton(), "rd_webrtc_pick_region") + pick_region_btn.clicked.connect(self._on_pick_region) + grid.addWidget(pick_region_btn, 11, 2) + self._cursor_check = self._tr(QCheckBox(), "rd_webrtc_show_cursor") + self._cursor_check.setChecked(True) + grid.addWidget(self._cursor_check, 3, 0, 1, 2) + self._blank_check = self._tr(QCheckBox(), "rd_webrtc_blank_screen") + self._blank_check.setChecked(False) + self._blank_check.toggled.connect(self._on_toggle_blanking) + grid.addWidget(self._blank_check, 4, 0, 1, 2) + self._readonly_check = self._tr(QCheckBox(), "rd_webrtc_read_only") + self._readonly_check.setChecked(False) + self._readonly_check.toggled.connect(self._on_toggle_readonly) + grid.addWidget(self._readonly_check, 5, 0, 1, 2) + self._adaptive_check = self._tr(QCheckBox(), "rd_webrtc_adaptive") + self._adaptive_check.setChecked(True) + self._adaptive_check.toggled.connect(self._on_toggle_adaptive) + grid.addWidget(self._adaptive_check, 6, 0, 1, 2) + self._mic_recv_check = self._tr(QCheckBox(), "rd_webrtc_recv_mic") + self._mic_recv_check.setChecked(False) + self._mic_recv_check.toggled.connect(self._on_toggle_mic_receive) + grid.addWidget(self._mic_recv_check, 7, 0, 1, 2) + self._host_voice_check = self._tr(QCheckBox(), "rd_webrtc_host_voice") + self._host_voice_check.setChecked(False) + grid.addWidget(self._host_voice_check, 13, 0, 1, 2) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_max_bitrate"), 10, 0) + self._max_bitrate_spin = QSpinBox() + self._max_bitrate_spin.setRange(0, 50000) + self._max_bitrate_spin.setSingleStep(500) + self._max_bitrate_spin.setSuffix(" kbps (0=∞)") + self._max_bitrate_spin.setValue(0) + grid.addWidget(self._max_bitrate_spin, 10, 1) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_ip_whitelist"), 12, 0) + self._ip_whitelist_edit = QTextEdit() + self._ip_whitelist_edit.setMaximumHeight(60) + self._tr(self._ip_whitelist_edit, "rd_webrtc_ip_whitelist_ph", + "setPlaceholderText") + grid.addWidget(self._ip_whitelist_edit, 12, 1) + self._accept_viewer_video_check = self._tr( + QCheckBox(), "rd_webrtc_accept_viewer_video", + ) + self._accept_viewer_video_check.setChecked(False) + self._accept_viewer_video_check.toggled.connect( + self._on_toggle_accept_viewer_video, + ) + grid.addWidget(self._accept_viewer_video_check, 8, 0, 1, 2) + self._accept_opus_audio_check = self._tr( + QCheckBox(), "rd_webrtc_accept_opus_audio", + ) + self._accept_opus_audio_check.setChecked(False) + self._accept_opus_audio_check.toggled.connect( + self._on_toggle_accept_opus_audio, + ) + grid.addWidget(self._accept_opus_audio_check, 9, 0, 1, 2) + group.setLayout(grid) + return group + + def _build_manual_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_manual_group") + layout = QVBoxLayout() + self._generate_btn = self._tr(QPushButton(), "rd_webrtc_generate_offer") + self._generate_btn.clicked.connect(self._on_generate_offer) + layout.addWidget(self._generate_btn) + layout.addWidget(self._tr(QLabel(), "rd_webrtc_offer_label")) + self._offer_view = QTextEdit() + self._offer_view.setReadOnly(True) + self._offer_view.setMinimumHeight(80) + layout.addWidget(self._offer_view) + layout.addWidget(self._tr(QLabel(), "rd_webrtc_answer_input_label")) + self._answer_input = QTextEdit() + self._answer_input.setMinimumHeight(80) + self._tr(self._answer_input, "rd_webrtc_paste_answer", "setPlaceholderText") + layout.addWidget(self._answer_input) + button_row = QHBoxLayout() + self._apply_btn = self._tr(QPushButton(), "rd_webrtc_apply_answer") + self._apply_btn.clicked.connect(self._on_apply_answer) + button_row.addWidget(self._apply_btn) + self._stop_btn = self._tr(QPushButton(), "rd_webrtc_stop_host") + self._stop_btn.clicked.connect(self._on_stop) + button_row.addWidget(self._stop_btn) + layout.addLayout(button_row) + group.setLayout(layout) + return group + + +__all__ = ["_HostUiMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel.py b/je_auto_control/gui/remote_desktop/webrtc_panel.py index c522f74a1..e96dfaaad 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel.py @@ -8,58 +8,50 @@ An advanced collapsible group below exposes STUN/TURN servers; defaults to Google's public STUN, which is enough for most LAN/home-network scenarios. Mobile / strict-NAT users will want to add a TURN server. + +The two panel classes own the state, the signal wiring and the handful of slots +other code patches by name here; each interaction group lives in a mixin module: +``webrtc_host_{ui,connection,sessions,media,trust}`` and +``webrtc_viewer_{ui,connection,media,files,address_book}``. Signals, defaults +and the config reader they share are in ``webrtc_panel_common``. """ from __future__ import annotations import logging from typing import TYPE_CHECKING, Optional -from PySide6.QtCore import QObject, Qt, QTimer, Signal -from PySide6.QtGui import QImage +from PySide6.QtCore import QTimer from PySide6.QtWidgets import ( - QAbstractItemView, QCheckBox, QComboBox, QFileDialog, QGridLayout, - QGroupBox, QHBoxLayout, QHeaderView, QInputDialog, QLabel, QLineEdit, - QMessageBox, QPushButton, QSpinBox, QTableWidget, QTableWidgetItem, - QTextEdit, QVBoxLayout, QWidget, + QMessageBox, QWidget, ) from je_auto_control.gui._i18n_helpers import TranslatableMixin from je_auto_control.gui.remote_desktop._helpers import ( - _CollapsibleSection, _read_import_entries, _t, -) -from je_auto_control.gui.remote_desktop.advanced_group import ( - build_advanced_group, + _t, ) from je_auto_control.gui.remote_desktop.blanking_overlay import BlankingOverlay -from je_auto_control.gui.remote_desktop.frame_display import _FrameDisplay from je_auto_control.gui.remote_desktop.remote_screen_window import ( RemoteScreenWindow, ) -from je_auto_control.gui.remote_desktop.sparkline import Sparkline from je_auto_control.gui.remote_desktop.annotation_overlay import ( HostAnnotationOverlay, ) from je_auto_control.gui.remote_desktop.tray_icon import install_host_tray -from je_auto_control.gui.remote_desktop.trusted_group import ( - build_trusted_group, -) from je_auto_control.gui.remote_desktop.viewer_screen_window import ( ViewerScreenWindow, ) from je_auto_control.gui.remote_desktop.webrtc_dialogs import ( - AddressBookList, AuditLogDialog, KnownHostsDialog, LanBrowseDialog, - PendingViewerDialog, RemoteFilesTable, + PendingViewerDialog, ) from je_auto_control.gui.remote_desktop.webrtc_workers import ( HostPublishLoopWorker, ViewerAnswerPushWorker, ViewerSignalingWorker, - generate_host_id, retire_worker, ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop import ( - MultiViewerHost, SessionRecorder, WebRTCConfig, WebRTCDesktopViewer, + MultiViewerHost, SessionRecorder, WebRTCDesktopViewer, default_address_book, - default_trust_list, install_hardware_codec, is_webrtc_available, - load_or_create_viewer_id, send_magic_packet, uninstall_hardware_codec, + default_trust_list, is_webrtc_available, + load_or_create_viewer_id, ) from je_auto_control.utils.remote_desktop.adaptive_bitrate import ( AdaptiveBitrateController, @@ -67,15 +59,22 @@ from je_auto_control.utils.remote_desktop.session_quality_cache import ( SessionQualityCache, ) -from je_auto_control.utils.remote_desktop.webrtc_inspector import ( - default_webrtc_inspector, -) from je_auto_control.utils.remote_desktop.webrtc_stats import ( - StatsPoller, StatsSnapshot, + StatsPoller, ) -from je_auto_control.utils.remote_desktop.webrtc_transport import ( - BANDWIDTH_PRESETS, fps_for_preset, +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelSignals, ) +from je_auto_control.gui.remote_desktop.webrtc_host_ui import _HostUiMixin +from je_auto_control.gui.remote_desktop.webrtc_host_trust import _HostTrustMixin +from je_auto_control.gui.remote_desktop.webrtc_host_media import _HostMediaMixin +from je_auto_control.gui.remote_desktop.webrtc_host_sessions import _HostSessionsMixin +from je_auto_control.gui.remote_desktop.webrtc_host_connection import _HostConnectionMixin +from je_auto_control.gui.remote_desktop.webrtc_viewer_ui import _ViewerUiMixin +from je_auto_control.gui.remote_desktop.webrtc_viewer_files import _ViewerFilesMixin +from je_auto_control.gui.remote_desktop.webrtc_viewer_address_book import _ViewerAddressBookMixin +from je_auto_control.gui.remote_desktop.webrtc_viewer_media import _ViewerMediaMixin +from je_auto_control.gui.remote_desktop.webrtc_viewer_connection import _ViewerConnectionMixin if TYPE_CHECKING: # imported lazily at runtime to keep startup cheap from je_auto_control.utils.remote_desktop.file_sync import ( @@ -86,107 +85,8 @@ ) -_DEFAULT_FPS = 24 -_DEFAULT_MONITOR = 1 -# Plain http:// is intentional: the bundled signaling server defaults -# to localhost without TLS, and operators put TLS in front via nginx / -# Caddy. Hotspot S5332 acknowledged on a per-line basis; see callers. -_DEFAULT_SIGNALING_URL = "http://127.0.0.1:8765" # NOSONAR python:S5332 - -_QUALITY_DOT_STYLE = "background-color: #555; border-radius: 7px;" -_JSON_FILE_FILTER = "JSON (*.json);;All (*)" - - -def _av_frame_to_qimage(frame) -> Optional[QImage]: - """Convert an aiortc/av video frame to a Qt-owned QImage.""" - try: - arr = frame.to_ndarray(format="rgb24") - except (ValueError, RuntimeError) as error: - autocontrol_logger.debug("av->QImage failed: %r", error) - return None - height, width, _ = arr.shape - image = QImage( - arr.tobytes(), width, height, width * 3, QImage.Format.Format_RGB888, - ) - return image.copy() - - -class _PanelSignals(QObject): - """Bridge so asyncio-thread callbacks reach Qt safely.""" - frame = Signal(QImage) - state = Signal(str) - auth = Signal(bool) - # Host-side: (session_id, viewer_id-or-None) per pending viewer prompt. - pending_viewer = Signal(str, object) - stats = Signal(object) # StatsSnapshot - session_count = Signal(int) - # Viewer-side file browser: list and op result. - inbox_listing = Signal(object) # list[dict] - inbox_op = Signal(str, bool, object) # name, ok, error - # Viewer-side: a file finished transferring (fired from the asyncio thread). - file_received = Signal(object) # path - # Host-side: incoming viewer-shared screen frame - viewer_video_frame = Signal(QImage) - # Host-side: incoming annotation event from viewer - annotation = Signal(object) # dict - - -def _checked_or(panel, attr: str, default: bool = False) -> bool: - """Return ``panel..isChecked()`` if the widget exists, else default.""" - widget = getattr(panel, attr, None) - return widget.isChecked() if widget is not None else default - - -def _read_region(panel) -> Optional[tuple]: - edit = getattr(panel, "_region_edit", None) - if edit is None: - return None - text = edit.text().strip() - if not text: - return None - try: - parts = [int(p.strip()) for p in text.split(",")] - except (ValueError, TypeError): - return None - return tuple(parts) if len(parts) == 4 else None - - -def _read_webrtc_config(panel) -> WebRTCConfig: - """Build a WebRTCConfig from the advanced group + monitor/fps fields.""" - from je_auto_control.utils.remote_desktop.webrtc_transport import ( - _DEFAULT_STUN_SERVERS, - ) - stun_field = panel._stun_edit.text().strip() - ice_servers = [stun_field] if stun_field else list(_DEFAULT_STUN_SERVERS) - monitor = ( - int(panel._monitor_combo.currentData() or _DEFAULT_MONITOR) - if hasattr(panel, "_monitor_combo") else _DEFAULT_MONITOR - ) - fps = (int(panel._fps_spin.value()) - if hasattr(panel, "_fps_spin") else _DEFAULT_FPS) - max_bitrate = ( - int(panel._max_bitrate_spin.value()) - if hasattr(panel, "_max_bitrate_spin") else 0 - ) - return WebRTCConfig( - ice_servers=ice_servers, - turn_url=panel._turn_edit.text().strip() or None, - turn_username=panel._turn_user_edit.text().strip() or None, - turn_credential=panel._turn_cred_edit.text() or None, - monitor_index=monitor, - fps=fps, - show_cursor=_checked_or(panel, "_cursor_check", default=True), - accept_viewer_video=_checked_or(panel, "_accept_viewer_video_check"), - accept_viewer_audio_opus=_checked_or(panel, "_accept_opus_audio_check"), - share_my_screen=_checked_or(panel, "_share_my_screen_check"), - share_my_audio_opus=_checked_or(panel, "_share_opus_mic_check"), - max_bitrate_kbps=max_bitrate, - region=_read_region(panel), - host_voice=_checked_or(panel, "_host_voice_check"), - ) - - -class _WebRTCHostPanel(TranslatableMixin, QWidget): +class _WebRTCHostPanel(_HostUiMixin, _HostTrustMixin, _HostMediaMixin, _HostSessionsMixin, + _HostConnectionMixin, TranslatableMixin, QWidget): """Host: stream this machine's screen and accept viewer input.""" def __init__(self, parent: Optional[QWidget] = None) -> None: @@ -225,431 +125,6 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._refresh_trusted_list() self._update_availability() - # --- UI construction --------------------------------------------------- - - def _build_ui(self) -> None: - layout = QVBoxLayout(self) - layout.addWidget(self._build_signaling_group()) - layout.addWidget(self._build_config_group()) - layout.addWidget(self._build_manual_group()) - layout.addWidget(build_advanced_group(self, include_hw_codec=True)) - layout.addWidget(build_trusted_group(self)) - self._status_label = QLabel(_t("rd_webrtc_status_idle")) - layout.addWidget(self._status_label) - sessions_row = QHBoxLayout() - self._host_quality_dot = QLabel() - self._host_quality_dot.setFixedSize(14, 14) - self._host_quality_dot.setStyleSheet( - _QUALITY_DOT_STYLE, - ) - self._host_quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) - sessions_row.addWidget(self._host_quality_dot) - self._sessions_label = QLabel(_t("rd_webrtc_sessions_count").format(n=0)) - sessions_row.addWidget(self._sessions_label, stretch=1) - layout.addLayout(sessions_row) - self._sessions_table = QTableWidget(0, 5) - self._sessions_table.setHorizontalHeaderLabels([ - "", # quality dot column - _t("rd_webrtc_sess_col_id"), - _t("rd_webrtc_sess_col_viewer"), - _t("rd_webrtc_sess_col_state"), - _t("rd_webrtc_sess_col_connected"), - ]) - self._sessions_table.setColumnWidth(0, 18) - self._sessions_table.horizontalHeader().setSectionResizeMode( - 2, QHeaderView.ResizeMode.Stretch, - ) - self._sessions_table.setEditTriggers( - QAbstractItemView.EditTrigger.NoEditTriggers, - ) - # Hint at a comfortable starting height, but let the table - # grow with the window instead of pinning it at 140 px even - # when the operator has a 4K monitor's worth of space. - self._sessions_table.setMinimumHeight(140) - self._sessions_table.setSelectionBehavior( - QAbstractItemView.SelectionBehavior.SelectRows, - ) - self._sessions_table.setSelectionMode( - QAbstractItemView.SelectionMode.SingleSelection, - ) - self._sessions_table.setContextMenuPolicy( - Qt.ContextMenuPolicy.CustomContextMenu, - ) - self._sessions_table.customContextMenuRequested.connect( - self._on_sessions_context_menu, - ) - layout.addWidget(self._sessions_table) - sessions_btn_row = QHBoxLayout() - self._disconnect_btn = self._tr( - QPushButton(), "rd_webrtc_disconnect_selected", - ) - self._disconnect_btn.clicked.connect(self._on_disconnect_selected) - sessions_btn_row.addWidget(self._disconnect_btn) - sessions_btn_row.addStretch() - layout.addLayout(sessions_btn_row) - push_row = QHBoxLayout() - self._push_file_btn = self._tr(QPushButton(), "rd_webrtc_push_file") - self._push_file_btn.clicked.connect(self._on_push_file) - push_row.addWidget(self._push_file_btn) - audit_btn = self._tr(QPushButton(), "rd_webrtc_view_audit") - audit_btn.clicked.connect(self._on_view_audit) - push_row.addWidget(audit_btn) - push_row.addStretch() - layout.addLayout(push_row) - - def _on_view_audit(self) -> None: - from je_auto_control.utils.remote_desktop.audit_log import ( - default_audit_log, - ) - AuditLogDialog(default_audit_log(), parent=self).exec() - - def _on_push_file(self) -> None: - if self._multi_host is None or self._multi_host.session_count() == 0: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_no_viewers"), - ) - return - path, _filter = QFileDialog.getOpenFileName( - self, _t("rd_webrtc_push_file"), "", - ) - if not path: - return - try: - sent = self._multi_host.broadcast_file(path) - QMessageBox.information( - self, "WebRTC", - _t("rd_webrtc_push_done").format(n=sent, name=path), - ) - except (RuntimeError, OSError, ValueError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_hw_codec_changed(self) -> None: - codec = self._hw_codec_combo.currentData() or "" - if not codec: - uninstall_hardware_codec() - self._status_label.setText(_t("rd_webrtc_hw_codec_off_status")) - return - if install_hardware_codec(codec): - self._status_label.setText( - _t("rd_webrtc_hw_codec_active").format(codec=codec), - ) - else: - self._status_label.setText( - _t("rd_webrtc_hw_codec_failed").format(codec=codec), - ) - - def _on_export_trust(self) -> None: - import json as _json - path, _filter = QFileDialog.getSaveFileName( - self, _t("rd_webrtc_trust_export"), "trusted_viewers.json", - _JSON_FILE_FILTER, - ) - if not path: - return - try: - with open(path, "w", encoding="utf-8") as fh: - _json.dump({"viewers": self._trust_list.list_entries()}, - fh, indent=2, ensure_ascii=False) - except OSError as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_import_trust(self) -> None: - path, _filter = QFileDialog.getOpenFileName( - self, _t("rd_webrtc_trust_import"), "", _JSON_FILE_FILTER, - ) - if not path: - return - try: - viewers = _read_import_entries(path, "viewers") - except (OSError, ValueError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - return - added = 0 - for entry in viewers: - vid, label = entry.get("viewer_id"), entry.get("label") - if isinstance(vid, str) and vid: - self._trust_list.add(vid, label=label if isinstance(label, str) else "") - added += 1 - QMessageBox.information( - self, "WebRTC", - _t("rd_webrtc_trust_import_done").format(n=added), - ) - self._refresh_trusted_list() - - def _refresh_trusted_list(self) -> None: - self._trusted_list.populate(self._trust_list.list_entries()) - - def _build_signaling_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_signaling_group") - grid = QGridLayout() - grid.addWidget(self._tr(QLabel(), "rd_webrtc_server_label"), 0, 0) - self._server_edit = QLineEdit(_DEFAULT_SIGNALING_URL) - grid.addWidget(self._server_edit, 0, 1, 1, 3) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_host_id_label"), 1, 0) - self._host_id_edit = QLineEdit(generate_host_id()) - grid.addWidget(self._host_id_edit, 1, 1, 1, 2) - regen = self._tr(QPushButton(), "rd_webrtc_regen_id") - regen.clicked.connect(self._on_regen_id) - grid.addWidget(regen, 1, 3) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_secret_label"), 2, 0) - self._secret_edit = QLineEdit() - self._secret_edit.setEchoMode(QLineEdit.EchoMode.Password) - grid.addWidget(self._secret_edit, 2, 1, 1, 3) - self._publish_btn = self._tr( - QPushButton(), "rd_webrtc_publish_via_server", - ) - self._publish_btn.clicked.connect(self._on_publish_via_server) - grid.addWidget(self._publish_btn, 3, 0, 1, 4) - # Read-only fingerprint label + copy button - from je_auto_control.utils.remote_desktop.fingerprint import ( - fingerprint_for_display, load_or_create_host_fingerprint, - ) - try: - fp = load_or_create_host_fingerprint() - except OSError: - fp = "" - grid.addWidget(self._tr(QLabel(), "rd_webrtc_my_fingerprint"), 4, 0) - self._fingerprint_label = QLabel( - fingerprint_for_display(fp) if fp else "", - ) - self._fingerprint_label.setStyleSheet( - "color: #888; font-family: 'Consolas', monospace; font-size: 10pt;", - ) - self._fingerprint_label.setTextInteractionFlags( - Qt.TextInteractionFlag.TextSelectableByMouse, - ) - grid.addWidget(self._fingerprint_label, 4, 1, 1, 2) - copy_fp_btn = self._tr(QPushButton(), "rd_webrtc_copy_fingerprint") - copy_fp_btn.clicked.connect(lambda: self._on_copy_fingerprint(fp)) - grid.addWidget(copy_fp_btn, 4, 3) - group.setLayout(grid) - return group - - def _on_copy_fingerprint(self, fp: str) -> None: - from PySide6.QtWidgets import QApplication - clipboard = QApplication.clipboard() - if clipboard is not None: - clipboard.setText(fp) - - def _build_config_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_config_group") - grid = QGridLayout() - grid.addWidget(self._tr(QLabel(), "rd_token_label"), 0, 0) - self._token_edit = self._tr(QLineEdit(), "rd_token_placeholder") - grid.addWidget(self._token_edit, 0, 1) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_monitor_label"), 1, 0) - self._monitor_combo = QComboBox() - self._populate_monitor_combo() - self._monitor_combo.currentIndexChanged.connect( - self._on_monitor_changed, - ) - grid.addWidget(self._monitor_combo, 1, 1) - grid.addWidget(self._tr(QLabel(), "rd_fps_label"), 2, 0) - self._fps_spin = QSpinBox() - self._fps_spin.setRange(1, 60) - self._fps_spin.setValue(_DEFAULT_FPS) - grid.addWidget(self._fps_spin, 2, 1) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_region_label"), 11, 0) - self._region_edit = QLineEdit() - self._tr(self._region_edit, "rd_webrtc_region_placeholder", - "setPlaceholderText") - grid.addWidget(self._region_edit, 11, 1) - pick_region_btn = self._tr(QPushButton(), "rd_webrtc_pick_region") - pick_region_btn.clicked.connect(self._on_pick_region) - grid.addWidget(pick_region_btn, 11, 2) - self._cursor_check = self._tr(QCheckBox(), "rd_webrtc_show_cursor") - self._cursor_check.setChecked(True) - grid.addWidget(self._cursor_check, 3, 0, 1, 2) - self._blank_check = self._tr(QCheckBox(), "rd_webrtc_blank_screen") - self._blank_check.setChecked(False) - self._blank_check.toggled.connect(self._on_toggle_blanking) - grid.addWidget(self._blank_check, 4, 0, 1, 2) - self._readonly_check = self._tr(QCheckBox(), "rd_webrtc_read_only") - self._readonly_check.setChecked(False) - self._readonly_check.toggled.connect(self._on_toggle_readonly) - grid.addWidget(self._readonly_check, 5, 0, 1, 2) - self._adaptive_check = self._tr(QCheckBox(), "rd_webrtc_adaptive") - self._adaptive_check.setChecked(True) - self._adaptive_check.toggled.connect(self._on_toggle_adaptive) - grid.addWidget(self._adaptive_check, 6, 0, 1, 2) - self._mic_recv_check = self._tr(QCheckBox(), "rd_webrtc_recv_mic") - self._mic_recv_check.setChecked(False) - self._mic_recv_check.toggled.connect(self._on_toggle_mic_receive) - grid.addWidget(self._mic_recv_check, 7, 0, 1, 2) - self._host_voice_check = self._tr(QCheckBox(), "rd_webrtc_host_voice") - self._host_voice_check.setChecked(False) - grid.addWidget(self._host_voice_check, 13, 0, 1, 2) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_max_bitrate"), 10, 0) - self._max_bitrate_spin = QSpinBox() - self._max_bitrate_spin.setRange(0, 50000) - self._max_bitrate_spin.setSingleStep(500) - self._max_bitrate_spin.setSuffix(" kbps (0=∞)") - self._max_bitrate_spin.setValue(0) - grid.addWidget(self._max_bitrate_spin, 10, 1) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_ip_whitelist"), 12, 0) - self._ip_whitelist_edit = QTextEdit() - self._ip_whitelist_edit.setMaximumHeight(60) - self._tr(self._ip_whitelist_edit, "rd_webrtc_ip_whitelist_ph", - "setPlaceholderText") - grid.addWidget(self._ip_whitelist_edit, 12, 1) - self._accept_viewer_video_check = self._tr( - QCheckBox(), "rd_webrtc_accept_viewer_video", - ) - self._accept_viewer_video_check.setChecked(False) - self._accept_viewer_video_check.toggled.connect( - self._on_toggle_accept_viewer_video, - ) - grid.addWidget(self._accept_viewer_video_check, 8, 0, 1, 2) - self._accept_opus_audio_check = self._tr( - QCheckBox(), "rd_webrtc_accept_opus_audio", - ) - self._accept_opus_audio_check.setChecked(False) - self._accept_opus_audio_check.toggled.connect( - self._on_toggle_accept_opus_audio, - ) - grid.addWidget(self._accept_opus_audio_check, 9, 0, 1, 2) - group.setLayout(grid) - return group - - def _on_toggle_accept_viewer_video(self, value: bool) -> None: - if self._multi_host is None: - return - with self._multi_host._lock: - sessions = list(self._multi_host._sessions.values()) - for host in sessions: - try: - if value: - host.set_viewer_video_callback( - self._on_viewer_video_av_frame, - ) - host.enable_accept_viewer_video() - else: - host.disable_accept_viewer_video() - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("toggle accept viewer video: %r", error) - if not value and self._viewer_screen_window is not None: - self._viewer_screen_window.set_image(None) - self._viewer_screen_window.hide() - - def _on_toggle_accept_opus_audio(self, value: bool) -> None: - if self._multi_host is None: - return - with self._multi_host._lock: - sessions = list(self._multi_host._sessions.values()) - for host in sessions: - try: - if value: - host.enable_accept_viewer_audio_opus() - else: - host.disable_accept_viewer_audio_opus() - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("toggle accept opus: %r", error) - - def _on_toggle_mic_receive(self, value: bool) -> None: - if self._multi_host is None: - return - # Apply to every active session. - with self._multi_host._lock: - sessions = list(self._multi_host._sessions.values()) - for host in sessions: - try: - if value: - host.enable_mic_receive() - else: - host.disable_mic_receive() - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("mic receive toggle: %r", error) - - def _on_toggle_adaptive(self, value: bool) -> None: - if value: - self._maybe_start_adaptive() - else: - self._stop_adaptive() - - def _populate_monitor_combo(self) -> None: - try: - from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber - with mss_grabber() as sct: - monitors = sct.monitors - for idx, mon in enumerate(monitors): - if idx == 0: - label = _t("rd_webrtc_monitor_all") - else: - label = f"#{idx}: {mon['width']}x{mon['height']} @"\ - f" ({mon['left']},{mon['top']})" - self._monitor_combo.addItem(label, idx) - except (ImportError, RuntimeError, OSError): - for idx in range(4): - self._monitor_combo.addItem(f"#{idx}", idx) - # Default to monitor #1 (the first real screen for mss) - idx_default = self._monitor_combo.findData(_DEFAULT_MONITOR) - if idx_default >= 0: - self._monitor_combo.setCurrentIndex(idx_default) - - def _on_pick_region(self) -> None: - try: - from je_auto_control.gui.selector import open_region_selector - region = open_region_selector(self) - except (ImportError, RuntimeError, OSError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - return - if region is None: - return - x, y, w, h = region - self._region_edit.setText(f"{x},{y},{w},{h}") - - def _on_monitor_changed(self, _i: int) -> None: - idx = self._monitor_combo.currentData() - if idx is None or self._multi_host is None: - return - track = self._multi_host.screen_track() - if track is None: - return - try: - track.set_target_monitor(int(idx)) - autocontrol_logger.info("monitor switched to #%d live", int(idx)) - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("set_target_monitor: %r", error) - - def _on_toggle_readonly(self, value: bool) -> None: - if self._multi_host is not None: - self._multi_host.set_read_only(value) - - def _on_toggle_blanking(self, checked: bool) -> None: - if checked: - if self._blanking is None: - self._blanking = BlankingOverlay() - self._blanking.show() - elif self._blanking is not None: - self._blanking.hide() - - def _build_manual_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_manual_group") - layout = QVBoxLayout() - self._generate_btn = self._tr(QPushButton(), "rd_webrtc_generate_offer") - self._generate_btn.clicked.connect(self._on_generate_offer) - layout.addWidget(self._generate_btn) - layout.addWidget(self._tr(QLabel(), "rd_webrtc_offer_label")) - self._offer_view = QTextEdit() - self._offer_view.setReadOnly(True) - self._offer_view.setMinimumHeight(80) - layout.addWidget(self._offer_view) - layout.addWidget(self._tr(QLabel(), "rd_webrtc_answer_input_label")) - self._answer_input = QTextEdit() - self._answer_input.setMinimumHeight(80) - self._tr(self._answer_input, "rd_webrtc_paste_answer", "setPlaceholderText") - layout.addWidget(self._answer_input) - button_row = QHBoxLayout() - self._apply_btn = self._tr(QPushButton(), "rd_webrtc_apply_answer") - self._apply_btn.clicked.connect(self._on_apply_answer) - button_row.addWidget(self._apply_btn) - self._stop_btn = self._tr(QPushButton(), "rd_webrtc_stop_host") - self._stop_btn.clicked.connect(self._on_stop) - button_row.addWidget(self._stop_btn) - layout.addLayout(button_row) - group.setLayout(layout) - return group - def _update_availability(self) -> None: if not is_webrtc_available(): for widget in (self._generate_btn, self._apply_btn, @@ -657,8 +132,6 @@ def _update_availability(self) -> None: widget.setEnabled(False) self._status_label.setText(_t("rd_webrtc_unavailable")) - # --- handlers ---------------------------------------------------------- - def prefill(self, token: Optional[str] = None, host_id: Optional[str] = None, signaling_url: Optional[str] = None) -> None: @@ -675,9 +148,6 @@ def prefill(self, token: Optional[str] = None, if signaling_url and hasattr(self, "_server_edit"): self._server_edit.setText(signaling_url) - def _on_regen_id(self) -> None: - self._host_id_edit.setText(generate_host_id()) - def _on_tray_open(self) -> None: win = self.window() if win is None: @@ -695,124 +165,6 @@ def _on_tray_quit(self) -> None: from PySide6.QtWidgets import QApplication QApplication.quit() - def _on_publish_via_server(self) -> None: - if not self._validate_required_fields(needs_server=True): - return - self._stop_host_if_any() - try: - self._multi_host = self._build_multi_host( - self._token_edit.text().strip(), - ) - except (ValueError, RuntimeError, OSError) as error: - self._show_error(error) - return - self._set_hosting(True) - self._publish_loop = HostPublishLoopWorker( - multi_host=self._multi_host, - server_url=self._server_edit.text().strip(), - host_id=self._host_id_edit.text().strip(), - secret=self._secret_edit.text() or None, - ) - self._publish_loop.offer_published.connect(self._on_loop_offer_published) - self._publish_loop.session_connected.connect(self._on_loop_session_connected) - self._publish_loop.failed.connect(self._on_signaling_failed) - self._status_label.setText(_t("rd_webrtc_publishing_offer")) - self._publish_loop.start() - self._start_lan_advertise() - - def _start_lan_advertise(self) -> None: - try: - from je_auto_control.utils.remote_desktop.lan_discovery import ( - HostAdvertiser, is_discovery_available, - ) - except ImportError: - return - if not is_discovery_available(): - return - try: - if self._lan_advertiser is not None: - self._lan_advertiser.stop() - self._lan_advertiser = HostAdvertiser( - host_id=self._host_id_edit.text().strip(), - signaling_url=self._server_edit.text().strip(), - ) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("lan advertise: %r", error) - - def _stop_lan_advertise(self) -> None: - if self._lan_advertiser is not None: - try: - self._lan_advertiser.stop() - except (RuntimeError, OSError): - pass - self._lan_advertiser = None - - def _on_loop_offer_published(self, session_id: str) -> None: - autocontrol_logger.debug("publish loop: offer for %s", session_id) - # Optional: surface the session in the UI; for now just log. - - def _on_loop_session_connected(self, session_id: str) -> None: - if self._multi_host is not None: - self._signals.session_count.emit(self._multi_host.session_count()) - - def _on_signaling_failed(self, message: str) -> None: - QMessageBox.warning(self, "WebRTC", message) - self._status_label.setText(_t("rd_webrtc_status_idle")) - - def _on_generate_offer(self) -> None: - token = self._token_edit.text().strip() - if not token: - QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_token_required")) - return - try: - if self._multi_host is None: - self._multi_host = self._build_multi_host(token) - except (ValueError, RuntimeError, OSError) as error: - self._show_error(error) - return - self._set_hosting(True) - self._status_label.setText(_t("rd_webrtc_generating_offer")) - self._offer_view.setPlainText("") - QTimer.singleShot(0, self, self._produce_offer) - - def _require_multi_host(self) -> MultiViewerHost: - """Return the running host, or say the session is not up yet.""" - host = self._multi_host - if host is None: - raise RuntimeError(_t("rd_webrtc_not_started")) - return host - - def _produce_offer(self) -> None: - try: - session_id, offer = self._require_multi_host().create_session_offer() - except (RuntimeError, OSError) as error: # PermissionError is an OSError - self._show_error(error) - return - self._manual_session_id = session_id - self._offer_view.setPlainText(offer) - self._status_label.setText(_t("rd_webrtc_offer_ready")) - - def _on_apply_answer(self) -> None: - if self._multi_host is None or not self._manual_session_id: - QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_no_offer_yet")) - return - answer = self._answer_input.toPlainText().strip() - if not answer: - QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_no_answer")) - return - try: - self._multi_host.accept_session_answer(self._manual_session_id, answer) - self._status_label.setText(_t("rd_webrtc_answer_applied")) - except (ValueError, RuntimeError, OSError, KeyError) as error: - self._show_error(error) - return - self._manual_session_id = None # consumed; next Generate creates new session - - def _on_stop(self) -> None: - self._stop_host_if_any() - self._status_label.setText(_t("rd_webrtc_status_idle")) - self._signals.session_count.emit(0) - def _on_pending_viewer(self, session_id: str, viewer_id) -> None: if self._multi_host is None: return @@ -838,406 +190,6 @@ def _on_pending_viewer(self, session_id: str, viewer_id) -> None: return self._signals.session_count.emit(host.session_count()) - def _on_session_count(self, count: int) -> None: - self._sessions_label.setText( - _t("rd_webrtc_sessions_count").format(n=count), - ) - if self._tray is not None: - self._tray.set_state(sessions=count) - # Color the badge by load: gray=0, green=1-3, yellow=4-10, red=>10 - if count == 0: - bg, fg = "#3a3a3a", "#888" - elif count <= 3: - bg, fg = "#1f4d1f", "#a6e3a6" - elif count <= 10: - bg, fg = "#5a4710", "#f5d99a" - else: - bg, fg = "#5a1010", "#ffaaaa" - self._sessions_label.setStyleSheet( - f"background: {bg}; color: {fg}; padding: 2px 8px;" - "border-radius: 8px; font-weight: bold;", - ) - self._sync_session_pollers() - self._refresh_sessions_table() - if count > 0: - self._maybe_start_adaptive() - else: - self._stop_adaptive() - self._reset_host_quality_dot() - - def _sync_session_pollers(self) -> None: - """Spawn StatsPoller for new sessions; stop pollers for gone ones.""" - if self._multi_host is None: - # Snapshot before clear() so a slow stop() doesn't race with the - # clear that follows. - for poller in list(self._session_pollers.values()): # NOSONAR python:S7504 - poller.stop() - self._session_pollers.clear() - self._session_cache.reset() - return - active_sids = {s["session_id"] for s in self._multi_host.list_sessions()} - # Stop pollers whose session is gone - # The loop deletes from self._session_pollers, so list() is required - # to avoid a RuntimeError. - for sid in list(self._session_pollers.keys()): # NOSONAR python:S7504 - if sid not in active_sids: - self._session_pollers[sid].stop() - del self._session_pollers[sid] - self._session_cache.drop(sid) - # Spawn pollers for new sessions - for sid in active_sids: - if sid in self._session_pollers: - continue - pc = self._multi_host.session_pc(sid) - if pc is None: - continue - poller = StatsPoller(pc, self._make_session_stats_handler(sid), - interval_s=1.0) - poller.start() - self._session_pollers[sid] = poller - - def _make_session_stats_handler(self, session_id: str): - """Closure capturing session_id for the per-session poller.""" - def _handle(snapshot: StatsSnapshot) -> None: - default_webrtc_inspector().record(snapshot) - color = self._quality_color(snapshot) - self._session_cache.set( - session_id, color=color, snapshot=snapshot, - ) - # Re-paint just the dot cell for this session_id (avoid full reflow). - # One read: the GUI thread can clear it between a check and a call. - host = self._multi_host - self._signals.session_count.emit(host.session_count() if host else 0) - return _handle - - @staticmethod - def _format_quality_tooltip(snapshot: Optional[StatsSnapshot]) -> str: - if snapshot is None: - return _t("rd_webrtc_quality_unknown") - parts = [] - if snapshot.rtt_ms is not None: - parts.append(f"RTT {snapshot.rtt_ms:.0f}ms") - if snapshot.packet_loss_pct is not None: - parts.append(f"loss {snapshot.packet_loss_pct:.1f}%") - if snapshot.fps is not None: - parts.append(f"FPS {snapshot.fps:.1f}") - if snapshot.bitrate_kbps is not None: - parts.append(f"{snapshot.bitrate_kbps:.0f}kbps") - return " | ".join(parts) if parts else _t("rd_webrtc_quality_unknown") - - @staticmethod - def _quality_color(snapshot: StatsSnapshot) -> str: - rtt = snapshot.rtt_ms - loss = snapshot.packet_loss_pct or 0.0 - if rtt is None: - return "#555" - if rtt < 80 and loss < 1.0: - return "#3a9c3a" - if rtt < 200 and loss < 5.0: - return "#c9a23a" - return "#cc4444" - - def _refresh_sessions_table(self) -> None: - from datetime import datetime - from PySide6.QtGui import QColor - if self._multi_host is None: - self._sessions_table.setRowCount(0) - return - sessions = self._multi_host.list_sessions() - self._sessions_table.setRowCount(len(sessions)) - for row, info in enumerate(sessions): - sid = info.get("session_id", "") - vid = info.get("pending_viewer_id") or "" - state = info.get("state", "") - connected = info.get("connected_at") or "" - if connected: - try: - dt = datetime.fromisoformat(connected) - connected = dt.astimezone().strftime("%H:%M:%S") - except (TypeError, ValueError): - pass - color = self._session_cache.get_color(sid) - dot_item = QTableWidgetItem("●") - dot_item.setForeground(QColor(color)) - dot_item.setTextAlignment(Qt.AlignmentFlag.AlignCenter) - dot_item.setToolTip(self._format_quality_tooltip( - self._session_cache.get_snapshot(sid), - )) - self._sessions_table.setItem(row, 0, dot_item) - id_item = QTableWidgetItem(sid[:8] if sid else "") - id_item.setData(Qt.ItemDataRole.UserRole, sid) - self._sessions_table.setItem(row, 1, id_item) - self._sessions_table.setItem( - row, 2, QTableWidgetItem(vid[:12] if vid else ""), - ) - self._sessions_table.setItem(row, 3, QTableWidgetItem(state)) - self._sessions_table.setItem(row, 4, QTableWidgetItem(connected)) - - def _on_sessions_context_menu(self, position) -> None: - from PySide6.QtWidgets import QMenu - if self._multi_host is None: - return - row = self._sessions_table.rowAt(position.y()) - if row < 0: - return - self._sessions_table.selectRow(row) - sid_item = self._sessions_table.item(row, 1) - viewer_item = self._sessions_table.item(row, 2) - if sid_item is None: - return - sid = sid_item.data(Qt.ItemDataRole.UserRole) or "" - viewer_id = viewer_item.text() if viewer_item is not None else "" - menu = QMenu(self._sessions_table) - actions = { - "disconnect": menu.addAction(_t("rd_webrtc_disconnect_selected")), - "trust": menu.addAction(_t("rd_webrtc_sess_trust_viewer")), - "copy": menu.addAction(_t("rd_webrtc_sess_copy_id")), - } - actions["trust"].setEnabled(bool(viewer_id)) - chosen = menu.exec( - self._sessions_table.viewport().mapToGlobal(position), - ) - self._dispatch_session_menu(chosen, actions, sid, viewer_id) - - def _dispatch_session_menu(self, chosen, actions: dict, - sid: str, viewer_id: str) -> None: - """Run the action chosen from the sessions context menu.""" - if chosen is actions["disconnect"]: - self._on_disconnect_selected() - elif chosen is actions["trust"] and viewer_id: - self._trust_session_viewer(sid) - elif chosen is actions["copy"] and sid: - self._copy_session_id_to_clipboard(sid) - - def _trust_session_viewer(self, sid: str) -> None: - try: - multi_host = self._require_multi_host() - with multi_host._lock: - host = multi_host._sessions.get(sid) - full_vid = host.pending_viewer_id if host is not None else None - if full_vid: - self._trust_list.add(full_vid, label=f"sess {sid[:6]}") - self._refresh_trusted_list() - except (RuntimeError, OSError, ValueError) as error: - autocontrol_logger.warning("trust viewer: %r", error) - - @staticmethod - def _copy_session_id_to_clipboard(sid: str) -> None: - from PySide6.QtWidgets import QApplication - clip = QApplication.clipboard() - if clip is not None: - clip.setText(sid) - - def _on_disconnect_selected(self) -> None: - if self._multi_host is None: - return - row = self._sessions_table.currentRow() - if row < 0: - return - item = self._sessions_table.item(row, 1) - if item is None: - return - sid = item.data(Qt.ItemDataRole.UserRole) - if not isinstance(sid, str) or not sid: - return - try: - self._multi_host.stop_session(sid) - except (KeyError, RuntimeError, OSError) as error: - autocontrol_logger.warning("disconnect session: %r", error) - self._signals.session_count.emit(self._multi_host.session_count()) - - def _maybe_start_adaptive(self) -> None: - # Always start a stats poller when sessions are active so the host - # quality dot updates; the adaptive controller is an optional - # consumer enabled via the checkbox. - if self._adaptive_poller is not None or self._multi_host is None: - return - track = self._multi_host.screen_track() - pc = self._multi_host.first_session_pc() - if pc is None: - return - if track is not None and self._adaptive_check.isChecked(): - max_fps = int(self._fps_spin.value()) - self._adaptive_controller = AdaptiveBitrateController( - track, max_fps=max_fps, - max_bitrate_kbps=int(self._max_bitrate_spin.value()), - ) - else: - self._adaptive_controller = None - self._adaptive_poller = StatsPoller( - pc, self._on_host_stats, interval_s=1.0, - ) - self._adaptive_poller.start() - autocontrol_logger.info( - "host stats poller active (adaptive=%s)", - self._adaptive_controller is not None, - ) - - def _on_host_stats(self, snapshot: StatsSnapshot) -> None: - # Fan-out: feed adaptive controller (if enabled) + update quality dot - if self._adaptive_controller is not None: - try: - self._adaptive_controller.on_stats(snapshot) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("adaptive on_stats: %r", error) - self._signals.stats.emit(snapshot) - - def _update_host_quality_dot(self, snapshot: StatsSnapshot) -> None: - rtt = snapshot.rtt_ms - loss = snapshot.packet_loss_pct or 0.0 - if rtt is None: - color = "#555" - tip_key = "rd_webrtc_quality_unknown" - elif rtt < 80 and loss < 1.0: - color = "#3a9c3a" - tip_key = "rd_webrtc_quality_good" - elif rtt < 200 and loss < 5.0: - color = "#c9a23a" - tip_key = "rd_webrtc_quality_fair" - else: - color = "#cc4444" - tip_key = "rd_webrtc_quality_poor" - self._host_quality_dot.setStyleSheet( - f"background-color: {color}; border-radius: 7px;", - ) - self._host_quality_dot.setToolTip(_t(tip_key)) - - def _reset_host_quality_dot(self) -> None: - self._host_quality_dot.setStyleSheet( - _QUALITY_DOT_STYLE, - ) - self._host_quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) - - def _stop_adaptive(self) -> None: - if self._adaptive_poller is not None: - self._adaptive_poller.stop() - self._adaptive_poller = None - self._adaptive_controller = None - - def _on_remove_trust(self, viewer_id: str) -> None: - self._trust_list.remove(viewer_id) - self._refresh_trusted_list() - - def _on_remove_trust_button(self) -> None: - item = self._trusted_list.currentItem() - if item is None: - return - viewer_id = item.data(Qt.ItemDataRole.UserRole) - if isinstance(viewer_id, str): - self._on_remove_trust(viewer_id) - - def _on_clear_trust(self) -> None: - result = QMessageBox.question( - self, "WebRTC", _t("rd_webrtc_clear_trust_confirm"), - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - if result != QMessageBox.StandardButton.Yes: - return - self._trust_list.clear() - self._refresh_trusted_list() - - # --- helpers ----------------------------------------------------------- - - def _validate_required_fields(self, *, needs_server: bool) -> bool: - token = self._token_edit.text().strip() - if not token: - QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_token_required")) - return False - if needs_server: - if not self._server_edit.text().strip(): - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_server_required"), - ) - return False - if not self._host_id_edit.text().strip(): - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_host_id_required"), - ) - return False - return True - - def _build_multi_host(self, token: str) -> MultiViewerHost: - whitelist_text = self._ip_whitelist_edit.toPlainText().strip() - whitelist = [line.strip() for line in whitelist_text.splitlines() - if line.strip() and not line.strip().startswith("#")] - host = MultiViewerHost( - token=token, - config=_read_webrtc_config(self), - trust_list=self._trust_list, - read_only=self._readonly_check.isChecked(), - ip_whitelist=whitelist, - on_annotation=self._signals.annotation.emit, - on_session_state=lambda _sid, state: self._signals.state.emit(state), - on_session_authenticated=self._on_session_authed, - on_pending_viewer=self._signals.pending_viewer.emit, - ) - return host - - def _on_annotation_event(self, data) -> None: - if not isinstance(data, dict): - return - if self._annotation_overlay is None: - self._annotation_overlay = HostAnnotationOverlay(parent=self) - self._annotation_overlay.apply(data) - - def _on_session_authed(self, session_id: str) -> None: - self._signals.auth.emit(True) - if (self._multi_host is None - or not self._accept_viewer_video_check.isChecked()): - return - # Wire viewer-video callback on this freshly-authed session - with self._multi_host._lock: - host = self._multi_host._sessions.get(session_id) - if host is None: - return - host.set_viewer_video_callback(self._on_viewer_video_av_frame) - - def _on_viewer_video_av_frame(self, frame) -> None: - image = _av_frame_to_qimage(frame) - if image is not None: - self._signals.viewer_video_frame.emit(image) - - def _on_viewer_video_image(self, image: QImage) -> None: - if self._viewer_screen_window is None: - self._viewer_screen_window = ViewerScreenWindow(parent=self) - self._viewer_screen_window.closed.connect( - self._on_viewer_screen_closed, - ) - if not self._viewer_screen_window.isVisible(): - self._viewer_screen_window.show() - self._viewer_screen_window.set_image(image) - - def _on_viewer_screen_closed(self) -> None: - if self._viewer_screen_window is not None: - self._viewer_screen_window.set_image(None) - - def _stop_host_if_any(self) -> None: - self._stop_adaptive() - self._stop_lan_advertise() - if self._annotation_overlay is not None: - self._annotation_overlay.clear() - self._annotation_overlay.hide() - # Snapshot before clear(), as in _refresh_session_pollers. - for poller in list(self._session_pollers.values()): # NOSONAR python:S7504 - poller.stop() - self._session_pollers.clear() - self._session_cache.reset() - retire_worker(self._publish_loop) - self._publish_loop = None - if self._viewer_screen_window is not None: - self._viewer_screen_window.set_image(None) - self._viewer_screen_window.hide() - if self._multi_host is None: - return - try: - self._multi_host.stop_all() - except (RuntimeError, OSError): - pass - finally: - self._multi_host = None - self._manual_session_id = None - self._set_hosting(False) - def _set_hosting(self, hosting: bool) -> None: if self._tray is not None: self._tray.set_hosting(hosting) @@ -1257,7 +209,8 @@ def retranslate(self) -> None: TranslatableMixin.retranslate(self) -class _WebRTCViewerPanel(TranslatableMixin, QWidget): +class _WebRTCViewerPanel(_ViewerUiMixin, _ViewerFilesMixin, _ViewerAddressBookMixin, _ViewerMediaMixin, + _ViewerConnectionMixin, TranslatableMixin, QWidget): """Viewer: receive screen and send input.""" def __init__(self, parent: Optional[QWidget] = None) -> None: @@ -1299,716 +252,6 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._refresh_address_book() self._update_availability() - def _build_ui(self) -> None: - layout = QVBoxLayout(self) - layout.setContentsMargins(12, 12, 12, 12) - layout.setSpacing(8) - # Essentials always visible: address book + recommended - # signaling-server flow + WebRTC config. - layout.addWidget(self._build_address_book_group()) - layout.addWidget(self._build_signaling_group()) - layout.addWidget(self._build_config_group()) - # Heavy / rarely-used groups now collapse by default so the - # tab fits on a normal display without scrolling. - layout.addWidget(self._wrap_collapsed( - self._build_manual_group(), - "rd_webrtc_manual_group", - )) - layout.addWidget(build_advanced_group(self)) - layout.addWidget(self._wrap_collapsed( - self._build_remote_files_group(), - "rd_webrtc_files_group", - )) - layout.addWidget(self._wrap_collapsed( - self._build_sync_group(), - "rd_webrtc_sync_group", - )) - self._status_label = QLabel(_t("rd_webrtc_status_idle")) - layout.addWidget(self._status_label) - action_row = QHBoxLayout() - self._cad_btn = self._tr(QPushButton(), "rd_webrtc_send_cad") - self._cad_btn.clicked.connect(self._on_send_cad) - action_row.addWidget(self._cad_btn) - self._wol_btn = self._tr(QPushButton(), "rd_webrtc_wake_on_lan") - self._wol_btn.clicked.connect(self._on_wake_on_lan) - action_row.addWidget(self._wol_btn) - self._mic_btn = self._tr(QPushButton(), "rd_webrtc_send_mic") - self._mic_btn.setCheckable(True) - self._mic_btn.clicked.connect(self._on_toggle_mic) - action_row.addWidget(self._mic_btn) - self._send_file_btn = self._tr(QPushButton(), "rd_webrtc_send_file") - self._send_file_btn.clicked.connect(self._on_send_file) - action_row.addWidget(self._send_file_btn) - self._record_btn = self._tr(QPushButton(), "rd_webrtc_start_recording") - self._record_btn.setCheckable(True) - self._record_btn.clicked.connect(self._on_toggle_recording) - action_row.addWidget(self._record_btn) - self._pen_btn = self._tr(QPushButton(), "rd_webrtc_pen_off") - self._pen_btn.setCheckable(True) - self._pen_btn.clicked.connect(self._on_toggle_pen) - action_row.addWidget(self._pen_btn) - self._pen_clear_btn = self._tr(QPushButton(), "rd_webrtc_pen_clear") - self._pen_clear_btn.clicked.connect(self._on_pen_clear) - action_row.addWidget(self._pen_clear_btn) - action_row.addStretch() - layout.addLayout(action_row) - stats_row = QHBoxLayout() - self._quality_dot = QLabel() - self._quality_dot.setFixedSize(14, 14) - self._quality_dot.setStyleSheet( - _QUALITY_DOT_STYLE, - ) - self._quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) - stats_row.addWidget(self._quality_dot) - self._stats_label = QLabel(_t("rd_webrtc_stats_idle")) - self._stats_label.setStyleSheet( - "color: #ccaa55; font-family: 'Consolas', monospace;", - ) - stats_row.addWidget(self._stats_label, stretch=1) - layout.addLayout(stats_row) - spark_row = QHBoxLayout() - self._rtt_spark = Sparkline(line_color="#3a9c3a") - self._rtt_spark.setToolTip("RTT (ms)") - spark_row.addWidget(self._rtt_spark, stretch=1) - self._bitrate_spark = Sparkline(line_color="#c97a00") - self._bitrate_spark.setToolTip("kbps") - spark_row.addWidget(self._bitrate_spark, stretch=1) - layout.addLayout(spark_row) - # Hidden _FrameDisplay placeholder kept around so the rest of - # the class (pen mode toggle, image setter) doesn't have to - # branch between "popup open" and "popup closed". It also lets - # the panel decode frames before the operator opens the popup. - # When the popup IS open, frames + input round-trip through - # the popup's display instead. - self._frame_display = _FrameDisplay() - self._frame_display.setVisible(False) - layout.addWidget(self._frame_display) - layout.addStretch(1) - self._wire_input_signals() - - def _build_sync_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_sync_group") - layout = QGridLayout() - layout.addWidget(self._tr(QLabel(), "rd_webrtc_sync_dir"), 0, 0) - self._sync_dir_edit = QLineEdit() - self._tr(self._sync_dir_edit, "rd_webrtc_sync_dir_ph", - "setPlaceholderText") - layout.addWidget(self._sync_dir_edit, 0, 1) - browse_btn = self._tr(QPushButton(), "rd_webrtc_browse") - browse_btn.clicked.connect(self._on_sync_browse) - layout.addWidget(browse_btn, 0, 2) - self._sync_btn = self._tr(QPushButton(), "rd_webrtc_sync_start") - self._sync_btn.setCheckable(True) - self._sync_btn.clicked.connect(self._on_toggle_sync) - layout.addWidget(self._sync_btn, 0, 3) - group.setLayout(layout) - return group - - def _on_sync_browse(self) -> None: - path = QFileDialog.getExistingDirectory( - self, _t("rd_webrtc_sync_dir"), - ) - if path: - self._sync_dir_edit.setText(path) - - def _on_toggle_sync(self, checked: bool) -> None: - if checked: - if self._viewer is None or not self._viewer.authenticated: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_cad_not_connected"), - ) - self._sync_btn.setChecked(False) - return - path = self._sync_dir_edit.text().strip() - if not path: - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_sync_dir_required"), - ) - self._sync_btn.setChecked(False) - return - from je_auto_control.utils.remote_desktop.file_sync import ( - FolderSyncEngine, - ) - from pathlib import Path as _Path - try: - viewer = self._require_viewer() - engine = FolderSyncEngine( - watch_dir=_Path(path), - sender=lambda local, name: viewer.send_file( - local, remote_name=name, - ), - ) - self._sync_engine = engine - engine.start() - except (RuntimeError, OSError) as error: # FileNotFoundError is an OSError - QMessageBox.warning(self, "WebRTC", str(error)) - self._sync_btn.setChecked(False) - return - self._sync_btn.setText(_t("rd_webrtc_sync_stop")) - else: - if self._sync_engine is not None: - try: - self._sync_engine.stop() - except (RuntimeError, OSError): - pass - self._sync_engine = None - self._sync_btn.setText(_t("rd_webrtc_sync_start")) - - def _build_remote_files_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_remote_files_group") - layout = QVBoxLayout() - button_row = QHBoxLayout() - refresh_btn = self._tr(QPushButton(), "rd_webrtc_browse_refresh") - refresh_btn.clicked.connect(self._on_browse_refresh) - button_row.addWidget(refresh_btn) - pull_btn = self._tr(QPushButton(), "rd_webrtc_browse_pull") - pull_btn.clicked.connect(self._on_browse_pull_button) - button_row.addWidget(pull_btn) - delete_btn = self._tr(QPushButton(), "rd_webrtc_browse_delete") - delete_btn.clicked.connect(self._on_browse_delete_button) - button_row.addWidget(delete_btn) - button_row.addStretch() - layout.addLayout(button_row) - self._remote_files_table = RemoteFilesTable() - self._remote_files_table.pull_requested.connect(self._on_pull_names) - self._remote_files_table.delete_requested.connect(self._on_delete_names) - self._remote_files_table.upload_requested.connect(self._on_upload_paths) - self._remote_files_table.copy_name_requested.connect( - self._on_copy_name, - ) - layout.addWidget(self._remote_files_table) - layout.addWidget(self._tr(QLabel(), "rd_webrtc_browse_dnd_hint")) - group.setLayout(layout) - return group - - def _on_browse_refresh(self) -> None: - if self._viewer is None or not self._viewer.authenticated: - return - try: - self._viewer.request_inbox_listing() - except (RuntimeError, OSError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_browse_pull_button(self) -> None: - names = self._remote_files_table.selected_names() - if not names: - return - self._on_pull_names(names) - - def _on_browse_delete_button(self) -> None: - names = self._remote_files_table.selected_names() - if not names: - return - self._on_delete_names(names) - - def _on_pull_names(self, names) -> None: - if self._viewer is None or not self._viewer.authenticated: - return - try: - for name in names: - self._viewer.request_inbox_file(name) - except (RuntimeError, OSError, ValueError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_delete_names(self, names) -> None: - if not names or self._viewer is None or not self._viewer.authenticated: - return - confirm_text = ( - _t("rd_webrtc_browse_delete_confirm").format(name=names[0]) - if len(names) == 1 - else _t("rd_webrtc_browse_delete_many_confirm").format(n=len(names)) - ) - result = QMessageBox.question( - self, "WebRTC", confirm_text, - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - if result != QMessageBox.StandardButton.Yes: - return - try: - for name in names: - self._viewer.delete_inbox_file(name) - except (RuntimeError, OSError, ValueError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_upload_paths(self, paths) -> None: - if self._viewer is None or not self._viewer.authenticated: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_cad_not_connected"), - ) - return - sent = 0 - last_error = None - for path in paths: - try: - self._viewer.send_file(path) - sent += 1 - except (RuntimeError, OSError, ValueError) as error: - last_error = error - autocontrol_logger.warning("upload %s: %r", path, error) - if sent: - self._status_label.setText( - _t("rd_webrtc_upload_done").format(n=sent), - ) - QTimer.singleShot(500, self, self._on_browse_refresh) - if last_error is not None and sent == 0: - QMessageBox.warning(self, "WebRTC", str(last_error)) - - def _on_copy_name(self, name: str) -> None: - from PySide6.QtWidgets import QApplication as _QApp - clipboard = _QApp.clipboard() - if clipboard is not None: - clipboard.setText(name) - - def _on_inbox_listing(self, files) -> None: - from datetime import datetime - if not isinstance(files, list): - return - def _format_mtime(value): - try: - return datetime.fromtimestamp(float(value)).strftime( - "%Y-%m-%d %H:%M:%S", - ) - except (TypeError, ValueError, OSError, OverflowError): - return str(value) - self._remote_files_table.populate(files, _format_mtime) - - def _on_inbox_op_result(self, name: str, ok: bool, error) -> None: - if ok: - self._status_label.setText( - _t("rd_webrtc_browse_op_ok").format(name=name), - ) - # Refresh listing so the table reflects the change - try: - if self._viewer is not None and self._viewer.authenticated: - self._viewer.request_inbox_listing() - except (RuntimeError, OSError): - pass - else: - QMessageBox.warning( - self, "WebRTC", - _t("rd_webrtc_browse_op_failed").format( - name=name, error=str(error or ""), - ), - ) - - def _on_send_cad(self) -> None: - if self._viewer is None or not self._viewer.authenticated: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_cad_not_connected"), - ) - return - try: - self._viewer.request_send_sas() - except (RuntimeError, OSError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_toggle_mic(self, checked: bool) -> None: - if self._viewer is None or not self._viewer.authenticated: - self._mic_btn.setChecked(False) - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_cad_not_connected"), - ) - return - try: - if checked: - self._viewer.enable_mic_send() - else: - self._viewer.disable_mic_send() - except (RuntimeError, OSError) as error: - self._mic_btn.setChecked(False) - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_send_file(self) -> None: - if self._viewer is None or not self._viewer.authenticated: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_cad_not_connected"), - ) - return - path, _filter = QFileDialog.getOpenFileName( - self, _t("rd_webrtc_send_file"), "", - ) - if not path: - return - try: - self._viewer.send_file(path) - self._status_label.setText( - _t("rd_webrtc_file_sent").format(name=path), - ) - except (RuntimeError, OSError, ValueError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - @staticmethod - def _wol_defaults(entry) -> tuple[str, str]: - """Return (mac, broadcast) pre-fill values from a book entry.""" - if entry is None: - return "", "" - return (entry.get("mac_address", "") or "", - entry.get("broadcast_address", "") or "") - - def _persist_wol_entry(self, entry, mac: str, broadcast: str) -> None: - """Save the MAC / broadcast just used back onto the book entry.""" - if entry is None: - return - self._address_book.upsert( - host_id=entry.get("host_id", ""), - server_url=entry.get("server_url", ""), - mac_address=mac.strip(), - broadcast_address=broadcast.strip() or None, - ) - self._refresh_address_book() - - def _on_wake_on_lan(self) -> None: - entry = self._address_list.selected_entry() - mac, broadcast = self._wol_defaults(entry) - mac, ok = QInputDialog.getText( - self, _t("rd_webrtc_wake_on_lan"), - _t("rd_webrtc_wol_mac_prompt"), text=mac, - ) - if not ok or not mac.strip(): - return - broadcast, ok2 = QInputDialog.getText( - self, _t("rd_webrtc_wake_on_lan"), - _t("rd_webrtc_wol_broadcast_prompt"), - text=broadcast or "255.255.255.255", - ) - if not ok2: - return - try: - send_magic_packet(mac.strip(), - broadcast_address=broadcast.strip() or None) - except (ValueError, OSError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - return - self._persist_wol_entry(entry, mac, broadcast) - QMessageBox.information( - self, _t("rd_webrtc_wake_on_lan"), _t("rd_webrtc_wol_sent"), - ) - - def _on_toggle_recording(self, checked: bool) -> None: - if checked: - if SessionRecorder is None: - QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_unavailable")) - self._record_btn.setChecked(False) - return - path, _filter = QFileDialog.getSaveFileName( - self, _t("rd_webrtc_recording_save_as"), "", - "MP4 (*.mp4);;WebM (*.webm);;Matroska (*.mkv);;All (*)", - ) - if not path: - self._record_btn.setChecked(False) - return - from je_auto_control.utils.remote_desktop.session_recorder import ( - preset_for_path, - ) - preset = preset_for_path(path) - self._recorder = SessionRecorder( - path, - fps=int(self._bandwidth_combo.currentData() and - fps_for_preset(self._bandwidth_combo.currentData()) - or 24), - codec=preset.get("codec", "libx264"), - pixel_format=preset.get("pixel_format", "yuv420p"), - ) - self._record_btn.setText(_t("rd_webrtc_stop_recording")) - else: - if self._recorder is not None: - self._recorder.stop() - # "Saved" was reported for a file that no frame ever created. - key = "rd_webrtc_recording_saved" if self._recorder.has_output else "rd_webrtc_recording_empty" - QMessageBox.information( - self, "WebRTC", _t(key).format(path=str(self._recorder.output_path)), - ) - self._recorder = None - self._record_btn.setText(_t("rd_webrtc_start_recording")) - - def _on_stats(self, snapshot: StatsSnapshot) -> None: - parts = [] - if snapshot.fps is not None: - parts.append(f"FPS {snapshot.fps:.1f}") - if snapshot.bitrate_kbps is not None: - parts.append(f"{snapshot.bitrate_kbps:.0f} kbps") - if snapshot.rtt_ms is not None: - parts.append(f"RTT {snapshot.rtt_ms:.0f} ms") - if snapshot.packet_loss_pct is not None: - parts.append(f"loss {snapshot.packet_loss_pct:.1f}%") - if snapshot.jitter_ms is not None: - parts.append(f"jitter {snapshot.jitter_ms:.1f}ms") - if not parts: - return - self._stats_label.setText(" | ".join(parts)) - self._update_quality_dot(snapshot) - if hasattr(self, "_rtt_spark"): - self._rtt_spark.push(snapshot.rtt_ms) - self._bitrate_spark.push(snapshot.bitrate_kbps) - - def _update_quality_dot(self, snapshot: StatsSnapshot) -> None: - rtt = snapshot.rtt_ms - loss = snapshot.packet_loss_pct or 0.0 - if rtt is None: - color = "#555" - tip_key = "rd_webrtc_quality_unknown" - elif rtt < 80 and loss < 1.0: - color = "#3a9c3a" - tip_key = "rd_webrtc_quality_good" - elif rtt < 200 and loss < 5.0: - color = "#c9a23a" - tip_key = "rd_webrtc_quality_fair" - else: - color = "#cc4444" - tip_key = "rd_webrtc_quality_poor" - self._quality_dot.setStyleSheet( - f"background-color: {color}; border-radius: 7px;", - ) - self._quality_dot.setToolTip(_t(tip_key)) - - def _wrap_collapsed(self, inner: QGroupBox, - title_key: str) -> _CollapsibleSection: - """Wrap an existing groupbox in a collapsed-by-default container. - - The inner group keeps its own translated title, so we just pass - it through the wrapper's body. Heavy / rarely-used groups - (manual SDP, remote files, sync) hide their bodies by default - so the panel doesn't scroll past the fold on a normal display. - """ - section = _CollapsibleSection() - # Translate the wrapper title; the inner groupbox already has - # its own header so we strip its frame to avoid double chrome. - self._tr(section, title_key, setter="setTitle") - inner.setStyleSheet("QGroupBox { border: none; margin-top: 0px; }") - body = QVBoxLayout() - body.setContentsMargins(0, 0, 0, 0) - body.addWidget(inner) - section.set_body_layout(body) - return section - - def _build_address_book_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_address_book_group") - layout = QVBoxLayout() - # Tag filter row - tag_row = QHBoxLayout() - tag_row.addWidget(self._tr(QLabel(), "rd_webrtc_tag_filter")) - self._tag_filter_combo = QComboBox() - self._tag_filter_combo.addItem(_t("rd_webrtc_tag_all"), "") - self._tag_filter_combo.currentIndexChanged.connect( - lambda _i: self._refresh_address_book(), - ) - tag_row.addWidget(self._tag_filter_combo, stretch=1) - layout.addLayout(tag_row) - self._address_list = AddressBookList() - self._address_list.chosen.connect(self._on_address_chosen) - self._address_list.deleted.connect(self._on_address_removed) - self._address_list.favorite_toggled.connect(self._on_address_favorite) - self._address_list.tags_edit_requested.connect(self._on_address_tags) - self._address_list.setMaximumHeight(120) - layout.addWidget(self._address_list) - button_row = QHBoxLayout() - connect_btn = self._tr(QPushButton(), "rd_webrtc_connect_selected") - connect_btn.clicked.connect(self._on_connect_selected_address) - button_row.addWidget(connect_btn) - save_btn = self._tr(QPushButton(), "rd_webrtc_save_current") - save_btn.clicked.connect(self._on_save_current_address) - button_row.addWidget(save_btn) - remove_btn = self._tr(QPushButton(), "rd_webrtc_remove_selected") - remove_btn.clicked.connect(self._on_remove_selected_address) - button_row.addWidget(remove_btn) - kh_btn = self._tr(QPushButton(), "rd_webrtc_manage_known_hosts") - kh_btn.clicked.connect(self._on_manage_known_hosts) - button_row.addWidget(kh_btn) - ab_export = self._tr(QPushButton(), "rd_webrtc_ab_export") - ab_export.clicked.connect(self._on_ab_export) - button_row.addWidget(ab_export) - ab_import = self._tr(QPushButton(), "rd_webrtc_ab_import") - ab_import.clicked.connect(self._on_ab_import) - button_row.addWidget(ab_import) - ab_clear = self._tr(QPushButton(), "rd_webrtc_ab_clear") - ab_clear.clicked.connect(self._on_ab_clear) - button_row.addWidget(ab_clear) - layout.addLayout(button_row) - group.setLayout(layout) - return group - - def _on_ab_export(self) -> None: - import json as _json - path, _filter = QFileDialog.getSaveFileName( - self, _t("rd_webrtc_ab_export"), "address_book.json", - _JSON_FILE_FILTER, - ) - if not path: - return - try: - with open(path, "w", encoding="utf-8") as fh: - _json.dump({"entries": self._address_book.list_entries()}, - fh, indent=2, ensure_ascii=False) - except OSError as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_ab_import(self) -> None: - path, _filter = QFileDialog.getOpenFileName( - self, _t("rd_webrtc_ab_import"), "", _JSON_FILE_FILTER, - ) - if not path: - return - try: - entries = _read_import_entries(path, "entries") - except (OSError, ValueError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - return - added = 0 - for entry in entries: - # Strings only: a numeric host_id was saved, then dropped on the - # next load, and a numeric MAC reached Wake-on-LAN. - text = {key: value for key, value in entry.items() if isinstance(value, str) and value} - if not ("host_id" in text and "server_url" in text): - continue - try: - self._address_book.upsert( - host_id=text["host_id"], server_url=text["server_url"], - label=text.get("label", ""), - mac_address=text.get("mac_address"), - broadcast_address=text.get("broadcast_address"), - ) - added += 1 - except (ValueError, OSError) as error: - autocontrol_logger.debug("ab import upsert: %r", error) - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_ab_import_done").format(n=added), - ) - self._refresh_address_book() - - def _on_ab_clear(self) -> None: - result = QMessageBox.question( - self, "WebRTC", _t("rd_webrtc_ab_clear_confirm"), - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - if result != QMessageBox.StandardButton.Yes: - return - self._address_book.clear() - self._refresh_address_book() - - def _on_manage_known_hosts(self) -> None: - dialog = KnownHostsDialog(self._known_hosts, parent=self) - dialog.exec() - - def _refresh_address_book(self) -> None: - # Refresh tag filter combo - current = self._tag_filter_combo.currentData() or "" - self._tag_filter_combo.blockSignals(True) - self._tag_filter_combo.clear() - self._tag_filter_combo.addItem(_t("rd_webrtc_tag_all"), "") - for tag in self._address_book.all_tags(): - self._tag_filter_combo.addItem(tag, tag) - idx = self._tag_filter_combo.findData(current) - if idx >= 0: - self._tag_filter_combo.setCurrentIndex(idx) - self._tag_filter_combo.blockSignals(False) - # Apply filter - active_tag = self._tag_filter_combo.currentData() or "" - self._address_list.populate( - self._address_book.list_entries(), tag_filter=active_tag, - ) - - def _on_address_tags(self, entry: dict) -> None: - existing = entry.get("tags", []) or [] - text, ok = QInputDialog.getText( - self, _t("rd_webrtc_edit_tags"), - _t("rd_webrtc_tags_prompt"), - text=", ".join(existing), - ) - if not ok: - return - new_tags = [t.strip() for t in text.split(",") if t.strip()] - try: - self._address_book.set_tags( - host_id=entry.get("host_id", ""), - server_url=entry.get("server_url", ""), - tags=new_tags, - ) - except (ValueError, OSError) as error: - autocontrol_logger.debug("set_tags: %r", error) - self._refresh_address_book() - - def _on_address_chosen(self, entry: dict) -> None: - self._server_edit.setText(entry.get("server_url", "")) - self._host_id_edit.setText(entry.get("host_id", "")) - self._on_connect_via_server() - - def _on_address_removed(self, entry: dict) -> None: - self._address_book.remove( - host_id=entry.get("host_id", ""), - server_url=entry.get("server_url", ""), - ) - self._refresh_address_book() - - def _on_address_favorite(self, entry: dict) -> None: - try: - self._address_book.toggle_favorite( - host_id=entry.get("host_id", ""), - server_url=entry.get("server_url", ""), - ) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("toggle favorite: %r", error) - self._refresh_address_book() - - def _on_connect_selected_address(self) -> None: - entry = self._address_list.selected_entry() - if entry is None: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_no_address_selected"), - ) - return - self._on_address_chosen(entry) - - def _on_save_current_address(self) -> None: - host_id = self._host_id_edit.text().strip() - server_url = self._server_edit.text().strip() - if not host_id or not server_url: - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_save_address_missing_fields"), - ) - return - self._address_book.upsert(host_id=host_id, server_url=server_url) - self._refresh_address_book() - - def _on_remove_selected_address(self) -> None: - entry = self._address_list.selected_entry() - if entry is not None: - self._on_address_removed(entry) - - def _build_signaling_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_signaling_group") - grid = QGridLayout() - grid.addWidget(self._tr(QLabel(), "rd_webrtc_server_label"), 0, 0) - self._server_edit = QLineEdit(_DEFAULT_SIGNALING_URL) - grid.addWidget(self._server_edit, 0, 1, 1, 3) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_host_id_label"), 1, 0) - self._host_id_edit = self._tr(QLineEdit(), "rd_webrtc_host_id_placeholder") - grid.addWidget(self._host_id_edit, 1, 1, 1, 3) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_secret_label"), 2, 0) - self._secret_edit = QLineEdit() - self._secret_edit.setEchoMode(QLineEdit.EchoMode.Password) - grid.addWidget(self._secret_edit, 2, 1, 1, 3) - self._connect_btn = self._tr(QPushButton(), "rd_webrtc_connect_via_server") - self._connect_btn.clicked.connect(self._on_connect_via_server) - grid.addWidget(self._connect_btn, 3, 0, 1, 3) - self._lan_browse_btn = self._tr(QPushButton(), "rd_webrtc_lan_browse") - self._lan_browse_btn.clicked.connect(self._on_lan_browse) - grid.addWidget(self._lan_browse_btn, 3, 3) - group.setLayout(grid) - return group - - def _on_lan_browse(self) -> None: - dialog = LanBrowseDialog(parent=self) - dialog.chosen.connect(self._on_lan_chosen) - dialog.exec() - - def _on_lan_chosen(self, svc: dict) -> None: - host_id = svc.get("host_id", "") - signaling = svc.get("signaling_url", "") - if host_id: - self._host_id_edit.setText(host_id) - if signaling: - self._server_edit.setText(signaling) - def prefill(self, host_id: Optional[str] = None, token: Optional[str] = None, signaling_url: Optional[str] = None) -> None: @@ -2026,92 +269,6 @@ def prefill(self, host_id: Optional[str] = None, if signaling_url: self._server_edit.setText(signaling_url) - def _build_config_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_config_group") - grid = QGridLayout() - grid.addWidget(self._tr(QLabel(), "rd_token_label"), 0, 0) - self._token_edit = self._tr(QLineEdit(), "rd_token_placeholder") - grid.addWidget(self._token_edit, 0, 1) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_bandwidth_label"), 1, 0) - self._bandwidth_combo = QComboBox() - for key, info in BANDWIDTH_PRESETS.items(): - self._bandwidth_combo.addItem(info["label"], key) - grid.addWidget(self._bandwidth_combo, 1, 1) - self._share_my_screen_check = self._tr( - QCheckBox(), "rd_webrtc_share_my_screen", - ) - self._share_my_screen_check.setChecked(False) - self._share_my_screen_check.toggled.connect( - self._on_toggle_share_my_screen, - ) - grid.addWidget(self._share_my_screen_check, 2, 0, 1, 2) - self._share_opus_mic_check = self._tr( - QCheckBox(), "rd_webrtc_share_opus_mic", - ) - self._share_opus_mic_check.setChecked(False) - self._share_opus_mic_check.toggled.connect( - self._on_toggle_share_opus_mic, - ) - grid.addWidget(self._share_opus_mic_check, 3, 0, 1, 2) - self._auto_reconnect_check = self._tr( - QCheckBox(), "rd_webrtc_auto_reconnect", - ) - self._auto_reconnect_check.setChecked(False) - grid.addWidget(self._auto_reconnect_check, 4, 0, 1, 2) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_reconnect_max"), 5, 0) - self._reconnect_max_spin = QSpinBox() - self._reconnect_max_spin.setRange(1, 50) - self._reconnect_max_spin.setValue(5) - grid.addWidget(self._reconnect_max_spin, 5, 1) - grid.addWidget(self._tr(QLabel(), "rd_webrtc_reconnect_delay"), 6, 0) - self._reconnect_delay_spin = QSpinBox() - self._reconnect_delay_spin.setRange(1, 60) - self._reconnect_delay_spin.setValue(1) - self._reconnect_delay_spin.setSuffix(" s") - grid.addWidget(self._reconnect_delay_spin, 6, 1) - group.setLayout(grid) - return group - - def _on_toggle_share_my_screen(self, value: bool) -> None: - if self._viewer is None: - return - try: - self._viewer.toggle_share_screen(value) - except (RuntimeError, OSError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_toggle_share_opus_mic(self, value: bool) -> None: - if self._viewer is None: - return - try: - self._viewer.toggle_opus_mic(value) - except (RuntimeError, OSError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _build_manual_group(self) -> QGroupBox: - group = self._tr(QGroupBox(), "rd_webrtc_manual_group") - layout = QVBoxLayout() - layout.addWidget(self._tr(QLabel(), "rd_webrtc_offer_input_label")) - self._offer_input = QTextEdit() - self._offer_input.setMinimumHeight(80) - self._tr(self._offer_input, "rd_webrtc_paste_offer", "setPlaceholderText") - layout.addWidget(self._offer_input) - button_row = QHBoxLayout() - self._answer_btn = self._tr(QPushButton(), "rd_webrtc_create_answer") - self._answer_btn.clicked.connect(self._on_create_answer) - button_row.addWidget(self._answer_btn) - self._stop_btn = self._tr(QPushButton(), "rd_webrtc_stop_viewer") - self._stop_btn.clicked.connect(self._on_stop) - button_row.addWidget(self._stop_btn) - layout.addLayout(button_row) - layout.addWidget(self._tr(QLabel(), "rd_webrtc_answer_label")) - self._answer_view = QTextEdit() - self._answer_view.setReadOnly(True) - self._answer_view.setMinimumHeight(80) - layout.addWidget(self._answer_view) - group.setLayout(layout) - return group - def _wire_input_signals(self) -> None: # Wire the hidden inline _FrameDisplay too, even though it # never gets focus while the popup is open — leaves the slots @@ -2149,148 +306,12 @@ def _wire_display_input(self, source) -> None: lambda text: self._send({"type": "type_text", "text": text})) source.annotation_event.connect(self._on_annotation_segment) - def _on_annotation_segment(self, action: str, x: int, y: int) -> None: - if self._viewer is None or not self._viewer.authenticated: - return - try: - self._viewer._send({ # noqa: SLF001 # reason: reuse internal sender - "type": "annotate", "action": action, - "x": int(x), "y": int(y), - "color": "#ff0000", "width": 3, - }) - except (RuntimeError, OSError): - pass - - def _on_toggle_pen(self, checked: bool) -> None: - self._frame_display.set_pen_mode(checked) - if self._screen_window is not None: - self._screen_window.set_pen_mode(checked) - self._pen_btn.setText(_t("rd_webrtc_pen_on" if checked - else "rd_webrtc_pen_off")) - - def _on_pen_clear(self) -> None: - if self._viewer is None or not self._viewer.authenticated: - return - try: - self._viewer._send({ # noqa: SLF001 - "type": "annotate", "action": "clear", - "x": 0, "y": 0, - }) - except (RuntimeError, OSError): - pass - def _update_availability(self) -> None: if not is_webrtc_available(): for widget in (self._answer_btn, self._connect_btn): widget.setEnabled(False) self._status_label.setText(_t("rd_webrtc_unavailable")) - # --- handlers ---------------------------------------------------------- - - def _on_connect_via_server(self) -> None: - if not self._validate_required_fields(needs_server=True): - return - self._user_initiated_disconnect = False - self._stop_viewer_if_any() - self._status_label.setText(_t("rd_webrtc_polling_offer")) - self._offer_worker = ViewerSignalingWorker( - server_url=self._server_edit.text().strip(), - host_id=self._host_id_edit.text().strip(), - secret=self._secret_edit.text() or None, - ) - self._offer_worker.offer_ready.connect(self._on_offer_received_from_server) - self._offer_worker.failed.connect(self._on_signaling_failed) - self._offer_worker.start() - - def _on_offer_received_from_server(self, offer_sdp: str) -> None: - try: - self._viewer = self._build_viewer(self._token_edit.text().strip()) - except (ValueError, RuntimeError, OSError) as error: - self._show_error(error) - return - self._status_label.setText(_t("rd_webrtc_creating_answer")) - QTimer.singleShot(0, self, lambda: self._answer_and_push(offer_sdp)) - - def _answer_and_push(self, offer_sdp: str) -> None: - host_id = self._host_id_edit.text().strip() - expected_dtls = self._known_hosts.dtls_fingerprint_for(host_id) if host_id else None - try: - answer = self._require_viewer().process_offer( - offer_sdp, expected_dtls_fingerprint=expected_dtls, - ) - except (ValueError, RuntimeError, OSError) as error: - self._show_error(error) - return - # First-time TOFU: stash the DTLS fingerprint we just observed - if host_id and not expected_dtls: - from je_auto_control.utils.remote_desktop.fingerprint import ( - extract_dtls_fingerprint, - ) - new_fp = extract_dtls_fingerprint(offer_sdp) - if new_fp: - self._known_hosts.remember_dtls_fingerprint(host_id, new_fp) - self._answer_view.setPlainText(answer) - self._status_label.setText(_t("rd_webrtc_pushing_answer")) - self._answer_worker = ViewerAnswerPushWorker( - server_url=self._server_edit.text().strip(), - host_id=self._host_id_edit.text().strip(), - secret=self._secret_edit.text() or None, - answer_sdp=answer, - ) - self._answer_worker.pushed.connect(self._on_answer_pushed) - self._answer_worker.failed.connect(self._on_signaling_failed) - self._answer_worker.start() - - def _on_answer_pushed(self) -> None: - self._status_label.setText(_t("rd_webrtc_waiting_auth")) - - def _on_signaling_failed(self, message: str) -> None: - QMessageBox.warning(self, "WebRTC", message) - self._status_label.setText(_t("rd_webrtc_status_idle")) - - def _on_create_answer(self) -> None: - if not self._validate_required_fields(needs_server=False): - return - offer = self._offer_input.toPlainText().strip() - if not offer: - QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_no_offer")) - return - try: - self._stop_viewer_if_any() - self._viewer = self._build_viewer(self._token_edit.text().strip()) - except (ValueError, RuntimeError, OSError) as error: - self._show_error(error) - return - self._status_label.setText(_t("rd_webrtc_creating_answer")) - QTimer.singleShot(0, self, lambda: self._produce_answer(offer)) - - def _require_viewer(self) -> WebRTCDesktopViewer: - """Return the live viewer, or say it is not connected yet.""" - viewer = self._viewer - if viewer is None: - raise RuntimeError(_t("rd_webrtc_not_connected")) - return viewer - - def _produce_answer(self, offer: str) -> None: - try: - answer = self._require_viewer().process_offer(offer) - except (ValueError, RuntimeError, OSError) as error: - self._show_error(error) - return - self._answer_view.setPlainText(answer) - self._status_label.setText(_t("rd_webrtc_answer_ready")) - - def _on_stop(self) -> None: - self._user_initiated_disconnect = True - self._auto_reconnect_attempts = 0 - self._reconnect_timer.stop() - self._stop_viewer_if_any() - self._frame_display.clear() - self._close_screen_window() - self._status_label.setText(_t("rd_webrtc_status_idle")) - - # --- pop-out screen window --------------------------------------------- - def _ensure_screen_window(self) -> RemoteScreenWindow: if self._screen_window is not None: return self._screen_window @@ -2327,187 +348,11 @@ def _on_screen_window_closed(self) -> None: if self._viewer is not None: self._on_stop() - # --- helpers ----------------------------------------------------------- - - def _validate_required_fields(self, *, needs_server: bool) -> bool: - token = self._token_edit.text().strip() - if not token: - QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_token_required")) - return False - if needs_server: - if not self._server_edit.text().strip(): - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_server_required"), - ) - return False - if not self._host_id_edit.text().strip(): - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_host_id_required"), - ) - return False - return True - - def _build_viewer(self, token: str) -> WebRTCDesktopViewer: - viewer = WebRTCDesktopViewer( - token=token, - config=_read_webrtc_config(self), - viewer_id=self._viewer_id, - on_frame=self._on_av_frame, - on_state_change=self._signals.state.emit, - on_auth_result=self._signals.auth.emit, - ) - viewer.set_file_received_callback(self._on_received_file) - viewer.set_inbox_listing_callback(self._signals.inbox_listing.emit) - viewer.set_inbox_op_result_callback(self._signals.inbox_op.emit) - return viewer - - def _on_received_file(self, path) -> None: - # Called from the asyncio thread, which has no Qt event loop: - # QTimer.singleShot would never fire. Emit a signal — Qt queues the - # status update onto the GUI thread. - self._signals.file_received.emit(path) - - def _on_file_received_ui(self, path) -> None: - self._status_label.setText( - _t("rd_webrtc_file_received").format(name=str(path)), - ) - - def _stop_viewer_if_any(self) -> None: - for worker in (self._offer_worker, self._answer_worker): - retire_worker(worker) - self._offer_worker = None - self._answer_worker = None - if self._sync_engine is not None: - try: - self._sync_engine.stop() - except (RuntimeError, OSError): - pass - self._sync_engine = None - if hasattr(self, "_sync_btn"): - self._sync_btn.setChecked(False) - self._sync_btn.setText(_t("rd_webrtc_sync_start")) - self._stop_stats_polling() - if self._recorder is not None: - self._recorder.stop() - self._recorder = None - self._record_btn.setChecked(False) - self._record_btn.setText(_t("rd_webrtc_start_recording")) - if self._viewer is None: - return - try: - self._viewer.stop() - except (RuntimeError, OSError): - pass - finally: - self._viewer = None - - # called from asyncio thread - def _on_av_frame(self, frame) -> None: - if self._recorder is not None: - try: - self._recorder.write_frame(frame) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("recorder write: %r", error) - image = _av_frame_to_qimage(frame) - if image is not None: - self._signals.frame.emit(image) - - def _on_frame_image(self, image: QImage) -> None: - # When the popup is open it owns the visible display; while - # closed (pre-auth or after stop), the hidden inline display - # still renders so debugging tools / screenshots work. - if self._screen_window is not None: - self._screen_window.set_image(image) - else: - self._frame_display.set_image(image) - def _on_state(self, state: str) -> None: self._status_label.setText(f"{_t('rd_webrtc_state_label')} {state}") if state in ("failed", "disconnected"): self._maybe_schedule_auto_reconnect() - def _on_auth(self, ok: bool) -> None: - key = "rd_webrtc_auth_ok" if ok else "rd_webrtc_auth_fail" - self._status_label.setText(_t(key)) - if ok: - self._auto_reconnect_attempts = 0 # reset on successful auth - host_id = self._host_id_edit.text().strip() - server_url = self._server_edit.text().strip() - if host_id and server_url: - try: - self._address_book.upsert( - host_id=host_id, server_url=server_url, - ) - self._refresh_address_book() - except (ValueError, OSError) as error: - autocontrol_logger.debug("address book upsert: %r", error) - if host_id: - try: - self._known_hosts.touch(host_id) - except OSError as error: - autocontrol_logger.debug("known_hosts touch: %r", error) - self._start_stats_polling() - # AnyDesk-style: surface the live screen in its own window - # so the workspace isn't fighting the control panel for - # vertical space. - window = self._ensure_screen_window() - window.show() - window.raise_() - window.activateWindow() - else: - self._stop_stats_polling() - - def _maybe_schedule_auto_reconnect(self) -> None: - if (not self._auto_reconnect_check.isChecked() - or self._user_initiated_disconnect): - return - max_attempts = int(self._reconnect_max_spin.value()) - base_delay_s = int(self._reconnect_delay_spin.value()) - if self._auto_reconnect_attempts >= max_attempts: - self._status_label.setText(_t("rd_webrtc_reconnect_giveup")) - return - if (not self._server_edit.text().strip() - or not self._host_id_edit.text().strip() - or not self._token_edit.text().strip()): - return - self._auto_reconnect_attempts += 1 - delay_ms = min( - 60000, 1000 * base_delay_s * (2 ** (self._auto_reconnect_attempts - 1)), - ) - self._status_label.setText( - _t("rd_webrtc_reconnecting").format( - n=self._auto_reconnect_attempts, max=max_attempts, - ), - ) - self._reconnect_timer.start(delay_ms) - - def _start_stats_polling(self) -> None: - if self._viewer is None or self._viewer._pc is None: - return - self._stop_stats_polling() - self._stats_poller = StatsPoller( - self._viewer._pc, self._on_viewer_stats_sample, - ) - self._stats_poller.start() - - def _on_viewer_stats_sample(self, snapshot: StatsSnapshot) -> None: - default_webrtc_inspector().record(snapshot) - self._signals.stats.emit(snapshot) - - def _stop_stats_polling(self) -> None: - if self._stats_poller is not None: - self._stats_poller.stop() - self._stats_poller = None - self._stats_label.setText(_t("rd_webrtc_stats_idle")) - if hasattr(self, "_quality_dot"): - self._quality_dot.setStyleSheet( - _QUALITY_DOT_STYLE, - ) - self._quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) - if hasattr(self, "_rtt_spark"): - self._rtt_spark.clear() - self._bitrate_spark.clear() - def _send(self, payload: dict) -> None: if self._viewer is None or not self._viewer.authenticated: return @@ -2524,4 +369,4 @@ def retranslate(self) -> None: TranslatableMixin.retranslate(self) -__all__ = ["_WebRTCHostPanel", "_WebRTCViewerPanel"] +__all__ = ["_PanelSignals", "_WebRTCHostPanel", "_WebRTCViewerPanel"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel_common.py b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py new file mode 100644 index 000000000..b745e9167 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py @@ -0,0 +1,113 @@ +"""Signals, defaults and config readers shared by the WebRTC host and viewer panels.""" +from __future__ import annotations + +from typing import Optional + +from PySide6.QtCore import QObject, Signal +from PySide6.QtGui import QImage + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop import ( + WebRTCConfig, +) + + +_DEFAULT_FPS = 24 +_DEFAULT_MONITOR = 1 +# Plain http:// is intentional: the bundled signaling server defaults +# to localhost without TLS, and operators put TLS in front via nginx / +# Caddy. Hotspot S5332 acknowledged on a per-line basis; see callers. +_DEFAULT_SIGNALING_URL = "http://127.0.0.1:8765" # NOSONAR python:S5332 + +_QUALITY_DOT_STYLE = "background-color: #555; border-radius: 7px;" +_JSON_FILE_FILTER = "JSON (*.json);;All (*)" + + +def _av_frame_to_qimage(frame) -> Optional[QImage]: + """Convert an aiortc/av video frame to a Qt-owned QImage.""" + try: + arr = frame.to_ndarray(format="rgb24") + except (ValueError, RuntimeError) as error: + autocontrol_logger.debug("av->QImage failed: %r", error) + return None + height, width, _ = arr.shape + image = QImage( + arr.tobytes(), width, height, width * 3, QImage.Format.Format_RGB888, + ) + return image.copy() + + +class _PanelSignals(QObject): + """Bridge so asyncio-thread callbacks reach Qt safely.""" + frame = Signal(QImage) + state = Signal(str) + auth = Signal(bool) + # Host-side: (session_id, viewer_id-or-None) per pending viewer prompt. + pending_viewer = Signal(str, object) + stats = Signal(object) # StatsSnapshot + session_count = Signal(int) + # Viewer-side file browser: list and op result. + inbox_listing = Signal(object) # list[dict] + inbox_op = Signal(str, bool, object) # name, ok, error + # Viewer-side: a file finished transferring (fired from the asyncio thread). + file_received = Signal(object) # path + # Host-side: incoming viewer-shared screen frame + viewer_video_frame = Signal(QImage) + # Host-side: incoming annotation event from viewer + annotation = Signal(object) # dict + + +def _checked_or(panel, attr: str, default: bool = False) -> bool: + """Return ``panel..isChecked()`` if the widget exists, else default.""" + widget = getattr(panel, attr, None) + return widget.isChecked() if widget is not None else default + + +def _read_region(panel) -> Optional[tuple]: + edit = getattr(panel, "_region_edit", None) + if edit is None: + return None + text = edit.text().strip() + if not text: + return None + try: + parts = [int(p.strip()) for p in text.split(",")] + except (ValueError, TypeError): + return None + return tuple(parts) if len(parts) == 4 else None + + +def _read_webrtc_config(panel) -> WebRTCConfig: + """Build a WebRTCConfig from the advanced group + monitor/fps fields.""" + from je_auto_control.utils.remote_desktop.webrtc_transport import ( + _DEFAULT_STUN_SERVERS, + ) + stun_field = panel._stun_edit.text().strip() + ice_servers = [stun_field] if stun_field else list(_DEFAULT_STUN_SERVERS) + monitor = ( + int(panel._monitor_combo.currentData() or _DEFAULT_MONITOR) + if hasattr(panel, "_monitor_combo") else _DEFAULT_MONITOR + ) + fps = (int(panel._fps_spin.value()) + if hasattr(panel, "_fps_spin") else _DEFAULT_FPS) + max_bitrate = ( + int(panel._max_bitrate_spin.value()) + if hasattr(panel, "_max_bitrate_spin") else 0 + ) + return WebRTCConfig( + ice_servers=ice_servers, + turn_url=panel._turn_edit.text().strip() or None, + turn_username=panel._turn_user_edit.text().strip() or None, + turn_credential=panel._turn_cred_edit.text() or None, + monitor_index=monitor, + fps=fps, + show_cursor=_checked_or(panel, "_cursor_check", default=True), + accept_viewer_video=_checked_or(panel, "_accept_viewer_video_check"), + accept_viewer_audio_opus=_checked_or(panel, "_accept_opus_audio_check"), + share_my_screen=_checked_or(panel, "_share_my_screen_check"), + share_my_audio_opus=_checked_or(panel, "_share_opus_mic_check"), + max_bitrate_kbps=max_bitrate, + region=_read_region(panel), + host_voice=_checked_or(panel, "_host_voice_check"), + ) + diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py new file mode 100644 index 000000000..c5a1d0de3 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py @@ -0,0 +1,239 @@ +"""Address book, known hosts, LAN browse and Wake-on-LAN for the WebRTC viewer panel. + +One interaction group of ``webrtc_panel._WebRTCViewerPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtWidgets import ( + QFileDialog, QInputDialog, QMessageBox, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _read_import_entries, _t, +) +from je_auto_control.gui.remote_desktop.webrtc_dialogs import ( + KnownHostsDialog, LanBrowseDialog, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop import ( + send_magic_packet, +) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _JSON_FILE_FILTER, +) + + +class _ViewerAddressBookMixin: + """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + + @staticmethod + def _wol_defaults(entry) -> tuple[str, str]: + """Return (mac, broadcast) pre-fill values from a book entry.""" + if entry is None: + return "", "" + return (entry.get("mac_address", "") or "", + entry.get("broadcast_address", "") or "") + + def _persist_wol_entry(self, entry, mac: str, broadcast: str) -> None: + """Save the MAC / broadcast just used back onto the book entry.""" + if entry is None: + return + self._address_book.upsert( + host_id=entry.get("host_id", ""), + server_url=entry.get("server_url", ""), + mac_address=mac.strip(), + broadcast_address=broadcast.strip() or None, + ) + self._refresh_address_book() + + def _on_wake_on_lan(self) -> None: + entry = self._address_list.selected_entry() + mac, broadcast = self._wol_defaults(entry) + mac, ok = QInputDialog.getText( + self, _t("rd_webrtc_wake_on_lan"), + _t("rd_webrtc_wol_mac_prompt"), text=mac, + ) + if not ok or not mac.strip(): + return + broadcast, ok2 = QInputDialog.getText( + self, _t("rd_webrtc_wake_on_lan"), + _t("rd_webrtc_wol_broadcast_prompt"), + text=broadcast or "255.255.255.255", + ) + if not ok2: + return + try: + send_magic_packet(mac.strip(), + broadcast_address=broadcast.strip() or None) + except (ValueError, OSError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + return + self._persist_wol_entry(entry, mac, broadcast) + QMessageBox.information( + self, _t("rd_webrtc_wake_on_lan"), _t("rd_webrtc_wol_sent"), + ) + + def _on_ab_export(self) -> None: + import json as _json + path, _filter = QFileDialog.getSaveFileName( + self, _t("rd_webrtc_ab_export"), "address_book.json", + _JSON_FILE_FILTER, + ) + if not path: + return + try: + with open(path, "w", encoding="utf-8") as fh: + _json.dump({"entries": self._address_book.list_entries()}, + fh, indent=2, ensure_ascii=False) + except OSError as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_ab_import(self) -> None: + path, _filter = QFileDialog.getOpenFileName( + self, _t("rd_webrtc_ab_import"), "", _JSON_FILE_FILTER, + ) + if not path: + return + try: + entries = _read_import_entries(path, "entries") + except (OSError, ValueError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + return + added = 0 + for entry in entries: + # Strings only: a numeric host_id was saved, then dropped on the + # next load, and a numeric MAC reached Wake-on-LAN. + text = {key: value for key, value in entry.items() if isinstance(value, str) and value} + if not ("host_id" in text and "server_url" in text): + continue + try: + self._address_book.upsert( + host_id=text["host_id"], server_url=text["server_url"], + label=text.get("label", ""), + mac_address=text.get("mac_address"), + broadcast_address=text.get("broadcast_address"), + ) + added += 1 + except (ValueError, OSError) as error: + autocontrol_logger.debug("ab import upsert: %r", error) + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_ab_import_done").format(n=added), + ) + self._refresh_address_book() + + def _on_ab_clear(self) -> None: + result = QMessageBox.question( + self, "WebRTC", _t("rd_webrtc_ab_clear_confirm"), + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + ) + if result != QMessageBox.StandardButton.Yes: + return + self._address_book.clear() + self._refresh_address_book() + + def _on_manage_known_hosts(self) -> None: + dialog = KnownHostsDialog(self._known_hosts, parent=self) + dialog.exec() + + def _refresh_address_book(self) -> None: + # Refresh tag filter combo + current = self._tag_filter_combo.currentData() or "" + self._tag_filter_combo.blockSignals(True) + self._tag_filter_combo.clear() + self._tag_filter_combo.addItem(_t("rd_webrtc_tag_all"), "") + for tag in self._address_book.all_tags(): + self._tag_filter_combo.addItem(tag, tag) + idx = self._tag_filter_combo.findData(current) + if idx >= 0: + self._tag_filter_combo.setCurrentIndex(idx) + self._tag_filter_combo.blockSignals(False) + # Apply filter + active_tag = self._tag_filter_combo.currentData() or "" + self._address_list.populate( + self._address_book.list_entries(), tag_filter=active_tag, + ) + + def _on_address_tags(self, entry: dict) -> None: + existing = entry.get("tags", []) or [] + text, ok = QInputDialog.getText( + self, _t("rd_webrtc_edit_tags"), + _t("rd_webrtc_tags_prompt"), + text=", ".join(existing), + ) + if not ok: + return + new_tags = [t.strip() for t in text.split(",") if t.strip()] + try: + self._address_book.set_tags( + host_id=entry.get("host_id", ""), + server_url=entry.get("server_url", ""), + tags=new_tags, + ) + except (ValueError, OSError) as error: + autocontrol_logger.debug("set_tags: %r", error) + self._refresh_address_book() + + def _on_address_chosen(self, entry: dict) -> None: + self._server_edit.setText(entry.get("server_url", "")) + self._host_id_edit.setText(entry.get("host_id", "")) + self._on_connect_via_server() + + def _on_address_removed(self, entry: dict) -> None: + self._address_book.remove( + host_id=entry.get("host_id", ""), + server_url=entry.get("server_url", ""), + ) + self._refresh_address_book() + + def _on_address_favorite(self, entry: dict) -> None: + try: + self._address_book.toggle_favorite( + host_id=entry.get("host_id", ""), + server_url=entry.get("server_url", ""), + ) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("toggle favorite: %r", error) + self._refresh_address_book() + + def _on_connect_selected_address(self) -> None: + entry = self._address_list.selected_entry() + if entry is None: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_no_address_selected"), + ) + return + self._on_address_chosen(entry) + + def _on_save_current_address(self) -> None: + host_id = self._host_id_edit.text().strip() + server_url = self._server_edit.text().strip() + if not host_id or not server_url: + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_save_address_missing_fields"), + ) + return + self._address_book.upsert(host_id=host_id, server_url=server_url) + self._refresh_address_book() + + def _on_remove_selected_address(self) -> None: + entry = self._address_list.selected_entry() + if entry is not None: + self._on_address_removed(entry) + + def _on_lan_browse(self) -> None: + dialog = LanBrowseDialog(parent=self) + dialog.chosen.connect(self._on_lan_chosen) + dialog.exec() + + def _on_lan_chosen(self, svc: dict) -> None: + host_id = svc.get("host_id", "") + signaling = svc.get("signaling_url", "") + if host_id: + self._host_id_edit.setText(host_id) + if signaling: + self._server_edit.setText(signaling) + + +__all__ = ["_ViewerAddressBookMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py new file mode 100644 index 000000000..ba5ef42dc --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py @@ -0,0 +1,252 @@ +"""Connecting, manual SDP exchange, auto-reconnect and shutdown for the WebRTC viewer panel. + +One interaction group of ``webrtc_panel._WebRTCViewerPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QMessageBox, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _t, +) +from je_auto_control.gui.remote_desktop.webrtc_workers import ( + ViewerAnswerPushWorker, ViewerSignalingWorker, + retire_worker, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop import ( + WebRTCDesktopViewer, +) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _read_webrtc_config, +) + + +class _ViewerConnectionMixin: + """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + + def _on_connect_via_server(self) -> None: + if not self._validate_required_fields(needs_server=True): + return + self._user_initiated_disconnect = False + self._stop_viewer_if_any() + self._status_label.setText(_t("rd_webrtc_polling_offer")) + self._offer_worker = ViewerSignalingWorker( + server_url=self._server_edit.text().strip(), + host_id=self._host_id_edit.text().strip(), + secret=self._secret_edit.text() or None, + ) + self._offer_worker.offer_ready.connect(self._on_offer_received_from_server) + self._offer_worker.failed.connect(self._on_signaling_failed) + self._offer_worker.start() + + def _on_offer_received_from_server(self, offer_sdp: str) -> None: + try: + self._viewer = self._build_viewer(self._token_edit.text().strip()) + except (ValueError, RuntimeError, OSError) as error: + self._show_error(error) + return + self._status_label.setText(_t("rd_webrtc_creating_answer")) + QTimer.singleShot(0, self, lambda: self._answer_and_push(offer_sdp)) + + def _answer_and_push(self, offer_sdp: str) -> None: + host_id = self._host_id_edit.text().strip() + expected_dtls = self._known_hosts.dtls_fingerprint_for(host_id) if host_id else None + try: + answer = self._require_viewer().process_offer( + offer_sdp, expected_dtls_fingerprint=expected_dtls, + ) + except (ValueError, RuntimeError, OSError) as error: + self._show_error(error) + return + # First-time TOFU: stash the DTLS fingerprint we just observed + if host_id and not expected_dtls: + from je_auto_control.utils.remote_desktop.fingerprint import ( + extract_dtls_fingerprint, + ) + new_fp = extract_dtls_fingerprint(offer_sdp) + if new_fp: + self._known_hosts.remember_dtls_fingerprint(host_id, new_fp) + self._answer_view.setPlainText(answer) + self._status_label.setText(_t("rd_webrtc_pushing_answer")) + self._answer_worker = ViewerAnswerPushWorker( + server_url=self._server_edit.text().strip(), + host_id=self._host_id_edit.text().strip(), + secret=self._secret_edit.text() or None, + answer_sdp=answer, + ) + self._answer_worker.pushed.connect(self._on_answer_pushed) + self._answer_worker.failed.connect(self._on_signaling_failed) + self._answer_worker.start() + + def _on_answer_pushed(self) -> None: + self._status_label.setText(_t("rd_webrtc_waiting_auth")) + + def _on_signaling_failed(self, message: str) -> None: + QMessageBox.warning(self, "WebRTC", message) + self._status_label.setText(_t("rd_webrtc_status_idle")) + + def _on_create_answer(self) -> None: + if not self._validate_required_fields(needs_server=False): + return + offer = self._offer_input.toPlainText().strip() + if not offer: + QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_no_offer")) + return + try: + self._stop_viewer_if_any() + self._viewer = self._build_viewer(self._token_edit.text().strip()) + except (ValueError, RuntimeError, OSError) as error: + self._show_error(error) + return + self._status_label.setText(_t("rd_webrtc_creating_answer")) + QTimer.singleShot(0, self, lambda: self._produce_answer(offer)) + + def _require_viewer(self) -> WebRTCDesktopViewer: + """Return the live viewer, or say it is not connected yet.""" + viewer = self._viewer + if viewer is None: + raise RuntimeError(_t("rd_webrtc_not_connected")) + return viewer + + def _produce_answer(self, offer: str) -> None: + try: + answer = self._require_viewer().process_offer(offer) + except (ValueError, RuntimeError, OSError) as error: + self._show_error(error) + return + self._answer_view.setPlainText(answer) + self._status_label.setText(_t("rd_webrtc_answer_ready")) + + def _on_stop(self) -> None: + self._user_initiated_disconnect = True + self._auto_reconnect_attempts = 0 + self._reconnect_timer.stop() + self._stop_viewer_if_any() + self._frame_display.clear() + self._close_screen_window() + self._status_label.setText(_t("rd_webrtc_status_idle")) + + def _validate_required_fields(self, *, needs_server: bool) -> bool: + token = self._token_edit.text().strip() + if not token: + QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_token_required")) + return False + if needs_server: + if not self._server_edit.text().strip(): + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_server_required"), + ) + return False + if not self._host_id_edit.text().strip(): + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_host_id_required"), + ) + return False + return True + + def _build_viewer(self, token: str) -> WebRTCDesktopViewer: + viewer = WebRTCDesktopViewer( + token=token, + config=_read_webrtc_config(self), + viewer_id=self._viewer_id, + on_frame=self._on_av_frame, + on_state_change=self._signals.state.emit, + on_auth_result=self._signals.auth.emit, + ) + viewer.set_file_received_callback(self._on_received_file) + viewer.set_inbox_listing_callback(self._signals.inbox_listing.emit) + viewer.set_inbox_op_result_callback(self._signals.inbox_op.emit) + return viewer + + def _stop_viewer_if_any(self) -> None: + for worker in (self._offer_worker, self._answer_worker): + retire_worker(worker) + self._offer_worker = None + self._answer_worker = None + if self._sync_engine is not None: + try: + self._sync_engine.stop() + except (RuntimeError, OSError): + pass + self._sync_engine = None + if hasattr(self, "_sync_btn"): + self._sync_btn.setChecked(False) + self._sync_btn.setText(_t("rd_webrtc_sync_start")) + self._stop_stats_polling() + if self._recorder is not None: + self._recorder.stop() + self._recorder = None + self._record_btn.setChecked(False) + self._record_btn.setText(_t("rd_webrtc_start_recording")) + if self._viewer is None: + return + try: + self._viewer.stop() + except (RuntimeError, OSError): + pass + finally: + self._viewer = None + + def _on_auth(self, ok: bool) -> None: + key = "rd_webrtc_auth_ok" if ok else "rd_webrtc_auth_fail" + self._status_label.setText(_t(key)) + if ok: + self._auto_reconnect_attempts = 0 # reset on successful auth + host_id = self._host_id_edit.text().strip() + server_url = self._server_edit.text().strip() + if host_id and server_url: + try: + self._address_book.upsert( + host_id=host_id, server_url=server_url, + ) + self._refresh_address_book() + except (ValueError, OSError) as error: + autocontrol_logger.debug("address book upsert: %r", error) + if host_id: + try: + self._known_hosts.touch(host_id) + except OSError as error: + autocontrol_logger.debug("known_hosts touch: %r", error) + self._start_stats_polling() + # AnyDesk-style: surface the live screen in its own window + # so the workspace isn't fighting the control panel for + # vertical space. + window = self._ensure_screen_window() + window.show() + window.raise_() + window.activateWindow() + else: + self._stop_stats_polling() + + def _maybe_schedule_auto_reconnect(self) -> None: + if (not self._auto_reconnect_check.isChecked() + or self._user_initiated_disconnect): + return + max_attempts = int(self._reconnect_max_spin.value()) + base_delay_s = int(self._reconnect_delay_spin.value()) + if self._auto_reconnect_attempts >= max_attempts: + self._status_label.setText(_t("rd_webrtc_reconnect_giveup")) + return + if (not self._server_edit.text().strip() + or not self._host_id_edit.text().strip() + or not self._token_edit.text().strip()): + return + self._auto_reconnect_attempts += 1 + delay_ms = min( + 60000, 1000 * base_delay_s * (2 ** (self._auto_reconnect_attempts - 1)), + ) + self._status_label.setText( + _t("rd_webrtc_reconnecting").format( + n=self._auto_reconnect_attempts, max=max_attempts, + ), + ) + self._reconnect_timer.start(delay_ms) + + +__all__ = ["_ViewerConnectionMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py new file mode 100644 index 000000000..1c5aab251 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py @@ -0,0 +1,214 @@ +"""Folder sync, the remote inbox and file transfer for the WebRTC viewer panel. + +One interaction group of ``webrtc_panel._WebRTCViewerPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QFileDialog, QMessageBox, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _t, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +class _ViewerFilesMixin: + """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + + def _on_sync_browse(self) -> None: + path = QFileDialog.getExistingDirectory( + self, _t("rd_webrtc_sync_dir"), + ) + if path: + self._sync_dir_edit.setText(path) + + def _on_toggle_sync(self, checked: bool) -> None: + if checked: + if self._viewer is None or not self._viewer.authenticated: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_cad_not_connected"), + ) + self._sync_btn.setChecked(False) + return + path = self._sync_dir_edit.text().strip() + if not path: + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_sync_dir_required"), + ) + self._sync_btn.setChecked(False) + return + from je_auto_control.utils.remote_desktop.file_sync import ( + FolderSyncEngine, + ) + from pathlib import Path as _Path + try: + viewer = self._require_viewer() + engine = FolderSyncEngine( + watch_dir=_Path(path), + sender=lambda local, name: viewer.send_file( + local, remote_name=name, + ), + ) + self._sync_engine = engine + engine.start() + except (RuntimeError, OSError) as error: # FileNotFoundError is an OSError + QMessageBox.warning(self, "WebRTC", str(error)) + self._sync_btn.setChecked(False) + return + self._sync_btn.setText(_t("rd_webrtc_sync_stop")) + else: + if self._sync_engine is not None: + try: + self._sync_engine.stop() + except (RuntimeError, OSError): + pass + self._sync_engine = None + self._sync_btn.setText(_t("rd_webrtc_sync_start")) + + def _on_browse_refresh(self) -> None: + if self._viewer is None or not self._viewer.authenticated: + return + try: + self._viewer.request_inbox_listing() + except (RuntimeError, OSError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_browse_pull_button(self) -> None: + names = self._remote_files_table.selected_names() + if not names: + return + self._on_pull_names(names) + + def _on_browse_delete_button(self) -> None: + names = self._remote_files_table.selected_names() + if not names: + return + self._on_delete_names(names) + + def _on_pull_names(self, names) -> None: + if self._viewer is None or not self._viewer.authenticated: + return + try: + for name in names: + self._viewer.request_inbox_file(name) + except (RuntimeError, OSError, ValueError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_delete_names(self, names) -> None: + if not names or self._viewer is None or not self._viewer.authenticated: + return + confirm_text = ( + _t("rd_webrtc_browse_delete_confirm").format(name=names[0]) + if len(names) == 1 + else _t("rd_webrtc_browse_delete_many_confirm").format(n=len(names)) + ) + result = QMessageBox.question( + self, "WebRTC", confirm_text, + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + ) + if result != QMessageBox.StandardButton.Yes: + return + try: + for name in names: + self._viewer.delete_inbox_file(name) + except (RuntimeError, OSError, ValueError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_upload_paths(self, paths) -> None: + if self._viewer is None or not self._viewer.authenticated: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_cad_not_connected"), + ) + return + sent = 0 + last_error = None + for path in paths: + try: + self._viewer.send_file(path) + sent += 1 + except (RuntimeError, OSError, ValueError) as error: + last_error = error + autocontrol_logger.warning("upload %s: %r", path, error) + if sent: + self._status_label.setText( + _t("rd_webrtc_upload_done").format(n=sent), + ) + QTimer.singleShot(500, self, self._on_browse_refresh) + if last_error is not None and sent == 0: + QMessageBox.warning(self, "WebRTC", str(last_error)) + + def _on_copy_name(self, name: str) -> None: + from PySide6.QtWidgets import QApplication as _QApp + clipboard = _QApp.clipboard() + if clipboard is not None: + clipboard.setText(name) + + def _on_inbox_listing(self, files) -> None: + from datetime import datetime + if not isinstance(files, list): + return + def _format_mtime(value): + try: + return datetime.fromtimestamp(float(value)).strftime( + "%Y-%m-%d %H:%M:%S", + ) + except (TypeError, ValueError, OSError, OverflowError): + return str(value) + self._remote_files_table.populate(files, _format_mtime) + + def _on_inbox_op_result(self, name: str, ok: bool, error) -> None: + if ok: + self._status_label.setText( + _t("rd_webrtc_browse_op_ok").format(name=name), + ) + # Refresh listing so the table reflects the change + try: + if self._viewer is not None and self._viewer.authenticated: + self._viewer.request_inbox_listing() + except (RuntimeError, OSError): + pass + else: + QMessageBox.warning( + self, "WebRTC", + _t("rd_webrtc_browse_op_failed").format( + name=name, error=str(error or ""), + ), + ) + + def _on_send_file(self) -> None: + if self._viewer is None or not self._viewer.authenticated: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_cad_not_connected"), + ) + return + path, _filter = QFileDialog.getOpenFileName( + self, _t("rd_webrtc_send_file"), "", + ) + if not path: + return + try: + self._viewer.send_file(path) + self._status_label.setText( + _t("rd_webrtc_file_sent").format(name=path), + ) + except (RuntimeError, OSError, ValueError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_received_file(self, path) -> None: + # Called from the asyncio thread, which has no Qt event loop: + # QTimer.singleShot would never fire. Emit a signal — Qt queues the + # status update onto the GUI thread. + self._signals.file_received.emit(path) + + def _on_file_received_ui(self, path) -> None: + self._status_label.setText( + _t("rd_webrtc_file_received").format(name=str(path)), + ) + + +__all__ = ["_ViewerFilesMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py new file mode 100644 index 000000000..3d05f221b --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py @@ -0,0 +1,236 @@ +"""Microphone, recording, statistics and annotation for the WebRTC viewer panel. + +One interaction group of ``webrtc_panel._WebRTCViewerPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtGui import QImage +from PySide6.QtWidgets import ( + QFileDialog, QMessageBox, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _t, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop import ( + SessionRecorder, +) +from je_auto_control.utils.remote_desktop.webrtc_inspector import ( + default_webrtc_inspector, +) +from je_auto_control.utils.remote_desktop.webrtc_stats import ( + StatsPoller, StatsSnapshot, +) +from je_auto_control.utils.remote_desktop.webrtc_transport import ( + fps_for_preset, +) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _QUALITY_DOT_STYLE, _av_frame_to_qimage, +) + + +class _ViewerMediaMixin: + """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + + def _on_send_cad(self) -> None: + if self._viewer is None or not self._viewer.authenticated: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_cad_not_connected"), + ) + return + try: + self._viewer.request_send_sas() + except (RuntimeError, OSError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_toggle_mic(self, checked: bool) -> None: + if self._viewer is None or not self._viewer.authenticated: + self._mic_btn.setChecked(False) + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_cad_not_connected"), + ) + return + try: + if checked: + self._viewer.enable_mic_send() + else: + self._viewer.disable_mic_send() + except (RuntimeError, OSError) as error: + self._mic_btn.setChecked(False) + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_toggle_recording(self, checked: bool) -> None: + if checked: + if SessionRecorder is None: + QMessageBox.warning(self, "WebRTC", _t("rd_webrtc_unavailable")) + self._record_btn.setChecked(False) + return + path, _filter = QFileDialog.getSaveFileName( + self, _t("rd_webrtc_recording_save_as"), "", + "MP4 (*.mp4);;WebM (*.webm);;Matroska (*.mkv);;All (*)", + ) + if not path: + self._record_btn.setChecked(False) + return + from je_auto_control.utils.remote_desktop.session_recorder import ( + preset_for_path, + ) + preset = preset_for_path(path) + self._recorder = SessionRecorder( + path, + fps=int(self._bandwidth_combo.currentData() and + fps_for_preset(self._bandwidth_combo.currentData()) + or 24), + codec=preset.get("codec", "libx264"), + pixel_format=preset.get("pixel_format", "yuv420p"), + ) + self._record_btn.setText(_t("rd_webrtc_stop_recording")) + else: + if self._recorder is not None: + self._recorder.stop() + # "Saved" was reported for a file that no frame ever created. + key = "rd_webrtc_recording_saved" if self._recorder.has_output else "rd_webrtc_recording_empty" + QMessageBox.information( + self, "WebRTC", _t(key).format(path=str(self._recorder.output_path)), + ) + self._recorder = None + self._record_btn.setText(_t("rd_webrtc_start_recording")) + + def _on_stats(self, snapshot: StatsSnapshot) -> None: + parts = [] + if snapshot.fps is not None: + parts.append(f"FPS {snapshot.fps:.1f}") + if snapshot.bitrate_kbps is not None: + parts.append(f"{snapshot.bitrate_kbps:.0f} kbps") + if snapshot.rtt_ms is not None: + parts.append(f"RTT {snapshot.rtt_ms:.0f} ms") + if snapshot.packet_loss_pct is not None: + parts.append(f"loss {snapshot.packet_loss_pct:.1f}%") + if snapshot.jitter_ms is not None: + parts.append(f"jitter {snapshot.jitter_ms:.1f}ms") + if not parts: + return + self._stats_label.setText(" | ".join(parts)) + self._update_quality_dot(snapshot) + if hasattr(self, "_rtt_spark"): + self._rtt_spark.push(snapshot.rtt_ms) + self._bitrate_spark.push(snapshot.bitrate_kbps) + + def _update_quality_dot(self, snapshot: StatsSnapshot) -> None: + rtt = snapshot.rtt_ms + loss = snapshot.packet_loss_pct or 0.0 + if rtt is None: + color = "#555" + tip_key = "rd_webrtc_quality_unknown" + elif rtt < 80 and loss < 1.0: + color = "#3a9c3a" + tip_key = "rd_webrtc_quality_good" + elif rtt < 200 and loss < 5.0: + color = "#c9a23a" + tip_key = "rd_webrtc_quality_fair" + else: + color = "#cc4444" + tip_key = "rd_webrtc_quality_poor" + self._quality_dot.setStyleSheet( + f"background-color: {color}; border-radius: 7px;", + ) + self._quality_dot.setToolTip(_t(tip_key)) + + def _on_toggle_share_my_screen(self, value: bool) -> None: + if self._viewer is None: + return + try: + self._viewer.toggle_share_screen(value) + except (RuntimeError, OSError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_toggle_share_opus_mic(self, value: bool) -> None: + if self._viewer is None: + return + try: + self._viewer.toggle_opus_mic(value) + except (RuntimeError, OSError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_annotation_segment(self, action: str, x: int, y: int) -> None: + if self._viewer is None or not self._viewer.authenticated: + return + try: + self._viewer._send({ # noqa: SLF001 # reason: reuse internal sender + "type": "annotate", "action": action, + "x": int(x), "y": int(y), + "color": "#ff0000", "width": 3, + }) + except (RuntimeError, OSError): + pass + + def _on_toggle_pen(self, checked: bool) -> None: + self._frame_display.set_pen_mode(checked) + if self._screen_window is not None: + self._screen_window.set_pen_mode(checked) + self._pen_btn.setText(_t("rd_webrtc_pen_on" if checked + else "rd_webrtc_pen_off")) + + def _on_pen_clear(self) -> None: + if self._viewer is None or not self._viewer.authenticated: + return + try: + self._viewer._send({ # noqa: SLF001 + "type": "annotate", "action": "clear", + "x": 0, "y": 0, + }) + except (RuntimeError, OSError): + pass + + # called from asyncio thread + def _on_av_frame(self, frame) -> None: + if self._recorder is not None: + try: + self._recorder.write_frame(frame) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("recorder write: %r", error) + image = _av_frame_to_qimage(frame) + if image is not None: + self._signals.frame.emit(image) + + def _on_frame_image(self, image: QImage) -> None: + # When the popup is open it owns the visible display; while + # closed (pre-auth or after stop), the hidden inline display + # still renders so debugging tools / screenshots work. + if self._screen_window is not None: + self._screen_window.set_image(image) + else: + self._frame_display.set_image(image) + + def _start_stats_polling(self) -> None: + if self._viewer is None or self._viewer._pc is None: + return + self._stop_stats_polling() + self._stats_poller = StatsPoller( + self._viewer._pc, self._on_viewer_stats_sample, + ) + self._stats_poller.start() + + def _on_viewer_stats_sample(self, snapshot: StatsSnapshot) -> None: + default_webrtc_inspector().record(snapshot) + self._signals.stats.emit(snapshot) + + def _stop_stats_polling(self) -> None: + if self._stats_poller is not None: + self._stats_poller.stop() + self._stats_poller = None + self._stats_label.setText(_t("rd_webrtc_stats_idle")) + if hasattr(self, "_quality_dot"): + self._quality_dot.setStyleSheet( + _QUALITY_DOT_STYLE, + ) + self._quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) + if hasattr(self, "_rtt_spark"): + self._rtt_spark.clear() + self._bitrate_spark.clear() + + +__all__ = ["_ViewerMediaMixin"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py new file mode 100644 index 000000000..68ff57f44 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py @@ -0,0 +1,328 @@ +"""Widget construction for the WebRTC viewer panel. + +One interaction group of ``webrtc_panel._WebRTCViewerPanel``, kept as a mixin so the panel class +still owns every widget and slot under its original name. +""" +from __future__ import annotations + + +from PySide6.QtWidgets import ( + QCheckBox, QComboBox, QGridLayout, + QGroupBox, QHBoxLayout, QLabel, QLineEdit, + QPushButton, QSpinBox, QTextEdit, QVBoxLayout, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _CollapsibleSection, _t, +) +from je_auto_control.gui.remote_desktop.advanced_group import ( + build_advanced_group, +) +from je_auto_control.gui.remote_desktop.frame_display import _FrameDisplay +from je_auto_control.gui.remote_desktop.sparkline import Sparkline +from je_auto_control.gui.remote_desktop.webrtc_dialogs import ( + AddressBookList, RemoteFilesTable, +) +from je_auto_control.utils.remote_desktop.webrtc_transport import ( + BANDWIDTH_PRESETS, +) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _DEFAULT_SIGNALING_URL, _QUALITY_DOT_STYLE, +) + + +class _ViewerUiMixin: + """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + + def _build_ui(self) -> None: + layout = QVBoxLayout(self) + layout.setContentsMargins(12, 12, 12, 12) + layout.setSpacing(8) + # Essentials always visible: address book + recommended + # signaling-server flow + WebRTC config. + layout.addWidget(self._build_address_book_group()) + layout.addWidget(self._build_signaling_group()) + layout.addWidget(self._build_config_group()) + # Heavy / rarely-used groups now collapse by default so the + # tab fits on a normal display without scrolling. + layout.addWidget(self._wrap_collapsed( + self._build_manual_group(), + "rd_webrtc_manual_group", + )) + layout.addWidget(build_advanced_group(self)) + layout.addWidget(self._wrap_collapsed( + self._build_remote_files_group(), + "rd_webrtc_files_group", + )) + layout.addWidget(self._wrap_collapsed( + self._build_sync_group(), + "rd_webrtc_sync_group", + )) + self._status_label = QLabel(_t("rd_webrtc_status_idle")) + layout.addWidget(self._status_label) + action_row = QHBoxLayout() + self._cad_btn = self._tr(QPushButton(), "rd_webrtc_send_cad") + self._cad_btn.clicked.connect(self._on_send_cad) + action_row.addWidget(self._cad_btn) + self._wol_btn = self._tr(QPushButton(), "rd_webrtc_wake_on_lan") + self._wol_btn.clicked.connect(self._on_wake_on_lan) + action_row.addWidget(self._wol_btn) + self._mic_btn = self._tr(QPushButton(), "rd_webrtc_send_mic") + self._mic_btn.setCheckable(True) + self._mic_btn.clicked.connect(self._on_toggle_mic) + action_row.addWidget(self._mic_btn) + self._send_file_btn = self._tr(QPushButton(), "rd_webrtc_send_file") + self._send_file_btn.clicked.connect(self._on_send_file) + action_row.addWidget(self._send_file_btn) + self._record_btn = self._tr(QPushButton(), "rd_webrtc_start_recording") + self._record_btn.setCheckable(True) + self._record_btn.clicked.connect(self._on_toggle_recording) + action_row.addWidget(self._record_btn) + self._pen_btn = self._tr(QPushButton(), "rd_webrtc_pen_off") + self._pen_btn.setCheckable(True) + self._pen_btn.clicked.connect(self._on_toggle_pen) + action_row.addWidget(self._pen_btn) + self._pen_clear_btn = self._tr(QPushButton(), "rd_webrtc_pen_clear") + self._pen_clear_btn.clicked.connect(self._on_pen_clear) + action_row.addWidget(self._pen_clear_btn) + action_row.addStretch() + layout.addLayout(action_row) + stats_row = QHBoxLayout() + self._quality_dot = QLabel() + self._quality_dot.setFixedSize(14, 14) + self._quality_dot.setStyleSheet( + _QUALITY_DOT_STYLE, + ) + self._quality_dot.setToolTip(_t("rd_webrtc_quality_unknown")) + stats_row.addWidget(self._quality_dot) + self._stats_label = QLabel(_t("rd_webrtc_stats_idle")) + self._stats_label.setStyleSheet( + "color: #ccaa55; font-family: 'Consolas', monospace;", + ) + stats_row.addWidget(self._stats_label, stretch=1) + layout.addLayout(stats_row) + spark_row = QHBoxLayout() + self._rtt_spark = Sparkline(line_color="#3a9c3a") + self._rtt_spark.setToolTip("RTT (ms)") + spark_row.addWidget(self._rtt_spark, stretch=1) + self._bitrate_spark = Sparkline(line_color="#c97a00") + self._bitrate_spark.setToolTip("kbps") + spark_row.addWidget(self._bitrate_spark, stretch=1) + layout.addLayout(spark_row) + # Hidden _FrameDisplay placeholder kept around so the rest of + # the class (pen mode toggle, image setter) doesn't have to + # branch between "popup open" and "popup closed". It also lets + # the panel decode frames before the operator opens the popup. + # When the popup IS open, frames + input round-trip through + # the popup's display instead. + self._frame_display = _FrameDisplay() + self._frame_display.setVisible(False) + layout.addWidget(self._frame_display) + layout.addStretch(1) + self._wire_input_signals() + + def _build_sync_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_sync_group") + layout = QGridLayout() + layout.addWidget(self._tr(QLabel(), "rd_webrtc_sync_dir"), 0, 0) + self._sync_dir_edit = QLineEdit() + self._tr(self._sync_dir_edit, "rd_webrtc_sync_dir_ph", + "setPlaceholderText") + layout.addWidget(self._sync_dir_edit, 0, 1) + browse_btn = self._tr(QPushButton(), "rd_webrtc_browse") + browse_btn.clicked.connect(self._on_sync_browse) + layout.addWidget(browse_btn, 0, 2) + self._sync_btn = self._tr(QPushButton(), "rd_webrtc_sync_start") + self._sync_btn.setCheckable(True) + self._sync_btn.clicked.connect(self._on_toggle_sync) + layout.addWidget(self._sync_btn, 0, 3) + group.setLayout(layout) + return group + + def _build_remote_files_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_remote_files_group") + layout = QVBoxLayout() + button_row = QHBoxLayout() + refresh_btn = self._tr(QPushButton(), "rd_webrtc_browse_refresh") + refresh_btn.clicked.connect(self._on_browse_refresh) + button_row.addWidget(refresh_btn) + pull_btn = self._tr(QPushButton(), "rd_webrtc_browse_pull") + pull_btn.clicked.connect(self._on_browse_pull_button) + button_row.addWidget(pull_btn) + delete_btn = self._tr(QPushButton(), "rd_webrtc_browse_delete") + delete_btn.clicked.connect(self._on_browse_delete_button) + button_row.addWidget(delete_btn) + button_row.addStretch() + layout.addLayout(button_row) + self._remote_files_table = RemoteFilesTable() + self._remote_files_table.pull_requested.connect(self._on_pull_names) + self._remote_files_table.delete_requested.connect(self._on_delete_names) + self._remote_files_table.upload_requested.connect(self._on_upload_paths) + self._remote_files_table.copy_name_requested.connect( + self._on_copy_name, + ) + layout.addWidget(self._remote_files_table) + layout.addWidget(self._tr(QLabel(), "rd_webrtc_browse_dnd_hint")) + group.setLayout(layout) + return group + + def _wrap_collapsed(self, inner: QGroupBox, + title_key: str) -> _CollapsibleSection: + """Wrap an existing groupbox in a collapsed-by-default container. + + The inner group keeps its own translated title, so we just pass + it through the wrapper's body. Heavy / rarely-used groups + (manual SDP, remote files, sync) hide their bodies by default + so the panel doesn't scroll past the fold on a normal display. + """ + section = _CollapsibleSection() + # Translate the wrapper title; the inner groupbox already has + # its own header so we strip its frame to avoid double chrome. + self._tr(section, title_key, setter="setTitle") + inner.setStyleSheet("QGroupBox { border: none; margin-top: 0px; }") + body = QVBoxLayout() + body.setContentsMargins(0, 0, 0, 0) + body.addWidget(inner) + section.set_body_layout(body) + return section + + def _build_address_book_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_address_book_group") + layout = QVBoxLayout() + # Tag filter row + tag_row = QHBoxLayout() + tag_row.addWidget(self._tr(QLabel(), "rd_webrtc_tag_filter")) + self._tag_filter_combo = QComboBox() + self._tag_filter_combo.addItem(_t("rd_webrtc_tag_all"), "") + self._tag_filter_combo.currentIndexChanged.connect( + lambda _i: self._refresh_address_book(), + ) + tag_row.addWidget(self._tag_filter_combo, stretch=1) + layout.addLayout(tag_row) + self._address_list = AddressBookList() + self._address_list.chosen.connect(self._on_address_chosen) + self._address_list.deleted.connect(self._on_address_removed) + self._address_list.favorite_toggled.connect(self._on_address_favorite) + self._address_list.tags_edit_requested.connect(self._on_address_tags) + self._address_list.setMaximumHeight(120) + layout.addWidget(self._address_list) + button_row = QHBoxLayout() + connect_btn = self._tr(QPushButton(), "rd_webrtc_connect_selected") + connect_btn.clicked.connect(self._on_connect_selected_address) + button_row.addWidget(connect_btn) + save_btn = self._tr(QPushButton(), "rd_webrtc_save_current") + save_btn.clicked.connect(self._on_save_current_address) + button_row.addWidget(save_btn) + remove_btn = self._tr(QPushButton(), "rd_webrtc_remove_selected") + remove_btn.clicked.connect(self._on_remove_selected_address) + button_row.addWidget(remove_btn) + kh_btn = self._tr(QPushButton(), "rd_webrtc_manage_known_hosts") + kh_btn.clicked.connect(self._on_manage_known_hosts) + button_row.addWidget(kh_btn) + ab_export = self._tr(QPushButton(), "rd_webrtc_ab_export") + ab_export.clicked.connect(self._on_ab_export) + button_row.addWidget(ab_export) + ab_import = self._tr(QPushButton(), "rd_webrtc_ab_import") + ab_import.clicked.connect(self._on_ab_import) + button_row.addWidget(ab_import) + ab_clear = self._tr(QPushButton(), "rd_webrtc_ab_clear") + ab_clear.clicked.connect(self._on_ab_clear) + button_row.addWidget(ab_clear) + layout.addLayout(button_row) + group.setLayout(layout) + return group + + def _build_signaling_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_signaling_group") + grid = QGridLayout() + grid.addWidget(self._tr(QLabel(), "rd_webrtc_server_label"), 0, 0) + self._server_edit = QLineEdit(_DEFAULT_SIGNALING_URL) + grid.addWidget(self._server_edit, 0, 1, 1, 3) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_host_id_label"), 1, 0) + self._host_id_edit = self._tr(QLineEdit(), "rd_webrtc_host_id_placeholder") + grid.addWidget(self._host_id_edit, 1, 1, 1, 3) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_secret_label"), 2, 0) + self._secret_edit = QLineEdit() + self._secret_edit.setEchoMode(QLineEdit.EchoMode.Password) + grid.addWidget(self._secret_edit, 2, 1, 1, 3) + self._connect_btn = self._tr(QPushButton(), "rd_webrtc_connect_via_server") + self._connect_btn.clicked.connect(self._on_connect_via_server) + grid.addWidget(self._connect_btn, 3, 0, 1, 3) + self._lan_browse_btn = self._tr(QPushButton(), "rd_webrtc_lan_browse") + self._lan_browse_btn.clicked.connect(self._on_lan_browse) + grid.addWidget(self._lan_browse_btn, 3, 3) + group.setLayout(grid) + return group + + def _build_config_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_config_group") + grid = QGridLayout() + grid.addWidget(self._tr(QLabel(), "rd_token_label"), 0, 0) + self._token_edit = self._tr(QLineEdit(), "rd_token_placeholder") + grid.addWidget(self._token_edit, 0, 1) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_bandwidth_label"), 1, 0) + self._bandwidth_combo = QComboBox() + for key, info in BANDWIDTH_PRESETS.items(): + self._bandwidth_combo.addItem(info["label"], key) + grid.addWidget(self._bandwidth_combo, 1, 1) + self._share_my_screen_check = self._tr( + QCheckBox(), "rd_webrtc_share_my_screen", + ) + self._share_my_screen_check.setChecked(False) + self._share_my_screen_check.toggled.connect( + self._on_toggle_share_my_screen, + ) + grid.addWidget(self._share_my_screen_check, 2, 0, 1, 2) + self._share_opus_mic_check = self._tr( + QCheckBox(), "rd_webrtc_share_opus_mic", + ) + self._share_opus_mic_check.setChecked(False) + self._share_opus_mic_check.toggled.connect( + self._on_toggle_share_opus_mic, + ) + grid.addWidget(self._share_opus_mic_check, 3, 0, 1, 2) + self._auto_reconnect_check = self._tr( + QCheckBox(), "rd_webrtc_auto_reconnect", + ) + self._auto_reconnect_check.setChecked(False) + grid.addWidget(self._auto_reconnect_check, 4, 0, 1, 2) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_reconnect_max"), 5, 0) + self._reconnect_max_spin = QSpinBox() + self._reconnect_max_spin.setRange(1, 50) + self._reconnect_max_spin.setValue(5) + grid.addWidget(self._reconnect_max_spin, 5, 1) + grid.addWidget(self._tr(QLabel(), "rd_webrtc_reconnect_delay"), 6, 0) + self._reconnect_delay_spin = QSpinBox() + self._reconnect_delay_spin.setRange(1, 60) + self._reconnect_delay_spin.setValue(1) + self._reconnect_delay_spin.setSuffix(" s") + grid.addWidget(self._reconnect_delay_spin, 6, 1) + group.setLayout(grid) + return group + + def _build_manual_group(self) -> QGroupBox: + group = self._tr(QGroupBox(), "rd_webrtc_manual_group") + layout = QVBoxLayout() + layout.addWidget(self._tr(QLabel(), "rd_webrtc_offer_input_label")) + self._offer_input = QTextEdit() + self._offer_input.setMinimumHeight(80) + self._tr(self._offer_input, "rd_webrtc_paste_offer", "setPlaceholderText") + layout.addWidget(self._offer_input) + button_row = QHBoxLayout() + self._answer_btn = self._tr(QPushButton(), "rd_webrtc_create_answer") + self._answer_btn.clicked.connect(self._on_create_answer) + button_row.addWidget(self._answer_btn) + self._stop_btn = self._tr(QPushButton(), "rd_webrtc_stop_viewer") + self._stop_btn.clicked.connect(self._on_stop) + button_row.addWidget(self._stop_btn) + layout.addLayout(button_row) + layout.addWidget(self._tr(QLabel(), "rd_webrtc_answer_label")) + self._answer_view = QTextEdit() + self._answer_view.setReadOnly(True) + self._answer_view.setMinimumHeight(80) + layout.addWidget(self._answer_view) + group.setLayout(layout) + return group + + +__all__ = ["_ViewerUiMixin"] From 6d3d02aa8c620ab8960c7ea1f7e941b08d113afe Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:29:40 +0800 Subject: [PATCH 03/44] Wait for the USB transfer the caller asked for and return channel errors to their request The viewer gave up after reply_timeout_s even when the transfer's own timeout_ms was longer, so a 30 s read was abandoned at 10 s while the host was still waiting on the device. A transfer now waits its timeout_ms (capped at the host's 60 s) plus reply_timeout_s. The ERRORs the channel adapter sends before a session exists (disabled, bad frame, backend unavailable, no session) echo the request id when the message carried a readable one, so the call fails with that message instead of timing out. viewer_client.py was two lines from the limit: the handle, the errors and the request bookkeeping move to their own modules and are re-exported from where they were. --- .../usb_passthrough_design.rst | 32 +- .../usb_passthrough_operator_guide.rst | 8 +- .../usb_passthrough_design.rst | 26 +- .../usb_passthrough_operator_guide.rst | 6 +- .../utils/usb/passthrough/_client_requests.py | 120 +++++++ .../utils/usb/passthrough/client_errors.py | 39 +++ .../utils/usb/passthrough/client_handle.py | 134 ++++++++ .../utils/usb/passthrough/viewer_client.py | 310 ++++-------------- .../utils/usb/passthrough/webrtc_channel.py | 39 ++- ...test_usb_client_wait_and_channel_errors.py | 304 +++++++++++++++++ 10 files changed, 750 insertions(+), 268 deletions(-) create mode 100644 je_auto_control/utils/usb/passthrough/_client_requests.py create mode 100644 je_auto_control/utils/usb/passthrough/client_errors.py create mode 100644 je_auto_control/utils/usb/passthrough/client_handle.py create mode 100644 test/unit_test/headless/test_usb_client_wait_and_channel_errors.py diff --git a/docs/source/Eng/doc/operations_layer/usb_passthrough_design.rst b/docs/source/Eng/doc/operations_layer/usb_passthrough_design.rst index 541afbed2..1e304c2a0 100644 --- a/docs/source/Eng/doc/operations_layer/usb_passthrough_design.rst +++ b/docs/source/Eng/doc/operations_layer/usb_passthrough_design.rst @@ -162,8 +162,8 @@ pending OPEN, the one pending LIST, the one pending exchange on a therefore handed to the *next* request of that kind -- ``open(bbbb)`` bound the claim the host had opened for ``aaaa``, and a bulk read returned the previous read's data. The host accepts a ``timeout_ms`` of -up to 60 s while the viewer gives up after 10 s by default, so this -happened in ordinary use. +up to 60 s while the viewer gave up after 10 s whatever it had asked +for, so this happened in ordinary use. Every JSON payload may now carry one extra, optional key:: @@ -231,6 +231,34 @@ the call it answers. And the tail of a reassembled message dropped for exceeding the 2 MiB cap was parsed as a message of its own; it is now skipped through its EOF frame. +How long the viewer waits +~~~~~~~~~~~~~~~~~~~~~~~~~ + +A transfer's reply cannot be expected before the device timeout the +request itself asked for, so the viewer waits ``timeout_ms`` (capped at +the host's 60 s) **plus** ``reply_timeout_s`` before it raises +``UsbClientTimeout``. ``reply_timeout_s`` (default 10 s) is therefore +the allowance for the host and the transport, not a ceiling on the whole +call: a transfer with ``timeout_ms=30000`` waits up to 40 s. ``OPEN``, +``RESUME``, ``LIST`` and ``CLOSE`` have no device timeout and wait +``reply_timeout_s``. Before this, any transfer asked to wait longer than +``reply_timeout_s`` was abandoned while the host was still waiting on the +device -- harmless against a host that echoes ids, a forced reconnect +against one that does not. + +Errors sent by the channel adapter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Four ``ERROR`` replies are decided by ``UsbChannelHost`` before any +session sees the frame: ``usb passthrough disabled``, ``bad frame: +...``, ``usb backend unavailable: ...`` and ``no usb session on host``. +They echo the request id whenever the offending message carried a +readable one -- four header bytes followed by a JSON object of at most +16 KiB with a valid ``request_id``, whether or not the frame as a whole +decodes -- so the call that sent it fails with that message instead of +timing out. A message with no readable id gets the same ``ERROR`` as +before, without the key. + Backpressure ------------ diff --git a/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst b/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst index bfb33fc12..0e7546078 100644 --- a/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst +++ b/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst @@ -237,10 +237,14 @@ Open + transfer Errors: - ``UsbClientTimeout`` — the host took longer than ``reply_timeout_s`` - (default 10s) to respond. Check the network / host process. + (default 10s) to respond; for a transfer, longer than its own + ``timeout_ms`` plus ``reply_timeout_s``. Check the network / host + process. - ``UsbClientError`` — the host replied with ``{ok: false, error: ...}``. The most common case is *denied by ACL policy* — go check the - prompt dialog or the ACL rule on the host. + prompt dialog or the ACL rule on the host. *usb passthrough disabled* + and *no usb session on host* mean the host has not opted in (or has + no backend); they are reported at once rather than as a timeout. - ``UsbClientClosed`` — the client or its handle was already shut down. - ``UsbClientDesynchronized`` — an earlier request timed out and the host does not echo request ids (it predates them), so a late reply diff --git a/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst b/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst index c30fc96e9..74a48463f 100644 --- a/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst +++ b/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst @@ -145,7 +145,7 @@ viewer 端用 ``UsbPassthroughClient.list_devices()`` 取得。讓列舉與傳 ``claim_id`` 上的那一次交換。請求逾時後才到的回覆因此會交給同種類的 *下一個*\ 請求——``open(bbbb)`` 綁到 host 為 ``aaaa`` 開的 claim，bulk 讀取拿到上一次讀取的資料。host 接受最長 60 秒的 ``timeout_ms``\ ， -viewer 預設 10 秒就放棄，所以正常使用就會遇到。 +viewer 不管要求多久都在 10 秒放棄，所以正常使用就會遇到。 現在每個 JSON payload 都可以多帶一個選用的 key:: @@ -206,6 +206,30 @@ viewer 從第一個帶回自己編號的回覆得知 host 是新版 而被丟棄的重組訊息，其尾段原本會被當成獨立的訊息解析；現在會一路 略過到它的 EOF frame。 +viewer 等多久 +~~~~~~~~~~~~~ + +傳輸的回覆不可能早於請求自己要求的裝置逾時，所以 viewer 會等 +``timeout_ms``\ （以 host 的 60 秒為上限）\ **再加上** +``reply_timeout_s``\ ，才丟出 ``UsbClientTimeout``\ 。 +``reply_timeout_s``\ （預設 10 秒）因此是留給 host 與傳輸層的餘裕， +不是整個呼叫的上限：``timeout_ms=30000`` 的傳輸最多等 40 秒。 +``OPEN``\ 、``RESUME``\ 、``LIST`` 與 ``CLOSE`` 沒有裝置逾時，只等 +``reply_timeout_s``\ 。在此之前，要求等待時間超過 ``reply_timeout_s`` +的傳輸會在 host 還在等裝置時就被放棄——對會帶回編號的 host 無害， +對不帶回的 host 則是被迫重新連線。 + +channel 轉接層送出的錯誤 +~~~~~~~~~~~~~~~~~~~~~~~~ + +有四種 ``ERROR`` 是 ``UsbChannelHost`` 在 session 看到 frame 之前就 +決定的：``usb passthrough disabled``\ 、``bad frame: ...``\ 、 +``usb backend unavailable: ...`` 與 ``no usb session on host``\ 。 +只要出問題的訊息帶著讀得出來的請求編號——四個 header 位元組之後是 +不超過 16 KiB、含有效 ``request_id`` 的 JSON 物件，不論整個 frame 能否 +解碼——這些錯誤就會帶回該編號，送出它的呼叫會以該訊息失敗而不是 +逾時。讀不出編號的訊息得到與以前相同的 ``ERROR``\ ，不含這個 key。 + Backpressure ------------- diff --git a/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst b/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst index 50fade05a..4278b983d 100644 --- a/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst +++ b/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst @@ -221,10 +221,12 @@ Open + transfer 錯誤： - ``UsbClientTimeout`` — host 超過 ``reply_timeout_s``\ （預設 10 秒） - 沒回。檢查網路 / host 程序。 + 沒回；傳輸則是超過它自己的 ``timeout_ms`` 加上 ``reply_timeout_s``\ 。 + 檢查網路 / host 程序。 - ``UsbClientError`` — host 回 ``{ok: false, error: ...}``\ 。最常見 情境是 *denied by ACL policy* — 去看 host 端的 prompt 對話框或 ACL - 規則。 + 規則。*usb passthrough disabled* 與 *no usb session on host* 表示 + host 沒有啟用（或沒有後端）；會立即回報，不再等到逾時。 - ``UsbClientClosed`` — client 或其 handle 已 shutdown。 - ``UsbClientDesynchronized`` — 先前有請求逾時，而 host 不帶回請求 編號（版本早於此欄位），遲到的回覆與下一個回覆無法分辨。此時不會 diff --git a/je_auto_control/utils/usb/passthrough/_client_requests.py b/je_auto_control/utils/usb/passthrough/_client_requests.py new file mode 100644 index 000000000..6aa60086b --- /dev/null +++ b/je_auto_control/utils/usb/passthrough/_client_requests.py @@ -0,0 +1,120 @@ +"""Request bookkeeping for the viewer-side USB passthrough client. + +What one outstanding request is, where it waits, how long it may wait, +and how a reply body is read. No locking and no transport here: the +client in ``viewer_client`` owns both. +""" +from __future__ import annotations + +import json +import threading +from dataclasses import dataclass, field +from typing import Any, Dict, Optional, Union + +from je_auto_control.utils.usb.passthrough.client_errors import UsbClientError +from je_auto_control.utils.usb.passthrough.protocol import Opcode + +#: Where a request waits: one OPEN/RESUME, one LIST, and one exchange per +#: claim id may be outstanding at a time. +Slot = Union[str, int] +OPEN_SLOT = "open" +LIST_SLOT = "list" + +#: The longest device timeout a host honours (``session.MAX_TIMEOUT_MS``). +#: A request asking for more is refused by the host at once, so the client +#: never needs to wait longer than this for the device itself. +MAX_DEVICE_TIMEOUT_MS = 60_000 + + +@dataclass +class PendingRequest: + """One outstanding viewer→host request awaiting a reply. + + The reply is kept as its decoded JSON body rather than a + :class:`Frame` because a reassembled payload (open question 2) may + exceed the per-frame cap that :class:`Frame` enforces. + """ + + send_op: Opcode + expected_op: Opcode + slot: Slot + event: threading.Event = field(default_factory=threading.Event) + request_id: str = "" + reply_op: Optional[Opcode] = None + reply_body: Dict[str, Any] = field(default_factory=dict) + cancelled: bool = False + + @property + def label(self) -> str: + """The request's opcode name, for messages.""" + return self.send_op.name + + @property + def claim_id(self) -> int: + """The claim the request is sent on; 0 for OPEN / RESUME / LIST.""" + return self.slot if isinstance(self.slot, int) else 0 + + def timeout_message(self) -> str: + """What :class:`UsbClientTimeout` says when this request expires.""" + if isinstance(self.slot, int): + return f"{self.label} timed out for claim {self.slot}" + return f"{self.label} timed out" + + +def desynchronized_message(slot: Slot) -> str: + """What :class:`UsbClientDesynchronized` says for ``slot``.""" + reason = ("an earlier request timed out and the host does not echo request " + "ids, so its late reply cannot be told from the next one") + if isinstance(slot, int): + return (f"claim {slot} cannot be reused: {reason}. " + "Close this handle and open the device again.") + return (f"no further {slot.upper()} on this client: {reason}. " + "Reconnect the USB channel and use a new client.") + + +def device_timeout_s(body: Dict[str, Any]) -> float: + """Seconds the host may spend on the device for the request ``body``. + + A transfer carries ``timeout_ms``, which the host hands to the device + and honours up to :data:`MAX_DEVICE_TIMEOUT_MS`; its reply cannot be + expected sooner. Requests without one (OPEN, LIST, CLOSE) and values + that are not a number count as zero. + """ + value = body.get("timeout_ms") + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0.0 + if value != value or value <= 0: # NaN compares unequal to itself + return 0.0 + return min(float(value), float(MAX_DEVICE_TIMEOUT_MS)) / 1000.0 + + +def accepted_body(request: PendingRequest, default_error: str) -> Dict[str, Any]: + """The reply's body, or :class:`UsbClientError` if the host refused.""" + body = request.reply_body + if request.reply_op == Opcode.ERROR: + raise UsbClientError(body.get("error", "host ERROR")) + if not body.get("ok"): + raise UsbClientError(body.get("error", default_error)) + return body + + +def claim_id_in(body: Dict[str, Any]) -> Optional[int]: + """The ``claim_id`` of an OPENED body if it is a usable one.""" + try: + claim_id = int(body["claim_id"]) + except (KeyError, TypeError, ValueError): + return None + return claim_id if 0 <= claim_id <= 0xFFFF else None + + +def decode_json(payload: bytes) -> Dict[str, Any]: + """The JSON object in ``payload``, or ``{}`` if it is not one.""" + if not payload: + return {} + try: + decoded = json.loads(payload.decode("utf-8")) + except ValueError: + return {} + if not isinstance(decoded, dict): + return {} + return decoded diff --git a/je_auto_control/utils/usb/passthrough/client_errors.py b/je_auto_control/utils/usb/passthrough/client_errors.py new file mode 100644 index 000000000..7b95f0749 --- /dev/null +++ b/je_auto_control/utils/usb/passthrough/client_errors.py @@ -0,0 +1,39 @@ +"""Errors the viewer-side USB passthrough client raises. + +Kept apart from ``viewer_client`` so the handle, the request bookkeeping +and the client can all raise them without importing each other. Every +name is re-exported from ``viewer_client``, which is where callers have +always imported them from. +""" +from __future__ import annotations + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +class UsbClientError(AutoControlException): + """The host reported a transfer or open failure.""" + + +class UsbClientTimeout(UsbClientError): + """A reply did not arrive within the configured timeout.""" + + +class UsbClientClosed(UsbClientError): + """The client / handle was shut down before a reply arrived.""" + + +class UsbClientDesynchronized(UsbClientError): + """A timed-out request left replies impossible to pair; reconnect. + + Raised instead of sending when an earlier request of the same kind + timed out and the host does not echo request ids: its late reply and + the answer to a new request would be indistinguishable. A claim is + recovered by closing its handle and opening the device again; OPEN / + RESUME and LIST by reconnecting the channel with a new client. + """ + + +__all__ = [ + "UsbClientClosed", "UsbClientDesynchronized", + "UsbClientError", "UsbClientTimeout", +] diff --git a/je_auto_control/utils/usb/passthrough/client_handle.py b/je_auto_control/utils/usb/passthrough/client_handle.py new file mode 100644 index 000000000..a4648ef3c --- /dev/null +++ b/je_auto_control/utils/usb/passthrough/client_handle.py @@ -0,0 +1,134 @@ +"""The viewer's handle on one claimed USB device. + +:class:`ClientHandle` is what a caller drives once +:meth:`UsbPassthroughClient.open` (or ``resume``) has returned: it turns +transfer calls into request bodies and leaves the wire, the pairing of +replies and the waiting to the client. Re-exported from ``viewer_client``. +""" +from __future__ import annotations + +import base64 +import threading +from typing import TYPE_CHECKING, Any, Dict + +from je_auto_control.utils.usb.passthrough.client_errors import UsbClientClosed +from je_auto_control.utils.usb.passthrough.protocol import Opcode + +if TYPE_CHECKING: + from je_auto_control.utils.usb.passthrough.viewer_client import ( + UsbPassthroughClient, + ) + + +class ClientHandle: + """One open USB device claim from the viewer's perspective. + + All transfer methods are blocking — they enqueue the right request + frame, wait for the host to send the matching reply (or ERROR), + and return ``bytes``. Backend errors raise :class:`UsbClientError`. + + ``timeout_ms`` is the device timeout the host applies (at most 60 000). + The call itself waits that long plus the client's ``reply_timeout_s`` + before raising :class:`UsbClientTimeout`. + """ + + def __init__(self, client: "UsbPassthroughClient", claim_id: int, + resume_token: str = "") -> None: # nosec B107 # reason: resume_token is a reconnect handle, not a credential; "" means "no token yet" + self._client = client + self._claim_id = claim_id + self._resume_token = resume_token + self._closed = False + self._lock = threading.Lock() + + @property + def claim_id(self) -> int: + return self._claim_id + + @property + def resume_token(self) -> str: + """Opaque token to re-bind this claim after a transport reconnect.""" + return self._resume_token + + @property + def closed(self) -> bool: + with self._lock: + return self._closed + + @property + def reusable(self) -> bool: + """False once the handle is closed or its claim is desynchronised. + + A transfer that timed out against a host that does not echo request + ids leaves the claim unable to pair replies; further transfers raise + :class:`UsbClientDesynchronized`. :meth:`close` still works. + """ + return not self.closed and self._client.slot_reusable(self._claim_id) + + def control_transfer(self, *, bm_request_type: int, b_request: int, + w_value: int = 0, w_index: int = 0, + data: bytes = b"", length: int = 0, + timeout_ms: int = 1000) -> bytes: + request: Dict[str, Any] = { + "bm_request_type": int(bm_request_type), + "b_request": int(b_request), + "w_value": int(w_value), "w_index": int(w_index), + "timeout_ms": int(timeout_ms), + } + if data: + request["data"] = base64.b64encode(bytes(data)).decode("ascii") + if length: + request["length"] = int(length) + return self._exchange(Opcode.CTRL, request) + + def bulk_transfer(self, *, endpoint: int, direction: str, + data: bytes = b"", length: int = 0, + timeout_ms: int = 1000) -> bytes: + return self._exchange(Opcode.BULK, _endpoint_request( + endpoint=endpoint, direction=direction, + data=data, length=length, timeout_ms=timeout_ms, + )) + + def interrupt_transfer(self, *, endpoint: int, direction: str, + data: bytes = b"", length: int = 0, + timeout_ms: int = 1000) -> bytes: + return self._exchange(Opcode.INT, _endpoint_request( + endpoint=endpoint, direction=direction, + data=data, length=length, timeout_ms=timeout_ms, + )) + + def close(self) -> None: + """Send CLOSE; block on CLOSED. Idempotent.""" + with self._lock: + if self._closed: + return + self._closed = True + try: + self._client._exchange_close(self._claim_id) + except UsbClientClosed: + # Client torn down concurrently; treat as success. + pass + + def _exchange(self, op: Opcode, body: Dict[str, Any]) -> bytes: + with self._lock: + if self._closed: + raise UsbClientClosed(f"handle for claim {self._claim_id} closed") + return self._client._exchange_transfer(self._claim_id, op, body) + + +def _endpoint_request(*, endpoint: int, direction: str, data: bytes, + length: int, timeout_ms: int) -> Dict[str, Any]: + if direction not in ("in", "out"): + raise ValueError(f"direction must be 'in' or 'out', got {direction!r}") + body: Dict[str, Any] = { + "endpoint": int(endpoint), + "direction": direction, + "timeout_ms": int(timeout_ms), + } + if data: + body["data"] = base64.b64encode(bytes(data)).decode("ascii") + if length: + body["length"] = int(length) + return body + + +__all__ = ["ClientHandle"] diff --git a/je_auto_control/utils/usb/passthrough/viewer_client.py b/je_auto_control/utils/usb/passthrough/viewer_client.py index 55e06df5c..b38aa59e6 100644 --- a/je_auto_control/utils/usb/passthrough/viewer_client.py +++ b/je_auto_control/utils/usb/passthrough/viewer_client.py @@ -24,9 +24,14 @@ handle.close() client.shutdown() +The handle, the errors and the request bookkeeping live in +``client_handle``, ``client_errors`` and ``_client_requests``; the public +names are re-exported here, where they have always been imported from. + Errors: -* ``UsbClientTimeout`` — peer did not reply within the timeout. +* ``UsbClientTimeout`` — peer did not reply within ``reply_timeout_s`` + plus, for a transfer, the ``timeout_ms`` the caller asked for. * ``UsbClientError`` — peer replied with ``{ok: false}`` or ERROR. * ``UsbClientClosed`` — the client (or its handle) was shut down. * ``UsbClientDesynchronized`` — an earlier request timed out against a @@ -56,11 +61,17 @@ import secrets import threading from collections import OrderedDict -from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple -from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.usb.passthrough._client_requests import ( + LIST_SLOT, OPEN_SLOT, PendingRequest, Slot, accepted_body, claim_id_in, + decode_json, desynchronized_message, device_timeout_s, +) +from je_auto_control.utils.usb.passthrough.client_errors import ( + UsbClientClosed, UsbClientDesynchronized, UsbClientError, UsbClientTimeout, +) +from je_auto_control.utils.usb.passthrough.client_handle import ClientHandle from je_auto_control.utils.usb.passthrough.protocol import ( FLAG_EOF, REQUEST_ID_KEY, Frame, Opcode, ProtocolError, valid_request_id, ) @@ -74,11 +85,6 @@ _DEFAULT_CREDIT_TIMEOUT_S = 30.0 _INITIAL_CREDIT_GUESS = 16 _CLIENT_SHUT_DOWN_MSG = "client is shut down" -#: Where a request waits: one OPEN/RESUME, one LIST, and one exchange per -#: claim id may be outstanding at a time. -_Slot = Union[str, int] -_OPEN_SLOT = "open" -_LIST_SLOT = "list" #: Replies a host may split across frames, and those that are always one. _FRAGMENTED_REPLIES = (Opcode.LIST, Opcode.CTRL, Opcode.BULK, Opcode.INT) _SINGLE_FRAME_REPLIES = (Opcode.OPENED, Opcode.CLOSED, Opcode.ERROR) @@ -87,175 +93,20 @@ _MAX_TOMBSTONES = 256 -class UsbClientError(AutoControlException): - """The host reported a transfer or open failure.""" - - -class UsbClientTimeout(UsbClientError): - """A reply did not arrive within the configured timeout.""" - - -class UsbClientClosed(UsbClientError): - """The client / handle was shut down before a reply arrived.""" - - -class UsbClientDesynchronized(UsbClientError): - """A timed-out request left replies impossible to pair; reconnect. - - Raised instead of sending when an earlier request of the same kind - timed out and the host does not echo request ids: its late reply and - the answer to a new request would be indistinguishable. A claim is - recovered by closing its handle and opening the device again; OPEN / - RESUME and LIST by reconnecting the channel with a new client. - """ - - -@dataclass -class _PendingRequest: - """One outstanding viewer→host request awaiting a reply. - - The reply is kept as its decoded JSON body rather than a - :class:`Frame` because a reassembled payload (open question 2) may - exceed the per-frame cap that :class:`Frame` enforces. - """ - - send_op: Opcode - expected_op: Opcode - slot: _Slot - event: threading.Event = field(default_factory=threading.Event) - request_id: str = "" - reply_op: Optional[Opcode] = None - reply_body: Dict[str, Any] = field(default_factory=dict) - cancelled: bool = False - - @property - def label(self) -> str: - return self.send_op.name - - @property - def claim_id(self) -> int: - return self.slot if isinstance(self.slot, int) else 0 - - def timeout_message(self) -> str: - """What :class:`UsbClientTimeout` says when this request expires.""" - if isinstance(self.slot, int): - return f"{self.label} timed out for claim {self.slot}" - return f"{self.label} timed out" - - -def _desynchronized_message(slot: _Slot) -> str: - reason = ("an earlier request timed out and the host does not echo request " - "ids, so its late reply cannot be told from the next one") - if isinstance(slot, int): - return (f"claim {slot} cannot be reused: {reason}. " - "Close this handle and open the device again.") - return (f"no further {slot.upper()} on this client: {reason}. " - "Reconnect the USB channel and use a new client.") - - -# --------------------------------------------------------------------------- -# ClientHandle — what the user actually drives once they hold a claim -# --------------------------------------------------------------------------- - - -class ClientHandle: - """One open USB device claim from the viewer's perspective. - - All transfer methods are blocking — they enqueue the right request - frame, wait for the host to send the matching reply (or ERROR), - and return ``bytes``. Backend errors raise :class:`UsbClientError`. - """ - - def __init__(self, client: "UsbPassthroughClient", claim_id: int, - resume_token: str = "") -> None: # nosec B107 # reason: resume_token is a reconnect handle, not a credential; "" means "no token yet" - self._client = client - self._claim_id = claim_id - self._resume_token = resume_token - self._closed = False - self._lock = threading.Lock() - - @property - def claim_id(self) -> int: - return self._claim_id - - @property - def resume_token(self) -> str: - """Opaque token to re-bind this claim after a transport reconnect.""" - return self._resume_token - - @property - def closed(self) -> bool: - with self._lock: - return self._closed - - @property - def reusable(self) -> bool: - """False once the handle is closed or its claim is desynchronised. - - A transfer that timed out against a host that does not echo request - ids leaves the claim unable to pair replies; further transfers raise - :class:`UsbClientDesynchronized`. :meth:`close` still works. - """ - return not self.closed and self._client.slot_reusable(self._claim_id) - - def control_transfer(self, *, bm_request_type: int, b_request: int, - w_value: int = 0, w_index: int = 0, - data: bytes = b"", length: int = 0, - timeout_ms: int = 1000) -> bytes: - request: Dict[str, Any] = { - "bm_request_type": int(bm_request_type), - "b_request": int(b_request), - "w_value": int(w_value), "w_index": int(w_index), - "timeout_ms": int(timeout_ms), - } - if data: - request["data"] = base64.b64encode(bytes(data)).decode("ascii") - if length: - request["length"] = int(length) - return self._exchange(Opcode.CTRL, request) - - def bulk_transfer(self, *, endpoint: int, direction: str, - data: bytes = b"", length: int = 0, - timeout_ms: int = 1000) -> bytes: - return self._exchange(Opcode.BULK, _endpoint_request( - endpoint=endpoint, direction=direction, - data=data, length=length, timeout_ms=timeout_ms, - )) - - def interrupt_transfer(self, *, endpoint: int, direction: str, - data: bytes = b"", length: int = 0, - timeout_ms: int = 1000) -> bytes: - return self._exchange(Opcode.INT, _endpoint_request( - endpoint=endpoint, direction=direction, - data=data, length=length, timeout_ms=timeout_ms, - )) - - def close(self) -> None: - """Send CLOSE; block on CLOSED. Idempotent.""" - with self._lock: - if self._closed: - return - self._closed = True - try: - self._client._exchange_close(self._claim_id) - except UsbClientClosed: - # Client torn down concurrently; treat as success. - pass - - def _exchange(self, op: Opcode, body: Dict[str, Any]) -> bytes: - with self._lock: - if self._closed: - raise UsbClientClosed(f"handle for claim {self._claim_id} closed") - return self._client._exchange_transfer(self._claim_id, op, body) - - # --------------------------------------------------------------------------- # UsbPassthroughClient — owns the protocol state machine and pending table # --------------------------------------------------------------------------- class UsbPassthroughClient: - """Symmetric counterpart of :class:`UsbPassthroughSession`.""" + """Symmetric counterpart of :class:`UsbPassthroughSession`. + + ``reply_timeout_s`` is how long the host and the transport may take to + answer, not counting the device: a transfer waits its own + ``timeout_ms`` (what the host hands the device, at most 60 s) plus + ``reply_timeout_s`` before :class:`UsbClientTimeout`. OPEN, RESUME, + LIST and CLOSE have no device timeout and wait ``reply_timeout_s``. + """ def __init__( self, @@ -270,11 +121,11 @@ def __init__( self._credit_timeout = float(credit_timeout_s) self._lock = threading.Lock() # One request per slot (pairing by kind, for hosts without ids)... - self._pending: Dict[_Slot, _PendingRequest] = {} + self._pending: Dict[Slot, PendingRequest] = {} # ...and the same requests by the id a current host echoes. - self._by_id: Dict[str, _PendingRequest] = {} + self._by_id: Dict[str, PendingRequest] = {} self._tombstones: "OrderedDict[str, Opcode]" = OrderedDict() - self._desynced: Set[_Slot] = set() + self._desynced: Set[Slot] = set() self._peer_echoes_ids = False self._id_prefix = secrets.token_hex(4) self._id_counter = 0 @@ -295,7 +146,7 @@ def shutdown(self) -> None: """Cancel every outstanding request; subsequent calls raise.""" with self._lock: self._closed = True - pending: List[_PendingRequest] = list(self._pending.values()) + pending: List[PendingRequest] = list(self._pending.values()) self._pending.clear() self._by_id.clear() self._tombstones.clear() @@ -323,9 +174,9 @@ def reusable(self) -> bool: :attr:`ClientHandle.reusable`. """ with self._lock: - return not self._closed and _OPEN_SLOT not in self._desynced + return not self._closed and OPEN_SLOT not in self._desynced - def slot_reusable(self, slot: _Slot) -> bool: + def slot_reusable(self, slot: Slot) -> bool: """Whether requests may still go out on a claim id, ``"open"`` or ``"list"``.""" with self._lock: return slot not in self._desynced @@ -392,9 +243,9 @@ def open(self, *, vendor_id: str, product_id: str, } if serial is not None: body["serial"] = serial - request = _PendingRequest(Opcode.OPEN, Opcode.OPENED, _OPEN_SLOT) + request = PendingRequest(Opcode.OPEN, Opcode.OPENED, OPEN_SLOT) self._round_trip(request, body, busy="another open is in progress") - return self._bind_claim(_accepted_body(request, "open failed")) + return self._bind_claim(accepted_body(request, "open failed")) def resume(self, resume_token: str) -> ClientHandle: """Re-bind a claim after a reconnect using a token from ``open``. @@ -404,13 +255,13 @@ def resume(self, resume_token: str) -> ClientHandle: for the same ``claim_id``; raises :class:`UsbClientError` if the token is unknown or expired. """ - request = _PendingRequest(Opcode.RESUME, Opcode.OPENED, _OPEN_SLOT) + request = PendingRequest(Opcode.RESUME, Opcode.OPENED, OPEN_SLOT) self._round_trip(request, {"resume_token": resume_token}, busy="another open is in progress") - return self._bind_claim(_accepted_body(request, "resume failed")) + return self._bind_claim(accepted_body(request, "resume failed")) def _bind_claim(self, body: Dict[str, Any]) -> ClientHandle: - claim_id = _claim_id_in(body) + claim_id = claim_id_in(body) if claim_id is None: raise UsbClientError(f"host reply has no valid claim_id: {body!r}") with self._lock: @@ -424,7 +275,7 @@ def list_devices(self) -> List[Dict[str, Any]]: Blocks until the host replies. Returns a list of dicts with ``vendor_id`` / ``product_id`` / ``serial`` / ``bus_location``. """ - request = _PendingRequest(Opcode.LIST, Opcode.LIST, _LIST_SLOT) + request = PendingRequest(Opcode.LIST, Opcode.LIST, LIST_SLOT) self._round_trip(request, {}, busy="another list is in progress") if request.reply_op == Opcode.ERROR: raise UsbClientError(request.reply_body.get("error", "host ERROR")) @@ -432,7 +283,7 @@ def list_devices(self) -> List[Dict[str, Any]]: return list(devices) if isinstance(devices, list) else [] def _exchange_close(self, claim_id: int) -> None: - request = _PendingRequest(Opcode.CLOSE, Opcode.CLOSED, int(claim_id)) + request = PendingRequest(Opcode.CLOSE, Opcode.CLOSED, int(claim_id)) with self._claim_lock(claim_id): self._round_trip(request, {}) self._forget_claim(claim_id) @@ -442,7 +293,7 @@ def _claim_lock(self, claim_id: int) -> threading.Lock: with self._lock: return self._claim_locks.setdefault(int(claim_id), threading.Lock()) - def _round_trip(self, request: "_PendingRequest", body: Dict[str, Any], + def _round_trip(self, request: "PendingRequest", body: Dict[str, Any], *, busy: Optional[str] = None) -> None: """Register ``request``, send it with its id, and wait for the reply. @@ -461,14 +312,18 @@ def _round_trip(self, request: "_PendingRequest", body: Dict[str, Any], with self._lock: self._unregister_locked(request) raise - if not request.event.wait(timeout=self._reply_timeout) and self._expire(request): + # The host may spend the request's own ``timeout_ms`` on the device + # before it has anything to say; ``reply_timeout_s`` is what the + # round trip may take on top of that. + wait_s = self._reply_timeout + device_timeout_s(body) + if not request.event.wait(timeout=wait_s) and self._expire(request): raise UsbClientTimeout(request.timeout_message()) if request.cancelled: raise UsbClientClosed(f"client shut down before {request.label} reply") if request.reply_op is None: raise UsbClientError("event signalled without a reply") - def _register(self, request: "_PendingRequest", busy: Optional[str]) -> None: + def _register(self, request: "PendingRequest", busy: Optional[str]) -> None: """Give ``request`` its id and its slot, or refuse to send it.""" with self._lock: if self._closed: @@ -476,7 +331,7 @@ def _register(self, request: "_PendingRequest", busy: Optional[str]) -> None: # CLOSE is exempt: its CLOSED reply is a kind no late transfer # reply can be mistaken for, and the device must be released. if request.send_op != Opcode.CLOSE and request.slot in self._desynced: - raise UsbClientDesynchronized(_desynchronized_message(request.slot)) + raise UsbClientDesynchronized(desynchronized_message(request.slot)) if busy is not None and request.slot in self._pending: raise UsbClientError(busy) request.request_id = self._next_request_id_locked() @@ -487,13 +342,13 @@ def _next_request_id_locked(self) -> str: self._id_counter += 1 return f"{self._id_prefix}-{self._id_counter}" - def _unregister_locked(self, request: "_PendingRequest") -> None: + def _unregister_locked(self, request: "PendingRequest") -> None: if self._pending.get(request.slot) is request: self._pending.pop(request.slot, None) if self._by_id.get(request.request_id) is request: self._by_id.pop(request.request_id, None) - def _expire(self, request: "_PendingRequest") -> bool: + def _expire(self, request: "PendingRequest") -> bool: """Give up on ``request``; False if its reply arrived just in time. A host that echoes ids only needs the tombstone. One that does not @@ -518,12 +373,12 @@ def _bury_locked(self, request_id: str, send_op: Opcode) -> None: def _exchange_transfer(self, claim_id: int, op: Opcode, body: Dict[str, Any]) -> bytes: - request = _PendingRequest(op, op, int(claim_id)) + request = PendingRequest(op, op, int(claim_id)) # Serialised per claim: a second transfer overwrote the first's # pending entry, and one caller received the other's data. with self._claim_lock(claim_id): self._round_trip(request, body) - reply = _accepted_body(request, "transfer failed") + reply = accepted_body(request, "transfer failed") try: return base64.b64decode(reply.get("data") or "", validate=True) except (TypeError, ValueError) as error: # binascii.Error is a ValueError @@ -533,7 +388,7 @@ def _exchange_transfer(self, claim_id: int, op: Opcode, def _route_reply(self, op: Opcode, claim_id: int, payload: bytes) -> None: """Hand a complete reply to the request it answers, or drop it.""" - body = _decode_json(payload) + body = decode_json(payload) request_id = body.get(REQUEST_ID_KEY) orphan: Optional[int] = None with self._lock: @@ -550,7 +405,7 @@ def _route_reply(self, op: Opcode, claim_id: int, payload: bytes) -> None: self._release_orphan(orphan) def _take_by_id_locked(self, request_id: str, op: Opcode, body: Dict[str, Any], - ) -> Tuple[Optional[_PendingRequest], Optional[int]]: + ) -> Tuple[Optional[PendingRequest], Optional[int]]: """The request ``request_id`` names, plus a claim to release if any.""" request = self._by_id.get(request_id) if request is None: @@ -581,7 +436,7 @@ def _discard_late_locked(self, request_id: str, op: Opcode, autocontrol_logger.debug( "passthrough client: late reply to timed-out %s discarded", sent_as.name) if sent_as == Opcode.OPEN and op == Opcode.OPENED and body.get("ok"): - return _claim_id_in(body) + return claim_id_in(body) return None def _peer_echoes_ids_locked(self) -> None: @@ -590,13 +445,13 @@ def _peer_echoes_ids_locked(self) -> None: self._desynced.clear() def _take_by_kind_locked(self, op: Opcode, claim_id: int, - payload: bytes) -> Optional[_PendingRequest]: + payload: bytes) -> Optional[PendingRequest]: """Pair a reply that has no id the way hosts without ids require.""" - slot: _Slot = claim_id + slot: Slot = claim_id if op == Opcode.OPENED: - slot = _OPEN_SLOT + slot = OPEN_SLOT elif op == Opcode.LIST: - slot = _LIST_SLOT + slot = LIST_SLOT request = self._pending.get(slot) if op == Opcode.ERROR: self._reasm.pop(claim_id, None) @@ -628,7 +483,7 @@ def _release_orphan(self, claim_id: int) -> None: def _on_credit(self, frame: Frame) -> None: try: - grant = int(_decode_json(frame.payload).get("credits", 0)) + grant = int(decode_json(frame.payload).get("credits", 0)) except (TypeError, ValueError, OverflowError): # null, a list, 1e999 return if grant <= 0: @@ -679,7 +534,7 @@ def credits_remaining(self, claim_id: int) -> int: def pending_count(self) -> int: with self._lock: - return sum(1 for slot in self._pending if slot != _LIST_SLOT) + return sum(1 for slot in self._pending if slot != LIST_SLOT) # --- Internal ---------------------------------------------------------- @@ -690,57 +545,6 @@ def _send(self, frame: Frame) -> None: raise UsbClientError(f"transport send failed: {error}") from error -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _endpoint_request(*, endpoint: int, direction: str, data: bytes, - length: int, timeout_ms: int) -> Dict[str, Any]: - if direction not in ("in", "out"): - raise ValueError(f"direction must be 'in' or 'out', got {direction!r}") - body: Dict[str, Any] = { - "endpoint": int(endpoint), - "direction": direction, - "timeout_ms": int(timeout_ms), - } - if data: - body["data"] = base64.b64encode(bytes(data)).decode("ascii") - if length: - body["length"] = int(length) - return body - - -def _accepted_body(request: _PendingRequest, default_error: str) -> Dict[str, Any]: - """The reply's body, or :class:`UsbClientError` if the host refused.""" - body = request.reply_body - if request.reply_op == Opcode.ERROR: - raise UsbClientError(body.get("error", "host ERROR")) - if not body.get("ok"): - raise UsbClientError(body.get("error", default_error)) - return body - - -def _claim_id_in(body: Dict[str, Any]) -> Optional[int]: - """The ``claim_id`` of an OPENED body if it is a usable one.""" - try: - claim_id = int(body["claim_id"]) - except (KeyError, TypeError, ValueError): - return None - return claim_id if 0 <= claim_id <= 0xFFFF else None - - -def _decode_json(payload: bytes) -> Dict[str, Any]: - if not payload: - return {} - try: - decoded = json.loads(payload.decode("utf-8")) - except ValueError: - return {} - if not isinstance(decoded, dict): - return {} - return decoded - __all__ = [ "ClientHandle", "UsbClientClosed", "UsbClientDesynchronized", diff --git a/je_auto_control/utils/usb/passthrough/webrtc_channel.py b/je_auto_control/utils/usb/passthrough/webrtc_channel.py index 1b05c5573..581538cb2 100644 --- a/je_auto_control/utils/usb/passthrough/webrtc_channel.py +++ b/je_auto_control/utils/usb/passthrough/webrtc_channel.py @@ -29,7 +29,8 @@ is_usb_passthrough_enabled, ) from je_auto_control.utils.usb.passthrough.protocol import ( - Frame, Opcode, ProtocolError, decode_frame, encode_frame, + HEADER_BYTES, MAX_PAYLOAD_BYTES, REQUEST_ID_KEY, Frame, Opcode, + ProtocolError, decode_frame, encode_frame, request_id_of, ) from je_auto_control.utils.usb.passthrough.session import UsbPassthroughSession from je_auto_control.utils.usb.passthrough.viewer_client import ( @@ -62,13 +63,34 @@ def _as_bytes(raw: Any) -> bytes: raise TypeError(f"usb channel expects binary frames, got {type(raw).__name__}") -def _error_bytes(message: str) -> bytes: +def _error_bytes(message: str, request_id: Optional[str] = None) -> bytes: + """An ERROR frame, echoing ``request_id`` so the viewer can pair it.""" + body = {"error": message} + if request_id is not None: + body[REQUEST_ID_KEY] = request_id return encode_frame(Frame( - op=Opcode.ERROR, - payload=json.dumps({"error": message}).encode("utf-8"), + op=Opcode.ERROR, payload=json.dumps(body).encode("utf-8"), )) +def _request_id_in(raw: Any) -> Optional[str]: + """The request id in a raw channel message, readable or not as a frame. + + The errors this adapter sends are decided before a session sees the + frame -- the feature is off, the frame does not decode, the host has + no session -- so the id has to be read here for the viewer to get the + error rather than a timeout. Never raises: a message that is not + binary, is shorter than a header, or whose payload is over the cap or + not a JSON object with an id simply has none. + """ + if not isinstance(raw, (bytes, bytearray, memoryview)): + return None + payload = bytes(raw)[HEADER_BYTES:] + if len(payload) > MAX_PAYLOAD_BYTES: + return None + return request_id_of(payload) + + class UsbChannelHost: """Carry a :class:`UsbPassthroughSession` over a host DataChannel.""" @@ -111,21 +133,22 @@ def _ensure_session(self) -> Optional[UsbPassthroughSession]: return self._session def _on_message(self, raw: Any) -> None: + request_id = _request_id_in(raw) if not self._enabled(): - self._reply(_error_bytes("usb passthrough disabled")) + self._reply(_error_bytes("usb passthrough disabled", request_id)) return try: frame = decode_frame(_as_bytes(raw)) except (ProtocolError, TypeError) as error: - self._reply(_error_bytes(f"bad frame: {error}")) + self._reply(_error_bytes(f"bad frame: {error}", request_id)) return try: session = self._ensure_session() except Exception as error: # noqa: BLE001 # pylint: disable=broad-except # reason: backend construction can fail many ways; report, don't crash the channel - self._reply(_error_bytes(f"usb backend unavailable: {error}")) + self._reply(_error_bytes(f"usb backend unavailable: {error}", request_id)) return if session is None: - self._reply(_error_bytes("no usb session on host")) + self._reply(_error_bytes("no usb session on host", request_id)) return for reply in session.handle_frame(frame): self._reply(encode_frame(reply)) diff --git a/test/unit_test/headless/test_usb_client_wait_and_channel_errors.py b/test/unit_test/headless/test_usb_client_wait_and_channel_errors.py new file mode 100644 index 000000000..9e7eced43 --- /dev/null +++ b/test/unit_test/headless/test_usb_client_wait_and_channel_errors.py @@ -0,0 +1,304 @@ +"""The viewer waits as long as the transfer it asked for, and hears channel errors. + +Two things a caller could only learn from a timeout: + +* a transfer with ``timeout_ms`` longer than the client's ``reply_timeout_s`` + was abandoned while the host was still, correctly, waiting on the device; +* an ERROR the channel adapter sends before any session sees the frame + (feature off, undecodable frame, no session, backend unavailable) carried + no request id, so the client could not hand it to the request it answered. + +Fake channels and a fake backend throughout: no aiortc, no USB device. +""" +import base64 +import json +import threading +import time + +import pytest + +from je_auto_control.utils.usb.passthrough import ( + ClientHandle, Frame, Opcode, UsbChannelClient, UsbChannelHost, + UsbClientClosed, UsbClientDesynchronized, UsbClientError, + UsbClientTimeout, UsbPassthroughClient, UsbPassthroughSession, + decode_frame, encode_frame, +) +from je_auto_control.utils.usb.passthrough import ( + _client_requests, client_errors, client_handle, viewer_client, +) +from je_auto_control.utils.usb.passthrough.backend import ( + BackendDevice, FakeUsbBackend, +) +from je_auto_control.utils.usb.passthrough.protocol import ( + FLAG_EOF, HEADER_BYTES, MAX_PAYLOAD_BYTES, +) + +_SAMPLE = BackendDevice(vendor_id="1050", product_id="0407", serial="ABC123") +_WAIT_S = 5.0 + + +# --- the wait follows the request ------------------------------------------ + + +class _Wire: + """Captures what the client sends; the test feeds the replies.""" + + def __init__(self, reply_timeout_s: float) -> None: + self.sent = [] + self._cond = threading.Condition() + self.client = UsbPassthroughClient( + send_frame=self._send, reply_timeout_s=reply_timeout_s) + + def _send(self, frame: Frame) -> None: + with self._cond: + self.sent.append(frame) + self._cond.notify_all() + + def wait_sent(self, count: int) -> Frame: + with self._cond: + if not self._cond.wait_for(lambda: len(self.sent) >= count, _WAIT_S): + raise AssertionError(f"client sent {len(self.sent)} frames, wanted {count}") + return self.sent[count - 1] + + def open_claim(self, claim_id: int = 1) -> ClientHandle: + outcome = {} + sent_before = len(self.sent) + self.client._reply_timeout, short = _WAIT_S, self.client._reply_timeout # noqa: SLF001 # reason: only the transfer under test gets the short timeout + thread = threading.Thread( + target=lambda: outcome.update(handle=self.client.open( + vendor_id="1050", product_id="0407")), daemon=True) + thread.start() + request = self.wait_sent(sent_before + 1) + self.client._reply_timeout = short # noqa: SLF001 # reason: see above + self.client.feed_frame(Frame(op=Opcode.OPENED, claim_id=claim_id, payload=_echo( + {"ok": True, "claim_id": claim_id}, request))) + thread.join(_WAIT_S) + return outcome["handle"] + + +def _echo(body: dict, request: Frame) -> bytes: + body = dict(body) + body["request_id"] = json.loads(request.payload)["request_id"] + return json.dumps(body).encode("utf-8") + + +def test_transfer_waits_for_the_timeout_it_asked_for(): + """A reply inside ``timeout_ms`` but after ``reply_timeout_s`` is the answer.""" + wire = _Wire(reply_timeout_s=0.1) + handle = wire.open_claim() + outcome = {} + + def read() -> None: + try: + outcome["data"] = handle.bulk_transfer( + endpoint=0x81, direction="in", length=4, timeout_ms=3000) + except UsbClientError as error: + outcome["error"] = error + + thread = threading.Thread(target=read, daemon=True) + thread.start() + request = wire.wait_sent(2) + time.sleep(0.4) # four times reply_timeout_s, well inside timeout_ms + wire.client.feed_frame(Frame( + op=Opcode.BULK, flags=FLAG_EOF, claim_id=1, payload=_echo( + {"ok": True, "data": base64.b64encode(b"late").decode("ascii")}, request))) + thread.join(_WAIT_S) + assert outcome == {"data": b"late"} + assert handle.reusable + + +def test_transfer_still_times_out_after_its_timeout_and_the_margin(): + wire = _Wire(reply_timeout_s=0.1) + handle = wire.open_claim() + started = time.monotonic() + with pytest.raises(UsbClientTimeout, match="BULK timed out for claim 1"): + handle.bulk_transfer(endpoint=0x81, direction="in", length=4, timeout_ms=200) + waited = time.monotonic() - started + assert 0.28 <= waited < _WAIT_S + + +def test_request_without_a_device_timeout_waits_reply_timeout_only(): + wire = _Wire(reply_timeout_s=0.1) + started = time.monotonic() + with pytest.raises(UsbClientTimeout, match="LIST timed out"): + wire.client.list_devices() + assert time.monotonic() - started < 1.0 + + +@pytest.mark.parametrize("body, expected", [ + ({}, 0.0), + ({"timeout_ms": 1000}, 1.0), + ({"timeout_ms": 30_000}, 30.0), + ({"timeout_ms": 60_000}, 60.0), + ({"timeout_ms": 600_000}, 60.0), # the host refuses more than 60 s + ({"timeout_ms": 0}, 0.0), + ({"timeout_ms": -5}, 0.0), + ({"timeout_ms": True}, 0.0), + ({"timeout_ms": "1000"}, 0.0), + ({"timeout_ms": None}, 0.0), + ({"timeout_ms": float("nan")}, 0.0), + ({"timeout_ms": float("inf")}, 60.0), +]) +def test_device_timeout_of_a_request_body(body, expected): + assert _client_requests.device_timeout_s(body) == expected + + +def test_client_cap_is_the_hosts_cap(): + from je_auto_control.utils.usb.passthrough import session + assert _client_requests.MAX_DEVICE_TIMEOUT_MS == session.MAX_TIMEOUT_MS + + +# --- channel-level errors reach the request -------------------------------- + + +class _ImmediateBridge: + @staticmethod + def call_soon(fn, *args): + fn(*args) + + +class _LinkedChannel: + """Fake RTCDataChannel whose ``send`` delivers to its peer's handler.""" + + def __init__(self): + self._peer = None + self.handlers = {} + + def link(self, peer): + self._peer = peer + + def on(self, event): + def _register(fn): + self.handlers[event] = fn + return fn + return _register + + def send(self, data): + handler = self._peer.handlers.get("message") + if handler is not None: + handler(data) + + +def _channel(**host_kwargs): + """A host adapter and a real client joined by fake channels.""" + host_ch, client_ch = _LinkedChannel(), _LinkedChannel() + host_ch.link(client_ch) + client_ch.link(host_ch) + host_kwargs.setdefault("enabled_check", lambda: True) + UsbChannelHost(host_ch, bridge=_ImmediateBridge(), **host_kwargs) + # Long enough that a test passing in well under it did not time out. + client = UsbChannelClient(client_ch, bridge=_ImmediateBridge(), reply_timeout_s=_WAIT_S) + return host_ch, client + + +def _refused(call) -> UsbClientError: + """Run ``call``; it must be refused by the host, promptly, not time out.""" + started = time.monotonic() + with pytest.raises(UsbClientError) as raised: + call() + assert not isinstance(raised.value, UsbClientTimeout) + assert time.monotonic() - started < _WAIT_S / 2 + return raised.value + + +def _session() -> UsbPassthroughSession: + return UsbPassthroughSession(FakeUsbBackend(devices=[_SAMPLE])) + + +@pytest.mark.parametrize("call", [ + lambda client: client.list_devices(), + lambda client: client.open(vendor_id="1050", product_id="0407"), + lambda client: client.resume("some-token"), +], ids=["list", "open", "resume"]) +def test_disabled_passthrough_is_reported_to_the_caller(call): + _host, client = _channel(session=_session(), enabled_check=lambda: False) + try: + assert "usb passthrough disabled" in str(_refused(lambda: call(client))) + assert client.client.reusable # nothing timed out, nothing to reconnect + finally: + client.shutdown() + + +def test_host_without_a_session_is_reported_to_the_caller(): + _host, client = _channel() + try: + error = _refused(lambda: client.open(vendor_id="1050", product_id="0407")) + assert "no usb session on host" in str(error) + finally: + client.shutdown() + + +def test_unavailable_backend_is_reported_to_the_caller(): + def factory(): + raise OSError("libusb not found") + + _host, client = _channel(session_factory=factory) + try: + error = _refused(client.list_devices) + assert "usb backend unavailable: libusb not found" in str(error) + finally: + client.shutdown() + + +def _raw_reply(host_ch: _LinkedChannel, raw) -> dict: + """Hand ``raw`` to the host adapter; return its one reply (ERROR) body.""" + sent = [] + host_ch._peer.handlers["message"] = sent.append + host_ch.handlers["message"](raw) + assert len(sent) == 1 + frame = decode_frame(sent[0]) + assert frame.op == Opcode.ERROR + return json.loads(frame.payload) + + +def test_bad_frame_error_echoes_the_id_it_can_read(): + host_ch, client = _channel(session=_session()) + client.shutdown() + payload = json.dumps({"request_id": "abc-7"}).encode("utf-8") + raw = bytes([0x7E, 0, 0, 0]) + payload # 0x7E is not an opcode + body = _raw_reply(host_ch, raw) + assert body["request_id"] == "abc-7" + assert body["error"].startswith("bad frame: unknown opcode") + + +@pytest.mark.parametrize("raw", [ + "not-binary", + b"\x02", # shorter than a header + bytes([0x7E, 0, 0, 0]) + b"not json", + bytes([0x7E, 0, 0, 0]) + b'["request_id"]', + bytes([0x7E, 0, 0, 0]) + json.dumps({"request_id": "x" * 65}).encode(), + bytes([0x7E, 0, 0, 0]) + json.dumps({"request_id": 7}).encode(), + bytes([0x02, 0, 0, 0]) + json.dumps( + {"request_id": "abc", "pad": "p" * MAX_PAYLOAD_BYTES}).encode(), # over the cap +], ids=["text", "short", "not-json", "not-object", "long-id", "int-id", "oversize"]) +def test_bad_frame_without_a_readable_id_gets_a_plain_error(raw): + host_ch, client = _channel(session=_session()) + client.shutdown() + body = _raw_reply(host_ch, raw) + assert "request_id" not in body + assert body["error"].startswith("bad frame") + + +def test_channel_error_for_a_request_without_an_id_is_unchanged(): + """A viewer older than request ids gets exactly the frame it always got.""" + host_ch, client = _channel(session=_session(), enabled_check=lambda: False) + client.shutdown() + raw = encode_frame(Frame(op=Opcode.LIST, payload=b"")) + assert len(raw) == HEADER_BYTES + assert _raw_reply(host_ch, raw) == {"error": "usb passthrough disabled"} + + +# --- the split keeps every name where it was ------------------------------- + + +def test_public_names_are_the_same_objects_wherever_they_are_imported_from(): + assert viewer_client.ClientHandle is client_handle.ClientHandle is ClientHandle + for name, exported in (("UsbClientError", UsbClientError), + ("UsbClientTimeout", UsbClientTimeout), + ("UsbClientClosed", UsbClientClosed), + ("UsbClientDesynchronized", UsbClientDesynchronized)): + assert getattr(viewer_client, name) is getattr(client_errors, name) is exported + assert set(viewer_client.__all__) == { + "ClientHandle", "UsbClientClosed", "UsbClientDesynchronized", + "UsbClientError", "UsbClientTimeout", "UsbPassthroughClient", + } From 8fe744404055944754f032ebab0c25a75680b71e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:37:40 +0800 Subject: [PATCH 04/44] Keep config-sync buckets in SQLite and refuse a push built on a stale revision The signaling server held buckets in a dict, so a restart lost them, and every PUT replaced the previous bucket, so two machines pushing together dropped one side's changes without either hearing about it. Buckets now live in ConfigStore, which compares the base revision and writes in one transaction and answers a repeated operation id with the original revision. The unconditional write stays only behind --allow-blind-config-writes. --- .../Eng/doc/config_sync/config_sync_doc.rst | 91 +++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/config_sync/config_sync_doc.rst | 80 ++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/utils/config_sync/client.py | 101 +++++++- je_auto_control/utils/config_sync/store.py | 230 +++++++++++++++++ .../utils/remote_desktop/signaling_server.py | 177 +++++++++---- test/unit_test/headless/test_config_sync.py | 67 ++++- .../headless/test_config_sync_persistence.py | 234 ++++++++++++++++++ .../headless/test_config_sync_tombstones.py | 5 +- .../headless/test_signaling_config_routes.py | 44 +++- 11 files changed, 958 insertions(+), 73 deletions(-) create mode 100644 docs/source/Eng/doc/config_sync/config_sync_doc.rst create mode 100644 docs/source/Zh/doc/config_sync/config_sync_doc.rst create mode 100644 je_auto_control/utils/config_sync/store.py create mode 100644 test/unit_test/headless/test_config_sync_persistence.py diff --git a/docs/source/Eng/doc/config_sync/config_sync_doc.rst b/docs/source/Eng/doc/config_sync/config_sync_doc.rst new file mode 100644 index 000000000..331a6f507 --- /dev/null +++ b/docs/source/Eng/doc/config_sync/config_sync_doc.rst @@ -0,0 +1,91 @@ +Cross-Machine Config Sync +========================= + +``je_auto_control.utils.config_sync`` keeps one user's hotkeys, triggers, +address book and other small settings the same on several machines. Each user +has one *bucket* on a sync server; the signaling server +(``python -m je_auto_control.utils.remote_desktop.signaling_server``, the +``[signaling]`` extra) serves it at ``GET`` / ``PUT /config/{user_id}``. + +Server: persistent, revision-checked buckets +-------------------------------------------- + +Buckets are stored in one SQLite file by +``je_auto_control.utils.config_sync.store.ConfigStore``, so a server restart +keeps them. The file is chosen by, in order, ``--config-db PATH``, the +``AC_SIGNALING_CONFIG_DB`` environment variable, and the default +``~/.je_auto_control/config_sync.sqlite3``; it is opened at the first +``/config`` request, not at start-up. When embedding the app, pass +``create_app(config_store_path=...)``. + +Every bucket has a server-assigned **revision** (``1`` for the first write, +then ``+1`` per commit). A write names the revision it was built on, and the +comparison and the write happen inside one SQLite transaction: + +.. code-block:: python + + from je_auto_control.utils.config_sync.store import ConfigStore, RevisionConflictError + + store = ConfigStore("buckets.sqlite3") + revision = store.commit("alice", bucket, base_revision=0, operation_id="op-1") # -> 1 + store.commit("alice", bucket, base_revision=0, operation_id="op-1") # -> 1 again + store.commit("alice", other, base_revision=0, operation_id="op-2") # RevisionConflictError + +A repeated ``operation_id`` returns the revision its first commit produced and +writes nothing, so a client whose reply was lost can simply send the same +request again. The last 256 operation ids per user are remembered. + +Wire format (version 2) +----------------------- + +.. list-table:: + :header-rows: 1 + + * - Request + - Reply + * - ``GET /config/{user_id}`` + - ``200`` with ``{"user_id", "sections", "revision": , "version": 2}``, + or ``404`` when the user has no bucket. + * - ``PUT /config/{user_id}`` with + ``{"version": 2, "base_revision": N, "operation_id": "...", "bucket": {...}}`` + - ``200 {"ok": true, "revision": N + 1, "version": 2}``; ``409 {"detail": "revision conflict", + "revision": }`` when the bucket is no longer at ``N`` (nothing is written); + ``400`` for a malformed envelope or bucket, or a bucket naming another user. + * - ``PUT /config/{user_id}`` with a bare bucket (the pre-version-2 body) + - ``428`` -- unless the server runs with ``--allow-blind-config-writes`` + (``create_app(allow_blind_config_writes=True)``), which overwrites without a check + and replies ``200 {"ok": true, "revision": ...}``. + +``base_revision`` ``0`` means "there is no bucket yet". The shared secret +(``X-Signaling-Secret``), the 1 MiB body cap (``413``, checked before the body +is read), the ``Content-Length`` requirement (``411``) and the 1,024-user limit +(``503``) are unchanged. The bucket is now validated on the way in: sections +must be mappings of entry id to an object. + +**Clients older than version 2** push a bare bucket. Against a server in its +default mode their ``push`` / ``sync`` fails with ``ConfigSyncError: config +sync PUT returned HTTP 428`` and nothing is written; their ``fetch`` keeps +working (the reply has the same fields plus ``version``, which they ignore). +Start the server with ``--allow-blind-config-writes`` to keep serving them, +accepting that two such clients pushing together can still overwrite each +other. A **version-2 client against an older server** is told so: +``push`` raises ``ConfigSyncError`` ("predates revision-checked writes") +because the reply carries no revision -- upgrade the server first. + +Client +------ + +.. code-block:: python + + from je_auto_control.utils.config_sync import ConfigBucket, ConfigSyncClient, ConfigSyncConflict + + client = ConfigSyncClient("https://sync.example", user_id="alice", secret="...") + merged, conflicts = client.sync(local_bucket) # fetch, merge, push on top of what was fetched + print(merged.revision) # the revision the server committed + +``sync`` pushes with the fetched revision as ``base_revision``; when another +machine pushed in between it fetches and merges again, up to ``max_attempts`` +(default 4) times, then raises ``ConfigSyncConflict``. ``push(bucket)`` uses +``bucket.revision`` as the base, returns the committed revision and raises +``ConfigSyncConflict`` (``.revision`` is the server's current one) instead of +overwriting. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e826e9bb4..d3d8cdbe2 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -24,6 +24,7 @@ Comprehensive guides for all AutoControl features. doc/ai_agents/ai_agents_doc doc/critical_exit/critical_exit_doc doc/cli/cli_doc + doc/config_sync/config_sync_doc doc/create_project/create_project_doc doc/new_features/new_features_doc doc/new_features/v2_features_doc diff --git a/docs/source/Zh/doc/config_sync/config_sync_doc.rst b/docs/source/Zh/doc/config_sync/config_sync_doc.rst new file mode 100644 index 000000000..9308eb001 --- /dev/null +++ b/docs/source/Zh/doc/config_sync/config_sync_doc.rst @@ -0,0 +1,80 @@ +跨機器設定同步 +============== + +``je_auto_control.utils.config_sync`` 讓同一位使用者的快捷鍵、觸發器、通訊錄與其他小型設定 +在多台機器上保持一致。每位使用者在同步 server 上有一個 *bucket*；signaling server +(``python -m je_auto_control.utils.remote_desktop.signaling_server``,需要 ``[signaling]`` extra) +以 ``GET`` / ``PUT /config/{user_id}`` 提供它。 + +Server:持久化、檢查修訂版的 bucket +----------------------------------- + +Bucket 由 ``je_auto_control.utils.config_sync.store.ConfigStore`` 存在一個 SQLite 檔案, +server 重啟後仍在。檔案位置依序取自 ``--config-db PATH``、環境變數 +``AC_SIGNALING_CONFIG_DB``、預設的 ``~/.je_auto_control/config_sync.sqlite3``; +在第一個 ``/config`` 請求時才開啟,不是啟動時。內嵌 app 時傳 +``create_app(config_store_path=...)``。 + +每個 bucket 有 server 指派的 **revision**(第一次寫入為 ``1``,之後每次 commit ``+1``)。 +寫入必須指明它所依據的 revision,比較與寫入在同一個 SQLite transaction 內完成: + +.. code-block:: python + + from je_auto_control.utils.config_sync.store import ConfigStore, RevisionConflictError + + store = ConfigStore("buckets.sqlite3") + revision = store.commit("alice", bucket, base_revision=0, operation_id="op-1") # -> 1 + store.commit("alice", bucket, base_revision=0, operation_id="op-1") # -> 仍是 1 + store.commit("alice", other, base_revision=0, operation_id="op-2") # RevisionConflictError + +重複的 ``operation_id`` 會回傳第一次 commit 產生的 revision 且不寫入任何東西, +所以沒收到回覆的 client 直接重送同一個請求即可。每位使用者保留最近 256 個 operation id。 + +傳輸格式(version 2) +--------------------- + +.. list-table:: + :header-rows: 1 + + * - 請求 + - 回覆 + * - ``GET /config/{user_id}`` + - ``200`` 與 ``{"user_id", "sections", "revision": <已提交>, "version": 2}``, + 使用者沒有 bucket 時為 ``404``。 + * - ``PUT /config/{user_id}``,body 為 + ``{"version": 2, "base_revision": N, "operation_id": "...", "bucket": {...}}`` + - ``200 {"ok": true, "revision": N + 1, "version": 2}``;bucket 已不在 ``N`` 時 + ``409 {"detail": "revision conflict", "revision": <目前>}``(不寫入); + envelope 或 bucket 格式錯誤、或 bucket 指名另一位使用者時 ``400``。 + * - ``PUT /config/{user_id}``,body 為裸 bucket(version 2 之前的格式) + - ``428`` —— 除非 server 以 ``--allow-blind-config-writes`` + (``create_app(allow_blind_config_writes=True)``)啟動,此時不檢查直接覆寫, + 回覆 ``200 {"ok": true, "revision": ...}``。 + +``base_revision`` 為 ``0`` 表示「還沒有 bucket」。共享密鑰(``X-Signaling-Secret``)、 +1 MiB body 上限(``413``,讀取 body 前檢查)、必須帶 ``Content-Length``(``411``)與 +1,024 位使用者上限(``503``)都不變。Bucket 現在在進入時驗證:section 必須是 +entry id 對應到物件的 mapping。 + +**早於 version 2 的 client** 送出裸 bucket。面對預設模式的 server,它們的 ``push`` / ``sync`` +會以 ``ConfigSyncError: config sync PUT returned HTTP 428`` 失敗且不寫入任何東西; +``fetch`` 照常運作(回覆欄位相同,多一個它們會忽略的 ``version``)。要繼續服務它們, +以 ``--allow-blind-config-writes`` 啟動 server,並接受兩個這種 client 同時 push 仍可能互相覆寫。 +**version-2 client 面對舊 server** 會被明確告知:回覆沒有 revision,``push`` 丟出 +``ConfigSyncError``(「predates revision-checked writes」)—— 請先升級 server。 + +Client +------ + +.. code-block:: python + + from je_auto_control.utils.config_sync import ConfigBucket, ConfigSyncClient, ConfigSyncConflict + + client = ConfigSyncClient("https://sync.example", user_id="alice", secret="...") + merged, conflicts = client.sync(local_bucket) # fetch、merge、在所 fetch 的版本上 push + print(merged.revision) # server 提交的 revision + +``sync`` 以 fetch 到的 revision 作為 ``base_revision`` push;若期間有另一台機器 push, +它會重新 fetch 與 merge,最多 ``max_attempts``(預設 4)次,之後丟出 ``ConfigSyncConflict``。 +``push(bucket)`` 以 ``bucket.revision`` 為基準,回傳已提交的 revision;落後時丟出 +``ConfigSyncConflict``(``.revision`` 為 server 目前的 revision)而不是覆寫。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index eb1f7a49f..23b4ad914 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -24,6 +24,7 @@ AutoControl 所有功能的完整使用指南。 doc/ai_agents/ai_agents_doc doc/critical_exit/critical_exit_doc doc/cli/cli_doc + doc/config_sync/config_sync_doc doc/create_project/create_project_doc doc/new_features/new_features_doc doc/new_features/v2_features_doc diff --git a/je_auto_control/utils/config_sync/client.py b/je_auto_control/utils/config_sync/client.py index 4128ee00f..8d5046557 100644 --- a/je_auto_control/utils/config_sync/client.py +++ b/je_auto_control/utils/config_sync/client.py @@ -5,12 +5,19 @@ import math import time import urllib.parse +import uuid from dataclasses import asdict, dataclass, field from typing import Any, Dict, List, Mapping, Optional, Tuple from je_auto_control.utils.exception.exceptions import AutoControlException _DEFAULT_TIMEOUT_S = 5.0 +#: The ``/config`` wire format: a PUT is an envelope naming the revision it +#: was built on and an operation id, and is refused when that revision is stale. +WIRE_VERSION = 2 +#: How often :meth:`ConfigSyncClient.sync` fetches, merges and pushes again +#: after losing a race to another machine before it gives up. +DEFAULT_SYNC_ATTEMPTS = 4 #: How long a deletion is remembered. A tombstone purged before every machine #: has synced lets a machine that still holds the entry bring it back, so this @@ -27,6 +34,23 @@ class ConfigSyncError(AutoControlException, RuntimeError): """Raised on network errors or schema validation failures.""" +class ConfigSyncConflict(ConfigSyncError): + """The server refused a push built on a revision that is no longer current. + + ``revision`` is the server's current revision when it reported one. + Nothing was written: fetch, merge and push again. + """ + + def __init__(self, message: str, revision: Optional[int] = None) -> None: + super().__init__(message) + self.revision = revision + + +def new_operation_id() -> str: + """A fresh id for one push; reuse it when retrying that same push.""" + return uuid.uuid4().hex + + @dataclass class ConflictRecord: """One entry that lost a last-modified race during the merge.""" @@ -258,6 +282,14 @@ def _request(self, method: str, *, status = int(response["status"]) if status == 404: return None + if status == 409: + raise ConfigSyncConflict( + f"config sync {method}: the server is at another revision", + _conflict_revision(response.get("text"))) + if status == 428: + raise ConfigSyncError( + f"config sync {method} returned HTTP 428: the server only accepts " + "revision-checked (version 2) writes") if not 200 <= status < 300: raise ConfigSyncError(f"config sync {method} returned HTTP {status}") if not response.get("text"): @@ -274,25 +306,72 @@ def fetch(self) -> Optional[ConfigBucket]: return None return ConfigBucket.from_dict(body) - def push(self, bucket: ConfigBucket) -> None: - """PUT the bucket to the server, replacing whatever's there.""" + def push(self, bucket: ConfigBucket, *, base_revision: Optional[int] = None, + operation_id: Optional[str] = None) -> int: + """PUT the bucket if the server is still at ``base_revision``. + + ``base_revision`` defaults to ``bucket.revision`` -- for a bucket + that came from :meth:`fetch` that is the revision it was read at; + ``0`` means "there is no bucket yet". Returns the committed revision + and stores it in ``bucket.revision``. Raises + :class:`ConfigSyncConflict` when another machine pushed in between; + nothing is overwritten. Pass the same ``operation_id`` when repeating + a push whose reply never arrived: the server answers with the + revision the first attempt produced instead of a conflict. + """ if bucket.user_id != self._user_id: raise ConfigSyncError( f"bucket user_id={bucket.user_id!r} mismatches client user_id" f"={self._user_id!r}", ) - self._request("PUT", body=bucket.to_dict()) + base = bucket.revision if base_revision is None else int(base_revision) + reply = self._request("PUT", body={ + "version": WIRE_VERSION, "base_revision": base, + "operation_id": operation_id or new_operation_id(), + "bucket": bucket.to_dict(), + }) + revision = (reply or {}).get("revision") + if isinstance(revision, bool) or not isinstance(revision, int): + # A server from before the version-2 format stores any JSON + # object and answers {"ok": true}: it did not check anything. + raise ConfigSyncError( + "config sync PUT: the server did not report a committed revision; " + "it predates revision-checked writes and must be upgraded") + bucket.revision = revision + return revision - def sync(self, local: ConfigBucket + def sync(self, local: ConfigBucket, *, max_attempts: int = DEFAULT_SYNC_ATTEMPTS, ) -> Tuple[ConfigBucket, List[ConflictRecord]]: - """One-shot bidirectional sync: fetch, merge, push the result.""" - remote = self.fetch() or ConfigBucket(user_id=self._user_id) - merged, conflicts = merge_buckets(local, remote) - self.push(merged) - return merged, conflicts + """Bidirectional sync: fetch, merge, push on top of what was fetched. + + When another machine pushes between the fetch and the push the + server refuses the write, and this fetches and merges again -- up to + ``max_attempts`` times, then :class:`ConfigSyncConflict`. The + returned bucket's ``revision`` is the one the server committed. + """ + for _attempt in range(max(1, int(max_attempts))): + remote = self.fetch() or ConfigBucket(user_id=self._user_id) + merged, conflicts = merge_buckets(local, remote) + try: + self.push(merged, base_revision=remote.revision) + except ConfigSyncConflict: + continue + return merged, conflicts + raise ConfigSyncConflict( + f"config sync: still behind the server after {max_attempts} attempts") + + +def _conflict_revision(text: Any) -> Optional[int]: + """The current revision a 409 reply names, when it names one.""" + try: + revision = json.loads(text or "").get("revision") + except (json.JSONDecodeError, RecursionError, AttributeError, TypeError): + return None + return revision if isinstance(revision, int) and not isinstance(revision, bool) else None __all__ = [ - "ConfigBucket", "ConflictRecord", "ConfigSyncClient", - "ConfigSyncError", "TOMBSTONE_RETENTION_S", "is_tombstone", "merge_buckets", + "ConfigBucket", "ConflictRecord", "ConfigSyncClient", "ConfigSyncConflict", + "ConfigSyncError", "DEFAULT_SYNC_ATTEMPTS", "TOMBSTONE_RETENTION_S", "WIRE_VERSION", + "is_tombstone", "merge_buckets", "new_operation_id", ] diff --git a/je_auto_control/utils/config_sync/store.py b/je_auto_control/utils/config_sync/store.py new file mode 100644 index 000000000..0e7a955f3 --- /dev/null +++ b/je_auto_control/utils/config_sync/store.py @@ -0,0 +1,230 @@ +"""SQLite-backed, revision-checked store for config-sync buckets. + +The signaling server kept each user's bucket in a dict: a restart lost every +bucket, and a ``PUT`` replaced whatever was there, so two machines pushing at +the same moment silently dropped one side's changes. :class:`ConfigStore` +keeps the buckets in one SQLite file and only accepts a write that names the +revision it was based on -- the comparison and the write happen inside one +``BEGIN IMMEDIATE`` transaction, so two writers (threads or processes) cannot +both pass the check. + +A write also carries an ``operation_id``. A client that never saw the reply +to a commit (timeout, dropped connection) repeats it with the same id and is +answered with the revision the first attempt produced instead of a conflict +against its own write. + +Pure standard library; imports no ``PySide6``. +""" +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any, ContextManager, Optional, Union + +from je_auto_control.utils.config_sync.client import ConfigBucket, ConfigSyncError +from je_auto_control.utils.sqlite_support import autocommit_connection, sqlite_errors_as + +#: How many distinct users may hold a bucket in one store. +DEFAULT_MAX_USERS = 1024 +#: Operation ids remembered per user for retry de-duplication. A retry older +#: than this many commits is answered as a revision conflict, which is safe: +#: the client fetches, merges and pushes again. +DEFAULT_MAX_OPERATIONS = 256 +_MAX_OPERATION_ID_CHARS = 128 + +_SCHEMA = ( + "CREATE TABLE IF NOT EXISTS buckets (" + " user_id TEXT PRIMARY KEY, revision INTEGER NOT NULL," + " body TEXT NOT NULL, updated_at REAL NOT NULL)", + "CREATE TABLE IF NOT EXISTS operations (" + " seq INTEGER PRIMARY KEY AUTOINCREMENT, user_id TEXT NOT NULL," + " operation_id TEXT NOT NULL, revision INTEGER NOT NULL," + " UNIQUE (user_id, operation_id))", +) + + +class ConfigStoreError(ConfigSyncError): + """The bucket database could not be opened, read or written.""" + + +class RevisionConflictError(ConfigStoreError): + """A commit named a ``base_revision`` that is no longer the current one.""" + + def __init__(self, user_id: str, base_revision: int, current_revision: int) -> None: + super().__init__( + f"bucket {user_id!r} is at revision {current_revision}, " + f"not the base revision {base_revision} this write was built on") + self.user_id = user_id + self.base_revision = base_revision + self.current_revision = current_revision + + +class StoreCapacityError(ConfigStoreError): + """A new user's bucket would exceed the store's user limit.""" + + +def default_store_path() -> Path: + """``~/.je_auto_control/config_sync.sqlite3``, resolved at call time.""" + return Path.home() / ".je_auto_control" / "config_sync.sqlite3" + + +def _checked_revision(base_revision: Any) -> int: + if isinstance(base_revision, bool) or not isinstance(base_revision, int) or base_revision < 0: + raise ConfigStoreError(f"base_revision must be an integer >= 0, got {base_revision!r}") + return base_revision + + +def _checked_operation_id(operation_id: Any) -> str: + if (not isinstance(operation_id, str) or not operation_id + or len(operation_id) > _MAX_OPERATION_ID_CHARS or not operation_id.isprintable()): + raise ConfigStoreError( + f"operation_id must be 1-{_MAX_OPERATION_ID_CHARS} printable characters") + return operation_id + + +class ConfigStore: + """Persistent map of user id -> bucket with compare-and-set commits. + + ``db_path`` of ``None`` means :func:`default_store_path`. Neither that + path nor the file is touched until the first read or write, so building + a store (as ``create_app`` does) has no side effect on disk. + """ + + def __init__(self, db_path: Union[str, Path, None] = None, *, + max_users: int = DEFAULT_MAX_USERS, + max_operations: int = DEFAULT_MAX_OPERATIONS) -> None: + self._configured_path = db_path + self._max_users = int(max_users) + self._max_operations = max(1, int(max_operations)) + self._resolved_path: Optional[str] = None + + @property + def path(self) -> Path: + """Where the database lives (resolving the default if none was given).""" + configured = self._configured_path + return Path(configured) if configured is not None else default_store_path() + + def _connection(self) -> ContextManager[Any]: + if self._resolved_path is None: + path = self.path + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as error: + raise ConfigStoreError(f"cannot create {path.parent}: {error}") from error + with autocommit_connection(str(path)) as connection: + for statement in _SCHEMA: + connection.execute(statement) + self._resolved_path = str(path) + return autocommit_connection(self._resolved_path) + + @sqlite_errors_as(ConfigStoreError) + def get(self, user_id: str) -> Optional[ConfigBucket]: + """The user's bucket with ``revision`` set to the committed revision.""" + with self._connection() as connection: + row = connection.execute( + "SELECT revision, body FROM buckets WHERE user_id = ?", (user_id,)).fetchone() + if row is None: + return None + try: + bucket = ConfigBucket.from_dict(json.loads(row["body"])) + except (json.JSONDecodeError, RecursionError) as error: + raise ConfigStoreError(f"stored bucket for {user_id!r} is not JSON") from error + bucket.revision = int(row["revision"]) + return bucket + + @sqlite_errors_as(ConfigStoreError) + def revision(self, user_id: str) -> int: + """The committed revision of the user's bucket; ``0`` when there is none.""" + with self._connection() as connection: + return self._current_revision(connection, user_id) + + @sqlite_errors_as(ConfigStoreError) + def commit(self, user_id: str, bucket: ConfigBucket, *, + base_revision: int, operation_id: str) -> int: + """Write ``bucket`` if the store is still at ``base_revision``. + + Returns the committed revision (``base_revision + 1``). A repeated + ``operation_id`` returns the revision its first commit produced and + writes nothing. Raises :class:`RevisionConflictError` when another + write got there first -- ``base_revision`` ``0`` means "no bucket + yet" -- and :class:`StoreCapacityError` when a new user would exceed + the user limit. + """ + base = _checked_revision(base_revision) + operation = _checked_operation_id(operation_id) + self._require_owner(user_id, bucket) + with self._connection() as connection: + connection.execute("BEGIN IMMEDIATE") + done = connection.execute( + "SELECT revision FROM operations WHERE user_id = ? AND operation_id = ?", + (user_id, operation)).fetchone() + if done is not None: + connection.execute("COMMIT") + return int(done["revision"]) + current = self._current_revision(connection, user_id) + if current != base: + raise RevisionConflictError(user_id, base, current) + revision = self._write(connection, user_id, bucket, current) + self._remember(connection, user_id, operation, revision) + connection.execute("COMMIT") + return revision + + @sqlite_errors_as(ConfigStoreError) + def overwrite(self, user_id: str, bucket: ConfigBucket) -> int: + """Replace the bucket without a revision check; returns the new revision. + + This is the pre-version-2 behaviour and can silently drop a + concurrent writer's changes; the signaling server only reaches it + when started with the explicit compatibility setting. + """ + self._require_owner(user_id, bucket) + with self._connection() as connection: + connection.execute("BEGIN IMMEDIATE") + revision = self._write( + connection, user_id, bucket, self._current_revision(connection, user_id)) + connection.execute("COMMIT") + return revision + + @staticmethod + def _require_owner(user_id: str, bucket: ConfigBucket) -> None: + if bucket.user_id != user_id: + raise ConfigStoreError( + f"bucket belongs to {bucket.user_id!r}, not to {user_id!r}") + + @staticmethod + def _current_revision(connection: Any, user_id: str) -> int: + row = connection.execute( + "SELECT revision FROM buckets WHERE user_id = ?", (user_id,)).fetchone() + return 0 if row is None else int(row["revision"]) + + def _write(self, connection: Any, user_id: str, bucket: ConfigBucket, current: int) -> int: + if current == 0: + # Revisions start at 1, so 0 means this user has no row yet. + users = connection.execute("SELECT COUNT(*) AS n FROM buckets").fetchone()["n"] + if int(users) >= self._max_users: + raise StoreCapacityError(f"store already holds {users} users") + revision = current + 1 + body = bucket.to_dict() + body["revision"] = revision + connection.execute( + "INSERT INTO buckets (user_id, revision, body, updated_at) VALUES (?, ?, ?, ?) " + "ON CONFLICT(user_id) DO UPDATE SET revision = excluded.revision, " + "body = excluded.body, updated_at = excluded.updated_at", + (user_id, revision, json.dumps(body, sort_keys=True), time.time())) + return revision + + def _remember(self, connection: Any, user_id: str, operation: str, revision: int) -> None: + connection.execute( + "INSERT INTO operations (user_id, operation_id, revision) VALUES (?, ?, ?)", + (user_id, operation, revision)) + connection.execute( + "DELETE FROM operations WHERE user_id = ? AND seq NOT IN (" + " SELECT seq FROM operations WHERE user_id = ? ORDER BY seq DESC LIMIT ?)", + (user_id, user_id, self._max_operations)) + + +__all__ = [ + "ConfigStore", "ConfigStoreError", "DEFAULT_MAX_OPERATIONS", "DEFAULT_MAX_USERS", + "RevisionConflictError", "StoreCapacityError", "default_store_path", +] diff --git a/je_auto_control/utils/remote_desktop/signaling_server.py b/je_auto_control/utils/remote_desktop/signaling_server.py index 78713ac2f..e9a119eb3 100644 --- a/je_auto_control/utils/remote_desktop/signaling_server.py +++ b/je_auto_control/utils/remote_desktop/signaling_server.py @@ -1,12 +1,27 @@ """Standalone rendezvous service for WebRTC SDP exchange. Hosts register an offer keyed by their host ID; viewers fetch the offer, -post an answer, and the host polls for it. The server is stateless beyond -an in-memory dict with TTL eviction — restart loses pending sessions. +post an answer, and the host polls for it. Sessions live in an in-memory +dict with TTL eviction — restart loses pending sessions. It also serves ``GET`` / ``PUT /config/{user_id}``, the per-user bucket that -:mod:`je_auto_control.utils.config_sync` pushes and pulls. Buckets are kept -in memory as sent (no TTL), so they too are lost on restart. +:mod:`je_auto_control.utils.config_sync` pushes and pulls. Buckets live in a +SQLite file (``--config-db``, ``AC_SIGNALING_CONFIG_DB``, default +``~/.je_auto_control/config_sync.sqlite3``; opened at the first ``/config`` +request), so they survive a restart. A ``PUT`` is a version-2 envelope:: + + {"version": 2, "base_revision": 3, "operation_id": "", + "bucket": {"user_id": "...", "sections": {...}}} + +and is committed only while the stored bucket is still at ``base_revision`` +(``0`` = no bucket yet): ``200 {"ok": true, "revision": 4}``, or ``409 +{"detail": "revision conflict", "revision": }`` when another write +got there first. Repeating a ``PUT`` with the same ``operation_id`` returns +the revision the first one produced. ``GET`` returns the bucket with +``revision`` set to the committed revision and ``"version": 2``. A bare +bucket (what clients sent before version 2) is answered ``428`` unless the +server runs with ``--allow-blind-config-writes``, which restores the +unconditional overwrite those clients expect. Run:: @@ -18,7 +33,9 @@ Deployment: drop behind nginx + TLS on a small VPS. The server itself is single-process; for HA put two instances behind a sticky load balancer -or swap the in-memory store for Redis (left as a follow-up). +or swap the in-memory session store for Redis (left as a follow-up); the +config buckets are already safe to share, as every commit is one SQLite +transaction. """ from __future__ import annotations @@ -29,7 +46,8 @@ import threading import time from dataclasses import dataclass, field -from typing import Annotated, Dict, List, Optional +from pathlib import Path +from typing import Annotated, Any, Dict, List, Optional, Tuple, Union try: from fastapi import Depends, FastAPI, Header, HTTPException, Request @@ -43,16 +61,23 @@ "pip install je_auto_control[signaling]" ) from exc +from je_auto_control.utils.config_sync.client import ConfigBucket, ConfigSyncError +from je_auto_control.utils.config_sync.store import ( + ConfigStore, RevisionConflictError, StoreCapacityError, +) + _DEFAULT_TTL_S = 120.0 _MAX_SDP_BYTES = 256 * 1024 # 256 KB; aiortc offers are typically ~4 KB # Live sessions at once. Any client allowed to post could otherwise create # sessions without limit -- 200 of them held 48 MB for the TTL. _MAX_SESSIONS = 1024 -# A config-sync bucket (hotkeys, triggers, address book) and how many users -# may hold one at once. +# A config-sync PUT body (the bucket plus its envelope) and how many users +# may hold a bucket at once. _MAX_CONFIG_BYTES = 1024 * 1024 _MAX_CONFIG_USERS = 1024 +#: The ``/config`` wire format this server speaks. +CONFIG_WIRE_VERSION = 2 _LOG = logging.getLogger("rd-signaling") _WEB_VIEWER_DIR = ( __import__("pathlib").Path(__file__).parent / "web_viewer" @@ -67,26 +92,6 @@ class _Session: updated_at: float = field(default_factory=time.monotonic) -class _ConfigStore: - """Thread-safe in-memory map of user id -> config-sync bucket.""" - - def __init__(self) -> None: - self._buckets: Dict[str, Dict] = {} - self._lock = threading.Lock() - - def get(self, user_id: str) -> Optional[Dict]: - with self._lock: - return self._buckets.get(user_id) - - def put(self, user_id: str, bucket: Dict) -> bool: - """Store ``bucket``; ``False`` when a new user would exceed the cap.""" - with self._lock: - if user_id not in self._buckets and len(self._buckets) >= _MAX_CONFIG_USERS: - return False - self._buckets[user_id] = bucket - return True - - class _SessionStore: """Thread-safe in-memory session map with TTL eviction.""" @@ -320,37 +325,101 @@ def _validate_user_id(user_id: str) -> None: _CONFIG_RESPONSES = { - 400: {"description": "invalid user_id or bucket"}, + 400: {"description": "invalid user_id, envelope or bucket"}, 404: {"description": "no bucket for this user"}, - 503: {"description": "too many users"}, + 409: {"description": "base_revision is not the current revision"}, + 428: {"description": "a write without base_revision / operation_id"}, + 503: {"description": "too many users, or the store is unavailable"}, **_AUTH_RESPONSES, } -def _register_config_routes(app: FastAPI, store: _ConfigStore, secret_dep) -> None: +def _config_bucket(user_id: str, body: Any) -> ConfigBucket: + """Validate ``body`` as the bucket of ``user_id`` (400 otherwise).""" + try: + bucket = ConfigBucket.from_dict(body) + except ConfigSyncError as error: + raise HTTPException(status_code=400, detail=f"invalid bucket: {error}") from error # NOSONAR + # Account isolation: the path names the account, the body cannot move it. + if bucket.user_id != user_id: + raise HTTPException(status_code=400, detail="bucket user_id mismatch") # NOSONAR + return bucket + + +def _config_envelope(body: Dict[str, Any]) -> Optional[Tuple[Any, int, str]]: + """``(bucket, base_revision, operation_id)`` of a version-2 PUT body. + + ``None`` for a bare bucket -- the pre-version-2 body, recognised by + having neither ``version`` nor ``bucket``. + """ + if "version" not in body and "bucket" not in body: + return None + base = body.get("base_revision") + operation = body.get("operation_id") + if (body.get("version") == CONFIG_WIRE_VERSION + and isinstance(base, int) and not isinstance(base, bool) and base >= 0 + and isinstance(operation, str) and 0 < len(operation) <= 128 + and operation.isprintable()): + return body.get("bucket"), base, operation + raise HTTPException( # NOSONAR — see _CONFIG_RESPONSES + status_code=400, + detail="expected {version: 2, base_revision: int >= 0, operation_id: str, bucket: {...}}") + + +def _commit_config(store: ConfigStore, user_id: str, body: Dict[str, Any], + allow_blind_writes: bool) -> Union[int, JSONResponse]: + """Commit a PUT body; the new revision, or the 409 reply on a conflict.""" + envelope = _config_envelope(body) + if envelope is None: + if not allow_blind_writes: + raise HTTPException( # NOSONAR — see _CONFIG_RESPONSES + status_code=428, + detail="this server requires a version-2 write (base_revision and operation_id); " + "unconditional writes need --allow-blind-config-writes") + return store.overwrite(user_id, _config_bucket(user_id, body)) + payload, base, operation = envelope + bucket = _config_bucket(user_id, payload) + try: + return store.commit(user_id, bucket, base_revision=base, operation_id=operation) + except RevisionConflictError as conflict: + return JSONResponse({"detail": "revision conflict", "revision": conflict.current_revision}, + status_code=409) + + +def _register_config_routes(app: FastAPI, store: ConfigStore, secret_dep, + allow_blind_writes: bool = False) -> None: """``GET`` / ``PUT /config/{user_id}`` for :mod:`je_auto_control.utils.config_sync`. - The client has always called these; nothing served them, so every sync - failed. A bucket is stored as sent -- the merge happens client-side. + The merge happens client-side; the server's part is to keep the bucket + and to refuse a write that was not built on the current revision. """ auth_only = [Depends(secret_dep)] @app.get("/config/{user_id}", responses=_CONFIG_RESPONSES, dependencies=auth_only) def _get_config(user_id: str) -> dict: _validate_user_id(user_id) - bucket = store.get(user_id) + try: + bucket = store.get(user_id) + except ConfigSyncError as error: + _LOG.error("config store read failed: %s", error) + raise HTTPException(status_code=503, detail="config store unavailable") from error # NOSONAR if bucket is None: raise HTTPException(status_code=404, detail="no bucket") # NOSONAR — see _CONFIG_RESPONSES - return bucket + return {**bucket.to_dict(), "version": CONFIG_WIRE_VERSION} @app.put("/config/{user_id}", responses=_CONFIG_RESPONSES, dependencies=auth_only) - def _put_config(user_id: str, bucket: Dict) -> dict: + def _put_config(user_id: str, body: Dict[str, Any]): _validate_user_id(user_id) - if bucket.get("user_id", user_id) != user_id: - raise HTTPException(status_code=400, detail="bucket user_id mismatch") # NOSONAR — see _CONFIG_RESPONSES - if not store.put(user_id, bucket): - raise HTTPException(status_code=503, detail="too many users") # NOSONAR — see _CONFIG_RESPONSES - return {"ok": True} + try: + outcome = _commit_config(store, user_id, body, allow_blind_writes) + except StoreCapacityError as error: + raise HTTPException(status_code=503, detail="too many users") from error # NOSONAR + except ConfigSyncError as error: + _LOG.error("config store write failed: %s", error) + raise HTTPException(status_code=503, detail="config store unavailable") from error # NOSONAR + if isinstance(outcome, JSONResponse): + return outcome + return {"ok": True, "revision": outcome, "version": CONFIG_WIRE_VERSION} def _register_request_logging(app: FastAPI) -> None: @@ -365,8 +434,17 @@ async def _log_request(request: Request, call_next): def create_app(shared_secret: Optional[str] = None, ttl_s: float = _DEFAULT_TTL_S, serve_web_viewer: bool = True, - cors_origins: Optional[list] = None) -> FastAPI: - """Build the FastAPI app. Importable for embedding in larger services.""" + cors_origins: Optional[list] = None, *, + config_store_path: Union[str, Path, None] = None, + allow_blind_config_writes: bool = False) -> FastAPI: + """Build the FastAPI app. Importable for embedding in larger services. + + ``config_store_path`` is the SQLite file behind ``/config`` (default + ``~/.je_auto_control/config_sync.sqlite3``); it is opened at the first + ``/config`` request, not here. ``allow_blind_config_writes`` accepts the + bare-bucket ``PUT`` of clients older than the version-2 envelope, which + overwrites without a revision check. + """ app = FastAPI(title="AutoControl Signaling", version="1.0.0") store = _SessionStore(ttl_s=ttl_s) # Before CORS, so CORS wraps it and a browser still sees the 401 / 413. @@ -375,7 +453,8 @@ def create_app(shared_secret: Optional[str] = None, _maybe_mount_viewer(app, serve_web_viewer) secret_dep = _build_secret_dependency(shared_secret) _register_routes(app, store, secret_dep) - _register_config_routes(app, _ConfigStore(), secret_dep) + _register_config_routes(app, ConfigStore(config_store_path, max_users=_MAX_CONFIG_USERS), + secret_dep, allow_blind_config_writes) _register_request_logging(app) return app @@ -398,6 +477,14 @@ def _build_arg_parser() -> argparse.ArgumentParser: help="don't mount the bundled web viewer at /viewer") parser.add_argument("--cors-origin", action="append", default=None, help="allowed CORS origin (repeatable; default: *)") + parser.add_argument("--config-db", default=None, + help="SQLite file for config-sync buckets (default: " + "$AC_SIGNALING_CONFIG_DB, else " + "~/.je_auto_control/config_sync.sqlite3)") + parser.add_argument("--allow-blind-config-writes", action="store_true", + help="accept the unconditional bucket PUT of clients older " + "than the version-2 envelope (no revision check: " + "concurrent pushes can overwrite each other)") return parser @@ -419,6 +506,8 @@ def main(argv: Optional[list] = None) -> None: ttl_s=args.ttl_seconds, serve_web_viewer=not args.no_web_viewer, cors_origins=args.cors_origin, + config_store_path=args.config_db or os.environ.get("AC_SIGNALING_CONFIG_DB") or None, + allow_blind_config_writes=args.allow_blind_config_writes, ) uvicorn.run(app, host=args.bind, port=args.port, log_level="info") diff --git a/test/unit_test/headless/test_config_sync.py b/test/unit_test/headless/test_config_sync.py index 9432e5f51..b3476a9fb 100644 --- a/test/unit_test/headless/test_config_sync.py +++ b/test/unit_test/headless/test_config_sync.py @@ -174,13 +174,24 @@ def test_push_round_trip_uses_put(): calls: list = [] local = ConfigBucket(user_id="alice") local.upsert("hotkeys", "hk1", {"combo": "ctrl+a"}) - with _patch_request({}, calls=calls): - client.push(local) + with _patch_request({"ok": True, "revision": 1}, calls=calls): + assert client.push(local) == 1 assert len(calls) == 1 method, body = calls[0] assert method == "PUT" - assert body["user_id"] == "alice" - assert "hk1" in body["sections"]["hotkeys"] + # Version 2: the bucket travels in an envelope naming what it was built on. + assert body["version"] == 2 and body["base_revision"] == 0 and body["operation_id"] + assert body["bucket"]["user_id"] == "alice" + assert "hk1" in body["bucket"]["sections"]["hotkeys"] + assert local.revision == 1 + + +def test_push_to_a_server_without_revision_checks_is_an_error(): + """A pre-version-2 server answers {"ok": true}: nothing was checked.""" + client = ConfigSyncClient("https://x", user_id="alice") + with _patch_request({"ok": True}): + with pytest.raises(ConfigSyncError, match="predates"): + client.push(ConfigBucket(user_id="alice")) def test_sync_merges_remote_into_local_and_pushes_result(): @@ -200,7 +211,7 @@ def test_sync_merges_remote_into_local_and_pushes_result(): def fake_request(self, method, body=None): fetched.append(method) - return remote_body if method == "GET" else {} + return remote_body if method == "GET" else {"ok": True, "revision": 1} with patch.object(ConfigSyncClient, "_request", new=fake_request): merged, conflicts = client.sync(local) @@ -208,3 +219,49 @@ def fake_request(self, method, body=None): assert merged.sections["hotkeys"]["hk1"]["combo"] == "ctrl+b" assert len(conflicts) == 1 assert fetched == ["GET", "PUT"] + assert merged.revision == 1 + + +def test_sync_refetches_and_merges_again_when_it_loses_the_race(): + """Another machine pushed between the fetch and the push: nothing is overwritten.""" + from je_auto_control.utils.config_sync.client import ConfigSyncConflict + client = ConfigSyncClient("https://x", user_id="alice") + local = ConfigBucket(user_id="alice") + local.upsert("hotkeys", "mine", {"combo": "ctrl+m", "last_modified": 100.0}) + server = {"revision": 1, "sections": {}} + bases: list = [] + + def fake_request(self, method, body=None): + if method == "GET": + return {"user_id": "alice", **server} + bases.append(body["base_revision"]) + if len(bases) == 1: + # The other machine's push lands first. + server.update(revision=2, sections={"hotkeys": { + "theirs": {"combo": "ctrl+t", "last_modified": 50.0}}}) + raise ConfigSyncConflict("behind", 2) + server.update(revision=3, sections=body["bucket"]["sections"]) + return {"ok": True, "revision": 3} + + with patch.object(ConfigSyncClient, "_request", new=fake_request): + merged, _conflicts = client.sync(local) + assert bases == [1, 2] + assert set(merged.entries("hotkeys")) == {"mine", "theirs"} + assert merged.revision == 3 + + +def test_sync_gives_up_after_a_bounded_number_of_lost_races(): + from je_auto_control.utils.config_sync.client import ConfigSyncConflict + client = ConfigSyncClient("https://x", user_id="alice") + attempts: list = [] + + def fake_request(self, method, body=None): + if method == "GET": + return None + attempts.append(body["operation_id"]) + raise ConfigSyncConflict("behind", 9) + + with patch.object(ConfigSyncClient, "_request", new=fake_request): + with pytest.raises(ConfigSyncConflict): + client.sync(ConfigBucket(user_id="alice"), max_attempts=3) + assert len(attempts) == 3 diff --git a/test/unit_test/headless/test_config_sync_persistence.py b/test/unit_test/headless/test_config_sync_persistence.py new file mode 100644 index 000000000..5caa1272f --- /dev/null +++ b/test/unit_test/headless/test_config_sync_persistence.py @@ -0,0 +1,234 @@ +"""Config-sync buckets survive a restart and refuse a write built on stale data. + +The signaling server kept buckets in a dict and let every ``PUT`` replace the +previous one: a restart lost them all, and two machines pushing together +dropped one side's changes without either hearing about it. +""" +import threading + +import pytest + +from je_auto_control.utils.config_sync.client import ConfigBucket, ConfigSyncError +from je_auto_control.utils.config_sync.store import ( + ConfigStore, ConfigStoreError, RevisionConflictError, StoreCapacityError, +) + +_SECRET = {"X-Signaling-Secret": "s3cret"} + + +def _bucket(user="alice", combo="ctrl+a"): + bucket = ConfigBucket(user_id=user) + bucket.upsert("hotkeys", "hk1", {"combo": combo, "last_modified": 100.0}) + return bucket + + +def _envelope(bucket, base, operation="op-1"): + return {"version": 2, "base_revision": base, "operation_id": operation, + "bucket": bucket.to_dict()} + + +@pytest.fixture +def db(tmp_path): + return tmp_path / "sync" / "buckets.sqlite3" + + +def _client(db, **options): + pytest.importorskip("fastapi") + testclient = pytest.importorskip("fastapi.testclient") + from je_auto_control.utils.remote_desktop.signaling_server import create_app + return testclient.TestClient(create_app( + shared_secret="s3cret", serve_web_viewer=False, config_store_path=db, **options)) + + +# --- the store ------------------------------------------------------------- + +def test_reopen_preserves_bucket(db): + user = "alice" + committed_revision = ConfigStore(db).commit( + user, _bucket(), base_revision=0, operation_id="op-1") + reopened = ConfigStore(db) + assert reopened.get(user).revision == committed_revision + assert reopened.get(user).sections["hotkeys"]["hk1"]["combo"] == "ctrl+a" + + +def test_a_stale_base_revision_is_refused_and_nothing_is_written(db): + store = ConfigStore(db) + store.commit("alice", _bucket(combo="first"), base_revision=0, operation_id="a") + with pytest.raises(RevisionConflictError) as raised: + store.commit("alice", _bucket(combo="second"), base_revision=0, operation_id="b") + assert raised.value.current_revision == 1 + assert store.get("alice").sections["hotkeys"]["hk1"]["combo"] == "first" + + +def test_operation_retry_is_idempotent(db): + store = ConfigStore(db) + original = store.commit("alice", _bucket(), base_revision=0, operation_id="op-1") + # The retry still names base 0, which is stale by now -- it is the + # operation id that tells the store this is the same write. + store_after_restart = ConfigStore(db) + repeated = store_after_restart.commit( + "alice", _bucket(), base_revision=0, operation_id="op-1") + assert repeated == original + assert store_after_restart.revision("alice") == original + + +def test_operation_ids_are_per_user(db): + store = ConfigStore(db) + store.commit("alice", _bucket("alice"), base_revision=0, operation_id="op-1") + assert store.commit("bob", _bucket("bob"), base_revision=0, operation_id="op-1") == 1 + assert store.get("bob").user_id == "bob" + + +def test_concurrent_commits_from_one_base_admit_exactly_one(db): + store = ConfigStore(db) + store.commit("alice", _bucket(), base_revision=0, operation_id="seed") + outcomes = [] + start = threading.Barrier(8) + + def push(index): + start.wait(timeout=10) + try: + outcomes.append(ConfigStore(db).commit( + "alice", _bucket(combo=f"writer-{index}"), + base_revision=1, operation_id=f"writer-{index}")) + except RevisionConflictError: + outcomes.append(None) + + threads = [threading.Thread(target=push, args=(index,)) for index in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + assert sorted(outcome for outcome in outcomes if outcome is not None) == [2] + assert outcomes.count(None) == 7 + + +def test_a_bucket_cannot_be_committed_under_another_user(db): + with pytest.raises(ConfigStoreError): + ConfigStore(db).commit("alice", _bucket("bob"), base_revision=0, operation_id="x") + + +def test_the_user_limit_is_enforced_inside_the_commit(db): + store = ConfigStore(db, max_users=1) + store.commit("alice", _bucket("alice"), base_revision=0, operation_id="a") + with pytest.raises(StoreCapacityError): + store.commit("bob", _bucket("bob"), base_revision=0, operation_id="b") + assert store.commit("alice", _bucket("alice"), base_revision=1, operation_id="c") == 2 + + +@pytest.mark.parametrize("base, operation", [(-1, "op"), (True, "op"), ("0", "op"), (0, ""), + (0, "x" * 200), (0, None)]) +def test_malformed_commit_arguments_are_refused(db, base, operation): + with pytest.raises(ConfigStoreError): + ConfigStore(db).commit("alice", _bucket(), base_revision=base, operation_id=operation) + + +def test_building_a_store_touches_nothing_until_first_use(db): + store = ConfigStore(db) + assert not db.parent.exists() + assert store.get("alice") is None + assert db.exists() + + +def test_the_default_path_is_resolved_at_call_time(tmp_path, monkeypatch): + from pathlib import Path + store = ConfigStore() + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + assert store.path == tmp_path / ".je_auto_control" / "config_sync.sqlite3" + + +def test_a_database_failure_is_a_framework_error(tmp_path): + not_a_database = tmp_path / "junk.sqlite3" + not_a_database.write_bytes(b"this is not sqlite" * 64) + with pytest.raises(ConfigSyncError): + ConfigStore(not_a_database).get("alice") + + +# --- the server ------------------------------------------------------------ + +def test_stale_revision_conflicts(db): + client = _client(db) + first = client.put("/config/alice", json=_envelope(_bucket(combo="first"), 0, "a"), + headers=_SECRET) + assert first.status_code == 200 and first.json()["revision"] == 1 + stale_response = client.put("/config/alice", json=_envelope(_bucket(combo="second"), 0, "b"), + headers=_SECRET) + assert stale_response.status_code == 409 + assert stale_response.json()["revision"] == 1 + stored = client.get("/config/alice", headers=_SECRET).json() + assert stored["sections"]["hotkeys"]["hk1"]["combo"] == "first" + + +def test_get_returns_the_committed_revision_not_the_one_the_client_sent(db): + client = _client(db) + bucket = _bucket() + bucket.revision = 41 + client.put("/config/alice", json=_envelope(bucket, 0), headers=_SECRET) + body = client.get("/config/alice", headers=_SECRET).json() + assert body["revision"] == 1 + assert body["version"] == 2 + + +def test_a_server_restart_keeps_the_bucket(db): + _client(db).put("/config/alice", json=_envelope(_bucket(), 0), headers=_SECRET) + body = _client(db).get("/config/alice", headers=_SECRET).json() + assert body["revision"] == 1 + assert body["sections"]["hotkeys"]["hk1"]["combo"] == "ctrl+a" + + +def test_a_retried_put_gets_the_original_revision(db): + client = _client(db) + envelope = _envelope(_bucket(), 0, "retry-me") + original = client.put("/config/alice", json=envelope, headers=_SECRET).json() + repeated = client.put("/config/alice", json=envelope, headers=_SECRET) + assert repeated.status_code == 200 + assert repeated.json()["revision"] == original["revision"] == 1 + + +def test_a_blind_write_is_refused_unless_the_compatibility_setting_is_on(db): + refused = _client(db).put("/config/alice", json=_bucket().to_dict(), headers=_SECRET) + assert refused.status_code == 428 + assert _client(db).get("/config/alice", headers=_SECRET).status_code == 404 + + compat = _client(db, allow_blind_config_writes=True) + assert compat.put("/config/alice", json=_bucket(combo="one").to_dict(), + headers=_SECRET).status_code == 200 + reply = compat.put("/config/alice", json=_bucket(combo="two").to_dict(), headers=_SECRET) + assert reply.status_code == 200 and reply.json()["revision"] == 2 + assert compat.get("/config/alice", headers=_SECRET).json()["revision"] == 2 + + +def test_accounts_stay_isolated(db): + client = _client(db) + client.put("/config/alice", json=_envelope(_bucket("alice"), 0), headers=_SECRET) + assert client.get("/config/bob", headers=_SECRET).status_code == 404 + crossed = client.put("/config/bob", json=_envelope(_bucket("alice"), 0), headers=_SECRET) + assert crossed.status_code == 400 + + +@pytest.mark.parametrize("envelope", [ + {"version": 2, "base_revision": "0", "operation_id": "a", "bucket": {"user_id": "alice"}}, + {"version": 2, "base_revision": 0, "operation_id": "", "bucket": {"user_id": "alice"}}, + {"version": 2, "base_revision": 0, "operation_id": "a", "bucket": []}, + {"version": 2, "base_revision": 0, "operation_id": "a", + "bucket": {"user_id": "alice", "sections": {"hotkeys": []}}}, + {"version": 3, "base_revision": 0, "operation_id": "a", "bucket": {"user_id": "alice"}}, +]) +def test_a_malformed_envelope_is_a_400(db, envelope): + assert _client(db).put("/config/alice", json=envelope, headers=_SECRET).status_code == 400 + + +def test_the_secret_and_the_body_cap_still_apply(db): + client = _client(db) + assert client.put("/config/alice", json=_envelope(_bucket(), 0)).status_code == 401 + body = b'{"x":"' + b"A" * (2 * 1024 * 1024) + b'"}' + oversized = client.put("/config/alice", content=body, + headers={**_SECRET, "Content-Type": "application/json"}) + assert oversized.status_code == 413 + assert not db.exists() + + +def test_creating_the_app_does_not_create_the_database(db): + client = _client(db) + assert client.get("/health").status_code == 200 + assert not db.parent.exists() diff --git a/test/unit_test/headless/test_config_sync_tombstones.py b/test/unit_test/headless/test_config_sync_tombstones.py index 929e92ef9..e6cb7aba9 100644 --- a/test/unit_test/headless/test_config_sync_tombstones.py +++ b/test/unit_test/headless/test_config_sync_tombstones.py @@ -23,8 +23,9 @@ def __init__(self) -> None: def request(self, method, body=None): if method == "PUT": - self.body = body - return {} + revision = body["base_revision"] + 1 + self.body = {**body["bucket"], "revision": revision} + return {"ok": True, "revision": revision} return self.body diff --git a/test/unit_test/headless/test_signaling_config_routes.py b/test/unit_test/headless/test_signaling_config_routes.py index 8825a99dc..adf4c525d 100644 --- a/test/unit_test/headless/test_signaling_config_routes.py +++ b/test/unit_test/headless/test_signaling_config_routes.py @@ -2,6 +2,7 @@ ``config_sync`` documented ``GET`` / ``PUT /config/{user_id}`` on the signaling server, but nothing served them, so every push and pull failed. +The revision check and persistence are in ``test_config_sync_persistence.py``. """ import socket import threading @@ -18,8 +19,13 @@ @pytest.fixture -def client(): - return testclient.TestClient(create_app(shared_secret="s3cret", serve_web_viewer=False)) +def client(tmp_path): + return testclient.TestClient(create_app(shared_secret="s3cret", serve_web_viewer=False, + config_store_path=tmp_path / "buckets.sqlite3")) + + +def _put(bucket, base=0, operation="op-1"): + return {"version": 2, "base_revision": base, "operation_id": operation, "bucket": bucket} def test_an_unknown_bucket_is_404(client): @@ -27,9 +33,12 @@ def test_an_unknown_bucket_is_404(client): def test_a_bucket_round_trips(client): - bucket = {"user_id": "alice", "revision": 3, "sections": {"hotkeys": []}} - assert client.put("/config/alice", json=bucket, headers=_SECRET).status_code == 200 - assert client.get("/config/alice", headers=_SECRET).json() == bucket + bucket = {"user_id": "alice", "revision": 3, + "sections": {"hotkeys": {"hk1": {"combo": "ctrl+a", "last_modified": 1.0}}}} + assert client.put("/config/alice", json=_put(bucket), headers=_SECRET).status_code == 200 + # The revision is the one the server committed, not the one the client sent. + assert client.get("/config/alice", headers=_SECRET).json() == { + **bucket, "revision": 1, "version": 2} def test_config_routes_need_the_secret(client): @@ -38,7 +47,7 @@ def test_config_routes_need_the_secret(client): def test_a_bucket_for_another_user_is_refused(client): - response = client.put("/config/alice", json={"user_id": "bob"}, headers=_SECRET) + response = client.put("/config/alice", json=_put({"user_id": "bob"}), headers=_SECRET) assert response.status_code == 400 @@ -55,12 +64,13 @@ def _free_port(): return probe.getsockname()[1] -def test_the_real_client_syncs_through_the_server(): +def test_the_real_client_syncs_through_the_server(tmp_path): uvicorn = pytest.importorskip("uvicorn") from je_auto_control.utils.config_sync import ConfigBucket, ConfigSyncClient port = _free_port() - server = uvicorn.Server(uvicorn.Config(create_app(shared_secret="s3cret", serve_web_viewer=False), - host="127.0.0.1", port=port, log_level="error")) + app = create_app(shared_secret="s3cret", serve_web_viewer=False, + config_store_path=tmp_path / "buckets.sqlite3") + server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error")) thread = threading.Thread(target=server.run, daemon=True) thread.start() try: @@ -69,8 +79,20 @@ def test_the_real_client_syncs_through_the_server(): time.sleep(0.05) sync = ConfigSyncClient(f"http://127.0.0.1:{port}", user_id="alice", secret="s3cret") assert sync.fetch() is None - sync.push(ConfigBucket(user_id="alice")) - assert sync.fetch().user_id == "alice" + assert sync.push(ConfigBucket(user_id="alice")) == 1 + fetched = sync.fetch() + assert fetched.user_id == "alice" and fetched.revision == 1 + # A second machine that never saw revision 1 cannot overwrite it ... + from je_auto_control.utils.config_sync.client import ConfigSyncConflict + other = ConfigSyncClient(f"http://127.0.0.1:{port}", user_id="alice", secret="s3cret") + stale = ConfigBucket(user_id="alice") + stale.upsert("hotkeys", "hk1", {"combo": "ctrl+a"}) + with pytest.raises(ConfigSyncConflict) as raised: + other.push(stale) + assert raised.value.revision == 1 + # ... but syncing merges on top of it. + merged, _conflicts = other.sync(stale) + assert merged.revision == 2 and "hk1" in sync.fetch().entries("hotkeys") finally: server.should_exit = True thread.join(timeout=10) From bdf631ec1fd3bed03bfffcfab887ea1d6d943b5e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:41:47 +0800 Subject: [PATCH 05/44] Give the dry run, observer callbacks, generated plans and state machines a variable scope of their own Four callers were still on the module executor's process-lifetime scope. A dry run now seeds --var into its own scope. An observer callback fires long after its registering run, so it keeps a snapshot of the variables visible at registration and each firing runs in a fresh scope seeded from it. A plan run from a description and a state machine are a run of their own from Python and a step of the calling run from an action list (run_level_scope). The state machine's default runner also took every on_enter entry for a list of actions, so the single action the documentation shows was rejected and never ran. Each daemon is now fired for real, with fakes around it, to show the isolation the shared helper was tested for. --- .../Eng/doc/new_features/new_features_doc.rst | 30 ++ .../Eng/doc/new_features/v6_features_doc.rst | 5 + .../Zh/doc/new_features/new_features_doc.rst | 25 ++ .../Zh/doc/new_features/v6_features_doc.rst | 4 + je_auto_control/cli.py | 9 +- .../utils/executor/action_executor.py | 16 +- je_auto_control/utils/llm/planner.py | 15 +- .../tools/_handlers_executor_bridge.py | 12 +- .../utils/script_vars/execution.py | 23 ++ je_auto_control/utils/state_machine/engine.py | 27 +- .../headless/test_daemon_scope_end_to_end.py | 260 ++++++++++++++ .../headless/test_scope_run_level_callers.py | 322 ++++++++++++++++++ 12 files changed, 732 insertions(+), 16 deletions(-) create mode 100644 test/unit_test/headless/test_daemon_scope_end_to_end.py create mode 100644 test/unit_test/headless/test_scope_run_level_callers.py diff --git a/docs/source/Eng/doc/new_features/new_features_doc.rst b/docs/source/Eng/doc/new_features/new_features_doc.rst index 560559b63..c4fc2a8a5 100644 --- a/docs/source/Eng/doc/new_features/new_features_doc.rst +++ b/docs/source/Eng/doc/new_features/new_features_doc.rst @@ -403,6 +403,36 @@ that called them, also inside an ``AC_parallel`` branch; a branch starts from a copy of its parent's variables and its own writes stay in the branch. An executor you construct yourself (``Executor()``) always owns its scope. +Four more callers have a scope chosen for them rather than the process scope: + +* ``je_auto_control run --dry-run --var name=value`` seeds the variables into + the dry run's own scope, as a real ``run --var`` does. +* **Observer callbacks** (``AC_observe_add`` and the MCP ``ac_observe_add``) + fire long after the run that registered them has ended, so they cannot + share its scope. A callback keeps a *snapshot* of the variables visible + when it was registered, and every firing runs on the module executor in a + fresh scope seeded from that snapshot -- whether the observer's thread or + an ``AC_observe_poll`` in some other run fired it. The actions see what + the registering run had set (later changes to those variables are not + seen); what they set reaches neither the process scope, the run that + polled, nor the next firing. A callback registered through MCP starts + with no variables, because a tool call's scope is empty. To keep state + between firings, keep it outside the scope (a file, the secrets vault). +* **A plan run from a description** (``run_from_description``, ``AC_llm_run``) + and **a state machine** (``run_state_machine``, ``AC_run_state_machine``) + are a run of their own when called from Python: each gets a fresh scope + -- shared by all of the machine's ``on_enter`` actions -- that is dropped + when the call returns, and neither reads what an earlier call left in the + process scope. Called from an action list, or inside an + ``execution_scope`` block, they are a step of that run and share its + variables; that is how you hand them values:: + + with ac.execution_scope({"user": "alice"}): + ac.run_state_machine(spec) # on_enter actions can use ${user} + + A state machine inside an ``AC_parallel`` branch uses the branch's + variables. + :: import je_auto_control as ac diff --git a/docs/source/Eng/doc/new_features/v6_features_doc.rst b/docs/source/Eng/doc/new_features/v6_features_doc.rst index c4dc237c3..c9e52c4d9 100644 --- a/docs/source/Eng/doc/new_features/v6_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v6_features_doc.rst @@ -61,6 +61,11 @@ loops / ifs for screen-flow automation:: } result = run_state_machine(spec) # {final_state, steps, elapsed_s} +Each ``on_enter`` entry is one action (``["AC_name", {...}]``) or a list +of actions. One ``run_state_machine`` call is one run: its ``on_enter`` +actions share a variable scope that is dropped when the machine stops (see +*Variable scope per run*). + Each state's ``on_enter`` actions run through the executor; transitions fire on guards: ``after`` (seconds in the state -- the machine waits for it), ``if_var_eq``, ``if_image_found`` (a template path, or diff --git a/docs/source/Zh/doc/new_features/new_features_doc.rst b/docs/source/Zh/doc/new_features/new_features_doc.rst index 9b96dd85b..d51ef710f 100644 --- a/docs/source/Zh/doc/new_features/new_features_doc.rst +++ b/docs/source/Zh/doc/new_features/new_features_doc.rst @@ -375,6 +375,31 @@ Variables 分頁顯示的、以及從 GUI 執行的腳本用的就是這個範 範圍裡，在 ``AC_parallel`` 分支內也一樣；分支從父層變數的一份複本開始， 自己寫入的值留在分支內。自行建立的 executor（``Executor()``）永遠擁有自己的範圍。 +另外四種呼叫者的範圍是刻意選定的，不再是行程層級的範圍： + +* ``je_auto_control run --dry-run --var name=value`` 把變數放進該次 dry run + 自己的範圍，與真正的 ``run --var`` 相同。 +* **observer 回呼**\ （``AC_observe_add`` 與 MCP 的 ``ac_observe_add``\ ） + 在註冊它的那次執行結束很久之後才觸發，無法共用那次執行的範圍。回呼會保留 + 註冊當下可見變數的一份\ *快照*\ ，每次觸發都在模組 executor 上、以該快照 + 為初值的全新範圍內執行——不論是 observer 的執行緒觸發，還是別的執行裡的 + ``AC_observe_poll`` 觸發。動作看得到註冊時那次執行設好的值（之後對那些 + 變數的修改看不到）；動作寫入的值不會進到行程範圍、不會進到呼叫 poll 的 + 那次執行，也不會留到下一次觸發。經 MCP 註冊的回呼一開始沒有任何變數， + 因為工具呼叫的範圍是空的。要在多次觸發之間保留狀態，請存在範圍之外 + （檔案、secrets vault）。 +* **由描述產生並執行的計畫**\ （``run_from_description``\ 、``AC_llm_run``\ ） + 與\ **狀態機**\ （``run_state_machine``\ 、``AC_run_state_machine``\ ） + 從 Python 呼叫時本身就是一次執行：各自拿到一個全新的範圍——狀態機所有 + ``on_enter`` 動作共用它——呼叫返回時丟棄，也讀不到先前的呼叫留在行程範圍 + 裡的值。從動作清單裡呼叫，或在 ``execution_scope`` 區塊內呼叫時，它們是 + 該次執行的一個步驟，共用其變數；要傳值給它們就用這個方式:: + + with ac.execution_scope({"user": "alice"}): + ac.run_state_machine(spec) # on_enter 的動作可以用 ${user} + + ``AC_parallel`` 分支內的狀態機使用該分支的變數。 + :: diff --git a/docs/source/Zh/doc/new_features/v6_features_doc.rst b/docs/source/Zh/doc/new_features/v6_features_doc.rst index 34a577d08..71958c7a5 100644 --- a/docs/source/Zh/doc/new_features/v6_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v6_features_doc.rst @@ -57,6 +57,10 @@ headless 測試(PIL 影像 / spec 以注入方式提供,完全不需真實螢幕 } result = run_state_machine(spec) # {final_state, steps, elapsed_s} +``on_enter`` 的每一項是一個動作(``["AC_name", {...}]``)或一串動作。 +一次 ``run_state_machine`` 呼叫就是一次執行:它的 ``on_enter`` 動作共用一個 +變數範圍,狀態機停止時丟棄(見\ *每次執行各自的變數範圍*\ )。 + 每個狀態的 ``on_enter`` 動作會透過執行器執行;transition 依 guard 觸發 (``after``:在該狀態停留的秒數,機器會等到它成立;``if_var_eq``; ``if_image_found``:範本路徑,或 ``{"image": ..., "detect_threshold": ...}``; diff --git a/je_auto_control/cli.py b/je_auto_control/cli.py index ed6e99c65..672b9e4bc 100644 --- a/je_auto_control/cli.py +++ b/je_auto_control/cli.py @@ -75,11 +75,14 @@ def cmd_run(args: argparse.Namespace) -> int: reset_recorded_failures() if args.dry_run: from je_auto_control.utils.executor.action_executor import executor + from je_auto_control.utils.script_vars.execution import execution_scope # Seeded, not interpolated over the whole tree: that failed on a loop # body's ${item} ("Unknown variable") and would print ${secrets.X} - # resolved into the dry-run keys. - executor.variables.update_many(variables) - result = executor.execute_action(actions, dry_run=True) + # resolved into the dry-run keys. Seeded into the run's own scope, as + # a real run's --var is: writing them into the module executor left + # them behind for whatever the same process ran next. + with execution_scope(variables): + result = executor.execute_action(actions, dry_run=True) elif variables: result = execute_action_with_vars(actions, variables) else: diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cc475cf85..22dd31e24 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3499,10 +3499,22 @@ def _seed_everything(seed: int = 0) -> Dict[str, Any]: def _observe_handler(actions: List[Any]) -> Callable[[str, Any], None]: - """Build an observer callback that runs an action list on each event.""" + """Build an observer callback that runs an action list on each event. + + The callback fires long after the run that registered it has ended -- on + the observer's thread, or inside whichever run calls ``AC_observe_poll`` + -- so it cannot share a run's scope. It keeps a snapshot of the variables + visible at registration, and every firing runs on the module executor in + a fresh scope seeded from that snapshot: the actions see what the + registering run had set, and what they set reaches neither the process + scope, the run that polled, nor the next firing. + """ + snapshot = _running_executor().variables.as_dict() + def handler(_event: str, _value: Any) -> None: if actions: - _running_executor().execute_action(list(actions)) + with execution_scope(dict(snapshot)): + executor.execute_action(list(actions)) return handler diff --git a/je_auto_control/utils/llm/planner.py b/je_auto_control/utils/llm/planner.py index b4c844223..afd7be1f6 100644 --- a/je_auto_control/utils/llm/planner.py +++ b/je_auto_control/utils/llm/planner.py @@ -14,6 +14,7 @@ from je_auto_control.utils.llm.backends import ( LLMBackend, LLMNotAvailableError, get_backend, ) +from je_auto_control.utils.script_vars.execution import run_level_scope _SYSTEM_PROMPT = ( "You translate plain-language automation instructions into a strict " @@ -73,7 +74,16 @@ def run_from_description(description: str, backend: Optional[LLMBackend] = None, model: Optional[str] = None, max_tokens: int = 2048) -> Dict[str, Any]: - """Plan a description and execute it on ``executor`` in one call.""" + """Plan a description and execute it on ``executor`` in one call. + + The plan is a run of its own: called from Python it gets a fresh variable + scope, so what a model-written ``AC_set_var`` sets is gone when the call + returns and the plan cannot read what an earlier call left behind. Called + from inside a run (``AC_llm_run`` in an action list, or a block wrapped + in ``execution_scope``) it is a step of that run and shares its variables + -- wrap the call in ``execution_scope({...})`` to hand the plan values. + An executor you constructed yourself always uses its own scope. + """ actions = plan_actions( description, known_commands=executor.known_commands(), @@ -84,7 +94,8 @@ def run_from_description(description: str, ) # A top-level run: ``_validated=True`` would mark it as a nested body, # and a stray AC_break from the model would escape as a raw LoopBreak. - record = executor.execute_action(actions) + with run_level_scope(): + record = executor.execute_action(actions) return {"actions": actions, "record": record} diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py b/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py index 049ba26c3..1b235eb16 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_executor_bridge.py @@ -165,12 +165,12 @@ def get_visible_text(name=None, role=None, app_name=None, automation_id=None): def _observe_handler(actions): - from je_auto_control.utils.executor.action_executor import executor - - def handler(_event, _value): - if actions: - executor.execute_action(list(actions)) - return handler + # The executor's own builder: each firing runs in a fresh variable scope + # seeded with what the registering tool call could see (normally nothing). + from je_auto_control.utils.executor.action_executor import ( + _observe_handler as build_handler, + ) + return build_handler(actions) def circuit_call(name, actions, threshold=5, reset_s=30.0): diff --git a/je_auto_control/utils/script_vars/execution.py b/je_auto_control/utils/script_vars/execution.py index a43df3a5a..c8a7048fb 100644 --- a/je_auto_control/utils/script_vars/execution.py +++ b/je_auto_control/utils/script_vars/execution.py @@ -17,6 +17,11 @@ Outside any ``execution_scope`` the module executor falls back to its own process-lifetime scope, which is what plain ``executor.execute_action(...)`` calls and the GUI's Variables tab use. + +Things that are a run of their own when called from Python but a nested step +when called from an action list -- a state machine, a plan generated from a +description -- use :func:`run_level_scope`: they join the enclosing run if +there is one and otherwise get a scope that lasts exactly as long as they do. """ from contextlib import contextmanager from contextvars import ContextVar @@ -43,6 +48,24 @@ def bound_scope(scope: VariableScope) -> Iterator[VariableScope]: _ACTIVE.reset(token) +@contextmanager +def run_level_scope() -> Iterator[VariableScope]: + """Join the enclosing run's scope, or open a fresh one for the block. + + Inside an :func:`execution_scope` (or a scope re-bound with + :func:`bound_scope`) the block is a step of that run and shares its + variables, as any nested action list does. With no run in progress the + block is the run: it gets an empty scope that is dropped when it ends, so + what it sets never reaches the process scope or the next caller. + """ + active = _ACTIVE.get() + if active is not None: + yield active + return + with execution_scope() as scope: + yield scope + + @contextmanager def execution_scope(variables: Optional[Mapping[str, Any]] = None ) -> Iterator[VariableScope]: diff --git a/je_auto_control/utils/state_machine/engine.py b/je_auto_control/utils/state_machine/engine.py index 25cd757a0..4af50bfb9 100644 --- a/je_auto_control/utils/state_machine/engine.py +++ b/je_auto_control/utils/state_machine/engine.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Dict, Mapping, Optional from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.script_vars.execution import run_level_scope # AutoControlException so `except AutoControlException` boundaries catch a @@ -30,6 +31,14 @@ class StateMachine: (a template path, or ``{"image": ..., "detect_threshold": ...}``) and ``predicate`` (a callable). Any other ``if_*`` key is an error rather than a guard that silently always passes. + + Variables: one :meth:`run` is one run. Started from Python it gets a + fresh variable scope that every ``on_enter`` action shares and that is + dropped when the machine stops; started from an action list + (``AC_run_state_machine``) it uses the scope of the list that called it, + as any nested step does. The default runner executes on the executor + running that list, so inside an ``AC_parallel`` branch the machine sees + the branch's variables. """ def __init__(self, spec: Mapping[str, Any], @@ -60,6 +69,10 @@ def run(self) -> Dict[str, Any]: Raises :class:`StateMachineError` on budget exhaustion or when a state has no fireable transition. """ + with run_level_scope(): + return self._drive() + + def _drive(self) -> Dict[str, Any]: current = self._spec["initial"] if current not in self._states: raise StateMachineError(f"initial state {current!r} undefined") @@ -148,9 +161,17 @@ def _validate_spec(spec: Mapping[str, Any]) -> None: def _default_execute_action(action: Any) -> Any: - """Lazy bridge to the main executor; isolates test imports.""" - from je_auto_control.utils.executor.action_executor import execute_action - return execute_action([action] if not isinstance(action, list) else action) + """Lazy bridge to the executor running the calling list, else the module's. + + An ``on_enter`` entry is one action -- ``["AC_name", {...}]`` -- or a + list of them. Every list used to be taken for a list of actions, so the + single action the documentation shows was rejected ("must be [name] or + [name, params]") and, the failure only being recorded, never ran. + """ + from je_auto_control.utils.executor.action_executor import _running_executor + is_action_list = (isinstance(action, list) and bool(action) + and not isinstance(action[0], str)) + return _running_executor().execute_action(action if is_action_list else [action]) def _after_ok(transition: Mapping[str, Any], _context: Mapping[str, Any], diff --git a/test/unit_test/headless/test_daemon_scope_end_to_end.py b/test/unit_test/headless/test_daemon_scope_end_to_end.py new file mode 100644 index 000000000..55af56ad3 --- /dev/null +++ b/test/unit_test/headless/test_daemon_scope_end_to_end.py @@ -0,0 +1,260 @@ +"""Every daemon really fires its script in a variable scope of its own. + +U-20261009-07 isolated scheduler jobs, triggers, hotkeys, webhooks and e-mail +triggers by wrapping their shared helper, and tested the helper. Here each +engine is driven for real -- its own thread, its own poll loop, its own HTTP +handler -- and three scripts are fired through it in turn: + +1. one that sets ``${user}``, +2. one that only reads ``${user}`` (it must fail: nothing set it in *this* + run), and +3. one that sets and then reads it (it must succeed, which shows the probe + works on that engine's thread and step 2 was not vacuous). + +Fakes stand in for everything outside the engine: the run-history store, the +error snapshot, the OS hotkey listener and the IMAP server. The webhook is +posted over loopback to a port the OS picks. The only commands that run are +variable commands and a probe, so no input reaches the desktop. +""" +import json +import queue +import threading +import time +import urllib.request +from email.message import EmailMessage + +import pytest + +from je_auto_control.utils.executor.action_executor import executor +from je_auto_control.utils.run_history.history_store import STATUS_ERROR, STATUS_OK + +PROBE = "AC_scope_probe" +_SET_USER = ["AC_set_var", {"name": "user", "value": "alice"}] +_READ_USER = [PROBE, {"value": "${user}"}] +_WAIT_S = 10.0 + + +class _History: + """A run-history store that keeps finished runs in memory and signals them.""" + + def __init__(self): + self._cond = threading.Condition() + self._next_id = 0 + self.finished = [] + + def start_run(self, _source, _target, _script_path): + with self._cond: + self._next_id += 1 + return self._next_id + + def finish_run(self, _run_id, status, error_text=None, artifact_path=None): + with self._cond: + self.finished.append((status, error_text)) + self._cond.notify_all() + + def wait_finished(self, count): + """Block until ``count`` runs have finished; return their statuses.""" + with self._cond: + if not self._cond.wait_for(lambda: len(self.finished) >= count, _WAIT_S): + raise AssertionError(f"{len(self.finished)} run(s) finished, wanted {count}") + return [status for status, _error in self.finished] + + +@pytest.fixture +def seen(): + """Register a probe command on the module executor; restore it afterwards.""" + values = [] + saved = executor.variables.as_dict() + executor.variables.clear() + executor.event_dict[PROBE] = lambda value=None: values.append(value) or value + yield values + executor.event_dict.pop(PROBE, None) + executor.variables.clear() + executor.variables.update_many(saved) + + +@pytest.fixture +def scripts(tmp_path): + """The three action files, by name.""" + paths = {} + for name, actions in (("set", [_SET_USER]), ("read", [_READ_USER]), + ("both", [_SET_USER, _READ_USER])): + path = tmp_path / f"{name}.json" + path.write_text(json.dumps(actions), encoding="utf-8") + paths[name] = str(path) + return paths + + +def _fake_history(monkeypatch, module) -> _History: + """Give ``module`` an in-memory history and no error snapshot.""" + history = _History() + monkeypatch.setattr(module, "default_history_store", history) + monkeypatch.setattr(module, "capture_error_snapshot", lambda _run_id: None) + return history + + +def _until(predicate) -> None: + deadline = time.monotonic() + _WAIT_S + while not predicate(): + if time.monotonic() > deadline: + raise AssertionError("condition not reached") + time.sleep(0.01) + + +def _assert_isolated(history: _History, seen: list) -> None: + """Three runs finished: set ok, read failed, set-then-read ok.""" + statuses = history.wait_finished(3) + assert statuses == [STATUS_OK, STATUS_ERROR, STATUS_OK] + assert "action(s) failed" in history.finished[1][1] + assert seen == ["alice"] # only the third run's own value + assert "user" not in executor.variables + + +def test_scheduler_jobs_do_not_share_variables(monkeypatch, seen, scripts): + from je_auto_control.utils.scheduler import scheduler as scheduler_module + history = _fake_history(monkeypatch, scheduler_module) + scheduler = scheduler_module.Scheduler(tick_seconds=0.1) + scheduler.start() + try: + for count, name in enumerate(("set", "read", "both"), start=1): + scheduler.add_job(scripts[name], 0.1, repeat=False) + history.wait_finished(count) + finally: + scheduler.stop() + _assert_isolated(history, seen) + + +def test_triggers_do_not_share_variables(monkeypatch, seen, scripts, tmp_path): + from je_auto_control.utils.triggers import trigger_engine + history = _fake_history(monkeypatch, trigger_engine) + engine = trigger_engine.TriggerEngine(tick_seconds=0.05) + engine.start() + try: + for count, name in enumerate(("set", "read", "both"), start=1): + watched = tmp_path / f"{name}.flag" + trigger = engine.add(trigger_engine.FilePathTrigger( + trigger_id=name, script_path=scripts[name], + watch_path=str(watched), cooldown_seconds=0.0)) + _until(lambda: trigger._primed) # noqa: SLF001, B023 # reason: the first poll only records a baseline + watched.write_text("x", encoding="utf-8") + history.wait_finished(count) + finally: + engine.stop() + _assert_isolated(history, seen) + + +class _FakeHotkeyBackend: + """Stands in for the OS listener: the test says which binding was pressed.""" + + name = "fake" + + def __init__(self): + self.pressed = queue.Queue() + + def run_forever(self, context) -> None: + while not context.stop_event.is_set(): + try: + binding_id = self.pressed.get(timeout=0.05) + except queue.Empty: + continue + context.fire(binding_id) + + +def test_hotkey_bindings_do_not_share_variables(monkeypatch, seen, scripts): + from je_auto_control.utils.hotkey import backends, hotkey_daemon + history = _fake_history(monkeypatch, hotkey_daemon) + backend = _FakeHotkeyBackend() + monkeypatch.setattr(backends, "get_backend", lambda: backend) + daemon = hotkey_daemon.HotkeyDaemon() + bindings = {name: daemon.bind(f"ctrl+alt+f{number}", scripts[name]) + for number, name in ((7, "set"), (8, "read"), (9, "both"))} + daemon.start() + try: + for count, name in enumerate(("set", "read", "both"), start=1): + backend.pressed.put(bindings[name].binding_id) + history.wait_finished(count) + finally: + daemon.stop() + _assert_isolated(history, seen) + assert [binding.fired for binding in bindings.values()] == [1, 1, 1] + + +def _post(port: int, path: str, body: dict) -> dict: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(request, timeout=_WAIT_S) as reply: # nosec B310 # reason: loopback, fixed scheme + return json.loads(reply.read()) + + +def test_webhooks_do_not_share_variables(monkeypatch, seen, scripts, tmp_path): + from je_auto_control.utils.triggers import webhook_server + history = _fake_history(monkeypatch, webhook_server) + echo = tmp_path / "echo.json" + echo.write_text(json.dumps([[PROBE, {"value": "${webhook.path}"}]]), encoding="utf-8") + server = webhook_server.WebhookTriggerServer() + for name in ("set", "read", "both"): + server.add(f"/{name}", scripts[name]) + server.add("/echo", str(echo)) + _host, port = server.start("127.0.0.1", 0) + try: + for name in ("set", "read", "both"): + assert _post(port, f"/{name}", {"user": "from-the-body"})["fired"] is True + _assert_isolated(history, seen) + # The payload is seeded into that firing's scope and nowhere else. + _post(port, "/echo", {}) + assert seen == ["alice", "/echo"] + assert "webhook.path" not in executor.variables + finally: + server.stop() + + +class _FakeImap: + """An IMAP server with one unread message, whatever the mailbox.""" + + def __init__(self, host, port, ssl_context=None, timeout=None): + self.host = host + + def login(self, _user, _password): + return ("OK", [b"logged in"]) + + def select(self, _mailbox, readonly=False): + return ("OK", [b"1"]) + + def response(self, code): + return (code, [b"1"] if code == "UIDVALIDITY" else [None]) + + def uid(self, command, *_args): + if command == "SEARCH": + return ("OK", [b"1"]) + if command == "FETCH": + message = EmailMessage() + message["Subject"] = f"hello {self.host}" + message["From"] = "sender@example.com" + message.set_content("body") + raw = message.as_bytes() + return ("OK", [(b"1 (BODY[] {%d}" % len(raw), raw)]) + return ("OK", [b"stored"]) + + def logout(self): + return ("BYE", [b"bye"]) + + +def test_email_triggers_do_not_share_variables(monkeypatch, seen, scripts, tmp_path): + from je_auto_control.utils.triggers import email_trigger + history = _fake_history(monkeypatch, email_trigger) + monkeypatch.setattr(email_trigger.imaplib, "IMAP4_SSL", _FakeImap) + monkeypatch.setattr(email_trigger.imaplib, "IMAP4", _FakeImap) + subject = tmp_path / "subject.json" + subject.write_text(json.dumps([[PROBE, {"value": "${email.subject}"}]]), encoding="utf-8") + watcher = email_trigger.EmailTriggerWatcher() + for name in ("set", "read", "both"): + watcher.add(f"{name}.example.com", "u", "p", scripts[name]) + # One pass per trigger: its one message fires, the earlier triggers' + # messages are already seen. + assert watcher.poll_once() == 1 + _assert_isolated(history, seen) + watcher.add("subject.example.com", "u", "p", str(subject)) + assert watcher.poll_once() == 1 + assert seen == ["alice", "hello subject.example.com"] + assert "email.subject" not in executor.variables diff --git a/test/unit_test/headless/test_scope_run_level_callers.py b/test/unit_test/headless/test_scope_run_level_callers.py new file mode 100644 index 000000000..ec3ccb803 --- /dev/null +++ b/test/unit_test/headless/test_scope_run_level_callers.py @@ -0,0 +1,322 @@ +"""The last callers that ran on the process-lifetime variable scope. + +U-20261009-07 gave every top-level run its own ``VariableScope`` but left +four callers on the module executor's process scope: ``run --dry-run --var``, +observer callbacks (``AC_observe_add`` and the MCP ``ac_observe_add``), a plan +run by ``run_from_description`` and a state machine's default runner. Each +has a deliberate scope now: + +* a dry run seeds ``--var`` into the run's own scope; +* an observer callback keeps a snapshot of the variables visible when it was + registered and every firing runs in a fresh scope seeded from it; +* a plan and a state machine are a run of their own from Python and a step of + the calling run from an action list. + +Only variable commands and a probe registered for the test run here; the +observer predicates are fakes, so nothing reads the screen. +""" +import json + +import pytest + +from je_auto_control.utils.executor import action_executor +from je_auto_control.utils.executor.action_executor import ( + Executor, execute_action, execute_action_with_vars, executor, +) +from je_auto_control.utils.script_vars import execution_scope +from je_auto_control.utils.script_vars.execution import current_scope + +PROBE = "AC_scope_probe" +_SET_USER = ["AC_set_var", {"name": "user", "value": "alice"}] +_READ_USER = [PROBE, {"value": "${user}"}] + + +@pytest.fixture +def seen(): + """Register a probe command on the module executor; restore it afterwards.""" + values = [] + saved = executor.variables.as_dict() + executor.variables.clear() + executor.event_dict[PROBE] = lambda value=None: values.append(value) or value + yield values + executor.event_dict.pop(PROBE, None) + executor.variables.clear() + executor.variables.update_many(saved) + + +def _failed(record): + return [value for value in record.values() if "Unknown variable" in str(value)] + + +# --- run_level_scope ------------------------------------------------------- + +def run_level_scope(): + """Imported late so the rest of the file runs against a tree without it.""" + from je_auto_control.utils.script_vars import execution + return execution.run_level_scope() + + +def test_run_level_scope_is_fresh_when_nothing_is_running(seen): + executor.variables.set("leftover", 1) + with run_level_scope() as scope: + assert current_scope() is scope + assert "leftover" not in scope + executor.execute_action([_SET_USER]) + assert scope.get_value("user") == "alice" + assert current_scope() is None + assert "user" not in executor.variables + + +def test_run_level_scope_joins_the_enclosing_run(seen): + with execution_scope({"user": "bob"}) as outer: + with run_level_scope() as scope: + assert scope is outer + executor.execute_action([["AC_set_var", {"name": "n", "value": 1}]]) + assert current_scope() is outer + assert outer.get_value("n") == 1 + + +def test_run_level_scope_is_dropped_after_an_error(seen): + with pytest.raises(RuntimeError): + with run_level_scope(): + executor.execute_action([_SET_USER]) + raise RuntimeError("boom") + assert current_scope() is None + assert "user" not in executor.variables + + +# --- je_auto_control run --dry-run --var ----------------------------------- + +def test_dry_run_vars_do_not_stay_in_the_process_scope(seen, tmp_path, capsys): + from je_auto_control.cli import main + script = tmp_path / "script.json" + script.write_text(json.dumps([_READ_USER]), encoding="utf-8") + assert main(["run", str(script), "--dry-run", "--var", "user=alice"]) == 0 + assert "Unknown variable" not in capsys.readouterr().out # the seed reached the run + assert seen == [] # ...which executes nothing + assert "user" not in executor.variables + + +def test_dry_run_with_vars_still_accepts_loop_bodies(seen, tmp_path, capsys): + from je_auto_control.cli import main + script = tmp_path / "script.json" + script.write_text(json.dumps([["AC_for_each", { + "items": "${names}", "as": "name", "body": [[PROBE, {"value": "${name}"}]], + }]]), encoding="utf-8") + assert main(["run", str(script), "--dry-run", "--var", 'names=["a","b"]']) == 0 + assert "Unknown variable" not in capsys.readouterr().out + assert "names" not in executor.variables + + +# --- observer callbacks ---------------------------------------------------- + +class _Flag: + """A fake predicate: the test says when the watched thing is there.""" + + def __init__(self): + self.value = False + + def __call__(self): + return self.value + + +@pytest.fixture +def watch(monkeypatch): + """A private observer and a fake predicate for both registration routes.""" + from je_auto_control.utils import observer as observer_package + from je_auto_control.utils.mcp_server.tools import _handlers_locators + from je_auto_control.utils.observer import ScreenObserver + flag = _Flag() + private = ScreenObserver() + monkeypatch.setattr(observer_package, "default_observer", private) + monkeypatch.setattr(action_executor, "_observe_predicate", lambda kind, params: flag) + monkeypatch.setattr(_handlers_locators, "_observe_predicate", lambda kind, params: flag) + return flag, private + + +def _appear(flag, private): + """One vanish-then-appear transition, polled on this thread.""" + flag.value = False + private.poll_once() + flag.value = True + return private.poll_once() + + +_COUNT_UP = [["AC_inc_var", {"name": "n"}], [PROBE, {"value": "${n}"}]] + + +def test_observer_callback_sees_the_variables_of_its_registration(seen, watch): + flag, private = watch + execute_action_with_vars([["AC_observe_add", { + "name": "w", "kind": "image", "actions": [_READ_USER]}]], {"user": "alice"}) + assert "user" not in executor.variables # the registering run is over + assert _appear(flag, private) + assert seen == ["alice"] + + +def test_observer_callback_writes_reach_nobody_else(seen, watch): + flag, private = watch + execute_action_with_vars([["AC_observe_add", { + "name": "w", "kind": "image", "actions": _COUNT_UP}]], {"n": 10}) + _appear(flag, private) + _appear(flag, private) + # Every firing starts from the snapshot: 11 twice, not 11 then 12. + assert seen == [11, 11] + assert "n" not in executor.variables + + +def test_observer_callback_does_not_read_or_write_the_polling_run(seen, watch): + flag, private = watch + execute_action_with_vars([["AC_observe_add", { + "name": "w", "kind": "image", "actions": [_READ_USER, _SET_USER]}]], {}) + flag.value = True + record = execute_action_with_vars( + [["AC_observe_poll", {}], [PROBE, {"value": "${user}"}]], {"user": "poller"}) + # The callback's ${user} is unknown (it was registered without one) and + # its AC_set_var did not overwrite the polling run's value. + assert seen == ["poller"] + assert not _failed(record) + + +def test_observer_callback_registered_outside_a_run_snapshots_the_process_scope(seen, watch): + flag, private = watch + executor.variables.set("user", "process") + execute_action([["AC_observe_add", { + "name": "w", "kind": "image", "actions": [_READ_USER, _SET_USER]}]]) + executor.variables.set("user", "changed-later") + _appear(flag, private) + assert seen == ["process"] + assert executor.variables.get_value("user") == "changed-later" + + +def test_observer_callback_fired_on_the_observer_thread_is_isolated(seen, watch): + import threading + flag, private = watch + execute_action_with_vars([["AC_observe_add", { + "name": "w", "kind": "image", "actions": _COUNT_UP}]], {"n": 1}) + flag.value = True + worker = threading.Thread(target=private.poll_once, daemon=True) + worker.start() + worker.join(5) + assert not worker.is_alive() + assert seen == [2] + assert "n" not in executor.variables + + +def test_mcp_observe_callback_runs_in_its_own_scope(seen, watch): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + flag, private = watch + tools = {tool.name: tool for tool in build_default_tool_registry()} + executor.variables.set("user", "process") + tools["ac_observe_add"].invoke({ + "name": "w", "kind": "image", "actions": [_SET_USER, _READ_USER, _COUNT_UP[0]]}) + _appear(flag, private) + _appear(flag, private) + # Registered inside a tool call, whose scope is empty: the callback does + # not see the process scope, and what it sets is gone between firings. + assert seen == ["alice", "alice"] + assert executor.variables.as_dict() == {"user": "process"} + + +# --- run_from_description -------------------------------------------------- + +class _Backend: + """An LLM backend that answers with a fixed plan.""" + + name = "fake" + available = True + + def __init__(self, plan): + self._plan = plan + + def complete(self, prompt, system=None, model=None, max_tokens=2048): + return json.dumps(self._plan) + + +def _plan(plan, on=executor): + from je_auto_control.utils.llm.planner import run_from_description + return run_from_description("do it", on, backend=_Backend(plan)) + + +def test_a_plan_run_from_python_gets_its_own_scope(seen): + executor.variables.set("user", "leftover") + outcome = _plan([_READ_USER]) + assert seen == [] + assert _failed(outcome["record"]) + _plan([_SET_USER, _READ_USER]) + assert seen == ["alice"] + assert executor.variables.get_value("user") == "leftover" + + +def test_a_plan_inside_a_run_is_a_step_of_that_run(seen): + with execution_scope({"user": "bob"}) as scope: + _plan([_READ_USER, ["AC_set_var", {"name": "made", "value": 1}]]) + assert scope.get_value("made") == 1 + assert seen == ["bob"] + + +def test_a_plan_on_a_private_executor_uses_that_executors_scope(seen): + private = Executor() + private.event_dict[PROBE] = executor.event_dict[PROBE] + private.variables.set("user", "private") + _plan([_READ_USER], on=private) + assert seen == ["private"] + + +# --- the state machine's default runner ------------------------------------ + +def _machine(*on_enter): + return {"initial": "a", "states": { + "a": {"on_enter": list(on_enter[:1]), "transitions": [{"go_to": "b"}]}, + "b": {"on_enter": list(on_enter[1:]), "final": True}, + }} + + +def test_a_state_machine_run_from_python_gets_its_own_scope(seen): + from je_auto_control.utils.state_machine import run_state_machine + executor.variables.set("user", "leftover") + run_state_machine(_machine(_READ_USER)) + assert seen == [] # the process scope is not read + # ...but states of one run share what they set + run_state_machine(_machine(_SET_USER, _READ_USER)) + assert seen == ["alice"] + assert executor.variables.get_value("user") == "leftover" + + +def test_a_state_machine_inside_a_run_uses_that_runs_scope(seen): + record = execute_action_with_vars([ + ["AC_run_state_machine", {"spec": _machine( + _READ_USER, ["AC_set_var", {"name": "user", "value": "machine"}])}], + _READ_USER, + ], {"user": "run"}) + assert not _failed(record) + assert seen == ["run", "machine"] + assert "user" not in executor.variables + + +def test_a_state_machine_in_a_parallel_branch_uses_the_branch_scope(seen): + branches = [[["AC_set_var", {"name": "user", "value": name}], + ["AC_run_state_machine", {"spec": _machine(_READ_USER)}]] + for name in ("left", "right")] + execute_action_with_vars([["AC_parallel", {"branches": branches}]], {"user": "parent"}) + assert sorted(seen) == ["left", "right"] + + +def test_default_runner_takes_one_action_or_a_list_of_actions(seen): + from je_auto_control.utils.state_machine import run_state_machine + run_state_machine({"initial": "a", "states": {"a": {"final": True, "on_enter": [ + [PROBE, {"value": "single"}], + [[PROBE, {"value": "first"}], [PROBE, {"value": "second"}]], + [PROBE], + ]}}}) + assert seen == ["single", "first", "second", None] + + +def test_a_custom_state_machine_runner_shares_the_machines_run_scope(seen): + from je_auto_control.utils.state_machine import StateMachine + machine = StateMachine( + _machine(_SET_USER, _READ_USER), + execute_action=lambda action: execute_action([action])) + machine.run() + assert seen == ["alice"] + assert "user" not in executor.variables From 4091d418eea76aa09f9645663113a782bb120d9a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:42:55 +0800 Subject: [PATCH 06/44] Give each mobile device its own session so parallel devices cannot reach each other A device matrix worker reached its device through process-wide defaults, so a step that left out the serial or URL went to whichever device adb or the default client picked. DeviceContext / open_device give each device an isolated session with its own transport, timeout and cancel signal; the matrix binds one per worker and the AC_android_* / AC_ios_* commands resolve to it. ADB, uiautomator2 and WDA errors now share DeviceError, with typed timeout, permission and missing-device errors, and a capability probe that sends no input. --- docs/source/Eng/doc/mobile/mobile_doc.rst | 132 ++++++ docs/source/Eng/eng_index.rst | 1 + docs/source/Zh/doc/mobile/mobile_doc.rst | 120 +++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 10 + je_auto_control/android/__init__.py | 15 +- je_auto_control/android/adb_client.py | 116 ++++- je_auto_control/android/client.py | 4 +- je_auto_control/android/input.py | 166 +++++++ je_auto_control/android/session.py | 131 ++++++ je_auto_control/ios/__init__.py | 3 +- je_auto_control/ios/client.py | 4 +- je_auto_control/ios/session.py | 91 ++++ je_auto_control/utils/device_matrix/matrix.py | 15 +- .../utils/executor/action_executor.py | 41 +- je_auto_control/wrapper/device_context.py | 443 ++++++++++++++++++ test/unit_test/headless/_mobile_doubles.py | 428 +++++++++++++++++ .../headless/test_mobile_context_isolation.py | 208 ++++++++ 18 files changed, 1898 insertions(+), 31 deletions(-) create mode 100644 docs/source/Eng/doc/mobile/mobile_doc.rst create mode 100644 docs/source/Zh/doc/mobile/mobile_doc.rst create mode 100644 je_auto_control/android/input.py create mode 100644 je_auto_control/android/session.py create mode 100644 je_auto_control/ios/session.py create mode 100644 je_auto_control/wrapper/device_context.py create mode 100644 test/unit_test/headless/_mobile_doubles.py create mode 100644 test/unit_test/headless/test_mobile_context_isolation.py diff --git a/docs/source/Eng/doc/mobile/mobile_doc.rst b/docs/source/Eng/doc/mobile/mobile_doc.rst new file mode 100644 index 000000000..25d241b1e --- /dev/null +++ b/docs/source/Eng/doc/mobile/mobile_doc.rst @@ -0,0 +1,132 @@ +================================ +Android and iOS devices +================================ + +AutoControl drives Android devices through ``adb`` (plus the optional +``uiautomator2`` daemon) and iOS devices through WebDriverAgent (the +``facebook-wda`` client). Both are optional: nothing here is imported until +it is used, so the package still imports on a host with neither. + +.. warning:: + + **Not verified on hardware.** Everything on this page is built and tested + against a fake ADB host and a fake WebDriverAgent client + (``test/unit_test/headless/_mobile_doubles.py``). Those tests prove the + commands AutoControl sends are the ones it means to send; they do not prove + a real emulator, phone or WebDriverAgent build accepts them. No CI job + attaches a device. Treat the setup notes below as the intended procedure, + not as a tested one. + +Device contexts and sessions +============================ + +A :class:`DeviceContext` is the frozen identity of one device, and +:func:`open_device` turns it into a :class:`DeviceSession` that owns its own +transport, timeout and cancellation signal:: + + from je_auto_control import DeviceContext, open_device + + left = open_device(DeviceContext("android", "emulator-5554")) + right = open_device(DeviceContext("ios", "http://192.168.1.20:8100")) + + print(left.capabilities()["unicode_text"].state) + left.cancel() # stops ``left`` only + assert right.connected + right.close() + +``device_id`` is the adb serial on Android and the WebDriverAgent URL on iOS. +Opening a session sends nothing: the adb client or WDA client is built on the +first operation, so a session can be opened on a host without ``adb``. + +* ``cancel()`` stops one session. A caller waiting on the device returns at + once with ``DeviceCancelledError`` and nothing further is sent. A call + already on its way to the device cannot be recalled. +* A call that outlives ``timeout_s`` (default 30 s) raises + ``DeviceTimeoutError`` **and closes the session**: the device is in a state + nobody observed, so the next step is refused (``DeviceClosedError``) instead + of being sent on top of it. Open a new session to continue. +* ``close()`` releases what the session built. A client handed in by the + caller (``open_device(context, adb=...)`` / ``device=...``) is used but not + owned. Closing an iOS session never terminates an app or deletes a remote + WDA session. + +Several devices in one process +------------------------------ + +``use_device(session)`` binds a session for the current thread; a mobile +command that names no ``serial`` / ``url`` then targets it. The device matrix +does this for every device it runs, so two workers cannot reach each other's +device by leaving the address out:: + + from je_auto_control import run_on_devices + + run_on_devices( + actions=[["AC_android_tap", {"x": 100, "y": 200}]], + devices=[{"platform": "android", "serial": "emulator-5554"}, + {"platform": "android", "serial": "emulator-5556"}], + ) + +An explicit ``serial`` / ``url`` in a step still wins over the bound device. +The older process-wide helpers (``default_ui_device()``, +``default_ios_device()``, the executor's per-serial adb cache) keep working +for single-device scripts; the matrix no longer touches them. + +Capabilities +============ + +``session.capabilities()`` returns one :class:`DeviceCapability` per feature — +``input``, ``unicode_text``, ``multi_touch``, ``screenshot``, ``ui_tree``, +``app_lifecycle``, ``alerts``, ``install``, ``files``, ``clipboard``, +``recording`` — each in one of four states: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - State + - Meaning + * - ``available`` + - Usable now. + * - ``needs_permission`` + - The device refused the host (USB debugging not authorised). + * - ``needs_dependency`` + - Something is missing; ``reason`` names it (``adb``, ``uiautomator2``, + ``facebook-wda``, an unreachable WDA endpoint, an unattached device, a + host-side adapter). + * - ``unsupported`` + - The backend cannot do it; ``alternative`` says what to use instead. + +Probing sends no input. On Android it runs ``adb devices`` and reads one +setting; on iOS it requests WDA's ``/status``. + +Errors +====== + +Every error derives from ``DeviceError`` (an ``AutoControlException`` and a +``RuntimeError``), whichever backend raised it: + +.. list-table:: + :header-rows: 1 + :widths: 35 65 + + * - Error + - Raised when + * - ``DeviceUnavailableError`` + - ``adb`` / the SDK is missing, or the device or WDA endpoint cannot be + reached (``AdbNotAvailable``, ``AdbDeviceMissingError``, + ``UIAutomatorUnavailableError``, ``IOSUnavailableError``). + * - ``DevicePermissionError`` + - The device has not authorised this host (``AdbUnauthorizedError``). + * - ``DeviceTimeoutError`` + - A call outlived the timeout (``AdbTimeoutError``); also a + ``TimeoutError``. + * - ``DeviceCancelledError`` + - The session was cancelled. + * - ``DeviceClosedError`` + - The session was closed or broken by a timeout. + * - ``DeviceUnsupportedError`` + - The backend cannot do what was asked; carries ``reason`` and + ``alternative``. + +``AdbError`` and the other pre-existing names are still raised and still +caught by existing ``except`` clauses; they now share this base. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e826e9bb4..0523dc20e 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -248,6 +248,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v221_features_doc doc/new_features/v222_features_doc doc/new_features/v223_features_doc + doc/mobile/mobile_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/mobile/mobile_doc.rst b/docs/source/Zh/doc/mobile/mobile_doc.rst new file mode 100644 index 000000000..528a02348 --- /dev/null +++ b/docs/source/Zh/doc/mobile/mobile_doc.rst @@ -0,0 +1,120 @@ +================================ +Android 與 iOS 裝置 +================================ + +AutoControl 透過 ``adb``\ （加上選用的 ``uiautomator2`` daemon）操作 Android 裝置， +透過 WebDriverAgent（``facebook-wda`` 用戶端）操作 iOS 裝置。兩者都是選用相依： +用到才匯入，所以沒裝的主機仍然可以匯入本套件。 + +.. warning:: + + **尚未在實機上驗證。** 本頁所有功能都是對著假的 ADB 主機與假的 WebDriverAgent + 用戶端（``test/unit_test/headless/_mobile_doubles.py``）開發與測試的。這些測試 + 證明 AutoControl 送出的指令就是它打算送的指令；它們不能證明真的模擬器、手機或 + WebDriverAgent 版本會接受這些指令。沒有任何 CI 工作接上裝置。下方的設定說明是 + 預期的流程，不是測過的流程。 + +裝置 context 與 session +======================= + +:class:`DeviceContext` 是一台裝置不可變的身分，\ :func:`open_device` 把它變成 +:class:`DeviceSession`\ ——擁有自己的傳輸層、逾時與取消訊號:: + + from je_auto_control import DeviceContext, open_device + + left = open_device(DeviceContext("android", "emulator-5554")) + right = open_device(DeviceContext("ios", "http://192.168.1.20:8100")) + + print(left.capabilities()["unicode_text"].state) + left.cancel() # 只停 ``left`` + assert right.connected + right.close() + +``device_id`` 在 Android 是 adb serial，在 iOS 是 WebDriverAgent 的 URL。 +開 session 不會送出任何東西：adb 用戶端或 WDA 用戶端在第一次操作時才建立， +所以沒有 ``adb`` 的主機也能開 session。 + +* ``cancel()`` 只停一個 session。正在等裝置回應的呼叫端會立刻收到 + ``DeviceCancelledError``\ ，之後不再送出任何東西。已經送往裝置的呼叫收不回來。 +* 超過 ``timeout_s``\ （預設 30 秒）的呼叫會丟 ``DeviceTimeoutError`` + **並關閉 session**\ ：裝置處於沒人觀察到的狀態，所以下一步會被拒絕 + （``DeviceClosedError``\ ），而不是疊上去送。要繼續請開新的 session。 +* ``close()`` 釋放 session 自己建立的東西。呼叫端交進來的用戶端 + （``open_device(context, adb=...)`` / ``device=...``\ ）只借用、不擁有。 + 關閉 iOS session 不會終止 App，也不會刪除遠端的 WDA session。 + +同一個行程操作多台裝置 +---------------------- + +``use_device(session)`` 在目前的執行緒綁定一個 session；沒寫 ``serial`` / ``url`` +的行動指令就會打到它。Device matrix 對每台裝置都這樣做，所以兩個 worker +不會因為省略位址而打到對方的裝置:: + + from je_auto_control import run_on_devices + + run_on_devices( + actions=[["AC_android_tap", {"x": 100, "y": 200}]], + devices=[{"platform": "android", "serial": "emulator-5554"}, + {"platform": "android", "serial": "emulator-5556"}], + ) + +步驟裡明確寫出的 ``serial`` / ``url`` 仍然優先於綁定的裝置。舊有的行程層級 +helper（``default_ui_device()``\ 、\ ``default_ios_device()``\ 、executor 的 +per-serial adb 快取）在單一裝置腳本裡照常運作；matrix 不再碰它們。 + +能力 +==== + +``session.capabilities()`` 對每項功能回傳一個 :class:`DeviceCapability`\ —— +``input``\ 、\ ``unicode_text``\ 、\ ``multi_touch``\ 、\ ``screenshot``\ 、 +``ui_tree``\ 、\ ``app_lifecycle``\ 、\ ``alerts``\ 、\ ``install``\ 、\ ``files``\ 、 +``clipboard``\ 、\ ``recording``\ ——每一項是下列四種狀態之一： + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - 狀態 + - 意義 + * - ``available`` + - 現在可用。 + * - ``needs_permission`` + - 裝置拒絕了這台主機（尚未授權 USB 偵錯）。 + * - ``needs_dependency`` + - 缺了東西；\ ``reason`` 會指出是什麼（``adb``\ 、\ ``uiautomator2``\ 、 + ``facebook-wda``\ 、連不上的 WDA 端點、沒接上的裝置、主機端的 adapter）。 + * - ``unsupported`` + - 後端做不到；\ ``alternative`` 說明可以改用什麼。 + +探測不會送出任何輸入。Android 只執行 ``adb devices`` 並讀一個設定值； +iOS 只請求 WDA 的 ``/status``\ 。 + +錯誤 +==== + +不論是哪個後端丟的，所有錯誤都繼承 ``DeviceError``\ （它是 +``AutoControlException``\ ，也是 ``RuntimeError``\ ）： + +.. list-table:: + :header-rows: 1 + :widths: 35 65 + + * - 錯誤 + - 何時丟出 + * - ``DeviceUnavailableError`` + - 缺少 ``adb`` / SDK，或連不上裝置或 WDA 端點（``AdbNotAvailable``\ 、 + ``AdbDeviceMissingError``\ 、\ ``UIAutomatorUnavailableError``\ 、 + ``IOSUnavailableError``\ ）。 + * - ``DevicePermissionError`` + - 裝置尚未授權這台主機（``AdbUnauthorizedError``\ ）。 + * - ``DeviceTimeoutError`` + - 呼叫超過逾時（``AdbTimeoutError``\ ）；同時也是 ``TimeoutError``\ 。 + * - ``DeviceCancelledError`` + - session 已被取消。 + * - ``DeviceClosedError`` + - session 已關閉，或因逾時而失效。 + * - ``DeviceUnsupportedError`` + - 後端做不到所要求的事；帶有 ``reason`` 與 ``alternative``\ 。 + +``AdbError`` 等既有名稱照樣丟出，既有的 ``except`` 也照樣接得到； +它們現在共用這個基底。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index eb1f7a49f..37926663d 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -248,6 +248,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v221_features_doc doc/new_features/v222_features_doc doc/new_features/v223_features_doc + doc/mobile/mobile_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 475c40ca2..6e4502f7d 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1038,6 +1038,12 @@ from je_auto_control.utils.device_matrix import ( DeviceResult, MatrixReport, run_on_devices, ) +# Mobile device contexts (one isolated session per Android / iOS device) +from je_auto_control.wrapper.device_context import ( + DeviceCancelledError, DeviceCapability, DeviceClosedError, DeviceContext, + DeviceError, DevicePermissionError, DeviceSession, DeviceTimeoutError, + DeviceUnavailableError, DeviceUnsupportedError, open_device, use_device, +) # Media assertions (audio activity, video motion) from je_auto_control.utils.media_assert import ( MediaAssertionResult, assert_audio_activity, assert_video_changes, @@ -1934,6 +1940,10 @@ def start_autocontrol_gui(*args, **kwargs): "wcag_audit", # Mobile device matrix "DeviceResult", "MatrixReport", "run_on_devices", + # Mobile device contexts + "DeviceCancelledError", "DeviceCapability", "DeviceClosedError", "DeviceContext", + "DeviceError", "DevicePermissionError", "DeviceSession", "DeviceTimeoutError", + "DeviceUnavailableError", "DeviceUnsupportedError", "open_device", "use_device", # Media assertions "MediaAssertionResult", "assert_audio_activity", "assert_video_changes", "measure_audio_rms", "video_segment_motion", diff --git a/je_auto_control/android/__init__.py b/je_auto_control/android/__init__.py index 3997f398a..e36216f42 100644 --- a/je_auto_control/android/__init__.py +++ b/je_auto_control/android/__init__.py @@ -25,12 +25,15 @@ The ADB binary is **not** bundled — install `Android Platform Tools `_ and -make sure ``adb`` is on ``PATH``. iOS support is deliberately not in -this phase; it needs a Mac + paid Apple Developer cert to sideload -WebDriverAgent, which is its own infrastructure problem. +make sure ``adb`` is on ``PATH``. iOS lives in :mod:`je_auto_control.ios`. + +To drive several devices from one process, open one session per device with +:func:`je_auto_control.wrapper.device_context.open_device` instead of relying +on a default serial: each :class:`AndroidSession` owns its own client. """ from je_auto_control.android.adb_client import ( - AdbClient, AdbError, AdbNotAvailable, AndroidDevice, + AdbClient, AdbDeviceMissingError, AdbError, AdbNotAvailable, + AdbTimeoutError, AdbUnauthorizedError, AndroidDevice, ) from je_auto_control.android.client import ( UIAutomatorDevice, UIAutomatorUnavailableError, @@ -39,9 +42,11 @@ from je_auto_control.android.find import ( ElementNotFoundError, click_element, dump_hierarchy, find_element, ) +from je_auto_control.android.session import AndroidSession __all__ = [ - "AdbClient", "AdbError", "AdbNotAvailable", "AndroidDevice", + "AdbClient", "AdbDeviceMissingError", "AdbError", "AdbNotAvailable", + "AdbTimeoutError", "AdbUnauthorizedError", "AndroidDevice", "AndroidSession", "ElementNotFoundError", "UIAutomatorDevice", "UIAutomatorUnavailableError", "click_element", "default_ui_device", "dump_hierarchy", diff --git a/je_auto_control/android/adb_client.py b/je_auto_control/android/adb_client.py index c354dd4b5..24b737c04 100644 --- a/je_auto_control/android/adb_client.py +++ b/je_auto_control/android/adb_client.py @@ -9,19 +9,64 @@ from pathlib import Path from typing import List, Optional, Sequence -from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.wrapper.device_context import ( + DeviceError, DevicePermissionError, DeviceTimeoutError, DeviceUnavailableError, + DeviceUnsupportedError, +) _DEFAULT_TIMEOUT_S = 30.0 -class AdbError(AutoControlException, RuntimeError): +class AdbError(DeviceError): """Raised when adb returns a non-zero exit code.""" -class AdbNotAvailable(AutoControlException, RuntimeError): +class AdbNotAvailable(DeviceUnavailableError): """Raised when the adb binary isn't on PATH and no path was supplied.""" +class AdbTimeoutError(AdbError, DeviceTimeoutError): + """Raised when adb did not answer within the client's timeout.""" + + +class AdbUnauthorizedError(AdbError, DevicePermissionError): + """Raised when the device has not authorised this host for USB debugging.""" + + +class AdbDeviceMissingError(AdbError, DeviceUnavailableError): + """Raised when the addressed device is offline or not attached.""" + + +class AdbUnsupportedError(AdbError, DeviceUnsupportedError): + """Raised when adb on this device cannot do what was asked.""" + + +def adb_text_safe(value: str) -> bool: + """Whether ``adb shell input text`` delivers ``value`` unchanged. + + It maps characters through the key character map, so only printable ASCII + arrives; and it turns every ``%s`` into a space with no escape for it. + """ + return "%s" not in value and all(" " <= char <= "~" for char in value) + + +#: adb's own wording for a refused or absent device, mapped to the typed error. +#: Anchored on "device": a shell's "sh: foo: not found" is not a missing device. +_STDERR_ERRORS = ( + (re.compile(r"device unauthorized|no permissions"), AdbUnauthorizedError), + (re.compile(r"device (?:'[^']*' )?not found|device offline|no devices/emulators found" + r"|more than one device"), AdbDeviceMissingError), +) + + +def _exit_error(args: Sequence[str], returncode: int, stderr: str) -> AdbError: + """The typed error for a non-zero adb exit.""" + lowered = stderr.lower() + kind = next((error for pattern, error in _STDERR_ERRORS if pattern.search(lowered)), + AdbError) + return kind(f"{_describe(args)} exited {returncode}: {stderr}") + + @dataclass class AndroidDevice: """One device row from ``adb devices -l``.""" @@ -72,6 +117,11 @@ def __init__(self, *, adb_path: Optional[str] = None, def adb_path(self) -> str: return self._adb + @property + def default_serial(self) -> Optional[str]: + """The serial every call targets when it names none.""" + return self._default_serial + # --- low-level command runner ------------------------------------- def run(self, args: Sequence[str], *, serial: Optional[str] = None, @@ -90,13 +140,15 @@ def run(self, args: Sequence[str], *, serial: Optional[str] = None, capture_output=True, timeout=timeout or self._timeout, check=False, ) + except subprocess.TimeoutExpired as error: + # Not str(error): TimeoutExpired quotes the whole argv, typed text included. + raise AdbTimeoutError( + f"{_describe(args)} timed out after {timeout or self._timeout:g}s") from error except (OSError, subprocess.SubprocessError) as error: raise AdbError(f"{_describe(args)} failed: {error}") from error if check and result.returncode != 0: stderr = result.stderr.decode("utf-8", errors="replace").strip() - raise AdbError( - f"{_describe(args)} exited {result.returncode}: {stderr}", - ) + raise _exit_error(args, result.returncode, stderr) return result def shell(self, command: str, *, serial: Optional[str] = None, @@ -139,6 +191,27 @@ def list_devices(self) -> List[AndroidDevice]: )) return devices + def device_state(self, serial: Optional[str] = None) -> str: + """The ``adb devices`` state of one device, or ``""`` when it is not listed. + + With no serial (and no default one) the answer is the single attached + device's state; several devices and no serial is ``""``. + """ + target = serial if serial is not None else self._default_serial + devices = self.list_devices() + if target: + return next((d.state for d in devices if d.serial == target), "") + return devices[0].state if len(devices) == 1 else "" + + def version(self) -> str: + """The adb build, e.g. ``"1.0.41 (35.0.2-12147458)"``.""" + out = self.run(["version"]).stdout.decode("utf-8", errors="replace") + release = re.search(r"Android Debug Bridge version (\S+)", out) + build = re.search(r"^Version (\S+)", out, re.MULTILINE) + parts = [release.group(1) if release else "", + f"({build.group(1)})" if build else ""] + return " ".join(part for part in parts if part) + # --- input ------------------------------------------------------- def tap(self, x: int, y: int, *, serial: Optional[str] = None) -> None: @@ -155,6 +228,19 @@ def swipe(self, x1: int, y1: int, x2: int, y2: int, serial=serial, ) + def input_command(self, command: str, *, serial: Optional[str] = None, + timeout: Optional[float] = None) -> None: + """Run ``adb shell input ``, raising when ``input`` rejected it. + + An ``input`` build that does not know a subcommand prints its usage + and can still exit 0. + """ + out = self.shell(f"input {command}", serial=serial, timeout=timeout) + if "Unknown command" in out or out.lstrip().startswith("Error:"): + raise AdbUnsupportedError( + f"this device's `input` does not support `{command.split()[0]}`", + alternative="uiautomator2 (pip install uiautomator2)") + def key_event(self, key: str, *, serial: Optional[str] = None) -> None: """Send a keycode — accepts ``KEYCODE_HOME`` or numeric codes.""" # ``input keyevent`` accepts both ``HOME`` and the full ``KEYCODE_HOME`` @@ -174,11 +260,19 @@ def key_event(self, key: str, *, serial: Optional[str] = None) -> None: def text(self, value: str, *, serial: Optional[str] = None) -> None: """Type ``value`` via ``input text``. Spaces are %s-escaped. - ``input text`` itself turns every ``%s`` into a space and offers no - escape for it, so a literal ``%s`` in ``value`` arrives as a space. + ``input text`` carries printable ASCII only and turns every ``%s`` + into a space with no escape for it, while reporting success either + way. Text it cannot deliver is refused here rather than sent; + :func:`je_auto_control.android.input.type_text` picks a path that can. """ if not isinstance(value, str): raise AdbError(f"text must be a string, got {type(value).__name__}") + if not adb_text_safe(value): + raise AdbUnsupportedError( + "adb input text cannot deliver this text (non-ASCII characters or a " + "literal %s); it would be dropped or altered while reporting success", + alternative="type_text() / AC_android_type_text, which use the " + "ADBKeyBoard IME or uiautomator2") # ``input text`` mangles spaces; the official workaround is to # replace them with %s before passing through the shell layer. escaped = value.replace(" ", "%s") @@ -210,4 +304,8 @@ def save_screenshot(self, file_path, return target -__all__ = ["AdbClient", "AdbError", "AdbNotAvailable", "AndroidDevice"] +__all__ = [ + "AdbClient", "AdbDeviceMissingError", "AdbError", "AdbNotAvailable", + "AdbTimeoutError", "AdbUnauthorizedError", "AdbUnsupportedError", + "AndroidDevice", "adb_text_safe", +] diff --git a/je_auto_control/android/client.py b/je_auto_control/android/client.py index 282e8cb7c..e02652695 100644 --- a/je_auto_control/android/client.py +++ b/je_auto_control/android/client.py @@ -13,10 +13,10 @@ class so the cheap adb-only path stays available when the daemon import threading from typing import Any, Callable, Optional, TypeVar -from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.wrapper.device_context import DeviceUnavailableError -class UIAutomatorUnavailableError(AutoControlException, RuntimeError): +class UIAutomatorUnavailableError(DeviceUnavailableError): """Raised when the ``uiautomator2`` SDK or a target device is missing.""" diff --git a/je_auto_control/android/input.py b/je_auto_control/android/input.py new file mode 100644 index 000000000..e55d935aa --- /dev/null +++ b/je_auto_control/android/input.py @@ -0,0 +1,166 @@ +"""Android text and gestures that say so when the device cannot receive them. + +``adb shell input text`` maps characters through the device's key character +map: it carries printable ASCII and nothing else, and it reports success +either way. ``adb shell input`` also has no multi-touch. This module picks a +path that can really deliver what was asked — the ADBKeyBoard IME or the +uiautomator2 daemon — and raises when there is none, instead of sending +something else and returning. +""" +from __future__ import annotations + +import base64 +import shlex +from importlib.util import find_spec +from typing import Any, Optional, Tuple + +from je_auto_control.android.adb_client import ( + AdbError, AdbUnsupportedError, adb_text_safe, +) +from je_auto_control.android.client import UIAutomatorDevice, translate_device_errors +from je_auto_control.wrapper.device_context import Drag, LongPress, Pinch, Swipe, Tap + +#: The IME of the ADBKeyBoard project, which accepts text as a broadcast. +ADB_KEYBOARD_IME = "com.android.adbkeyboard/.AdbIME" +#: uiautomator injects a two-pointer gesture in steps of about 5 ms. +_STEPS_PER_SECOND = 200 + + +def current_input_method(adb: Any) -> str: + """The id of the device's selected IME. A settings read: no input is sent.""" + return str(adb.shell("settings get secure default_input_method")).strip() + + +def _ui_device_for(adb: Any, ui_device: Optional[UIAutomatorDevice]) -> Optional[UIAutomatorDevice]: + """The uiautomator2 wrapper to use: the given one, else a new one if the SDK is installed.""" + if ui_device is not None: + return ui_device + if find_spec("uiautomator2") is None: + return None + return UIAutomatorDevice(serial=getattr(adb, "default_serial", None)) + + +def _broadcast_text(adb: Any, text: str) -> None: + """Hand ``text`` to the selected ADBKeyBoard IME as base64.""" + payload = base64.b64encode(text.encode("utf-8")).decode("ascii") + out = adb.shell("am broadcast -a ADB_INPUT_B64 --es msg " + shlex.quote(payload)) + if "Broadcast completed" not in out: + raise AdbError("the ADBKeyBoard broadcast was not delivered") + + +@translate_device_errors +def _send_keys(ui_device: UIAutomatorDevice, text: str) -> None: + ui_device.handle.send_keys(text) + + +def type_text(adb: Any, text: str, *, + ui_device: Optional[UIAutomatorDevice] = None) -> None: + """Type ``text`` into the focused field through a path that can carry it. + + Printable ASCII goes through ``adb shell input text``. Anything else goes + to the ADBKeyBoard IME when it is the selected input method, else to + uiautomator2 when it is installed. With neither, this raises + :class:`AdbUnsupportedError`: nothing is sent, because ``input text`` would + drop the characters and still exit 0. + """ + if not isinstance(text, str): + raise AdbError(f"text must be a string, got {type(text).__name__}") + if not text: + return + if adb_text_safe(text): + adb.text(text) + return + if current_input_method(adb) == ADB_KEYBOARD_IME: + _broadcast_text(adb, text) + return + chosen = _ui_device_for(adb, ui_device) + if chosen is None: + raise AdbUnsupportedError( + "this text has characters `adb shell input text` cannot deliver, and " + "neither Unicode path is available", + reason="non-ASCII text needs the ADBKeyBoard IME selected on the device " + "or uiautomator2 installed on the host", + alternative="pip install uiautomator2, or install ADBKeyBoard and run " + f"`adb shell ime set {ADB_KEYBOARD_IME}`") + _send_keys(chosen, text) + + +def tap(adb: Any, gesture: Tap) -> None: + """Tap.""" + adb.tap(gesture.x, gesture.y) + + +def long_press(adb: Any, gesture: LongPress) -> None: + """Hold in place: a swipe that does not move.""" + adb.swipe(gesture.x, gesture.y, gesture.x, gesture.y, + duration_ms=max(1, round(gesture.duration_s * 1000))) + + +def swipe(adb: Any, gesture: Swipe) -> None: + """Swipe.""" + adb.swipe(gesture.x1, gesture.y1, gesture.x2, gesture.y2, + duration_ms=max(1, round(gesture.duration_s * 1000))) + + +@translate_device_errors +def _ui_drag(ui_device: UIAutomatorDevice, gesture: Drag) -> None: + ui_device.handle.drag(gesture.x1, gesture.y1, gesture.x2, gesture.y2, + duration=float(gesture.hold_s + gesture.duration_s)) + + +def drag(adb: Any, gesture: Drag, *, + ui_device: Optional[UIAutomatorDevice] = None) -> None: + """Drag and drop. + + ``input draganddrop`` holds before it moves, which is what lifts an item; + a device whose ``input`` lacks it falls back to uiautomator2, and raises + when that is not installed either. + """ + total_ms = max(1, round((gesture.hold_s + gesture.duration_s) * 1000)) + try: + adb.input_command( + f"draganddrop {int(gesture.x1)} {int(gesture.y1)} " + f"{int(gesture.x2)} {int(gesture.y2)} {total_ms}") + except AdbUnsupportedError: + chosen = _ui_device_for(adb, ui_device) + if chosen is None: + raise + _ui_drag(chosen, gesture) + + +def pinch_points(gesture: Pinch) -> Tuple[Tuple[int, int], Tuple[int, int], + Tuple[int, int], Tuple[int, int]]: + """``(start1, start2, end1, end2)`` for two fingers either side of the centre.""" + wide = gesture.span / 2.0 + narrow = wide / max(gesture.scale, 1.0 / gesture.scale) + begin, end = (narrow, wide) if gesture.scale > 1 else (wide, narrow) + x, y = int(gesture.x), int(gesture.y) + return ((round(x - begin), y), (round(x + begin), y), + (round(x - end), y), (round(x + end), y)) + + +@translate_device_errors +def _ui_pinch(ui_device: UIAutomatorDevice, gesture: Pinch) -> None: + start1, start2, end1, end2 = pinch_points(gesture) + steps = max(2, round(gesture.duration_s * _STEPS_PER_SECOND)) + # A two-pointer gesture is performed on a UI object but takes absolute + # points, so the empty selector (the root node) serves for any position. + ui_device.handle().gesture(start1, start2, end1, end2, steps=steps) + + +def pinch(adb: Any, gesture: Pinch, *, + ui_device: Optional[UIAutomatorDevice] = None) -> None: + """Two-finger pinch. ``adb shell input`` has one pointer, so this needs uiautomator2.""" + chosen = _ui_device_for(adb, ui_device) + if chosen is None: + raise AdbUnsupportedError( + "a pinch needs two pointers and `adb shell input` has one", + reason="multi-touch needs uiautomator2 installed on the host", + alternative="pip install uiautomator2") + _ui_pinch(chosen, gesture) + + +__all__ = [ + "ADB_KEYBOARD_IME", "current_input_method", "drag", "long_press", "pinch", + "pinch_points", "swipe", "tap", "type_text", +] diff --git a/je_auto_control/android/session.py b/je_auto_control/android/session.py new file mode 100644 index 000000000..ef3150ddf --- /dev/null +++ b/je_auto_control/android/session.py @@ -0,0 +1,131 @@ +"""One Android device as a :class:`DeviceSession`: its own adb client and widget daemon.""" +from __future__ import annotations + +from importlib.util import find_spec +from typing import Any, Dict, Optional, Tuple + +from je_auto_control.android.adb_client import AdbClient, AdbError +from je_auto_control.android.client import UIAutomatorDevice +from je_auto_control.android.input import ADB_KEYBOARD_IME, current_input_method +from je_auto_control.wrapper.device_context import ( + CAPABILITY_NAMES, STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, STATE_NEEDS_PERMISSION, + DeviceCapability, DeviceContext, DeviceError, DeviceSession, bound_session, +) + +#: Capabilities plain ``adb`` provides once the device is authorised. +_ADB_CAPABILITIES = ("input", "screenshot", "app_lifecycle", "install", "files", "recording") +#: Capabilities that need the uiautomator2 daemon on top of adb. +_UI_CAPABILITIES = ("ui_tree", "multi_touch", "alerts", "clipboard") +_NEEDS_UI = "uiautomator2 is not installed: pip install uiautomator2" +_UNAUTHORIZED = ("the device has not authorised this host: accept the USB debugging " + "prompt on the device, then check `adb devices`") + + +class AndroidSession(DeviceSession): + """An Android device addressed by adb serial. + + The adb client and the uiautomator2 wrapper are built on first use and + belong to this session alone; one handed in by the caller is used but not + owned. + """ + + def __init__(self, context: DeviceContext, *, adb: Optional[Any] = None, + ui_device: Optional[UIAutomatorDevice] = None) -> None: + super().__init__(context) + self._adb = adb + self._ui_device = ui_device + self._ui_injected = ui_device is not None + + @property + def adb(self) -> Any: + """This session's :class:`AdbClient`, bound to the context's serial.""" + self.check_usable() + with self._state_lock: + if self._adb is None: + self._adb = AdbClient( + adb_path=self.context.adb_path, + default_serial=self.device_id or None, + timeout_s=self.context.timeout_s, + ) + return self._adb + + @property + def ui_device(self) -> UIAutomatorDevice: + """This session's uiautomator2 wrapper (connects on first use of its handle).""" + self.check_usable() + with self._state_lock: + if self._ui_device is None: + self._ui_device = UIAutomatorDevice(serial=self.device_id or None) + return self._ui_device + + @property + def has_ui_automator(self) -> bool: + """Whether the uiautomator2 path can be used (installed, or handed in).""" + return self._ui_injected or find_spec("uiautomator2") is not None + + def capabilities(self) -> Dict[str, DeviceCapability]: + """What this device can do right now; only ``adb devices`` and a settings read are sent.""" + return self.invoke("capabilities", self._probe) + + def _probe(self) -> Dict[str, DeviceCapability]: + blocker = self._blocker() + if blocker is not None: + state, reason = blocker + return {name: DeviceCapability(name, state, reason) for name in CAPABILITY_NAMES} + found = {name: DeviceCapability(name, STATE_AVAILABLE) for name in _ADB_CAPABILITIES} + for name in _UI_CAPABILITIES: + found[name] = self._ui_capability(name) + found["unicode_text"] = self._unicode_capability() + return {name: found[name] for name in CAPABILITY_NAMES} + + def _blocker(self) -> Optional[Tuple[str, str]]: + """Why nothing can be used: ``(state, reason)``, or ``None`` when adb sees the device.""" + try: + state = self.adb.device_state() + except DeviceError as error: + return STATE_NEEDS_DEPENDENCY, str(error) + if state == "device": + return None + if state in ("unauthorized", "no permissions"): + return STATE_NEEDS_PERMISSION, _UNAUTHORIZED + name = self.device_id or "(default)" + if not state: + return STATE_NEEDS_DEPENDENCY, ( + f"no device {name!r} is attached: start the emulator or connect the " + "device, then check `adb devices`") + return STATE_NEEDS_DEPENDENCY, f"device {name!r} is {state}, not ready" + + def _ui_capability(self, name: str) -> DeviceCapability: + if self.has_ui_automator: + return DeviceCapability(name, STATE_AVAILABLE) + return DeviceCapability(name, STATE_NEEDS_DEPENDENCY, _NEEDS_UI) + + def _unicode_capability(self) -> DeviceCapability: + name = "unicode_text" + if self.has_ui_automator: + return DeviceCapability(name, STATE_AVAILABLE) + try: + ime = current_input_method(self.adb) + except AdbError as error: + return DeviceCapability(name, STATE_NEEDS_DEPENDENCY, str(error)) + if ime == ADB_KEYBOARD_IME: + return DeviceCapability(name, STATE_AVAILABLE) + return DeviceCapability( + name, STATE_NEEDS_DEPENDENCY, + "`adb shell input text` carries printable ASCII only; other text needs " + "uiautomator2 on the host or the ADBKeyBoard IME selected on the device", + alternative="ASCII text works without either") + + def _release(self) -> None: + self._adb = None + if not self._ui_injected: + self._ui_device = None + + +def bound_android_session(serial: Optional[str] = None) -> Optional[AndroidSession]: + """The Android session bound by ``use_device`` when ``serial`` addresses it.""" + session = bound_session("android", serial) + return session if isinstance(session, AndroidSession) else None + + +__all__ = ["AndroidSession", "bound_android_session"] diff --git a/je_auto_control/ios/__init__.py b/je_auto_control/ios/__init__.py index 8d9bc80ca..008e21344 100644 --- a/je_auto_control/ios/__init__.py +++ b/je_auto_control/ios/__init__.py @@ -25,10 +25,11 @@ long_press, press_key, swipe, tap, type_text, ) from je_auto_control.ios.screen import screen_size, screenshot +from je_auto_control.ios.session import IOSSession __all__ = [ - "ElementNotFoundError", "IOSDevice", "IOSUnavailableError", + "ElementNotFoundError", "IOSDevice", "IOSSession", "IOSUnavailableError", "click_element", "default_ios_device", "dump_source", "find_element", "long_press", "press_key", "reset_default_ios_device", "screen_size", "screenshot", "swipe", diff --git a/je_auto_control/ios/client.py b/je_auto_control/ios/client.py index 33ddebd36..a8248dad8 100644 --- a/je_auto_control/ios/client.py +++ b/je_auto_control/ios/client.py @@ -20,10 +20,10 @@ import threading from typing import Any, Callable, Optional, TypeVar -from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.wrapper.device_context import DeviceUnavailableError -class IOSUnavailableError(AutoControlException, RuntimeError): +class IOSUnavailableError(DeviceUnavailableError): """Raised when the ``wda`` SDK is missing or the device can't be reached.""" diff --git a/je_auto_control/ios/session.py b/je_auto_control/ios/session.py new file mode 100644 index 000000000..544dfbec8 --- /dev/null +++ b/je_auto_control/ios/session.py @@ -0,0 +1,91 @@ +"""One iOS device as a :class:`DeviceSession`: its own WebDriverAgent client.""" +from __future__ import annotations + +from typing import Any, Dict, Optional, Tuple + +from je_auto_control.ios.client import IOSDevice, translate_device_errors +from je_auto_control.wrapper.device_context import ( + CAPABILITY_NAMES, STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, + DeviceCapability, DeviceContext, DeviceError, DeviceSession, bound_session, +) + +#: Capabilities WebDriverAgent itself provides. +_WDA_CAPABILITIES = ("input", "unicode_text", "multi_touch", "screenshot", "ui_tree", + "app_lifecycle", "alerts") +#: Capabilities WebDriverAgent has no endpoint for. +_ADAPTER_CAPABILITIES = ("install", "files", "clipboard", "recording") +_NEEDS_ADAPTER = ("WebDriverAgent has no API for this; it needs a host-side adapter " + "(for example tidevice, pymobiledevice3 or Appium)") + + +@translate_device_errors +def _wda_status(handle: Any) -> Dict[str, Any]: + """``GET /status`` — a read; nothing is sent to the app under test.""" + return dict(handle.status() or {}) + + +class IOSSession(DeviceSession): + """An iOS device addressed by its WebDriverAgent URL. + + The client is built on first use and belongs to this session alone; an + :class:`IOSDevice` handed in by the caller is used but not owned. Closing + the session never terminates an app or deletes a remote WDA session: it + releases the local client only. + """ + + def __init__(self, context: DeviceContext, *, + device: Optional[IOSDevice] = None) -> None: + super().__init__(context) + self._device = device + self._injected = device is not None + + @property + def device(self) -> IOSDevice: + """This session's :class:`IOSDevice`, bound to the context's URL.""" + self.check_usable() + with self._state_lock: + if self._device is None: + self._device = IOSDevice(url=self.device_id or None) + return self._device + + def status(self) -> Dict[str, Any]: + """WebDriverAgent's ``/status`` document.""" + return self.invoke("status", lambda: _wda_status(self.device.handle)) + + def capabilities(self) -> Dict[str, DeviceCapability]: + """What this device can do right now; only ``GET /status`` is sent.""" + return self.invoke("capabilities", self._probe) + + def _probe(self) -> Dict[str, DeviceCapability]: + blocker = self._blocker() + if blocker is not None: + state, reason = blocker + return {name: DeviceCapability(name, state, reason) for name in CAPABILITY_NAMES} + found = {name: DeviceCapability(name, STATE_AVAILABLE) for name in _WDA_CAPABILITIES} + for name in _ADAPTER_CAPABILITIES: + found[name] = self._adapter_capability(name) + return {name: found[name] for name in CAPABILITY_NAMES} + + def _blocker(self) -> Optional[Tuple[str, str]]: + """Why nothing can be used: ``(state, reason)``, or ``None`` when WDA answers.""" + try: + _wda_status(self.device.handle) + except (DeviceError, OSError) as error: + return STATE_NEEDS_DEPENDENCY, str(error) + return None + + def _adapter_capability(self, name: str) -> DeviceCapability: + return DeviceCapability(name, STATE_NEEDS_DEPENDENCY, _NEEDS_ADAPTER) + + def _release(self) -> None: + if not self._injected: + self._device = None + + +def bound_ios_session(url: Optional[str] = None) -> Optional[IOSSession]: + """The iOS session bound by ``use_device`` when ``url`` addresses it.""" + session = bound_session("ios", url) + return session if isinstance(session, IOSSession) else None + + +__all__ = ["IOSSession", "bound_ios_session"] diff --git a/je_auto_control/utils/device_matrix/matrix.py b/je_auto_control/utils/device_matrix/matrix.py index 477baa5af..1e7ebc5c8 100644 --- a/je_auto_control/utils/device_matrix/matrix.py +++ b/je_auto_control/utils/device_matrix/matrix.py @@ -6,6 +6,12 @@ runtime variable scopes never collide between threads) and aggregating the per-device pass/fail outcome. +Each mobile device also gets its own :class:`DeviceSession`, opened from an +explicit :class:`DeviceContext` and bound for the duration of that device's +run. An ``AC_android_*`` / ``AC_ios_*`` step that names no ``serial`` / ``url`` +therefore reaches the device its worker is running, never a process-wide +default another worker could have set. + The action list addresses the current device through a bound variable, e.g. ``${device.serial}`` / ``${device.url}``, so the same script targets every device:: @@ -25,6 +31,7 @@ from typing import Any, Dict, List, Optional from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.wrapper.device_context import DeviceContext, open_device, use_device @dataclass @@ -87,7 +94,13 @@ def _run_one_device(actions: List[Any], device: Dict[str, Any], platform = str(device.get("platform", "")) started = time.monotonic() try: - runner.execute_action(actions, raise_on_error=True) + context = DeviceContext.from_spec(device) + if context is None: + # Not a mobile spec: nothing to bind, the list runs as written. + runner.execute_action(actions, raise_on_error=True) + else: + with open_device(context) as session, use_device(session): + runner.execute_action(actions, raise_on_error=True) return DeviceResult(device_id, platform, True, time.monotonic() - started) # The executor's own containment set: an ImageNotFoundException or a diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cc475cf85..79a2358a6 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -1654,7 +1654,15 @@ def _ac_web_current_url() -> Any: def _android_client(serial: Optional[str] = None, adb_path: Optional[str] = None) -> Any: - """Build (or return) a cached :class:`AdbClient` for ``serial``.""" + """Build (or return) a cached :class:`AdbClient` for ``serial``. + + Inside ``use_device`` (a device-matrix worker) the bound session's own + client is used unless ``serial`` names another device. + """ + from je_auto_control.android.session import bound_android_session + session = bound_android_session(serial) + if session is not None: + return session.adb key = (serial, adb_path) cached = _android_client_cache.get(key) if cached is not None: @@ -1665,6 +1673,16 @@ def _android_client(serial: Optional[str] = None, return cached +def _android_ui_device(serial: Optional[str] = None) -> Any: + """The uiautomator2 wrapper for ``serial``: the bound session's, else a new one.""" + from je_auto_control.android.session import bound_android_session + session = bound_android_session(serial) + if session is not None: + return session.ui_device + from je_auto_control.android import UIAutomatorDevice + return UIAutomatorDevice(serial=serial) + + def _ac_android_tap(x: int, y: int, serial: Optional[str] = None, adb_path: Optional[str] = None) -> None: @@ -1731,10 +1749,8 @@ def _ac_android_find_element(text: Optional[str] = None, serial: Optional[str] = None, ) -> Dict[str, int]: """Find an Android widget via uiautomator2; return its bounding rect.""" - from je_auto_control.android import ( - UIAutomatorDevice, find_element, - ) - device = UIAutomatorDevice(serial=serial) + from je_auto_control.android import find_element + device = _android_ui_device(serial) x1, y1, x2, y2 = find_element( text=text, resource_id=resource_id, description=description, class_name=class_name, timeout_s=float(timeout_s), device=device, @@ -1750,10 +1766,8 @@ def _ac_android_click_element(text: Optional[str] = None, serial: Optional[str] = None, ) -> Dict[str, int]: """Tap the first widget matching the selectors; return click centre.""" - from je_auto_control.android import ( - UIAutomatorDevice, click_element, - ) - device = UIAutomatorDevice(serial=serial) + from je_auto_control.android import click_element + device = _android_ui_device(serial) cx, cy = click_element( text=text, resource_id=resource_id, description=description, class_name=class_name, timeout_s=float(timeout_s), device=device, @@ -1763,14 +1777,19 @@ def _ac_android_click_element(text: Optional[str] = None, def _ac_android_dump_hierarchy(serial: Optional[str] = None) -> str: """Return the device's widget tree as an XML string.""" - from je_auto_control.android import UIAutomatorDevice, dump_hierarchy - device = UIAutomatorDevice(serial=serial) + from je_auto_control.android import dump_hierarchy + device = _android_ui_device(serial) return dump_hierarchy(device=device) # === iOS executor adapters (WebDriverAgent / facebook-wda) ================== def _ios_device(url: Optional[str]) -> Any: + """The WDA wrapper for ``url``: the bound session's, else a new one.""" + from je_auto_control.ios.session import bound_ios_session + session = bound_ios_session(url) + if session is not None: + return session.device from je_auto_control.ios import IOSDevice return IOSDevice(url=url) diff --git a/je_auto_control/wrapper/device_context.py b/je_auto_control/wrapper/device_context.py new file mode 100644 index 000000000..73e4c5869 --- /dev/null +++ b/je_auto_control/wrapper/device_context.py @@ -0,0 +1,443 @@ +"""Explicit device contexts for Android and iOS automation. + +The mobile backends grew up around process-wide defaults: one cached +``AdbClient`` per serial, one default ``UIAutomatorDevice``, one default +``IOSDevice``. That is fine for one phone on a desk and wrong for a device +matrix, where two workers must never be able to reach each other's device by +leaving an argument out. + +A :class:`DeviceContext` is the frozen identity and configuration of one +device; :func:`open_device` turns it into a :class:`DeviceSession` that owns +its own transport, its own timeout and its own cancellation signal. Nothing in +a session reads or writes a module-level default, and a session bound with +:func:`use_device` is visible only to the thread (and context) that bound it. + +Every error a session raises derives from :class:`DeviceError`, itself an +``AutoControlException``, so the executor's containment boundaries see one +family whichever backend failed. +""" +from __future__ import annotations + +import contextvars +import threading +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from time import monotonic +from typing import Any, Callable, Dict, Iterator, Mapping, Optional, TypeVar, Union + +from je_auto_control.utils.exception.exceptions import AutoControlException + +PLATFORM_ANDROID = "android" +PLATFORM_IOS = "ios" +MOBILE_PLATFORMS = (PLATFORM_ANDROID, PLATFORM_IOS) + +STATE_AVAILABLE = "available" +STATE_NEEDS_PERMISSION = "needs_permission" +STATE_NEEDS_DEPENDENCY = "needs_dependency" +STATE_UNSUPPORTED = "unsupported" + +#: Every capability a session reports, in report order. +CAPABILITY_NAMES = ( + "input", "unicode_text", "multi_touch", "screenshot", "ui_tree", + "app_lifecycle", "alerts", "install", "files", "clipboard", "recording", +) + +DEFAULT_TIMEOUT_S = 30.0 +_POLL_S = 0.02 + +_Result = TypeVar("_Result") + + +class DeviceError(AutoControlException, RuntimeError): + """Base of every error a mobile device session raises.""" + + +class DeviceUnavailableError(DeviceError): + """The backend SDK, the transport or the device itself cannot be reached.""" + + +class DevicePermissionError(DeviceError): + """The device refused the host (USB debugging not authorised, no udev access).""" + + +class DeviceTimeoutError(DeviceError, TimeoutError): + """A device call outlived the session's timeout; its outcome is unknown.""" + + +class DeviceCancelledError(DeviceError): + """The session was cancelled; nothing more is sent to the device.""" + + +class DeviceClosedError(DeviceError): + """The session was closed (or broken by a timeout) and must be reopened.""" + + +class DeviceUnsupportedError(DeviceError): + """The backend cannot do what was asked; says why and what to use instead.""" + + def __init__(self, message: str, *, reason: str = "", + alternative: str = "") -> None: + super().__init__(message) + self.reason = reason or message + self.alternative = alternative + + +@dataclass(frozen=True) +class DeviceCapability: + """Whether one feature can be used on one device, and if not, why.""" + + name: str + state: str + reason: str = "" + alternative: str = "" + + @property + def available(self) -> bool: + """Whether the feature can be used right now.""" + return self.state == STATE_AVAILABLE + + def to_dict(self) -> Dict[str, Any]: + """JSON-safe form for executor, MCP and GUI consumers.""" + return asdict(self) + + +@dataclass(frozen=True) +class DeviceContext: + """The identity and configuration of one device. + + ``device_id`` is the adb serial on Android and the WebDriverAgent URL on + iOS; empty means "whatever the backend picks when only one is attached". + """ + + platform: str + device_id: str = "" + adb_path: Optional[str] = None + timeout_s: float = DEFAULT_TIMEOUT_S + label: str = "" + + def __post_init__(self) -> None: + if self.platform not in MOBILE_PLATFORMS: + raise DeviceError( + f"unknown device platform {self.platform!r}; expected one of {MOBILE_PLATFORMS}") + if not isinstance(self.timeout_s, (int, float)) or self.timeout_s <= 0: + raise DeviceError(f"timeout_s must be a positive number, got {self.timeout_s!r}") + + @property + def name(self) -> str: + """A label for reports: the explicit label, else the id, else the platform.""" + return self.label or self.device_id or f"{self.platform}-default" + + @classmethod + def from_spec(cls, spec: Mapping[str, Any]) -> Optional["DeviceContext"]: + """Build a context from a device-matrix spec; ``None`` if it is not a mobile one. + + A spec is ``{"platform": "android", "serial": ...}`` or + ``{"platform": "ios", "url": ...}``, optionally with ``adb_path``, + ``timeout_s`` and ``label``. + """ + platform = str(spec.get("platform", "")).lower() + if platform not in MOBILE_PLATFORMS: + return None + key = "serial" if platform == PLATFORM_ANDROID else "url" + adb_path = spec.get("adb_path") + return cls( + platform=platform, + device_id=str(spec.get(key) or spec.get("device_id") or ""), + adb_path=str(adb_path) if adb_path else None, + timeout_s=float(spec.get("timeout_s") or DEFAULT_TIMEOUT_S), + label=str(spec.get("label") or ""), + ) + + +def _non_negative(name: str, value: float) -> None: + if not isinstance(value, (int, float)) or value < 0: + raise DeviceError(f"{name} must be a non-negative number, got {value!r}") + + +@dataclass(frozen=True) +class Tap: + """A single tap at ``(x, y)`` in device input coordinates.""" + + x: int + y: int + + +@dataclass(frozen=True) +class LongPress: + """Press and hold at ``(x, y)`` for ``duration_s`` seconds.""" + + x: int + y: int + duration_s: float = 1.0 + + def __post_init__(self) -> None: + _non_negative("duration_s", self.duration_s) + + +@dataclass(frozen=True) +class Swipe: + """A quick stroke from ``(x1, y1)`` to ``(x2, y2)`` (a scroll or a flick).""" + + x1: int + y1: int + x2: int + y2: int + duration_s: float = 0.25 + + def __post_init__(self) -> None: + _non_negative("duration_s", self.duration_s) + + +@dataclass(frozen=True) +class Drag: + """Press at ``(x1, y1)``, hold ``hold_s`` so the item lifts, move to ``(x2, y2)``.""" + + x1: int + y1: int + x2: int + y2: int + hold_s: float = 0.5 + duration_s: float = 0.5 + + def __post_init__(self) -> None: + _non_negative("hold_s", self.hold_s) + _non_negative("duration_s", self.duration_s) + + +@dataclass(frozen=True) +class Pinch: + """A two-finger pinch about ``(x, y)``: ``scale`` above 1 zooms in, below 1 zooms out. + + ``span`` is the distance between the fingers at the wide end of the motion. + """ + + x: int + y: int + scale: float + duration_s: float = 0.5 + span: int = 400 + + def __post_init__(self) -> None: + _non_negative("duration_s", self.duration_s) + if not isinstance(self.scale, (int, float)) or self.scale <= 0 or self.scale == 1: + raise DeviceError(f"pinch scale must be positive and not 1, got {self.scale!r}") + if self.span <= 0: + raise DeviceError(f"pinch span must be positive, got {self.span!r}") + + +Gesture = Union[Tap, LongPress, Swipe, Drag, Pinch] + + +class _Call: + """One device call running on its own thread, so the caller can stop waiting.""" + + def __init__(self, function: Callable[[], Any]) -> None: + self._function = function + self.done = threading.Event() + self.result: Any = None + self.error: Optional[BaseException] = None + + def run(self) -> None: + """Run the call and keep whatever it returned or raised.""" + try: + self.result = self._function() + except BaseException as error: # noqa: BLE001 # reason: re-raised on the calling thread by DeviceSession.invoke + self.error = error + finally: + self.done.set() + + +class DeviceSession: + """One open device: its transport, its timeout and its cancellation signal. + + Subclasses add the platform's operations and route every device call + through :meth:`invoke`, which is what makes timeout, cancellation and the + closed state behave the same on ADB, uiautomator2 and WebDriverAgent. + """ + + def __init__(self, context: DeviceContext) -> None: + self._context = context + self._cancel = threading.Event() + self._closed = threading.Event() + self._state_lock = threading.Lock() + self._broken_reason = "" + + @property + def context(self) -> DeviceContext: + """The context this session was opened from.""" + return self._context + + @property + def platform(self) -> str: + """``"android"`` or ``"ios"``.""" + return self._context.platform + + @property + def device_id(self) -> str: + """The adb serial or WebDriverAgent URL (may be empty).""" + return self._context.device_id + + @property + def cancelled(self) -> bool: + """Whether :meth:`cancel` was called.""" + return self._cancel.is_set() + + @property + def connected(self) -> bool: + """Whether the session still accepts operations.""" + return not self._closed.is_set() and not self._cancel.is_set() + + def cancel(self) -> None: + """Stop this session: waiting callers return at once and nothing more is sent. + + Only this session is affected. A call already on its way to the device + cannot be recalled; the session stops waiting for it and refuses every + later one, so a cancelled run cannot keep typing into the device. + """ + self._cancel.set() + + def close(self) -> None: + """Release what the session owns. Safe to call more than once.""" + with self._state_lock: + if self._closed.is_set(): + return + self._closed.set() + self._release() + + def __enter__(self) -> "DeviceSession": + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def capabilities(self) -> Dict[str, DeviceCapability]: + """What this device can do right now. Sends no input to the device.""" + raise NotImplementedError + + def capture(self) -> Any: + """The current screen as a :class:`~je_auto_control.wrapper.device_frame.DeviceFrame`.""" + raise NotImplementedError + + def type_text(self, text: str) -> None: + """Type ``text`` into the focused field, or raise if the device cannot receive it.""" + raise NotImplementedError + + def press_key(self, key: str) -> None: + """Press one hardware or system key.""" + raise NotImplementedError + + def perform(self, gesture: Gesture) -> None: + """Perform one touch gesture, in device input coordinates.""" + handler = self._gesture_handlers().get(type(gesture)) + if handler is None: + raise DeviceUnsupportedError( + f"{type(gesture).__name__} is not a gesture this session can perform", + alternative="use Tap, LongPress, Swipe, Drag or Pinch") + self.invoke(type(gesture).__name__, lambda: handler(gesture)) + + def _gesture_handlers(self) -> Dict[type, Callable[[Any], None]]: + """Gesture type to the backend call that performs it; overridden per platform.""" + return {} + + def invoke(self, operation: str, function: Callable[[], _Result], + *, timeout_s: Optional[float] = None) -> _Result: + """Run one device call under the session's timeout and cancel signal. + + A call that outlives the timeout leaves the device in a state nobody + observed, so the session is closed rather than left to send the next + step on top of it. + """ + self.check_usable() + budget = float(timeout_s) if timeout_s is not None else float(self._context.timeout_s) + call = _Call(function) + threading.Thread(target=call.run, name=f"device-{self._context.name}", + daemon=True).start() + deadline = monotonic() + budget + while not call.done.wait(_POLL_S): + if self._cancel.is_set(): + raise DeviceCancelledError( + f"{operation} on {self._context.name}: session cancelled") + if monotonic() >= deadline: + self._break(f"{operation} did not finish within {budget:g}s") + raise DeviceTimeoutError( + f"{operation} on {self._context.name} timed out after {budget:g}s; " + "the session is closed because the device state is unknown") + if call.error is not None: + raise call.error + return call.result + + def check_usable(self) -> None: + """Raise unless the session can still send operations.""" + if self._cancel.is_set(): + raise DeviceCancelledError(f"session for {self._context.name} was cancelled") + if self._closed.is_set(): + detail = f" ({self._broken_reason})" if self._broken_reason else "" + raise DeviceClosedError(f"session for {self._context.name} is closed{detail}") + + def _break(self, reason: str) -> None: + self._broken_reason = reason + self.close() + + def _release(self) -> None: + """Drop the transports this session owns; overridden per platform.""" + + +def open_device(context: DeviceContext, **transports: Any) -> DeviceSession: + """Open a session for ``context``. + + Nothing is sent to the device here: the transport is built on first use, + so opening a session needs neither ``adb`` nor a reachable WebDriverAgent. + ``transports`` lets a caller (or a test) hand in an already-built client — + ``adb=`` / ``ui_device=`` on Android, ``device=`` on iOS — which the + session then uses but does not own. + """ + if not isinstance(context, DeviceContext): + raise DeviceError(f"open_device needs a DeviceContext, got {type(context).__name__}") + if context.platform == PLATFORM_ANDROID: + from je_auto_control.android.session import AndroidSession + return AndroidSession(context, **transports) + from je_auto_control.ios.session import IOSSession + return IOSSession(context, **transports) + + +_BOUND: "contextvars.ContextVar[Optional[DeviceSession]]" = contextvars.ContextVar( + "je_auto_control_device_session", default=None) + + +@contextmanager +def use_device(session: DeviceSession) -> Iterator[DeviceSession]: + """Make ``session`` the device that mobile commands without an address target. + + The binding lives in a context variable, so it is visible only to the + thread that made it: two device-matrix workers each see their own device. + """ + token = _BOUND.set(session) + try: + yield session + finally: + _BOUND.reset(token) + + +def bound_session(platform: str, device_id: Optional[str] = None) -> Optional[DeviceSession]: + """The session bound by :func:`use_device` when it is the one being addressed. + + ``None`` when nothing is bound, the platform differs, or ``device_id`` + names another device — the caller then opens its own. + """ + session = _BOUND.get() + if session is None or session.platform != platform: + return None + if device_id and device_id != session.device_id: + return None + return session + + +__all__ = [ + "CAPABILITY_NAMES", "DEFAULT_TIMEOUT_S", "DeviceCancelledError", "DeviceCapability", + "DeviceClosedError", "DeviceContext", "DeviceError", "DevicePermissionError", + "DeviceSession", "DeviceTimeoutError", "DeviceUnavailableError", + "DeviceUnsupportedError", "Drag", "Gesture", "LongPress", "MOBILE_PLATFORMS", + "PLATFORM_ANDROID", "PLATFORM_IOS", "Pinch", "STATE_AVAILABLE", + "STATE_NEEDS_DEPENDENCY", "STATE_NEEDS_PERMISSION", "STATE_UNSUPPORTED", + "Swipe", "Tap", "bound_session", "open_device", "use_device", +] diff --git a/test/unit_test/headless/_mobile_doubles.py b/test/unit_test/headless/_mobile_doubles.py new file mode 100644 index 000000000..48b825754 --- /dev/null +++ b/test/unit_test/headless/_mobile_doubles.py @@ -0,0 +1,428 @@ +"""Fake ADB host and fake WebDriverAgent client for the mobile tests. + +No ``adb`` binary, emulator, iPhone or WebDriverAgent exists on a CI runner +(or on the machine these tests were written on), so the mobile code is driven +against these doubles: + +* :class:`FakeAdbHost` stands in for ``subprocess.run`` inside + ``android.adb_client``. It parses the real argv the client builds — so the + ``-s `` routing, the shell quoting and the exit-code handling are the + production code's — and answers from one :class:`FakeAndroidDevice` per + serial. +* :class:`FakeWda` stands in for ``wda.Client``. It records every call and + keeps just enough state (apps, alert, clipboard, orientation) to answer. + +They prove the code sends the commands it means to send. They do not prove a +real device accepts them. +""" +from __future__ import annotations + +import base64 +import io +import shlex +import subprocess +import threading +from typing import Any, Callable, Dict, List, Optional, Tuple + +ADB_KEYBOARD_IME = "com.android.adbkeyboard/.AdbIME" +DEFAULT_IME = "com.google.android.inputmethod.latin/.LatinIME" + +#: Shell command prefixes that put input into the device. +_INPUT_PREFIXES = ("input ", "am broadcast", "am start", "am force-stop", "monkey ", + "pm ", "screenrecord", "pkill") + + +def png_bytes(width: int, height: int, color: Tuple[int, int, int] = (0, 0, 0), + patch: Optional[Tuple[int, int, int, int]] = None) -> bytes: + """A PNG of ``width`` x ``height``; ``patch`` is a white ``(x, y, w, h)`` block.""" + from PIL import Image + image = Image.new("RGB", (width, height), color) + if patch is not None: + x, y, w, h = patch + image.paste(Image.new("RGB", (w, h), (255, 255, 255)), (x, y)) + # A dark dot inside the block, so a template of it has structure to match. + image.paste(Image.new("RGB", (max(1, w // 3), max(1, h // 3)), (10, 40, 90)), + (x + w // 4, y + h // 4)) + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +class FakeAndroidDevice: + """State of one pretend Android device.""" + + def __init__(self, serial: str, *, state: str = "device", + size: Tuple[int, int] = (1080, 1920), rotation: int = 0, + ime: str = DEFAULT_IME) -> None: + self.serial = serial + self.state = state + self.size = size + self.rotation = rotation + self.ime = ime + self.shell_commands: List[str] = [] + self.typed: List[str] = [] + self.running: set = set() + self.installed: set = set() + self.files: Dict[str, bytes] = {} + self.screen: Optional[bytes] = None + self.recording = False + self.gate: Optional[threading.Event] = None + #: False imitates an old ``input`` build without ``draganddrop``. + self.has_draganddrop = True + + @property + def device_id(self) -> str: + """The serial, under the name a device context uses.""" + return self.serial + + @property + def input_calls(self) -> List[str]: + """Every shell command that would have acted on the device.""" + return [command for command in self.shell_commands + if command.startswith(_INPUT_PREFIXES)] + + def screencap(self) -> bytes: + """The current screen as PNG, in the display's current orientation.""" + if self.screen is not None: + return self.screen + width, height = self.size + if self.rotation % 2: + width, height = height, width + return png_bytes(width, height) + + def shell(self, command: str) -> Tuple[int, str]: + """Answer one ``adb shell`` command: ``(exit code, stdout)``.""" + self.shell_commands.append(command) + if self.gate is not None: + self.gate.wait(5) + for prefix, handler in self._handlers(): + if command.startswith(prefix): + return handler(command) + return 0, "" + + def _handlers(self) -> List[Tuple[str, Callable[[str], Tuple[int, str]]]]: + return [ + ("settings get secure default_input_method", lambda _c: (0, self.ime + "\n")), + ("wm size", lambda _c: (0, f"Physical size: {self.size[0]}x{self.size[1]}\n")), + ("dumpsys input", lambda _c: (0, f" SurfaceOrientation: {self.rotation}\n")), + ("getprop ro.build.version.release", lambda _c: (0, "14\n")), + ("am broadcast -a ADB_INPUT_B64", self._ime_broadcast), + ("input text ", self._input_text), + ("input draganddrop ", self._draganddrop), + ("monkey -p ", self._launch), + ("am force-stop ", self._stop), + ("pidof ", self._pidof), + ("dumpsys activity activities", self._resumed), + ("pkill -INT screenrecord", self._stop_recording), + ("rm ", lambda _c: (0, "")), + ] + + def _ime_broadcast(self, command: str) -> Tuple[int, str]: + # The reply is the same whether or not an IME received it, as on a device. + if self.ime == ADB_KEYBOARD_IME: + payload = shlex.split(command)[-1] + self.typed.append(base64.b64decode(payload).decode("utf-8")) + return 0, "Broadcasting: Intent { act=ADB_INPUT_B64 }\nBroadcast completed: result=0\n" + + def _input_text(self, command: str) -> Tuple[int, str]: + # What ``input text`` does: %s becomes a space, non-ASCII is dropped, exit 0. + raw = shlex.split(command)[2].replace("%s", " ") + self.typed.append("".join(char for char in raw if " " <= char <= "~")) + return 0, "" + + def _draganddrop(self, _command: str) -> Tuple[int, str]: + if self.has_draganddrop: + return 0, "" + return 0, "Error: Unknown command: draganddrop\nUsage: input [] \n" + + def _launch(self, command: str) -> Tuple[int, str]: + package = shlex.split(command)[2] + if package not in self.installed: + return 0, "** No activities found to run, monkey aborted.\n" + self.running.add(package) + self.foreground = package + return 0, "Events injected: 1\n" + + def _stop(self, command: str) -> Tuple[int, str]: + package = shlex.split(command)[-1] + self.running.discard(package) + if getattr(self, "foreground", "") == package: + self.foreground = "" + return 0, "" + + def _pidof(self, command: str) -> Tuple[int, str]: + package = shlex.split(command)[-1] + return (0, "4242\n") if package in self.running else (1, "") + + def _resumed(self, _command: str) -> Tuple[int, str]: + package = getattr(self, "foreground", "") + if not package: + return 0, "" + return 0, f" topResumedActivity=ActivityRecord{{1 u0 {package}/.Main t7}}\n" + + def _stop_recording(self, _command: str) -> Tuple[int, str]: + self.recording = False + return 0, "" + + +class FakeAdbHost: + """Replacement for ``subprocess.run`` in ``android.adb_client``.""" + + def __init__(self) -> None: + self.devices: Dict[str, FakeAndroidDevice] = {} + self.calls: List[Tuple[Optional[str], List[str]]] = [] + self.host_files: Dict[str, bytes] = {} + self._lock = threading.Lock() + + def add(self, serial: str, **kwargs: Any) -> FakeAndroidDevice: + """Attach a device and return it.""" + device = FakeAndroidDevice(serial, **kwargs) + self.devices[serial] = device + return device + + def calls_for(self, serial: str) -> List[List[str]]: + """The adb argv (after ``-s serial``) of every call addressed to ``serial``.""" + return [args for target, args in self.calls if target == serial] + + def run(self, cmd: List[str], **_kwargs: Any) -> subprocess.CompletedProcess: + """Answer one adb invocation.""" + serial, args = self._split(cmd) + with self._lock: + self.calls.append((serial, list(args))) + if args[:1] == ["devices"]: + return self._done(0, self._listing()) + if args[:1] == ["version"]: + return self._done(0, "Android Debug Bridge version 1.0.41\nVersion 35.0.2-12147458\n") + device = self._target(serial) + if isinstance(device, subprocess.CompletedProcess): + return device + return self._dispatch(device, args) + + def _dispatch(self, device: FakeAndroidDevice, + args: List[str]) -> subprocess.CompletedProcess: + if args[0] == "shell": + code, out = device.shell(args[1]) + return self._done(code, out) + if args[:2] == ["exec-out", "screencap"]: + return self._done(0, device.screencap()) + if args[0] == "install": + device.installed.add(args[-1]) + return self._done(0, "Performing Streamed Install\nSuccess\n") + if args[0] == "push": + device.files[args[2]] = self.host_files.get(args[1], b"") + return self._done(0, "1 file pushed\n") + if args[0] == "pull": + if args[1] not in device.files: + return self._done(1, "", f"adb: error: failed to stat remote object '{args[1]}'") + self.host_files[args[2]] = device.files[args[1]] + return self._done(0, "1 file pulled\n") + return self._done(0, "") + + def _target(self, serial: Optional[str]) -> Any: + if serial is None: + ready = list(self.devices.values()) + if len(ready) != 1: + return self._done(1, "", "adb: more than one device/emulator") + return ready[0] + device = self.devices.get(serial) + if device is None: + return self._done(1, "", f"adb: device '{serial}' not found") + if device.state == "unauthorized": + return self._done( + 1, "", "adb: device unauthorized.\nThis adb server's $ADB_VENDOR_KEYS is not set") + if device.state != "device": + return self._done(1, "", f"adb: device {device.state}") + return device + + def _listing(self) -> str: + lines = ["List of devices attached"] + lines.extend(f"{d.serial}\t{d.state} product:fake model:Fake_{d.serial} transport_id:1" + for d in self.devices.values()) + return "\n".join(lines) + "\n" + + @staticmethod + def _split(cmd: List[str]) -> Tuple[Optional[str], List[str]]: + rest = list(cmd[1:]) + if rest[:1] == ["-s"]: + return rest[1], rest[2:] + return None, rest + + @staticmethod + def _done(code: int, out: Any, err: str = "") -> subprocess.CompletedProcess: + stdout = out if isinstance(out, bytes) else str(out).encode("utf-8") + # nosemgrep # reason: a CompletedProcess value object, no process is started + return subprocess.CompletedProcess(args=[], returncode=code, stdout=stdout, + stderr=err.encode("utf-8")) + + +class FakeU2Object: + """What ``device(**selector)`` returns.""" + + def __init__(self, owner: "FakeU2", selector: Dict[str, Any]) -> None: + self._owner = owner + self._selector = selector + + @property + def exists(self) -> bool: + return self._selector.get("resourceId") in self._owner.buttons + + def click(self) -> None: + self._owner.record("click", **self._selector) + self._owner.buttons.discard(self._selector.get("resourceId")) + + def gesture(self, start1: Any, start2: Any, end1: Any, end2: Any, + steps: int = 100) -> None: + self._owner.record("gesture", start1=start1, start2=start2, end1=end1, + end2=end2, steps=steps) + + +class FakeU2: + """Replacement for ``uiautomator2.Device``.""" + + def __init__(self) -> None: + self.calls: List[Dict[str, Any]] = [] + self.typed: List[str] = [] + self.buttons: set = set() + self.clipboard = "" + + def record(self, op: str, **details: Any) -> None: + """Note one call.""" + self.calls.append({"op": op, **details}) + + def __call__(self, **selector: Any) -> FakeU2Object: + return FakeU2Object(self, selector) + + def send_keys(self, text: str, clear: bool = False) -> None: + self.record("send_keys", clear=clear) + self.typed.append(text) + + def drag(self, sx: int, sy: int, ex: int, ey: int, duration: float = 0.5) -> None: + self.record("drag", sx=sx, sy=sy, ex=ex, ey=ey, duration=duration) + + def set_clipboard(self, text: str, label: Optional[str] = None) -> None: + self.record("set_clipboard", label=label) + self.clipboard = text + + +class FakeWdaAlert: + """The ``client.alert`` object.""" + + def __init__(self, owner: "FakeWda") -> None: + self._owner = owner + + @property + def exists(self) -> bool: + return self._owner.alert_text is not None + + @property + def text(self) -> str: + return self._owner.alert_text or "" + + def accept(self) -> None: + self._owner.record("alert.accept") + self._owner.alert_text = None + + def dismiss(self) -> None: + self._owner.record("alert.dismiss") + self._owner.alert_text = None + + +class FakeWdaElement: + """What a selector resolves to.""" + + def __init__(self, owner: "FakeWda") -> None: + self._owner = owner + + def pinch(self, scale: float, velocity: float) -> None: + self._owner.record("pinch", scale=scale, velocity=velocity) + + +class FakeWda: + """Replacement for ``wda.Client``: records calls, keeps a little state.""" + + def __init__(self, *, points: Tuple[int, int] = (390, 844), scale: int = 3, + orientation: str = "PORTRAIT") -> None: + self.points = points + self.scale = scale + self.orientation = orientation + self.calls: List[Dict[str, Any]] = [] + self.typed: List[str] = [] + self.app_states: Dict[str, int] = {} + self.alert_text: Optional[str] = None + self.clipboard = "" + self.screen: Optional[bytes] = None + self.gate: Optional[threading.Event] = None + self.alert = FakeWdaAlert(self) + + def record(self, op: str, **details: Any) -> None: + """Note one call.""" + self.calls.append({"op": op, **details}) + + @property + def input_calls(self) -> List[Dict[str, Any]]: + """Every call that would have acted on the device.""" + passive = {"status", "window_size", "screenshot", "orientation", "app_state", "select"} + return [call for call in self.calls if call["op"] not in passive] + + def __call__(self, **selectors: Any) -> FakeWdaElement: + self.record("select", **selectors) + return FakeWdaElement(self) + + def status(self) -> Dict[str, Any]: + self.record("status") + return {"build": {"productBundleIdentifier": "com.facebook.WebDriverAgentRunner", + "version": "8.5.2"}, + "os": {"name": "iOS", "version": "17.4"}, "ready": True} + + def window_size(self) -> Tuple[int, int]: + self.record("window_size") + width, height = self.points + if self.orientation != "PORTRAIT": + width, height = height, width + return width, height + + def screenshot(self, *args: Any, **kwargs: Any) -> Any: + self.record("screenshot") + from PIL import Image + if self.screen is not None: + image = Image.open(io.BytesIO(self.screen)) + else: + width, height = self.window_size() + image = Image.new("RGB", (width * self.scale, height * self.scale)) + if args and isinstance(args[0], str): + image.save(args[0], format="PNG") + return image + + def tap(self, x: int, y: int) -> None: + if self.gate is not None: + self.gate.wait(5) + self.record("tap", x=x, y=y) + + def tap_hold(self, x: int, y: int, duration: float) -> None: + self.record("tap_hold", x=x, y=y, duration=duration) + + def swipe(self, x1: int, y1: int, x2: int, y2: int, duration: float) -> None: + self.record("swipe", x1=x1, y1=y1, x2=x2, y2=y2, duration=duration) + + def send_keys(self, text: str) -> None: + self.record("send_keys", text=text) + self.typed.append(text) + + def press(self, name: str) -> None: + self.record("press", name=name) + + def app_launch(self, bundle_id: str, **_kwargs: Any) -> None: + self.record("app_launch", bundle_id=bundle_id) + self.app_states[bundle_id] = 4 + + def app_terminate(self, bundle_id: str) -> None: + self.record("app_terminate", bundle_id=bundle_id) + self.app_states[bundle_id] = 1 + + def app_state(self, bundle_id: str) -> Dict[str, Any]: + self.record("app_state", bundle_id=bundle_id) + return {"value": self.app_states.get(bundle_id, 1)} + + def set_clipboard(self, content: str, *_args: Any) -> None: + self.record("set_clipboard") + self.clipboard = content diff --git a/test/unit_test/headless/test_mobile_context_isolation.py b/test/unit_test/headless/test_mobile_context_isolation.py new file mode 100644 index 000000000..54fb16f9d --- /dev/null +++ b/test/unit_test/headless/test_mobile_context_isolation.py @@ -0,0 +1,208 @@ +"""Device contexts: two devices in one process never reach each other. + +Driven against the fake ADB host and fake WebDriverAgent client in +``_mobile_doubles`` — no device is involved, so these prove the routing and +the session state machine, not that a phone accepts the commands. +""" +import threading + +import pytest + +from headless._mobile_doubles import FakeAdbHost, FakeWda +from je_auto_control.android import AdbError, client as android_client +from je_auto_control.ios import IOSDevice, client as ios_client +from je_auto_control.utils.device_matrix import run_on_devices +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.wrapper.device_context import ( + DeviceCancelledError, DeviceClosedError, DeviceContext, DeviceError, + DevicePermissionError, DeviceTimeoutError, DeviceUnavailableError, + bound_session, open_device, use_device, +) + + +@pytest.fixture +def adb_host(monkeypatch): + """A fake ADB host with two attached devices, ``left`` and ``right``.""" + host = FakeAdbHost() + host.add("left") + host.add("right") + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", host.run) + monkeypatch.setattr("je_auto_control.android.adb_client.shutil.which", + lambda _name: "fake-adb") + from je_auto_control.utils.executor import action_executor + monkeypatch.setattr(action_executor, "_android_client_cache", {}) + return host + + +def _android(serial: str, **kwargs) -> DeviceContext: + return DeviceContext("android", serial, adb_path="fake-adb", **kwargs) + + +def test_parallel_devices_do_not_change_default(adb_host): + android_client.reset_default_ui_device() + ios_client.reset_default_ios_device() + from je_auto_control.utils.executor import action_executor + # No serial in the action: each worker must reach its own device anyway. + report = run_on_devices( + [["AC_android_tap", {"x": 11, "y": 22}]], + [{"platform": "android", "serial": "left", "adb_path": "fake-adb"}, + {"platform": "android", "serial": "right", "adb_path": "fake-adb"}], + max_parallel=2, + ) + assert report.passed == 2, report.to_dict() + left_calls = adb_host.devices["left"] + assert left_calls.device_id == "left" + assert adb_host.calls_for("left") == [["shell", "input tap 11 22"]] + assert adb_host.calls_for("right") == [["shell", "input tap 11 22"]] + # Nothing went to "whichever device adb picks", and no default was created. + assert adb_host.calls_for(None) == [] + assert android_client._DEFAULT_DEVICE is None + assert ios_client._DEFAULT_DEVICE is None + assert action_executor._android_client_cache == {} + assert bound_session("android") is None + + +def test_explicit_serial_beats_the_bound_device(adb_host): + from je_auto_control.utils.executor.action_executor import executor + with open_device(_android("left")) as left, use_device(left): + executor.event_dict["AC_android_tap"](1, 2, serial="right", adb_path="fake-adb") + executor.event_dict["AC_android_tap"](3, 4) + assert adb_host.calls_for("right") == [["shell", "input tap 1 2"]] + assert adb_host.calls_for("left") == [["shell", "input tap 3 4"]] + + +def test_session_cancel_only_target(adb_host): + left_session = open_device(_android("left")) + right_session = open_device(_android("right")) + left_session.cancel() + assert left_session.connected is False + assert right_session.connected is True + with pytest.raises(DeviceCancelledError): + left_session.invoke("tap", lambda: left_session.adb.tap(1, 1)) + right_session.invoke("tap", lambda: right_session.adb.tap(5, 6)) + assert adb_host.devices["left"].input_calls == [] + assert adb_host.devices["right"].input_calls == ["input tap 5 6"] + + +def test_cancel_releases_a_caller_waiting_on_the_device(adb_host): + gate = threading.Event() + adb_host.devices["left"].gate = gate + session = open_device(_android("left")) + raised = [] + + def blocked() -> None: + try: + session.invoke("tap", lambda: session.adb.tap(1, 1)) + except DeviceError as error: + raised.append(error) + + worker = threading.Thread(target=blocked) + worker.start() + session.cancel() + worker.join(2) + gate.set() + assert not worker.is_alive() + assert isinstance(raised[0], DeviceCancelledError) + + +def test_timeout_closes_the_session(adb_host): + gate = threading.Event() + adb_host.devices["left"].gate = gate + session = open_device(_android("left", timeout_s=0.1)) + with pytest.raises(DeviceTimeoutError): + session.invoke("tap", lambda: session.adb.tap(1, 1)) + gate.set() + # The tap's outcome is unknown, so the next step must not be sent on top. + assert session.connected is False + with pytest.raises(DeviceClosedError): + session.invoke("tap", lambda: session.adb.tap(2, 2)) + + +def test_device_capability_probe_has_no_input(adb_host): + session = open_device(_android("left")) + capabilities = session.capabilities() + probe_input_calls = adb_host.devices["left"].input_calls + assert probe_input_calls == [] + assert capabilities["input"].state == "available" + assert capabilities["screenshot"].available + assert capabilities["ui_tree"].state == "needs_dependency" + assert "uiautomator2" in capabilities["ui_tree"].reason + + +def test_ios_capability_probe_has_no_input(): + handle = FakeWda() + session = open_device(DeviceContext("ios", "http://wda.test:8100"), + device=IOSDevice(handle=handle)) + capabilities = session.capabilities() + assert handle.input_calls == [] + assert capabilities["input"].state == "available" + + +def test_unauthorized_device_needs_permission(adb_host): + adb_host.add("locked", state="unauthorized") + session = open_device(_android("locked")) + capability = session.capabilities()["input"] + assert capability.state == "needs_permission" + assert "USB debugging" in capability.reason + with pytest.raises(DevicePermissionError) as caught: + session.adb.tap(1, 1) + # Still the type existing callers catch. + assert isinstance(caught.value, AdbError) + assert adb_host.devices["locked"].input_calls == [] + + +def test_missing_device_and_missing_adb_are_reported(adb_host, monkeypatch): + gone = open_device(_android("gone")).capabilities()["input"] + assert gone.state == "needs_dependency" + assert "gone" in gone.reason + monkeypatch.setattr("je_auto_control.android.adb_client.shutil.which", lambda _name: None) + no_adb = open_device(DeviceContext("android", "left")).capabilities()["input"] + assert no_adb.state == "needs_dependency" + assert "platform-tools" in no_adb.reason + + +def test_adb_timeout_is_a_framework_timeout(monkeypatch): + import subprocess + + def slow(cmd, **kwargs): + raise subprocess.TimeoutExpired(cmd, kwargs.get("timeout")) + + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", slow) + session = open_device(_android("left")) + with pytest.raises(DeviceTimeoutError) as caught: + session.adb.tap(1, 1) + assert isinstance(caught.value, AdbError) + assert isinstance(caught.value, AutoControlException) + + +def test_every_backend_error_is_a_device_error(): + from je_auto_control.android import AdbNotAvailable, UIAutomatorUnavailableError + from je_auto_control.ios import IOSUnavailableError + for error in (AdbNotAvailable, UIAutomatorUnavailableError, IOSUnavailableError): + assert issubclass(error, DeviceUnavailableError) + assert issubclass(error, RuntimeError) + assert issubclass(AdbError, DeviceError) + + +def test_session_owns_only_the_transport_it_built(): + handle = FakeWda() + device = IOSDevice(handle=handle) + session = open_device(DeviceContext("ios", "http://wda.test:8100"), device=device) + session.close() + session.close() + assert session.connected is False + # A transport the caller handed in is the caller's to keep using. + assert device.handle is handle + + +def test_context_is_frozen_and_validated(): + context = DeviceContext("ios", "http://wda.test:8100") + with pytest.raises(AttributeError): + context.device_id = "other" + with pytest.raises(DeviceError): + DeviceContext("windows", "x") + with pytest.raises(DeviceError): + DeviceContext("android", "x", timeout_s=0) + assert DeviceContext.from_spec({"platform": "desktop"}) is None + spec = DeviceContext.from_spec({"platform": "ios", "url": "http://a:8100"}) + assert spec == DeviceContext("ios", "http://a:8100") From beddc5f29313a3ef48afe3ca46c8004859b8716e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:44:34 +0800 Subject: [PATCH 07/44] Free the USB prompt dialog by reference count so a collection cannot destroy it mid-scan The Python 3.10 Linux and macOS squares died with SIGSEGV inside the prompt tests' dialog driver, on QApplication.topLevelWidgets(). The fault dumps show the worker parked in Event.wait and the GUI thread inside the dialog's modal loop, so the hand-off was not at fault. TranslatableMixin._tr(self, ...) stored the widget in its own registry, which made every prompt dialog a reference cycle: dropped, it stayed a live parentless window until the cycle collector ran. On 3.10/3.11 the collector runs inside any allocation, including the wrapper PySide allocates while converting the returned widget list, and it then destroys widgets whose pointers are still in that list. The registry now records the widget itself as None, the prompt tests ask Qt for the active modal widget instead of scanning every window, and CI sets PYTHONFAULTHANDLER so a crash outside pytest's own window still leaves a dump. --- .github/workflows/quality.yml | 10 + je_auto_control/gui/_i18n_helpers.py | 17 +- je_auto_control/gui/usb_passthrough_prompt.py | 19 +- .../unit_test/headless/test_usb_acl_prompt.py | 52 +++-- .../headless/test_usb_prompt_lifetime.py | 187 ++++++++++++++++++ 5 files changed, 255 insertions(+), 30 deletions(-) create mode 100644 test/unit_test/headless/test_usb_prompt_lifetime.py diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index b994e1950..c2b1231f3 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -162,8 +162,18 @@ jobs: # worst-hit files were the biggest ones (`action_executor` +786, # `_handlers` +684). A package that registers a pytest plugin cannot # measure itself with `pytest --cov`; `coverage run` starts first. + # + # PYTHONFAULTHANDLER: pytest's own faulthandler covers only the window + # between its configure and unconfigure, and a native crash here (Qt, + # aiortc, a platform backend) is as likely while plugins load or the + # interpreter shuts down. With the variable the dump is installed before + # `coverage` starts, pytest leaves it in place, and the step log — the + # only evidence a SIGSEGV leaves — names the Python line every thread + # was on. - name: Run headless pytest suite under coverage shell: bash + env: + PYTHONFAULTHANDLER: "1" run: >- ${{ runner.os == 'Linux' && 'xvfb-run -a -s "-screen 0 1280x800x24"' || '' }} python -m coverage run -m pytest -v --tb=short --timeout=120 diff --git a/je_auto_control/gui/_i18n_helpers.py b/je_auto_control/gui/_i18n_helpers.py index 081f2b00f..9681f8510 100644 --- a/je_auto_control/gui/_i18n_helpers.py +++ b/je_auto_control/gui/_i18n_helpers.py @@ -5,8 +5,15 @@ ``self.retranslate()`` re-pulls every key from the language wrapper and re-applies it through the recorded setter. Destroyed widgets are skipped silently so removing a row never breaks a later language switch. + +A widget that registers *itself* (``self._tr(self, "title_key", +setter="setWindowTitle")``) is recorded as ``None``, not as a reference. Holding +it made the widget a reference cycle, so dropping one never freed it: it stayed +a live window until the cycle collector ran, and the collector destroyed the +C++ object wherever it happened to be — on Python 3.10/3.11 that can be in the +middle of a PySide call that is still walking a list of widgets. """ -from typing import List, Tuple +from typing import List, Optional, Tuple from PySide6.QtWidgets import ( QAbstractButton, QGroupBox, QLabel, QLineEdit, QTabWidget, QWidget, @@ -31,7 +38,7 @@ class TranslatableMixin: """Provides ``_tr(...)`` / ``retranslate()`` for a widget-building class.""" def _tr_init(self) -> None: - self._tr_registry: List[Tuple[QWidget, str, str]] = [] + self._tr_registry: List[Tuple[Optional[QWidget], str, str]] = [] self._tr_tabs: List[Tuple[QTabWidget, int, str]] = [] def _tr(self, widget: QWidget, key: str, setter: str = "") -> QWidget: @@ -41,7 +48,8 @@ def _tr(self, widget: QWidget, key: str, setter: str = "") -> QWidget: resolved = setter or _default_setter(widget) translated = language_wrapper.translate(key, key) getattr(widget, resolved)(translated) - self._tr_registry.append((widget, key, resolved)) + # None stands for ``self``: see the module docstring. + self._tr_registry.append((None if widget is self else widget, key, resolved)) return widget def _tr_tab(self, tab_widget: QTabWidget, index: int, key: str) -> None: @@ -54,8 +62,9 @@ def _tr_tab(self, tab_widget: QTabWidget, index: int, key: str) -> None: def retranslate(self) -> None: """Re-apply every registered translation key.""" for widget, key, setter in getattr(self, "_tr_registry", []): + target = self if widget is None else widget try: - getattr(widget, setter)(language_wrapper.translate(key, key)) + getattr(target, setter)(language_wrapper.translate(key, key)) except RuntimeError: # Widget destroyed; leave it for a future cleanup pass. continue diff --git a/je_auto_control/gui/usb_passthrough_prompt.py b/je_auto_control/gui/usb_passthrough_prompt.py index c0a2ff372..2bbc2c236 100644 --- a/je_auto_control/gui/usb_passthrough_prompt.py +++ b/je_auto_control/gui/usb_passthrough_prompt.py @@ -10,9 +10,14 @@ synchronous — it must return ``True`` / ``False`` from a non-GUI thread (the callback runs on the WebRTC/asyncio bridge thread, not the Qt main thread). :class:`PromptBridge` does the cross-thread -marshalling: the worker thread calls ``decide()``, which posts a -``QMetaObject.invokeMethod`` to the GUI thread, waits on a -``threading.Event``, and returns the operator's verdict. +marshalling: the worker thread calls ``decide()``, which emits a queued +signal to the GUI thread, waits on a ``threading.Event``, and returns the +operator's verdict. + +The dialog is built, shown and destroyed on the GUI thread, and it must not be +a reference cycle: its last reference goes when the slot returns, so the C++ +window dies there and then. A dialog left to the cycle collector is destroyed +wherever the collector runs — another thread, or the middle of a PySide call. """ from __future__ import annotations @@ -126,11 +131,9 @@ def decide(self, vendor_id: str, product_id: str, ) if not done.wait(timeout=wait_timeout_s): return False - # Sonar can't see through the cross-thread QMetaObject - # .invokeMethod + queued slot above: ``result`` is mutated by - # ``_show_dialog`` on the GUI thread before ``done`` is set, - # so neither key is guaranteed False at this point. - # (Cross-thread mutation through Q_ARG(object, result).) + # Sonar can't see through the queued signal above: ``result`` is + # mutated by ``_show_dialog`` on the GUI thread before ``done`` is + # set, so neither key is guaranteed False at this point. if result["allow"] and result["remember"] and self._acl is not None: # NOSONAR self._acl.add_rule(AclRule( vendor_id=vendor_id, product_id=product_id, diff --git a/test/unit_test/headless/test_usb_acl_prompt.py b/test/unit_test/headless/test_usb_acl_prompt.py index bf25b8c0a..661e72871 100644 --- a/test/unit_test/headless/test_usb_acl_prompt.py +++ b/test/unit_test/headless/test_usb_acl_prompt.py @@ -73,13 +73,30 @@ def test_dialog_remember_reflects_checkbox(qapp): # outlives the test that started it: the qapp fixture below reuses # QApplication.instance(), which in a full-suite run was created by an earlier # module and lives for the whole session. An orphaned chain therefore keeps -# scanning topLevelWidgets() indefinitely and can press accept()/reject() on a +# looking for a prompt indefinitely and can press accept()/reject() on a # later test's prompt. 2s of retries comfortably covers the 3s decide() # timeout these tests use. _DIALOG_RETRY_INTERVAL_MS = 20 _DIALOG_MAX_ATTEMPTS = 100 +def _open_prompt(): + """The prompt whose modal loop is running, or None. + + Asked of Qt as one pointer (``activeModalWidget``), never by scanning + ``QApplication.topLevelWidgets()``: that scan is what took the Python 3.10 + Linux and macOS squares down with SIGSEGV. PySide converts the returned + ``QList`` element by element, allocating a wrapper for any window + Python has not seen; on 3.10/3.11 an allocation can run the cycle collector + on the spot, the collector destroys whatever window garbage the suite has + left, and the next pointer in the list is a freed widget. + """ + widget = QApplication.activeModalWidget() + if isinstance(widget, UsbPassthroughPromptDialog) and widget.isVisible(): + return widget + return None + + def _drive_dialog_when_visible(action: str) -> None: """Schedule a bounded Qt timer chain that finds the modal dialog and presses Allow / Deny / cancel on it. @@ -87,18 +104,18 @@ def _drive_dialog_when_visible(action: str) -> None: remaining = {"attempts": _DIALOG_MAX_ATTEMPTS} def attempt(): - for widget in QApplication.topLevelWidgets(): - if isinstance(widget, UsbPassthroughPromptDialog) and widget.isVisible(): - if action == "allow": - widget.accept() - elif action == "deny": - widget.reject() - elif action == "remember-allow": - widget._remember_check.setChecked(True) - widget.accept() - else: - widget.reject() - return + widget = _open_prompt() + if widget is not None: + if action == "allow": + widget.accept() + elif action == "deny": + widget.reject() + elif action == "remember-allow": + widget._remember_check.setChecked(True) + widget.accept() + else: + widget.reject() + return # Try again shortly if the dialog hasn't appeared yet — but give up # rather than outlive this test. remaining["attempts"] -= 1 @@ -114,11 +131,10 @@ def _close_dialog_after(ms: int) -> None: remaining = {"attempts": _DIALOG_MAX_ATTEMPTS} def shut(): - for widget in QApplication.topLevelWidgets(): - if (isinstance(widget, UsbPassthroughPromptDialog) - and widget.isVisible()): - widget.reject() - return + widget = _open_prompt() + if widget is not None: + widget.reject() + return remaining["attempts"] -= 1 if remaining["attempts"] > 0: QTimer.singleShot(_DIALOG_RETRY_INTERVAL_MS, shut) diff --git a/test/unit_test/headless/test_usb_prompt_lifetime.py b/test/unit_test/headless/test_usb_prompt_lifetime.py new file mode 100644 index 000000000..49bbfede2 --- /dev/null +++ b/test/unit_test/headless/test_usb_prompt_lifetime.py @@ -0,0 +1,187 @@ +"""The USB prompt dialog must die by reference count, never by the cycle collector. + +``TranslatableMixin._tr(self, ...)`` stored the widget in its own registry, so +every ``UsbPassthroughPromptDialog`` was a reference cycle: dropped, it stayed +alive as garbage — a parentless top-level window Python still owned — until the +cycle collector happened to run, and the collector then ran the C++ destructor +wherever it was standing. + +On Python 3.10 and 3.11 the collector runs *inside* any allocation, including +the ones PySide makes while it converts the ``QList`` returned by +``QApplication.topLevelWidgets()`` into a Python list. A collection there +destroys widgets whose pointers are still in the list being converted, and the +next element is a freed object: SIGSEGV with no Python-level cause. That killed +the 3.10 Linux and macOS squares inside ``test_usb_acl_prompt.py``'s dialog +driver, which scanned ``topLevelWidgets()`` from a timer. (3.12+ only collects +between bytecodes, so the same garbage is harmless there.) +""" +import gc +import os +import subprocess # nosec B404 # reason: runs this test's own probe script +import sys +import textwrap +import threading +import time +import weakref +from pathlib import Path + +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) +# gui/__init__.py loads main_window -> webrtc_panel -> aiortc. +pytest.importorskip("av") +pytest.importorskip("aiortc") + +from PySide6.QtWidgets import QApplication, QDialog, QWidget # noqa: E402 + +from je_auto_control.gui._i18n_helpers import TranslatableMixin # noqa: E402 +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( # noqa: E402 + language_wrapper, +) +from je_auto_control.gui.usb_passthrough_prompt import ( # noqa: E402 + PromptBridge, UsbPassthroughPromptDialog, +) +from je_auto_control.utils.usb.passthrough import UsbAcl # noqa: E402 + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_QUALITY_YML = _REPO_ROOT / ".github" / "workflows" / "quality.yml" + + +@pytest.fixture(scope="module") +def qapp(): + app = QApplication.instance() or QApplication([]) + yield app + + +@pytest.fixture +def no_cycle_collector(): + """Reference counting only, so 'it was freed' cannot mean 'the collector got to it'.""" + was_enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + if was_enabled: + gc.enable() + + +class _Titled(TranslatableMixin, QWidget): + """The smallest widget that registers itself for retranslation.""" + + def __init__(self): + super().__init__() + self._tr_init() + self._tr(self, "usb_prompt_title", setter="setWindowTitle") + + +def test_registering_itself_does_not_make_a_widget_a_cycle(qapp, no_cycle_collector): + widget = _Titled() + alive = weakref.ref(widget) + del widget + assert alive() is None + + +def test_a_widget_that_registered_itself_is_still_retranslated(qapp, monkeypatch): + widget = _Titled() + monkeypatch.setattr(language_wrapper, "translate", lambda key, default=None: f"<{key}>") + widget.retranslate() + assert widget.windowTitle() == "" + + +def test_a_dropped_prompt_dialog_is_freed_at_once(qapp, no_cycle_collector): + dialog = UsbPassthroughPromptDialog( + vendor_id="1050", product_id="0407", serial=None, viewer_id=None, + ) + alive = weakref.ref(dialog) + del dialog + assert alive() is None + + +_WORKERS = 4 +_DECISIONS_EACH = 25 + + +def _answer_by_product(seen): + """An ``exec`` that allows even product ids, remembers them, and records the dialog.""" + def answer(dialog): + seen.append(weakref.ref(dialog)) + allow = int(dialog._product_id, 16) % 2 == 0 # noqa: SLF001 + dialog._remember_check.setChecked(allow) # noqa: SLF001 + return QDialog.DialogCode.Accepted if allow else QDialog.DialogCode.Rejected + return answer + + +def test_many_workers_get_their_own_verdicts_and_leave_no_dialog( + qapp, monkeypatch, tmp_path, no_cycle_collector): + seen = [] + monkeypatch.setattr(UsbPassthroughPromptDialog, "exec", _answer_by_product(seen)) + acl = UsbAcl(path=tmp_path / "acl.json") + bridge = PromptBridge(acl=acl) + verdicts = {} + + def work(worker_index): + for step in range(_DECISIONS_EACH): + product = f"{worker_index * _DECISIONS_EACH + step:04x}" + verdicts[product] = bridge.decide( + "1050", product, None, viewer_id=f"vw-{worker_index}", wait_timeout_s=20.0, + ) + + workers = [threading.Thread(target=work, args=(index,), daemon=True) + for index in range(_WORKERS)] + for worker in workers: + worker.start() + deadline = time.monotonic() + 30.0 + while any(worker.is_alive() for worker in workers) and time.monotonic() < deadline: + qapp.processEvents() + workers[0].join(0.005) + assert not any(worker.is_alive() for worker in workers), "a decision never came back" + + total = _WORKERS * _DECISIONS_EACH + assert verdicts == {f"{n:04x}": n % 2 == 0 for n in range(total)} + assert sorted(rule.product_id for rule in acl.list_rules()) == \ + [f"{n:04x}" for n in range(0, total, 2)] + # No flush of deferred deletes and no collection has run: each dialog went + # when its slot returned, on the GUI thread, not later and not elsewhere. + assert len(seen) == total + assert [ref for ref in seen if ref() is not None] == [] + + +# What the 3.10 squares died of, made deterministic: prompt dialogs dropped the +# way a test or a caller drops them, a top-level window Python has never +# wrapped (so converting the list allocates), and a collection that is due. +_COLLECTION_PROBE = textwrap.dedent(""" + import faulthandler, gc, os + faulthandler.enable() + os.environ["QT_QPA_PLATFORM"] = "offscreen" + from PySide6.QtWidgets import QApplication, QComboBox + from je_auto_control.gui.usb_passthrough_prompt import UsbPassthroughPromptDialog + app = QApplication([]) + combos = [QComboBox() for _ in range(20)] + for combo in combos: + combo.view() # Qt builds the popup container; no Python wrapper exists + gc.collect() + gc.disable() + for _ in range(400): + UsbPassthroughPromptDialog(vendor_id="1050", product_id="0407", serial=None, viewer_id=None) + scan = QApplication.topLevelWidgets + gc.enable() + widgets = scan() + gc.disable() + print("prompts-left", sum(isinstance(w, UsbPassthroughPromptDialog) for w in widgets)) +""") + + +def test_a_due_collection_inside_a_widget_scan_finds_no_prompt_to_destroy(): + env = dict(os.environ, PYTHONPATH=str(_REPO_ROOT), QT_QPA_PLATFORM="offscreen") + argv = [sys.executable, "-c", _COLLECTION_PROBE] + done = subprocess.run(argv, capture_output=True, text=True, timeout=180, env=env, cwd=str(_REPO_ROOT), check=False) # nosec B603 # nosemgrep # reason: this test's own probe, fixed argv + assert done.returncode == 0, done.stderr[-2000:] + assert done.stdout.strip().splitlines()[-1] == "prompts-left 0" + + +def test_ci_keeps_the_fault_dump_outside_pytests_own_window(): + """pytest's faulthandler is off before collection and after unconfigure; the variable is not.""" + workflow = _QUALITY_YML.read_text(encoding="utf-8") + assert 'PYTHONFAULTHANDLER: "1"' in workflow From 7149a02ec160b545ef8cefac5e277dbccdfbba2b Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:44:45 +0800 Subject: [PATCH 08/44] Serialise changes to one UsbAcl so concurrent prompt approvals keep every rule PromptBridge.decide writes a remembered approval from whichever worker thread asked. Refresh, change and save were three separately locked steps, so two threads on one instance interleaved: one re-read the file between the other's data and signature writes and refused it as tampered, or reloaded over a rule that had been appended but not yet saved. Four threads adding 60 rules kept 43. One lock now spans the whole change; the prompt stress test in the previous commit depends on it. --- je_auto_control/utils/usb/passthrough/acl.py | 88 +++++++++++-------- test/unit_test/headless/test_usb_acl_audit.py | 27 +++++- 2 files changed, 75 insertions(+), 40 deletions(-) diff --git a/je_auto_control/utils/usb/passthrough/acl.py b/je_auto_control/utils/usb/passthrough/acl.py index 43863b3e6..5526745f6 100644 --- a/je_auto_control/utils/usb/passthrough/acl.py +++ b/je_auto_control/utils/usb/passthrough/acl.py @@ -179,6 +179,12 @@ def __init__(self, *, path: Optional[Path] = None, self._require_signature = bool(require_signature) self._integrity_ok = True self._lock = threading.Lock() + # Held across refresh -> change -> save. ``_lock`` only guards the + # state object, so two threads changing one instance interleaved: the + # second re-read the file between the first's data and signature + # writes (a "signature mismatch" on a file nobody tampered with), or + # reloaded over a rule the first had appended but not yet saved. + self._change_lock = threading.Lock() if default_policy not in _VALID_DEFAULTS: raise ValueError( f"default_policy must be one of {_VALID_DEFAULTS}", @@ -206,31 +212,33 @@ def list_rules(self) -> List[AclRule]: return list(self._state.rules) def add_rule(self, rule: AclRule, *, persist: bool = True) -> None: - if persist: - self._refresh() - with self._lock: - self._state.rules.append(rule) - if persist: - self._save() + with self._change_lock: + if persist: + self._refresh() + with self._lock: + self._state.rules.append(rule) + if persist: + self._save() def remove_rule(self, *, vendor_id: str, product_id: str, serial: Optional[str] = None, persist: bool = True) -> bool: - if persist: - self._refresh() - with self._lock: - # The same id comparison as matches(): a rule a device matches - # ("0x1050" for 1050) must also be removable by that id. - new_rules = [ - r for r in self._state.rules - if not (_same_id(r.vendor_id, vendor_id) - and _same_id(r.product_id, product_id) - and r.serial == serial) - ] - removed = len(new_rules) != len(self._state.rules) - self._state.rules = new_rules - if removed and persist: - self._save() + with self._change_lock: + if persist: + self._refresh() + with self._lock: + # The same id comparison as matches(): a rule a device matches + # ("0x1050" for 1050) must also be removable by that id. + new_rules = [ + r for r in self._state.rules + if not (_same_id(r.vendor_id, vendor_id) + and _same_id(r.product_id, product_id) + and r.serial == serial) + ] + removed = len(new_rules) != len(self._state.rules) + self._state.rules = new_rules + if removed and persist: + self._save() return removed def export_rules(self) -> dict: @@ -262,18 +270,19 @@ def import_rules(self, payload: dict, *, replace: bool = False, raise ValueError("'rules' must be a list") imported = [AclRule.from_dict(r) for r in raw_rules if isinstance(r, dict)] - if persist: - self._refresh() - with self._lock: - if replace: - default = str(payload.get("default", self._state.default)) - if default not in _VALID_DEFAULTS: - default = "deny" - self._state = _AclState(default=default, rules=list(imported)) - else: - self._state.rules.extend(imported) - if persist: - self._save() + with self._change_lock: + if persist: + self._refresh() + with self._lock: + if replace: + default = str(payload.get("default", self._state.default)) + if default not in _VALID_DEFAULTS: + default = "deny" + self._state = _AclState(default=default, rules=list(imported)) + else: + self._state.rules.extend(imported) + if persist: + self._save() return len(imported) def set_default_policy(self, policy: str, *, persist: bool = True) -> None: @@ -281,12 +290,13 @@ def set_default_policy(self, policy: str, *, persist: bool = True) -> None: raise ValueError( f"default_policy must be one of {_VALID_DEFAULTS}", ) - if persist: - self._refresh() - with self._lock: - self._state.default = policy - if persist: - self._save() + with self._change_lock: + if persist: + self._refresh() + with self._lock: + self._state.default = policy + if persist: + self._save() def decide(self, *, vendor_id: str, product_id: str, serial: Optional[str]) -> str: diff --git a/test/unit_test/headless/test_usb_acl_audit.py b/test/unit_test/headless/test_usb_acl_audit.py index be43a747a..3ac1edbe1 100644 --- a/test/unit_test/headless/test_usb_acl_audit.py +++ b/test/unit_test/headless/test_usb_acl_audit.py @@ -3,9 +3,11 @@ Deleting the ``.sig`` next to a rewritten file passed as a legacy unsigned ACL; a damaged file was replaced by the next save; malformed rule ids were stored and never matched; ``remove_rule`` compared ids differently from -``matches``; and two instances overwrote each other's rules. +``matches``; and two instances overwrote each other's rules. Found later, by +the prompt stress test: threads changing one instance lost each other's rules. """ import json +import threading import pytest @@ -69,6 +71,29 @@ def test_two_instances_keep_each_others_rules(tmp_path): assert sorted(rule["vendor_id"] for rule in saved["rules"]) == ["1111", "2222"] +def test_threads_changing_one_instance_keep_every_rule(tmp_path): + # Refresh, append and save were three separately locked steps: a thread + # re-read the file between another's data and signature writes ("signature + # mismatch", load refused) or reloaded over a rule not yet saved. + path = tmp_path / "usb_acl.json" + acl = UsbAcl(path=path) + + def add(worker): + for step in range(15): + acl.add_rule(AclRule(vendor_id="1050", product_id=f"{worker * 15 + step:04x}", allow=True)) + + workers = [threading.Thread(target=add, args=(index,)) for index in range(4)] + for worker in workers: + worker.start() + for worker in workers: + worker.join(30.0) + expected = [f"{n:04x}" for n in range(60)] + assert sorted(rule.product_id for rule in acl.list_rules()) == expected + assert acl.integrity_ok is True + reloaded = UsbAcl(path=path) + assert sorted(rule.product_id for rule in reloaded.list_rules()) == expected + + def test_a_usb_error_without_errno_is_eio(): from je_auto_control.utils.usbip.libusb_backend import _translate_error usb_error = type("USBError", (Exception,), {})("Other error") From 9c86e5ed21a1d5f21b7427ff4255f3eaa9d9c451 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:47:03 +0800 Subject: [PATCH 09/44] Merge synced entries by causality and queue offline changes durably The merge compared wall-clock stamps, so a fast clock won every disagreement and two machines editing one entry apart lost an edit without a trace; a change made offline lived only in memory; and a tombstone was dropped after thirty days whether or not every machine had seen it. Entries now carry a version vector and concurrent edits are kept as a conflict, pending operations sit in a SQLite outbox and are resent by operation id with bounded, cancellable back-off, and a tombstone is collected only once every active peer has acknowledged it, with a retired peer required to resync in full. --- .../Eng/doc/config_sync/config_sync_doc.rst | 79 ++- .../Eng/doc/new_features/v81_features_doc.rst | 4 +- .../Zh/doc/config_sync/config_sync_doc.rst | 65 ++- .../Zh/doc/new_features/v81_features_doc.rst | 2 +- je_auto_control/utils/config_sync/__init__.py | 74 ++- je_auto_control/utils/config_sync/client.py | 458 +++++++++++++++--- je_auto_control/utils/config_sync/outbox.py | 272 +++++++++++ je_auto_control/utils/config_sync/versions.py | 384 +++++++++++++++ .../headless/test_config_sync_convergence.py | 364 ++++++++++++++ .../headless/test_signaling_config_routes.py | 2 +- 10 files changed, 1617 insertions(+), 87 deletions(-) create mode 100644 je_auto_control/utils/config_sync/outbox.py create mode 100644 je_auto_control/utils/config_sync/versions.py create mode 100644 test/unit_test/headless/test_config_sync_convergence.py diff --git a/docs/source/Eng/doc/config_sync/config_sync_doc.rst b/docs/source/Eng/doc/config_sync/config_sync_doc.rst index 331a6f507..1e493e98e 100644 --- a/docs/source/Eng/doc/config_sync/config_sync_doc.rst +++ b/docs/source/Eng/doc/config_sync/config_sync_doc.rst @@ -44,7 +44,7 @@ Wire format (version 2) * - Request - Reply * - ``GET /config/{user_id}`` - - ``200`` with ``{"user_id", "sections", "revision": , "version": 2}``, + - ``200`` with ``{"user_id", "sections", "peers", "revision": , "version": 2}``, or ``404`` when the user has no bucket. * - ``PUT /config/{user_id}`` with ``{"version": 2, "base_revision": N, "operation_id": "...", "bucket": {...}}`` @@ -89,3 +89,80 @@ machine pushed in between it fetches and merges again, up to ``max_attempts`` ``bucket.revision`` as the base, returns the committed revision and raises ``ConfigSyncConflict`` (``.revision`` is the server's current one) instead of overwriting. + +Causal merge: no clock picks a winner +------------------------------------- + +An entry written with a device id is *versioned* +(``je_auto_control.utils.config_sync.versions.SyncEntry``): it carries a +version vector (per device, how many of that device's changes it includes), +the ``origin`` device and an ``operation_id``. ``merge_entries(left, right)`` +returns a ``MergeDecision``: + +* the entry made *knowing* the other supersedes it -- nothing is reported; +* two changes to the same key made apart are **both kept**: ``decision.conflict`` + is a ``SyncConflict(key, local, remote)`` and ``decision.entry`` holds the + candidates as ``siblings`` until someone calls + ``entry.resolved(value, origin)``. An edit made apart from a delete is a + conflict too; +* changes to different keys never conflict (``merge_collections``). + +``modified_at`` is carried for display only. Whatever the clocks of the +machines say, the decision is the same, and it is the same on every machine. + +.. code-block:: python + + bucket.upsert("hotkeys", "hk1", {"combo": "ctrl+a"}, origin="laptop") # versioned + bucket.remove("hotkeys", "hk1", origin="laptop") # versioned tombstone + bucket.values("hotkeys") # live values; entries still in conflict are left out + bucket.conflicts() # [(section, SyncEntry with .siblings), ...] + +``upsert`` / ``remove`` *without* ``origin`` still write the older flat entry +stamped with ``last_modified``, and two flat entries still merge by "later +wins" (reported in ``ConflictRecord``); a versioned entry supersedes a flat +copy of the same id. ``ConflictRecord.unresolved`` is true for a causal +conflict, where nothing was dropped. + +Deletions and retired devices +----------------------------- + +A deletion is a tombstone. A versioned tombstone is **never dropped by age**: +``collect_tombstones`` removes it only when every active device recorded +under the bucket's ``peers`` has acknowledged a revision that includes it. +``ConfigSyncClient.push_operations`` maintains those acknowledgements. + +A device that will not come back is retired with +``client.retire_peer(device_id)`` (or automatically with +``push_operations(..., max_offline_s=...)``, which compares ``last_seen`` +stamps -- the only use of a clock, and it never decides between two values). +Tombstones stop waiting for a retired device, so when it does return its +pushes are refused with ``FullResyncRequired``; it must call +``client.full_resync(device_id=...)``, replace its local state with the +returned bucket and discard its pending operations. Merging instead could +bring deleted entries back. + +Offline outbox +-------------- + +``SyncOutbox(db_path, account=..., endpoint=...)`` is a SQLite queue of +``SyncOperation(section, entry)`` not yet on the server (default file +``~/.je_auto_control/config_sync_outbox.sqlite3``; one file can hold several +accounts and endpoints, each isolated). + +.. code-block:: python + + from je_auto_control.utils.config_sync import SyncEntry, SyncOperation, SyncOutbox + + outbox = SyncOutbox(account="alice", endpoint="https://sync.example") + outbox.enqueue(SyncOperation("hotkeys", SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop"))) + report = outbox.drain( + lambda batch: client.push_operations(batch, device_id="laptop"), cancel=stop_event) + # DrainReport(sent, pending, attempts, offline, cancelled, error) + +Operations are queued under their operation id (queueing one twice is a +no-op), survive a restart, and are resent with the same ids -- applying a +batch the server already has changes nothing. A failed send backs off +2 s, 4 s, 8 s ... up to 300 s (``base_delay_s`` / ``max_delay_s``), a drain +makes at most ``max_attempts`` sends (default 5), ``wait=False`` returns +instead of sleeping through a back-off, and setting ``cancel`` ends a drain +even in the middle of a wait. diff --git a/docs/source/Eng/doc/new_features/v81_features_doc.rst b/docs/source/Eng/doc/new_features/v81_features_doc.rst index 92dcd9f9b..8a0eee52f 100644 --- a/docs/source/Eng/doc/new_features/v81_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v81_features_doc.rst @@ -2,8 +2,8 @@ Layered Configuration Resolver ============================== ``json_patch.merge_patch`` merges exactly two documents, ``config_sync`` -resolves by last-write-wins timestamp (a tie goes to a deletion, then to the -same entry on every client, and is reported as a conflict), and ``AssetStore`` is flat per +merges per entry by version vector (concurrent edits of one entry are kept as +a conflict; see :doc:`../config_sync/config_sync_doc`), and ``AssetStore`` is flat per environment. None of them compose an ordered ``defaults < file < env < CLI`` precedence stack with a deep dict merge, nor report *which layer won each key*. This adds that 12-factor resolver. diff --git a/docs/source/Zh/doc/config_sync/config_sync_doc.rst b/docs/source/Zh/doc/config_sync/config_sync_doc.rst index 9308eb001..86c355471 100644 --- a/docs/source/Zh/doc/config_sync/config_sync_doc.rst +++ b/docs/source/Zh/doc/config_sync/config_sync_doc.rst @@ -39,7 +39,7 @@ server 重啟後仍在。檔案位置依序取自 ``--config-db PATH``、環境 * - 請求 - 回覆 * - ``GET /config/{user_id}`` - - ``200`` 與 ``{"user_id", "sections", "revision": <已提交>, "version": 2}``, + - ``200`` 與 ``{"user_id", "sections", "peers", "revision": <已提交>, "version": 2}``, 使用者沒有 bucket 時為 ``404``。 * - ``PUT /config/{user_id}``,body 為 ``{"version": 2, "base_revision": N, "operation_id": "...", "bucket": {...}}`` @@ -78,3 +78,66 @@ Client 它會重新 fetch 與 merge,最多 ``max_attempts``(預設 4)次,之後丟出 ``ConfigSyncConflict``。 ``push(bucket)`` 以 ``bucket.revision`` 為基準,回傳已提交的 revision;落後時丟出 ``ConfigSyncConflict``(``.revision`` 為 server 目前的 revision)而不是覆寫。 + +因果合併:時鐘不決定勝負 +------------------------ + +帶裝置 id 寫入的 entry 是 *versioned* +(``je_auto_control.utils.config_sync.versions.SyncEntry``):它帶有 version vector +(每台裝置的變更有幾次已包含在內)、``origin`` 裝置與 ``operation_id``。 +``merge_entries(left, right)`` 回傳 ``MergeDecision``: + +* *知道* 對方而做出的那一筆取代對方 —— 不回報任何東西; +* 分開進行、針對同一 key 的兩個變更 **兩份都保留**:``decision.conflict`` 是 + ``SyncConflict(key, local, remote)``,``decision.entry`` 以 ``siblings`` 保存候選, + 直到有人呼叫 ``entry.resolved(value, origin)``。與刪除並行的編輯同樣是衝突; +* 不同 key 的變更不會衝突(``merge_collections``)。 + +``modified_at`` 只供顯示。無論各機器的時鐘怎麼說,決定都相同,而且每台機器算出的結果一致。 + +.. code-block:: python + + bucket.upsert("hotkeys", "hk1", {"combo": "ctrl+a"}, origin="laptop") # versioned + bucket.remove("hotkeys", "hk1", origin="laptop") # versioned tombstone + bucket.values("hotkeys") # 有效值;仍在衝突中的 entry 不列入 + bucket.conflicts() # [(section, 帶 .siblings 的 SyncEntry), ...] + +*不帶* ``origin`` 的 ``upsert`` / ``remove`` 仍寫入舊式、以 ``last_modified`` 蓋章的扁平 entry, +兩筆扁平 entry 仍以「較晚者勝」合併(回報於 ``ConflictRecord``);versioned entry 會取代 +同 id 的扁平副本。因果衝突的 ``ConflictRecord.unresolved`` 為 true,此時沒有任何東西被丟棄。 + +刪除與退休裝置 +-------------- + +刪除是一個 tombstone。Versioned tombstone **絕不因時間而丟棄**:``collect_tombstones`` +只在 bucket 的 ``peers`` 記錄的每台有效裝置都已確認包含它的 revision 之後才移除。 +``ConfigSyncClient.push_operations`` 負責維護這些確認。 + +不會再回來的裝置以 ``client.retire_peer(device_id)`` 退休(或以 +``push_operations(..., max_offline_s=...)`` 自動退休,它比較 ``last_seen`` —— +這是唯一用到時鐘的地方,而且從不在兩個值之間做決定)。Tombstone 不再等待已退休的裝置, +所以它回來時 push 會被 ``FullResyncRequired`` 拒絕;它必須呼叫 +``client.full_resync(device_id=...)``,以回傳的 bucket 取代本機狀態並丟棄待送操作。 +改用合併可能讓已刪除的 entry 復活。 + +離線 outbox +----------- + +``SyncOutbox(db_path, account=..., endpoint=...)`` 是尚未送達 server 的 +``SyncOperation(section, entry)`` 的 SQLite 佇列(預設檔案 +``~/.je_auto_control/config_sync_outbox.sqlite3``;一個檔案可容納多個帳號與 endpoint,彼此隔離)。 + +.. code-block:: python + + from je_auto_control.utils.config_sync import SyncEntry, SyncOperation, SyncOutbox + + outbox = SyncOutbox(account="alice", endpoint="https://sync.example") + outbox.enqueue(SyncOperation("hotkeys", SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop"))) + report = outbox.drain( + lambda batch: client.push_operations(batch, device_id="laptop"), cancel=stop_event) + # DrainReport(sent, pending, attempts, offline, cancelled, error) + +操作以 operation id 入列(同一筆入列兩次不會重複)、重啟後仍在,並以相同 id 重送 —— +套用 server 已有的批次不會改變任何東西。送出失敗時退避 2 秒、4 秒、8 秒……最多 300 秒 +(``base_delay_s`` / ``max_delay_s``),一次 drain 最多送 ``max_attempts`` 次(預設 5), +``wait=False`` 會直接返回而不在退避期間睡眠,設定 ``cancel`` 即使在等待中途也會結束 drain。 diff --git a/docs/source/Zh/doc/new_features/v81_features_doc.rst b/docs/source/Zh/doc/new_features/v81_features_doc.rst index e03ed8239..c9e534116 100644 --- a/docs/source/Zh/doc/new_features/v81_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v81_features_doc.rst @@ -1,7 +1,7 @@ 分層設定解析器 ============ -``json_patch.merge_patch`` 只合併兩份文件,``config_sync`` 以 last-write-wins 時間戳解析(時間戳相同時刪除優先,其餘每個客戶端都選同一筆,並記為衝突), +``json_patch.merge_patch`` 只合併兩份文件,``config_sync`` 逐筆以 version vector 合併(同一筆的並行編輯保留為衝突,見 :doc:`../config_sync/config_sync_doc`), ``AssetStore`` 則是每環境的扁平結構。它們都無法組成一個有序的 ``defaults < file < env < CLI`` 優先序 堆疊並做深度 dict 合併,也無法回報*每個鍵由哪一層勝出*。本功能補上這個 12-factor 解析器。 diff --git a/je_auto_control/utils/config_sync/__init__.py b/je_auto_control/utils/config_sync/__init__.py index f47ee5ad6..14c8b0aa4 100644 --- a/je_auto_control/utils/config_sync/__init__.py +++ b/je_auto_control/utils/config_sync/__init__.py @@ -1,34 +1,64 @@ -"""Phase 7.4: cross-machine config sync through a small bucket server. +"""Cross-machine config sync through a small bucket server. Operators running AutoControl on several machines (work desktop, home -desktop, demo laptop) currently have to copy hotkey bindings, trigger -definitions, and address-book entries by hand. The sync module gives -each user a small namespaced bucket on a sync server and a -deterministic merge strategy: every entry carries a ``last_modified`` -timestamp; the newer entry wins on conflict. +desktop, demo laptop) would otherwise copy hotkey bindings, trigger +definitions, and address-book entries by hand. Each user has one namespaced +bucket on a sync server, and every machine merges its changes into it. -This module is the **headless client**. It speaks HTTP to a sync endpoint, +The server side is the signaling server +(:mod:`je_auto_control.utils.remote_desktop.signaling_server`), which serves ``GET`` / ``PUT /config/{user_id}`` with an optional ``X-Signaling-Secret`` -header, that stores the bucket as an opaque JSON document; this package does -not ship that endpoint yet -- the signaling server has no ``/config`` routes. +header and keeps the buckets in SQLite (:mod:`.store`). A ``PUT`` is a +version-2 envelope naming the revision it was built on and an operation id; +the server refuses it with ``409`` when that revision is no longer current, +and answers a repeated operation id with the revision of the first attempt. -Each section maps an entry id to a dict carrying ``last_modified``. Section -names are free-form; ``hotkeys``, ``triggers``, ``address_book`` and -``custom`` are the conventional ones, and the syncer treats every section the -same way. +This package is the **headless client** side: -Conflicts always resolve to "later wins"; the loser is preserved in -``ConflictRecord`` so callers can show a "merged 3 entries, dropped 1 -older copy" notification. A removed entry is kept as a tombstone so the -deletion reaches every machine; read live entries with -``ConfigBucket.entries(section)``. +* :mod:`.client` -- :class:`ConfigBucket`, :class:`ConfigSyncClient` + (``fetch`` / ``push`` / ``sync`` / ``push_operations`` / ``full_resync``). +* :mod:`.versions` -- :class:`SyncEntry` with a version vector, and + :func:`merge_entries`: the change made knowing the other wins; two changes + to one key made apart are *both kept* as a conflict. No clock is consulted. +* :mod:`.outbox` -- :class:`SyncOutbox`, the durable queue of changes not + yet on the server (resent by operation id, bounded back-off, cancellable). + +Each section maps an entry id to an entry. Section names are free-form; +``hotkeys``, ``triggers``, ``address_book`` and ``custom`` are the +conventional ones, and the syncer treats every section the same way. + +A removed entry is kept as a tombstone so the deletion reaches every +machine; read live entries with ``ConfigBucket.entries(section)`` or +``values(section)``. A versioned tombstone is dropped only once every +participating device has acknowledged it -- a device retired for staying +away must do an explicit full resync. Entries written without a device id +keep the older rule: the later ``last_modified`` wins and the loser is +reported in a ``ConflictRecord``. """ from je_auto_control.utils.config_sync.client import ( - TOMBSTONE_RETENTION_S, ConflictRecord, ConfigBucket, ConfigSyncClient, - ConfigSyncError, is_tombstone, merge_buckets, + DEFAULT_SYNC_ATTEMPTS, TOMBSTONE_RETENTION_S, WIRE_VERSION, ConflictRecord, ConfigBucket, + ConfigSyncClient, ConfigSyncConflict, ConfigSyncError, FullResyncRequired, SyncResult, + batch_operation_id, is_tombstone, merge_buckets, new_operation_id, +) +from je_auto_control.utils.config_sync.outbox import ( + DrainReport, OutboxError, SyncOutbox, default_outbox_path, +) +from je_auto_control.utils.config_sync.store import ( + ConfigStore, ConfigStoreError, RevisionConflictError, StoreCapacityError, + default_store_path, +) +from je_auto_control.utils.config_sync.versions import ( + MergeDecision, PeerState, SyncConflict, SyncEntry, SyncOperation, collect_tombstones, + collectable_revision, compare_vectors, merge_collections, merge_entries, ) __all__ = [ - "ConfigBucket", "ConflictRecord", "ConfigSyncClient", - "ConfigSyncError", "TOMBSTONE_RETENTION_S", "is_tombstone", "merge_buckets", + "ConfigBucket", "ConfigStore", "ConfigStoreError", "ConflictRecord", "ConfigSyncClient", + "ConfigSyncConflict", "ConfigSyncError", "DEFAULT_SYNC_ATTEMPTS", "DrainReport", + "FullResyncRequired", "MergeDecision", "OutboxError", "PeerState", "RevisionConflictError", + "StoreCapacityError", "SyncConflict", "SyncEntry", "SyncOperation", "SyncOutbox", + "SyncResult", "TOMBSTONE_RETENTION_S", "WIRE_VERSION", "batch_operation_id", + "collect_tombstones", "collectable_revision", "compare_vectors", "default_outbox_path", + "default_store_path", "is_tombstone", "merge_buckets", "merge_collections", + "merge_entries", "new_operation_id", ] diff --git a/je_auto_control/utils/config_sync/client.py b/je_auto_control/utils/config_sync/client.py index 8d5046557..975f7488c 100644 --- a/je_auto_control/utils/config_sync/client.py +++ b/je_auto_control/utils/config_sync/client.py @@ -1,16 +1,22 @@ """HTTP client + deterministic merge for the config-sync bucket.""" from __future__ import annotations +import hashlib import json import math import time import urllib.parse import uuid -from dataclasses import asdict, dataclass, field -from typing import Any, Dict, List, Mapping, Optional, Tuple +from dataclasses import asdict, dataclass, field, replace +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Sequence, Tuple from je_auto_control.utils.exception.exceptions import AutoControlException +if TYPE_CHECKING: # reason: versions imports this module for ConfigSyncError + from je_auto_control.utils.config_sync.versions import ( + PeerState, SyncConflict, SyncEntry, SyncOperation, + ) + _DEFAULT_TIMEOUT_S = 5.0 #: The ``/config`` wire format: a PUT is an envelope naming the revision it #: was built on and an operation id, and is refused when that revision is stale. @@ -19,9 +25,9 @@ #: after losing a race to another machine before it gives up. DEFAULT_SYNC_ATTEMPTS = 4 -#: How long a deletion is remembered. A tombstone purged before every machine -#: has synced lets a machine that still holds the entry bring it back, so this -#: bounds how long a machine may stay offline without that happening. +#: How long an *unversioned* deletion is remembered. Entries written with a +#: device id carry a version vector instead, and their tombstones are never +#: dropped by age -- see :func:`~je_auto_control.utils.config_sync.versions.collect_tombstones`. TOMBSTONE_RETENTION_S = 30 * 24 * 3600.0 @@ -46,6 +52,16 @@ def __init__(self, message: str, revision: Optional[int] = None) -> None: self.revision = revision +class FullResyncRequired(ConfigSyncError): + """This device was retired from the group and may not push its old state. + + While it was away the others stopped waiting for it and collected the + tombstones it never saw, so merging its copy could bring deleted entries + back. Call :meth:`ConfigSyncClient.full_resync` and replace the local + state with what it returns. + """ + + def new_operation_id() -> str: """A fresh id for one push; reuse it when retrying that same push.""" return uuid.uuid4().hex @@ -53,32 +69,43 @@ def new_operation_id() -> str: @dataclass class ConflictRecord: - """One entry that lost a last-modified race during the merge.""" + """One entry on which the two buckets disagreed. + + For unversioned entries ``dropped`` lost a last-modified race to ``kept``. + With ``unresolved`` true nothing was dropped: the entries were changed + concurrently on two devices, ``kept`` holds both as ``siblings`` and + ``dropped`` is the remote side's copy for reference. + """ section: str entry_id: str dropped: Dict[str, Any] kept: Dict[str, Any] + unresolved: bool = False @dataclass class ConfigBucket: """JSON-shaped bucket persisted on the sync server. - Each section maps an opaque ``entry_id`` to a dict that must carry - a ``last_modified`` epoch timestamp. Unknown sections are passed - through untouched so callers can extend the schema without - touching the syncer. - - A removed entry stays in ``sections`` as a tombstone - (``{"deleted": True, "last_modified": ...}``) so the deletion wins the - merge against an older copy on another machine instead of being undone - by it; read entries through :meth:`entries`, which leaves tombstones out. + Each section maps an opaque ``entry_id`` to an entry. An entry written + with an ``origin`` device id is *versioned*: it has the + :class:`~je_auto_control.utils.config_sync.versions.SyncEntry` shape + (``value`` / ``vector`` / ``origin`` / ``operation_id``) and merges by + causality. An entry written without one is the older flat dict stamped + with ``last_modified``, which merges by "later wins". Unknown sections + are passed through untouched. + + A removed entry stays in ``sections`` as a tombstone (``"deleted": true``) + so the deletion reaches the other machines; read entries through + :meth:`entries` or :meth:`values`, which leave tombstones out. ``peers`` + records which devices take part and the revision each has merged. """ user_id: str sections: Dict[str, Dict[str, Dict[str, Any]]] = field( default_factory=dict, ) revision: int = 0 + peers: Dict[str, Dict[str, Any]] = field(default_factory=dict) def to_dict(self) -> Dict[str, Any]: return asdict(self) @@ -99,40 +126,109 @@ def from_dict(cls, body: Mapping[str, Any]) -> "ConfigBucket": user_id=body["user_id"], sections={str(name): _section(name, sec) for name, sec in sections.items()}, revision=int(_finite(body.get("revision", 0), "revision")), + peers=_peers(body.get("peers") or {}), ) def upsert(self, section: str, entry_id: str, - entry: Mapping[str, Any]) -> None: - """Add or replace an entry, stamping it with the current time. + entry: Mapping[str, Any], *, origin: Optional[str] = None) -> None: + """Add or replace an entry. + + With ``origin`` (this device's id) the entry is versioned: the change + is recorded as made by that device on top of whatever the bucket + holds for ``entry_id``, and no clock decides later merges. - A ``last_modified`` already in ``entry`` is kept, so drop it when - editing an entry read back from a bucket -- otherwise the edit keeps + Without it the entry is stamped with the current time, and a + ``last_modified`` already in ``entry`` is kept -- so drop it when + editing an entry read back from a bucket, otherwise the edit keeps its old stamp and loses the merge to any newer remote copy. """ + if origin is not None: + from je_auto_control.utils.config_sync.versions import SyncEntry + current = self.get_entry(section, entry_id) + now = time.time() + self.put_entry(section, ( + SyncEntry.create(entry_id, entry, origin, modified_at=now) if current is None + else current.edited(entry, origin, modified_at=now))) + return body = dict(entry) body["last_modified"] = float(body.get("last_modified", time.time())) self.sections.setdefault(section, {})[entry_id] = body - def remove(self, section: str, entry_id: str) -> bool: + def remove(self, section: str, entry_id: str, *, origin: Optional[str] = None) -> bool: """Replace a live entry with a tombstone; False when there was none. Dropping the entry outright let the next sync bring it straight back - from the server, where it still existed. The tombstone is stamped no - earlier than the entry it deletes, so a clock running behind cannot - make the deletion lose to the value it removed. + from the server, where it still existed. With ``origin`` (required + for a versioned entry) the tombstone is a new version made by that + device. Without it the tombstone is stamped no earlier than the entry + it deletes, so a clock running behind cannot make the deletion lose + to the value it removed. """ entry = self.sections.get(section, {}).get(entry_id) if entry is None or is_tombstone(entry): return False + versioned = self.get_entry(section, entry_id) + if versioned is not None: + if origin is None: + raise ConfigSyncError( + f"{section}/{entry_id} is versioned: removing it needs origin=") + self.put_entry(section, versioned.removed(origin, modified_at=time.time())) + return True stamp = max(time.time(), float(entry.get("last_modified", 0))) self.sections[section][entry_id] = {"deleted": True, "last_modified": stamp} return True def entries(self, section: str) -> Dict[str, Dict[str, Any]]: - """The live entries of ``section``: everything except tombstones.""" + """The live entries of ``section`` as stored: everything except tombstones.""" return {entry_id: entry for entry_id, entry in self.sections.get(section, {}).items() if not is_tombstone(entry)} + def values(self, section: str) -> Dict[str, Dict[str, Any]]: + """The live values of ``section``, whichever shape they are stored in. + + A versioned entry yields its ``value``; an entry still in conflict is + left out (see :meth:`conflicts`); a flat entry yields itself. + """ + values: Dict[str, Dict[str, Any]] = {} + for entry_id, entry in self.entries(section).items(): + if "vector" not in entry: + values[entry_id] = entry + elif isinstance(entry.get("value"), Mapping): + values[entry_id] = dict(entry["value"]) + return values + + def get_entry(self, section: str, entry_id: str) -> Optional["SyncEntry"]: + """The versioned entry stored for ``entry_id``; ``None`` if absent or flat.""" + from je_auto_control.utils.config_sync.versions import SyncEntry, is_versioned + body = self.sections.get(section, {}).get(entry_id) + if body is None or not is_versioned(body): + return None + return SyncEntry.from_dict(entry_id, body) + + def put_entry(self, section: str, entry: "SyncEntry") -> None: + """Store a versioned entry under its key.""" + self.sections.setdefault(section, {})[entry.key] = entry.to_dict() + + def sync_entries(self, section: str) -> Dict[str, "SyncEntry"]: + """Every versioned entry of ``section``, tombstones and conflicts included.""" + from je_auto_control.utils.config_sync.versions import SyncEntry, is_versioned + return {entry_id: SyncEntry.from_dict(entry_id, body) + for entry_id, body in self.sections.get(section, {}).items() + if is_versioned(body)} + + def conflicts(self) -> List[Tuple[str, "SyncEntry"]]: + """``(section, entry)`` for every entry still holding concurrent changes.""" + found: List[Tuple[str, "SyncEntry"]] = [] + for section in sorted(self.sections): + found.extend((section, entry) for _key, entry in sorted(self.sync_entries(section).items()) + if entry.in_conflict) + return found + + def peer_states(self) -> Dict[str, "PeerState"]: + """The devices taking part in this bucket, by device id.""" + from je_auto_control.utils.config_sync.versions import PeerState + return {device: PeerState.from_dict(device, body) for device, body in self.peers.items()} + def _finite(value: Any, what: str) -> float: """``value`` as a finite float, or :class:`ConfigSyncError`.""" @@ -146,6 +242,7 @@ def _finite(value: Any, what: str) -> float: def _section(name: Any, section: Any) -> Dict[str, Dict[str, Any]]: + from je_auto_control.utils.config_sync.versions import SyncEntry, is_versioned if not isinstance(section, Mapping): raise ConfigSyncError(f"section {name!r} must be a mapping") entries: Dict[str, Dict[str, Any]] = {} @@ -156,58 +253,113 @@ def _section(name: Any, section: Any) -> Dict[str, Dict[str, Any]]: _finite(entry.get("last_modified", 0), f"{name}/{entry_id} last_modified") if not isinstance(entry.get("deleted", False), bool): raise ConfigSyncError(f"entry {name!r}/{entry_id!r} deleted must be a boolean") + if is_versioned(entry): + SyncEntry.from_dict(str(entry_id), entry) return entries +def _peers(peers: Any) -> Dict[str, Dict[str, Any]]: + from je_auto_control.utils.config_sync.versions import PeerState + if not isinstance(peers, Mapping): + raise ConfigSyncError("bucket peers must be a mapping") + return {str(device): PeerState.from_dict(str(device), body).to_dict() + for device, body in peers.items()} + + def merge_buckets(local: ConfigBucket, remote: ConfigBucket, *, now: Optional[float] = None, tombstone_retention_s: float = TOMBSTONE_RETENTION_S, ) -> Tuple[ConfigBucket, List[ConflictRecord]]: - """Last-write-wins merge across every section. Returns merged + conflicts. - - Tombstones take part like any other entry, so a deletion newer than a - remote copy removes it and an edit newer than a deletion restores it. - Tombstones older than ``tombstone_retention_s`` (measured from ``now``, - default the current time) are left out of the result. + """Merge every section of two buckets. Returns merged + conflicts. + + Versioned entries (see :meth:`ConfigBucket.upsert`) merge by causality: + the one made knowing the other wins, and two made apart are both kept as + an unresolved conflict -- no clock is consulted. Their tombstones are + never dropped here. + + Unversioned entries keep the last-write-wins rule: tombstones take part + like any other entry, so a deletion newer than a remote copy removes it + and an edit newer than a deletion restores it, and unversioned tombstones + older than ``tombstone_retention_s`` (measured from ``now``, default the + current time) are left out of the result. A versioned entry supersedes + an unversioned copy of the same id, which is reported as dropped. """ if local.user_id != remote.user_id: raise ConfigSyncError( f"user_id mismatch: local={local.user_id!r} remote={remote.user_id!r}", ) - merged = ConfigBucket(user_id=local.user_id) + merged = ConfigBucket(user_id=local.user_id, peers=_merged_peers(local, remote)) conflicts: List[ConflictRecord] = [] - sections = set(local.sections) | set(remote.sections) - for name in sections: - merged_section: Dict[str, Dict[str, Any]] = {} - local_sec = local.sections.get(name, {}) - remote_sec = remote.sections.get(name, {}) - ids = set(local_sec) | set(remote_sec) - for entry_id in ids: - local_entry = local_sec.get(entry_id) - remote_entry = remote_sec.get(entry_id) - if local_entry is None: - if remote_entry is not None: - merged_section[entry_id] = remote_entry - continue - if remote_entry is None: - merged_section[entry_id] = local_entry - continue - kept, dropped = _winner(local_entry, remote_entry) - merged_section[entry_id] = kept - if dropped is not None: - conflicts.append(ConflictRecord( - section=name, entry_id=entry_id, dropped=dropped, kept=kept, - )) - merged.sections[name] = _without_expired( - merged_section, (time.time() if now is None else now) - tombstone_retention_s) + cutoff = (time.time() if now is None else now) - tombstone_retention_s + for name in set(local.sections) | set(remote.sections): + merged_section, found = _merge_section( + name, local.sections.get(name, {}), remote.sections.get(name, {})) + conflicts.extend(found) + merged.sections[name] = _without_expired(merged_section, cutoff) merged.revision = max(local.revision, remote.revision) + 1 return merged, conflicts -def _winner(local_entry: Dict[str, Any], remote_entry: Dict[str, Any], +def _merged_peers(local: ConfigBucket, remote: ConfigBucket) -> Dict[str, Dict[str, Any]]: + # The server's record of the peers is the shared one; local-only devices + # are ones this machine has not announced yet. + return {**{device: dict(body) for device, body in local.peers.items()}, + **{device: dict(body) for device, body in remote.peers.items()}} + + +def _merge_section(name: str, local_sec: Mapping[str, Dict[str, Any]], + remote_sec: Mapping[str, Dict[str, Any]], + ) -> Tuple[Dict[str, Dict[str, Any]], List[ConflictRecord]]: + merged: Dict[str, Dict[str, Any]] = {} + conflicts: List[ConflictRecord] = [] + for entry_id in set(local_sec) | set(remote_sec): + local_entry = local_sec.get(entry_id) + remote_entry = remote_sec.get(entry_id) + if local_entry is None or remote_entry is None: + present = local_entry if remote_entry is None else remote_entry + if present is not None: + merged[entry_id] = present + continue + kept, dropped, unresolved = _winner(entry_id, local_entry, remote_entry) + merged[entry_id] = kept + if dropped is not None: + conflicts.append(ConflictRecord(section=name, entry_id=entry_id, dropped=dropped, + kept=kept, unresolved=unresolved)) + return merged, conflicts + + +def _winner(entry_id: str, local_entry: Dict[str, Any], remote_entry: Dict[str, Any], + ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], bool]: + """``(kept, dropped, unresolved)`` for two copies of one entry.""" + local_versioned = "vector" in local_entry + remote_versioned = "vector" in remote_entry + if local_versioned and remote_versioned: + return _causal_winner(entry_id, local_entry, remote_entry) + if local_versioned != remote_versioned: + # The unversioned copy carries no history to compare; the versioned + # one was written by a client that had seen the bucket. + if local_versioned: + return local_entry, remote_entry, False + return remote_entry, local_entry, False + kept, dropped = _latest(local_entry, remote_entry) + return kept, dropped, False + + +def _causal_winner(entry_id: str, local_entry: Dict[str, Any], remote_entry: Dict[str, Any], + ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], bool]: + from je_auto_control.utils.config_sync.versions import SyncEntry, merge_entries + decision = merge_entries(SyncEntry.from_dict(entry_id, local_entry), + SyncEntry.from_dict(entry_id, remote_entry)) + if decision.conflict is not None: + return decision.entry.to_dict(), remote_entry, True + # One side already included the other: nothing was lost, nothing to report. + return decision.entry.to_dict(), None, False + + +def _latest(local_entry: Dict[str, Any], remote_entry: Dict[str, Any], ) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]: - """The entry that wins and the one it beat (``None`` when they are equal). + """The unversioned entry that wins and the one it beat (``None`` when equal). The later ``last_modified`` wins. At a tie a deletion wins -- ``delete`` stamps its tombstone no earlier than the entry it removes -- and then the @@ -233,8 +385,99 @@ def _tie_rank(entry: Dict[str, Any]) -> Tuple[bool, str]: def _without_expired(section: Dict[str, Dict[str, Any]], cutoff: float) -> Dict[str, Dict[str, Any]]: + # Only unversioned tombstones expire by age. A versioned one is collected + # when the peers have acknowledged it, however long that takes. return {entry_id: entry for entry_id, entry in section.items() - if not (is_tombstone(entry) and float(entry.get("last_modified", 0)) < cutoff)} + if not (is_tombstone(entry) and "vector" not in entry + and float(entry.get("last_modified", 0)) < cutoff)} + + +@dataclass +class SyncResult: + """What :meth:`ConfigSyncClient.push_operations` left on the server. + + ``bucket`` is the server's state including this device's operations, + ``revision`` its committed revision, ``conflicts`` the keys where an + operation met a concurrent change (both kept), and ``pushed`` whether a + write was needed at all. + """ + bucket: ConfigBucket + revision: int + conflicts: List["SyncConflict"] = field(default_factory=list) + pushed: bool = False + + +def batch_operation_id(base_revision: int, operations: Sequence["SyncOperation"]) -> str: + """The id of the push that applies ``operations`` on top of ``base_revision``. + + Derived from the operations themselves, so the same batch retried after + a lost reply -- or after a restart -- names the same server-side write. + """ + digest = hashlib.sha256(str(int(base_revision)).encode("ascii")) + for operation_id in sorted(operation.operation_id for operation in operations): + digest.update(b"\0" + operation_id.encode("utf-8")) + return "batch-" + digest.hexdigest()[:48] + + +def _apply_operation(bucket: ConfigBucket, operation: "SyncOperation", + conflicts: List["SyncConflict"]) -> bool: + """Merge one operation into ``bucket``; whether the bucket changed.""" + from je_auto_control.utils.config_sync.versions import SyncConflict, merge_entries + entry = operation.entry + current = bucket.get_entry(operation.section, entry.key) + if current is None: + # No entry, or an unversioned one this change supersedes. + bucket.put_entry(operation.section, entry) + return True + decision = merge_entries(current, entry) + if decision.conflict is not None: + conflicts.append(SyncConflict(key=entry.key, local=entry, remote=current, + section=operation.section)) + if decision.entry == current: + return False + bucket.put_entry(operation.section, decision.entry) + return True + + +def _awaits_ack(bucket: ConfigBucket, device_id: str) -> bool: + """Whether a tombstone is still waiting for ``device_id`` to acknowledge it.""" + peer = bucket.peer_states().get(device_id) + if peer is None: + return True + return any(entry.deleted and entry.deleted_revision > peer.acked_revision + for section in bucket.sections + for entry in bucket.sync_entries(section).values()) + + +def _settle(bucket: ConfigBucket, device_id: str, revision: int, now: float, + max_offline_s: Optional[float]) -> None: + """Prepare ``bucket`` to be committed as ``revision`` by ``device_id``. + + Stamps new tombstones with the revision that first carries them, records + that this device has merged up to it, retires peers unseen for longer + than ``max_offline_s`` (when given), and drops the tombstones every + remaining peer has acknowledged. + """ + from je_auto_control.utils.config_sync.versions import PeerState, collect_tombstones + peers = bucket.peer_states() + peers[device_id] = PeerState(device_id, acked_revision=revision, last_seen=now) + if max_offline_s is not None: + peers = {device: (replace(peer, retired=True) + if device != device_id and peer.last_seen < now - max_offline_s else peer) + for device, peer in peers.items()} + bucket.peers = {device: peer.to_dict() for device, peer in peers.items()} + for section in list(bucket.sections): + versioned = _stamped(bucket.sync_entries(section), revision) + kept = collect_tombstones(versioned, peers.values()) + flat = {key: body for key, body in bucket.sections[section].items() if key not in versioned} + bucket.sections[section] = {**flat, **{key: entry.to_dict() for key, entry in kept.items()}} + + +def _stamped(entries: Mapping[str, "SyncEntry"], revision: int) -> Dict[str, "SyncEntry"]: + """``entries`` with each new tombstone marked as first carried by ``revision``.""" + return {key: (replace(entry, deleted_revision=revision) + if entry.deleted and not entry.deleted_revision else entry) + for key, entry in entries.items()} class ConfigSyncClient: @@ -256,6 +499,16 @@ def __init__(self, server_url: str, *, self._secret = secret self._timeout = float(timeout_s) + @property + def user_id(self) -> str: + """The account this client syncs.""" + return self._user_id + + @property + def server_url(self) -> str: + """The server this client talks to, without a trailing slash.""" + return self._server_url + def _endpoint(self, suffix: str = "") -> str: encoded = urllib.parse.quote(self._user_id, safe="") path = f"/config/{encoded}{suffix}" @@ -360,6 +613,92 @@ def sync(self, local: ConfigBucket, *, max_attempts: int = DEFAULT_SYNC_ATTEMPTS raise ConfigSyncConflict( f"config sync: still behind the server after {max_attempts} attempts") + def push_operations(self, operations: Sequence["SyncOperation"], *, device_id: str, + max_attempts: int = DEFAULT_SYNC_ATTEMPTS, now: Optional[float] = None, + max_offline_s: Optional[float] = None) -> SyncResult: + """Apply this device's pending changes on top of the server's state. + + Fetches the bucket, merges each operation into it by causality + (a concurrent change to the same key is kept beside it as a + conflict), records that ``device_id`` has merged the result, and + commits against the fetched revision -- fetching again when another + device got there first, up to ``max_attempts`` times. Applying the + same operations twice changes nothing, so a batch can be resent + after an uncertain failure. With no operations this still pushes + when the bucket holds a deletion this device has not acknowledged. + + Raises :class:`FullResyncRequired` when the group has retired this + device. ``max_offline_s`` retires peers not seen for that long. + """ + if not device_id: + raise ConfigSyncError("device_id is required") + stamp = time.time() if now is None else float(now) + for _attempt in range(max(1, int(max_attempts))): + remote = self.fetch() or ConfigBucket(user_id=self._user_id) + self._require_active(remote, device_id) + merged = ConfigBucket.from_dict(remote.to_dict()) + conflicts: List["SyncConflict"] = [] + changed = [_apply_operation(merged, operation, conflicts) for operation in operations] + if not any(changed) and not _awaits_ack(remote, device_id): + return SyncResult(bucket=remote, revision=remote.revision, conflicts=conflicts) + _settle(merged, device_id, remote.revision + 1, stamp, max_offline_s) + try: + revision = self.push(merged, base_revision=remote.revision, + operation_id=batch_operation_id(remote.revision, operations)) + except ConfigSyncConflict: + continue + return SyncResult(bucket=merged, revision=revision, conflicts=conflicts, pushed=True) + raise ConfigSyncConflict( + f"config sync: still behind the server after {max_attempts} attempts") + + def _require_active(self, remote: ConfigBucket, device_id: str) -> None: + peer = remote.peer_states().get(device_id) + if peer is not None and peer.retired: + raise FullResyncRequired( + f"device {device_id!r} was retired from the bucket of {self._user_id!r}; " + "call full_resync() and replace the local state") + + def retire_peer(self, device_id: str, *, max_attempts: int = DEFAULT_SYNC_ATTEMPTS) -> int: + """Stop waiting for ``device_id``; returns the committed revision. + + Tombstones no longer wait for a retired device, and it is refused + with :class:`FullResyncRequired` until it does a full resync. + """ + from je_auto_control.utils.config_sync.versions import PeerState + for _attempt in range(max(1, int(max_attempts))): + remote = self.fetch() or ConfigBucket(user_id=self._user_id) + known = remote.peer_states().get(device_id) or PeerState(device_id) + remote.peers[device_id] = PeerState( + device_id, known.acked_revision, known.last_seen, retired=True).to_dict() + try: + return self.push(remote) + except ConfigSyncConflict: + continue + raise ConfigSyncConflict( + f"config sync: still behind the server after {max_attempts} attempts") + + def full_resync(self, *, device_id: str, max_attempts: int = DEFAULT_SYNC_ATTEMPTS, + now: Optional[float] = None) -> ConfigBucket: + """Re-admit ``device_id`` and return the server's state to adopt. + + The caller must replace its local state with the returned bucket and + discard its pending operations -- they were made against state the + group has moved past -- rather than merge the two. + """ + from je_auto_control.utils.config_sync.versions import PeerState + stamp = time.time() if now is None else float(now) + for _attempt in range(max(1, int(max_attempts))): + remote = self.fetch() or ConfigBucket(user_id=self._user_id) + remote.peers[device_id] = PeerState( + device_id, acked_revision=remote.revision + 1, last_seen=stamp).to_dict() + try: + self.push(remote) + except ConfigSyncConflict: + continue + return remote + raise ConfigSyncConflict( + f"config sync: still behind the server after {max_attempts} attempts") + def _conflict_revision(text: Any) -> Optional[int]: """The current revision a 409 reply names, when it names one.""" @@ -372,6 +711,7 @@ def _conflict_revision(text: Any) -> Optional[int]: __all__ = [ "ConfigBucket", "ConflictRecord", "ConfigSyncClient", "ConfigSyncConflict", - "ConfigSyncError", "DEFAULT_SYNC_ATTEMPTS", "TOMBSTONE_RETENTION_S", "WIRE_VERSION", + "ConfigSyncError", "DEFAULT_SYNC_ATTEMPTS", "FullResyncRequired", "SyncResult", + "TOMBSTONE_RETENTION_S", "WIRE_VERSION", "batch_operation_id", "is_tombstone", "merge_buckets", "new_operation_id", ] diff --git a/je_auto_control/utils/config_sync/outbox.py b/je_auto_control/utils/config_sync/outbox.py new file mode 100644 index 000000000..3f382ada5 --- /dev/null +++ b/je_auto_control/utils/config_sync/outbox.py @@ -0,0 +1,272 @@ +"""Durable queue of local changes that have not reached the sync server yet. + +A change made while the server is unreachable used to exist only in the +caller's memory: closing the program lost it, and the caller had no way to +know which pushes had landed. :class:`SyncOutbox` keeps every pending +:class:`~je_auto_control.utils.config_sync.versions.SyncOperation` in SQLite +under its operation id, so it survives a restart and is sent again -- with +the same id, which makes the resend harmless when the first attempt did +arrive -- until the server has taken it. + +Failed attempts back off exponentially up to a ceiling, a drain makes a +bounded number of attempts, and every wait can be cancelled. One database +can serve several accounts and servers: each ``(account, endpoint)`` pair has +its own queue and its own remembered state. + +Pure standard library; imports no ``PySide6``. +""" +from __future__ import annotations + +import json +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, ContextManager, List, Optional, Sequence, Union + +from je_auto_control.utils.config_sync.client import ( + ConfigBucket, ConfigSyncError, FullResyncRequired, +) +from je_auto_control.utils.config_sync.versions import SyncOperation +from je_auto_control.utils.sqlite_support import autocommit_connection, sqlite_errors_as + +DEFAULT_BASE_DELAY_S = 2.0 +DEFAULT_MAX_DELAY_S = 300.0 +DEFAULT_DRAIN_ATTEMPTS = 5 +_BASELINE = "baseline" +_GO = "go" +_EMPTY = "empty" +_CANCELLED = "" + +_SCHEMA = ( + "CREATE TABLE IF NOT EXISTS operations (" + " seq INTEGER PRIMARY KEY AUTOINCREMENT, account TEXT NOT NULL, endpoint TEXT NOT NULL," + " operation_id TEXT NOT NULL, body TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0," + " next_attempt REAL NOT NULL DEFAULT 0, last_error TEXT NOT NULL DEFAULT ''," + " UNIQUE (account, endpoint, operation_id))", + "CREATE TABLE IF NOT EXISTS state (" + " account TEXT NOT NULL, endpoint TEXT NOT NULL, name TEXT NOT NULL, value TEXT NOT NULL," + " PRIMARY KEY (account, endpoint, name))", +) + + +class OutboxError(ConfigSyncError): + """The outbox database could not be opened, read or written.""" + + +def default_outbox_path() -> Path: + """``~/.je_auto_control/config_sync_outbox.sqlite3``, resolved at call time.""" + return Path.home() / ".je_auto_control" / "config_sync_outbox.sqlite3" + + +@dataclass(frozen=True) +class DrainReport: + """How a :meth:`SyncOutbox.drain` ended. + + ``sent`` operations were accepted, ``pending`` are still queued, + ``attempts`` sends were made. ``offline`` is true when the last attempt + failed (``error`` says why); ``cancelled`` when the cancel event ended it. + """ + sent: int = 0 + pending: int = 0 + attempts: int = 0 + offline: bool = False + cancelled: bool = False + error: str = "" + + +class SyncOutbox: + """The pending operations of one account on one sync endpoint.""" + + def __init__(self, db_path: Union[str, Path, None] = None, *, account: str, endpoint: str, + base_delay_s: float = DEFAULT_BASE_DELAY_S, + max_delay_s: float = DEFAULT_MAX_DELAY_S) -> None: + if not account or not endpoint: + raise OutboxError("an outbox needs an account and an endpoint") + self._configured_path = db_path + self._scope = (str(account), str(endpoint).rstrip("/")) + self._base_delay = max(0.0, float(base_delay_s)) + self._max_delay = max(self._base_delay, float(max_delay_s)) + self._resolved_path: Optional[str] = None + + def _connection(self) -> ContextManager[Any]: + if self._resolved_path is None: + configured = self._configured_path + path = Path(configured) if configured is not None else default_outbox_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as error: + raise OutboxError(f"cannot create {path.parent}: {error}") from error + with autocommit_connection(str(path)) as connection: + for statement in _SCHEMA: + connection.execute(statement) + self._resolved_path = str(path) + return autocommit_connection(self._resolved_path) + + @sqlite_errors_as(OutboxError) + def enqueue(self, operation: SyncOperation) -> None: + """Queue ``operation``; queueing the same operation id again is a no-op.""" + if not operation.operation_id: + raise OutboxError("an operation needs an operation id to be queued") + with self._connection() as connection: + connection.execute( + "INSERT OR IGNORE INTO operations (account, endpoint, operation_id, body) " + "VALUES (?, ?, ?, ?)", + (*self._scope, operation.operation_id, json.dumps(operation.to_dict()))) + + @sqlite_errors_as(OutboxError) + def pending(self) -> List[SyncOperation]: + """Every queued operation, oldest first.""" + with self._connection() as connection: + rows = connection.execute( + "SELECT body FROM operations WHERE account = ? AND endpoint = ? ORDER BY seq", + self._scope).fetchall() + return [SyncOperation.from_dict(json.loads(row["body"])) for row in rows] + + @sqlite_errors_as(OutboxError) + def acknowledge(self, operation_ids: Sequence[str]) -> None: + """Remove operations the server has accepted.""" + with self._connection() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.executemany( + "DELETE FROM operations WHERE account = ? AND endpoint = ? AND operation_id = ?", + [(*self._scope, operation_id) for operation_id in operation_ids]) + connection.execute("COMMIT") + + def retry_delay(self, attempts: int) -> float: + """Seconds to wait after ``attempts`` failures: doubling, never above the ceiling.""" + if attempts <= 0: + return 0.0 + return min(self._max_delay, self._base_delay * (2 ** min(attempts - 1, 32))) + + @sqlite_errors_as(OutboxError) + def record_failure(self, operation_ids: Sequence[str], *, now: float, error: str = "") -> None: + """Count a failed send and schedule the next attempt for those operations.""" + with self._connection() as connection: + connection.execute("BEGIN IMMEDIATE") + for operation_id in operation_ids: + row = connection.execute( + "SELECT attempts FROM operations WHERE account = ? AND endpoint = ? " + "AND operation_id = ?", (*self._scope, operation_id)).fetchone() + if row is None: + continue + attempts = int(row["attempts"]) + 1 + connection.execute( + "UPDATE operations SET attempts = ?, next_attempt = ?, last_error = ? " + "WHERE account = ? AND endpoint = ? AND operation_id = ?", + (attempts, now + self.retry_delay(attempts), error[:500], + *self._scope, operation_id)) + connection.execute("COMMIT") + + @sqlite_errors_as(OutboxError) + def seconds_until_due(self, now: float) -> Optional[float]: + """How long until the queue may be sent again; ``None`` when it is empty. + + The whole queue goes out together and in order, so it waits for the + operation that has to wait longest. + """ + with self._connection() as connection: + row = connection.execute( + "SELECT COUNT(*) AS n, MAX(next_attempt) AS due FROM operations " + "WHERE account = ? AND endpoint = ?", self._scope).fetchone() + if not row["n"]: + return None + return max(0.0, float(row["due"]) - now) + + @sqlite_errors_as(OutboxError) + def clear(self) -> List[SyncOperation]: + """Empty the queue and return what was in it (for a full resync).""" + discarded = self.pending() + with self._connection() as connection: + connection.execute( + "DELETE FROM operations WHERE account = ? AND endpoint = ?", self._scope) + return discarded + + @sqlite_errors_as(OutboxError) + def get_state(self, name: str) -> Optional[Any]: + """A remembered JSON value for this account and endpoint, or ``None``.""" + with self._connection() as connection: + row = connection.execute( + "SELECT value FROM state WHERE account = ? AND endpoint = ? AND name = ?", + (*self._scope, name)).fetchone() + return None if row is None else json.loads(row["value"]) + + @sqlite_errors_as(OutboxError) + def set_state(self, name: str, value: Any) -> None: + """Remember a JSON value for this account and endpoint.""" + with self._connection() as connection: + connection.execute( + "INSERT INTO state (account, endpoint, name, value) VALUES (?, ?, ?, ?) " + "ON CONFLICT(account, endpoint, name) DO UPDATE SET value = excluded.value", + (*self._scope, name, json.dumps(value))) + + def load_baseline(self) -> Optional[ConfigBucket]: + """The bucket this device last merged, or ``None`` before its first sync.""" + body = self.get_state(_BASELINE) + return None if body is None else ConfigBucket.from_dict(body) + + def save_baseline(self, bucket: ConfigBucket) -> None: + """Remember ``bucket`` as the state local changes are measured against.""" + self.set_state(_BASELINE, bucket.to_dict()) + + def drain(self, send: Callable[[List[SyncOperation]], Any], *, + cancel: Optional[threading.Event] = None, + max_attempts: int = DEFAULT_DRAIN_ATTEMPTS, wait: bool = True, + clock: Callable[[], float] = time.time) -> DrainReport: + """Send the queue through ``send`` until it is empty or attempts run out. + + ``send(operations)`` must raise :class:`ConfigSyncError` when the + operations did not (or may not have) arrived; they stay queued under + the same ids and are tried again after the back-off delay. With + ``wait`` false a queue that is still backing off is left alone. + Setting ``cancel`` ends the drain at the next check, including in the + middle of a wait. :class:`FullResyncRequired` is not retried. + """ + stop = cancel if cancel is not None else threading.Event() + sent = attempts = 0 + error = "" + while attempts < max(1, int(max_attempts)): + turn = self._await_turn(stop, wait, clock()) + if turn == _EMPTY: + return DrainReport(sent=sent, attempts=attempts) + if turn != _GO: + error = error or turn + break + batch = self.pending() + attempts += 1 + error = self._send(send, batch, clock) + if not error: + sent += len(batch) + return DrainReport(sent=sent, pending=len(self.pending()), attempts=attempts, + offline=bool(error), cancelled=stop.is_set(), error=error) + + def _await_turn(self, stop: threading.Event, wait: bool, now: float) -> str: + """Wait out the back-off: ``_GO``, ``_EMPTY``, or why the drain must stop.""" + delay = self.seconds_until_due(now) + if delay is None: + return _EMPTY + if delay > 0 and not wait: + return "waiting to retry after an earlier failure" + if stop.is_set() or (delay > 0 and stop.wait(delay)): + return _CANCELLED + return _GO + + def _send(self, send: Callable[[List[SyncOperation]], Any], batch: List[SyncOperation], + clock: Callable[[], float]) -> str: + """Send one batch; ``""`` when it was accepted, else the failure text.""" + ids = [operation.operation_id for operation in batch] + try: + send(batch) + except FullResyncRequired: + raise + except ConfigSyncError as failure: + self.record_failure(ids, now=clock(), error=str(failure)) + return str(failure) or "send failed" + self.acknowledge(ids) + return "" + + +__all__ = [ + "DEFAULT_BASE_DELAY_S", "DEFAULT_DRAIN_ATTEMPTS", "DEFAULT_MAX_DELAY_S", + "DrainReport", "OutboxError", "SyncOutbox", "default_outbox_path", +] diff --git a/je_auto_control/utils/config_sync/versions.py b/je_auto_control/utils/config_sync/versions.py new file mode 100644 index 000000000..1e333d4ce --- /dev/null +++ b/je_auto_control/utils/config_sync/versions.py @@ -0,0 +1,384 @@ +"""Causal versions for synced entries: who changed what, on top of what. + +The first merge compared ``last_modified`` wall-clock stamps, so a machine +whose clock ran ahead won every disagreement, and two machines editing the +same entry while apart silently lost one edit. A :class:`SyncEntry` instead +carries a *version vector* -- per device, how many changes of that device the +entry includes. Comparing two vectors says whether one entry was made +knowing the other (it supersedes it) or neither knew the other (a real +conflict). Clocks take no part: ``modified_at`` is carried for display only. + +Concurrent changes to one key are not resolved here. :func:`merge_entries` +returns both in a :class:`SyncConflict`, and the merged entry holds them as +``siblings`` until someone picks a value with :meth:`SyncEntry.resolved`. +Every machine computes the same conflicted entry, so the state still +converges. + +A deletion is a tombstone (``deleted`` true) with a vector of its own, so it +competes like any other change. :func:`collect_tombstones` drops one only +when every peer that is still part of the group has acknowledged a revision +that includes it; a peer that stayed away too long is *retired* and must do +an explicit full resync, so it cannot bring deleted entries back. + +Pure standard library; imports no ``PySide6``. +""" +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple + +from je_auto_control.utils.config_sync.client import ConfigSyncError + +#: Relations :func:`compare_vectors` reports. +EQUAL = "equal" +BEFORE = "before" +AFTER = "after" +CONCURRENT = "concurrent" + +#: Outcomes of :func:`merge_entries`. +KEPT_LEFT = "left" +KEPT_RIGHT = "right" +CONFLICT = "conflict" + + +def compare_vectors(left: Mapping[str, int], right: Mapping[str, int]) -> str: + """How ``left`` relates to ``right``: equal, before, after or concurrent.""" + devices = set(left) | set(right) + behind = any(left.get(device, 0) < right.get(device, 0) for device in devices) + ahead = any(left.get(device, 0) > right.get(device, 0) for device in devices) + if behind and ahead: + return CONCURRENT + if behind: + return BEFORE + return AFTER if ahead else EQUAL + + +def join_vectors(*vectors: Mapping[str, int]) -> Dict[str, int]: + """The smallest vector that includes every change in ``vectors``.""" + joined: Dict[str, int] = {} + for vector in vectors: + for device, count in vector.items(): + joined[device] = max(joined.get(device, 0), int(count)) + return joined + + +def _checked_origin(origin: str) -> str: + if not isinstance(origin, str) or not origin: + raise ConfigSyncError("a change needs the id of the device that made it") + return origin + + +@dataclass(frozen=True) +class SyncEntry: + """One synced value (or its deletion) with its causal history. + + ``vector`` maps device id to the number of that device's changes this + entry includes; ``origin`` and ``operation_id`` name the change that + produced it. ``siblings`` is non-empty while the entry is an unresolved + conflict -- then ``value`` is ``None`` and the candidates are there. + """ + key: str + value: Optional[Mapping[str, Any]] = None + vector: Mapping[str, int] = field(default_factory=dict) + origin: str = "" + operation_id: str = "" + deleted: bool = False + modified_at: float = 0.0 + deleted_revision: int = 0 + siblings: Tuple["SyncEntry", ...] = () + + @property + def is_deleted(self) -> bool: + """Whether this entry records a deletion.""" + return self.deleted + + @property + def in_conflict(self) -> bool: + """Whether concurrent changes are still waiting for a choice.""" + return bool(self.siblings) + + @classmethod + def create(cls, key: str, value: Mapping[str, Any], origin: str, *, + modified_at: float = 0.0) -> "SyncEntry": + """A brand-new entry: the first change ``origin`` makes to ``key``.""" + return cls(key=key, value=dict(value), vector={_checked_origin(origin): 1}, + origin=origin, operation_id=uuid.uuid4().hex, modified_at=modified_at) + + def _successor(self, origin: str, **changes: Any) -> "SyncEntry": + vector = dict(self.vector) + vector[_checked_origin(origin)] = vector.get(origin, 0) + 1 + return SyncEntry(key=self.key, vector=vector, origin=origin, + operation_id=uuid.uuid4().hex, **changes) + + def edited(self, value: Mapping[str, Any], origin: str, *, + modified_at: float = 0.0) -> "SyncEntry": + """This entry with a new value, made by ``origin`` knowing this one.""" + return self._successor(origin, value=dict(value), modified_at=modified_at) + + def removed(self, origin: str, *, modified_at: float = 0.0) -> "SyncEntry": + """The tombstone that deletes this entry.""" + return self._successor(origin, deleted=True, modified_at=modified_at) + + def resolved(self, value: Optional[Mapping[str, Any]], origin: str, *, + modified_at: float = 0.0) -> "SyncEntry": + """Settle a conflict: ``value`` (``None`` = delete) supersedes every sibling.""" + if value is None: + return self.removed(origin, modified_at=modified_at) + return self.edited(value, origin, modified_at=modified_at) + + def to_dict(self) -> Dict[str, Any]: + """The JSON shape stored in a bucket section. + + ``deleted`` and ``last_modified`` keep the names the unversioned + format used, so code that only asks "is this a tombstone" still works. + """ + body: Dict[str, Any] = { + "value": None if self.value is None else dict(self.value), + "vector": dict(self.vector), "origin": self.origin, + "operation_id": self.operation_id, "deleted": self.deleted, + "last_modified": float(self.modified_at), + } + if self.deleted_revision: + body["deleted_revision"] = int(self.deleted_revision) + if self.siblings: + body["siblings"] = [sibling.to_dict() for sibling in self.siblings] + return body + + @classmethod + def from_dict(cls, key: str, body: Mapping[str, Any], *, _depth: int = 0) -> "SyncEntry": + """Parse the :meth:`to_dict` shape; :class:`ConfigSyncError` if it is not one.""" + if not is_versioned(body): + raise ConfigSyncError(f"entry {key!r} carries no version vector") + value = body.get("value") + siblings = body.get("siblings") or () + if value is not None and not isinstance(value, Mapping): + raise ConfigSyncError(f"entry {key!r}: value must be an object or null") + if not isinstance(siblings, (list, tuple)) or (siblings and _depth): + raise ConfigSyncError(f"entry {key!r}: malformed siblings") + return cls( + key=str(key), value=None if value is None else dict(value), + vector=_checked_vector(key, body["vector"]), + origin=str(body.get("origin", "")), operation_id=str(body.get("operation_id", "")), + deleted=body.get("deleted") is True, + modified_at=_number(key, body.get("last_modified", 0.0)), + deleted_revision=int(_number(key, body.get("deleted_revision", 0))), + siblings=tuple(cls.from_dict(key, sibling, _depth=1) for sibling in siblings), + ) + + +def _number(key: str, value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value: + raise ConfigSyncError(f"entry {key!r}: expected a number, got {value!r}") + return float(value) + + +def _checked_vector(key: str, vector: Any) -> Dict[str, int]: + if not isinstance(vector, Mapping): + raise ConfigSyncError(f"entry {key!r}: vector must be an object") + checked: Dict[str, int] = {} + for device, count in vector.items(): + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + raise ConfigSyncError(f"entry {key!r}: vector counts must be integers >= 0") + checked[str(device)] = count + return checked + + +def is_versioned(body: Any) -> bool: + """Whether a stored entry is in the versioned (:class:`SyncEntry`) shape.""" + return isinstance(body, Mapping) and isinstance(body.get("vector"), Mapping) + + +@dataclass(frozen=True) +class SyncConflict: + """Two changes to one key, neither made knowing the other.""" + key: str + local: SyncEntry + remote: SyncEntry + section: str = "" + + +@dataclass(frozen=True) +class SyncOperation: + """One local change waiting to reach the server: an entry in a section. + + Its identity is the entry's ``operation_id``, so queueing or sending the + same change twice is the same operation. + """ + section: str + entry: SyncEntry + + @property + def operation_id(self) -> str: + """The id of the change this operation carries.""" + return self.entry.operation_id + + def to_dict(self) -> Dict[str, Any]: + """The JSON shape the outbox stores.""" + return {"section": self.section, "key": self.entry.key, "entry": self.entry.to_dict()} + + @classmethod + def from_dict(cls, body: Mapping[str, Any]) -> "SyncOperation": + """Parse the :meth:`to_dict` shape.""" + if not isinstance(body, Mapping) or not isinstance(body.get("entry"), Mapping): + raise ConfigSyncError("a sync operation must be an object with an entry") + return cls(section=str(body.get("section", "")), + entry=SyncEntry.from_dict(str(body.get("key", "")), body["entry"])) + + +@dataclass(frozen=True) +class MergeDecision: + """What merging two versions of one key produced. + + ``entry`` is what both sides should now hold. ``outcome`` says which + side it came from; on ``"conflict"`` it is a new entry whose ``siblings`` + are the candidates and ``conflict`` names the two inputs. + """ + entry: SyncEntry + outcome: str + conflict: Optional[SyncConflict] = None + + +def _candidates(entry: SyncEntry) -> Tuple[SyncEntry, ...]: + return entry.siblings if entry.siblings else (entry,) + + +def _same_change(left: SyncEntry, right: SyncEntry) -> bool: + return (left.operation_id == right.operation_id and left.value == right.value + and left.deleted == right.deleted and left.siblings == right.siblings) + + +def _conflicted(key: str, left: SyncEntry, right: SyncEntry) -> SyncEntry: + """The entry holding every candidate neither side has superseded.""" + unique: Dict[str, SyncEntry] = {} + for candidate in (*_candidates(left), *_candidates(right)): + unique.setdefault(_identity(candidate), candidate) + pool = list(unique.values()) + survivors = [candidate for candidate in pool + if not any(compare_vectors(candidate.vector, other.vector) == BEFORE + for other in pool)] + survivors.sort(key=_identity) + vector = join_vectors(left.vector, right.vector) + if len(survivors) == 1: + only = survivors[0] + return SyncEntry(key=key, value=only.value, vector=vector, origin=only.origin, + operation_id=only.operation_id, deleted=only.deleted, + modified_at=only.modified_at, deleted_revision=only.deleted_revision) + return SyncEntry( + key=key, vector=vector, siblings=tuple(survivors), + operation_id="conflict:" + "+".join(_identity(candidate) for candidate in survivors)) + + +def _identity(entry: SyncEntry) -> str: + # Two devices configured with one id would produce equal vectors for + # different changes; the operation id still tells those apart. + return f"{entry.origin}/{entry.operation_id}" + + +def merge_entries(left: SyncEntry, right: SyncEntry) -> MergeDecision: + """Merge two versions of one key by causality alone. + + The version made knowing the other wins. When neither knew the other the + result is a conflict that keeps both: no timestamp, device name or value + is used to pick one, so the decision is the same whatever the clocks say + and the same on every machine. An edit made concurrently with a delete is + a conflict too -- the deletion does not silently win. + """ + if left.key != right.key: + raise ConfigSyncError(f"cannot merge {left.key!r} with {right.key!r}") + relation = compare_vectors(left.vector, right.vector) + if relation == AFTER or (relation == EQUAL and _same_change(left, right)): + return MergeDecision(entry=left, outcome=KEPT_LEFT) + if relation == BEFORE: + return MergeDecision(entry=right, outcome=KEPT_RIGHT) + merged = _conflicted(left.key, left, right) + if not merged.in_conflict: + outcome = KEPT_LEFT if _same_change(merged, left) else KEPT_RIGHT + return MergeDecision(entry=merged, outcome=outcome) + return MergeDecision(entry=merged, outcome=CONFLICT, + conflict=SyncConflict(key=left.key, local=left, remote=right)) + + +def merge_collections(local: Mapping[str, SyncEntry], remote: Mapping[str, SyncEntry], + ) -> Tuple[Dict[str, SyncEntry], List[SyncConflict]]: + """Merge two maps of key -> entry; returns the merged map and its conflicts. + + Changes to different keys never conflict. A key only one side has is + taken as it is -- which is why a deletion must stay as a tombstone until + :func:`collect_tombstones` says every peer has seen it. + """ + merged: Dict[str, SyncEntry] = {} + conflicts: List[SyncConflict] = [] + for key in sorted(set(local) | set(remote)): + mine, theirs = local.get(key), remote.get(key) + if mine is None or theirs is None: + merged[key] = mine if theirs is None else theirs # type: ignore[assignment] + continue + decision = merge_entries(mine, theirs) + merged[key] = decision.entry + if decision.conflict is not None: + conflicts.append(decision.conflict) + return merged, conflicts + + +@dataclass(frozen=True) +class PeerState: + """What the group knows about one device. + + ``acked_revision`` is the newest server revision the device has merged + into its own state. A ``retired`` device no longer holds back tombstone + collection and must do a full resync before it may push again. + """ + device_id: str + acked_revision: int = 0 + last_seen: float = 0.0 + retired: bool = False + + def to_dict(self) -> Dict[str, Any]: + """The JSON shape stored under ``peers`` in a bucket.""" + return {"acked_revision": int(self.acked_revision), "last_seen": float(self.last_seen), + "retired": bool(self.retired)} + + @classmethod + def from_dict(cls, device_id: str, body: Any) -> "PeerState": + """Parse the :meth:`to_dict` shape.""" + if not isinstance(body, Mapping): + raise ConfigSyncError(f"peer {device_id!r} must be an object") + return cls(device_id=str(device_id), + acked_revision=int(_number(device_id, body.get("acked_revision", 0))), + last_seen=_number(device_id, body.get("last_seen", 0.0)), + retired=body.get("retired") is True) + + +def collectable_revision(peers: Iterable[PeerState]) -> int: + """The newest revision every active (non-retired) peer has acknowledged. + + ``0`` when no active peer is known: with nobody to confirm, nothing may + be collected. + """ + acked = [peer.acked_revision for peer in peers if not peer.retired] + return min(acked) if acked else 0 + + +def collect_tombstones(entries: Mapping[str, SyncEntry], peers: Iterable[PeerState], + ) -> Dict[str, SyncEntry]: + """``entries`` without the tombstones every active peer has acknowledged. + + A tombstone is dropped only when its ``deleted_revision`` -- the server + revision that first carried it -- is at or below what each active peer + has merged. Age is never a reason: a tombstone dropped while a machine + that still holds the entry is merely offline lets that machine bring the + entry back. A tombstone not yet stamped with a revision is always kept. + """ + horizon = collectable_revision(peers) + return {key: entry for key, entry in entries.items() + if not (entry.deleted and not entry.siblings + and 0 < entry.deleted_revision <= horizon)} + + +__all__ = [ + "AFTER", "BEFORE", "CONCURRENT", "CONFLICT", "EQUAL", "KEPT_LEFT", "KEPT_RIGHT", + "MergeDecision", "PeerState", "SyncConflict", "SyncEntry", "SyncOperation", + "collect_tombstones", "collectable_revision", "compare_vectors", "is_versioned", + "join_vectors", "merge_collections", "merge_entries", +] diff --git a/test/unit_test/headless/test_config_sync_convergence.py b/test/unit_test/headless/test_config_sync_convergence.py new file mode 100644 index 000000000..b46ea2605 --- /dev/null +++ b/test/unit_test/headless/test_config_sync_convergence.py @@ -0,0 +1,364 @@ +"""Config sync converges without trusting clocks, across restarts and deletions. + +The first merge picked the later ``last_modified``: a machine whose clock ran +ahead won every disagreement, two machines editing one entry while apart lost +an edit silently, a change made offline lived only in memory, and a tombstone +was dropped after thirty days whether or not every machine had seen it. +""" +import threading +from dataclasses import replace +from unittest.mock import patch + +import pytest + +from je_auto_control.utils.config_sync import ( + ConfigBucket, ConfigSyncClient, ConfigSyncConflict, ConfigSyncError, FullResyncRequired, + PeerState, SyncEntry, SyncOperation, SyncOutbox, collect_tombstones, compare_vectors, + merge_buckets, merge_collections, merge_entries, +) + + +class _Server: + """The revision-checked bucket endpoint, in memory.""" + + def __init__(self): + self.body = None + self.revision = 0 + self.offline = False + self.puts = [] + + def request(self, method, body=None): + if self.offline: + raise ConfigSyncError("config sync: connection refused") + if method == "GET": + return self.body + self.puts.append(body) + if body["base_revision"] != self.revision: + raise ConfigSyncConflict("behind", self.revision) + self.revision += 1 + self.body = {**body["bucket"], "revision": self.revision} + return {"ok": True, "revision": self.revision} + + +@pytest.fixture +def server(): + endpoint = _Server() + with patch.object(ConfigSyncClient, "_request", + new=lambda _client, method, body=None: endpoint.request(method, body)): + yield endpoint + + +def _client(): + return ConfigSyncClient("https://sync.invalid", user_id="alice") + + +def _op(entry, section="hotkeys"): + return SyncOperation(section=section, entry=entry) + + +# --- merge_entries --------------------------------------------------------- + +def test_parallel_edits_preserve_conflict(): + base = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + left = base.edited({"combo": "ctrl+l"}, "laptop") + right = base.edited({"combo": "ctrl+r"}, "desktop") + decision = merge_entries(left, right) + conflict = decision.conflict + assert conflict.local == left and conflict.remote == right + siblings = decision.entry.siblings + assert decision.entry.in_conflict and len(siblings) == 2 + assert left in siblings and right in siblings + # Both machines compute the same conflicted entry, so they still converge. + assert merge_entries(right, left).entry == decision.entry + + +def test_clock_skew_does_not_choose_winner(): + base = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop", modified_at=1_000.0) + newer = base.edited({"combo": "ctrl+b"}, "laptop", modified_at=5.0) # clock far behind + stale = replace(base, modified_at=9_999_999_999.0) # clock far ahead + assert merge_entries(stale, newer).entry == newer + assert merge_entries(newer, stale).entry == newer + + skewed = merge_entries(base.edited({"combo": "l"}, "laptop", modified_at=1.0), + base.edited({"combo": "r"}, "desktop", modified_at=9e12)) + wall_clock_does_not_change_merge = skewed.conflict is not None and skewed.entry.in_conflict + assert wall_clock_does_not_change_merge is True + + +def test_a_change_made_knowing_the_other_supersedes_it_and_reports_nothing(): + base = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + edited = base.edited({"combo": "ctrl+b"}, "desktop") + assert merge_entries(base, edited).entry == edited + assert merge_entries(edited, base).conflict is None + assert compare_vectors(base.vector, edited.vector) == "before" + + +def test_an_edit_made_apart_from_a_delete_is_a_conflict_not_a_silent_loss(): + base = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + tombstone = base.removed("laptop") + assert tombstone.is_deleted is True + decision = merge_entries(tombstone, base.edited({"combo": "ctrl+z"}, "desktop")) + assert decision.conflict is not None + # A delete made after seeing the value just wins. + assert merge_entries(base, tombstone).entry == tombstone + + +def test_resolving_a_conflict_supersedes_every_sibling(): + base = SyncEntry.create("hk1", {"combo": "a"}, "laptop") + left, right = base.edited({"combo": "l"}, "laptop"), base.edited({"combo": "r"}, "desktop") + conflicted = merge_entries(left, right).entry + chosen = conflicted.resolved({"combo": "r"}, "laptop") + assert not chosen.in_conflict + for other in (left, right, conflicted): + assert merge_entries(other, chosen).entry == chosen + + +def test_changes_to_different_keys_merge_without_conflict(): + one = SyncEntry.create("hk1", {"combo": "a"}, "laptop") + two = SyncEntry.create("hk2", {"combo": "b"}, "desktop") + merged, conflicts = merge_collections({"hk1": one}, {"hk2": two}) + assert merged == {"hk1": one, "hk2": two} and conflicts == [] + + +def test_an_entry_round_trips_through_its_bucket_shape(): + base = SyncEntry.create("hk1", {"combo": "a"}, "laptop") + conflicted = merge_entries(base.edited({"combo": "l"}, "laptop"), + base.removed("desktop")).entry + assert SyncEntry.from_dict("hk1", conflicted.to_dict()) == conflicted + with pytest.raises(ConfigSyncError): + SyncEntry.from_dict("hk1", {"value": {}, "vector": {"laptop": -1}}) + with pytest.raises(ConfigSyncError): + SyncEntry.from_dict("hk1", {"combo": "flat entry"}) + + +# --- buckets --------------------------------------------------------------- + +def test_a_bucket_written_with_a_device_id_merges_by_causality(): + laptop = ConfigBucket(user_id="alice") + laptop.upsert("hotkeys", "hk1", {"combo": "ctrl+a"}, origin="laptop") + desktop = ConfigBucket.from_dict(laptop.to_dict()) + laptop.upsert("hotkeys", "hk1", {"combo": "ctrl+l"}, origin="laptop") + desktop.upsert("hotkeys", "hk1", {"combo": "ctrl+r"}, origin="desktop") + + merged, conflicts = merge_buckets(laptop, desktop) + assert len(conflicts) == 1 and conflicts[0].unresolved is True + assert merged.values("hotkeys") == {} + [(section, entry)] = merged.conflicts() + assert section == "hotkeys" + assert {sibling.value["combo"] for sibling in entry.siblings} == {"ctrl+l", "ctrl+r"} + + +def test_a_versioned_tombstone_is_not_dropped_by_age(): + bucket = ConfigBucket(user_id="alice") + bucket.upsert("hotkeys", "hk1", {"combo": "ctrl+a"}, origin="laptop") + assert bucket.remove("hotkeys", "hk1", origin="laptop") is True + merged, _conflicts = merge_buckets(bucket, ConfigBucket(user_id="alice"), now=9e12) + assert merged.get_entry("hotkeys", "hk1").is_deleted is True + with pytest.raises(ConfigSyncError): + other = ConfigBucket(user_id="alice") + other.upsert("hotkeys", "hk1", {"combo": "ctrl+a"}, origin="laptop") + other.remove("hotkeys", "hk1") + + +# --- tombstone collection -------------------------------------------------- + +def test_a_tombstone_waits_for_every_active_peer(): + tombstone = replace(SyncEntry.create("hk1", {"c": 1}, "laptop").removed("laptop"), + deleted_revision=5) + entries = {"hk1": tombstone} + behind = [PeerState("laptop", acked_revision=6), PeerState("desktop", acked_revision=4)] + assert collect_tombstones(entries, behind) == entries + caught_up = [PeerState("laptop", acked_revision=6), PeerState("desktop", acked_revision=5)] + assert collect_tombstones(entries, caught_up) == {} + retired = [PeerState("laptop", acked_revision=6), + PeerState("desktop", acked_revision=4, retired=True)] + assert collect_tombstones(entries, retired) == {} + assert collect_tombstones(entries, []) == entries + + +def test_a_deletion_reaches_an_offline_machine_however_long_it_was_away(server): + client = _client() + entry = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + client.push_operations([_op(entry)], device_id="laptop", now=0.0) + client.push_operations([], device_id="desktop", now=0.0) + + client.push_operations([_op(entry.removed("laptop"))], device_id="laptop", now=1.0) + # Years pass on the laptop's clock; the desktop has not been back. + for year in range(1, 4): + result = client.push_operations([], device_id="laptop", now=year * 3.2e7) + assert result.bucket.get_entry("hotkeys", "hk1").is_deleted is True + + seen = client.push_operations([], device_id="desktop", now=1e9) + assert seen.bucket.values("hotkeys") == {} + # Now that every peer has acknowledged it, the tombstone can go. + final = client.push_operations([], device_id="laptop", now=1e9) + assert final.bucket.sections["hotkeys"] == {} + # ... and nobody keeps pushing acknowledgements at each other afterwards. + puts = len(server.puts) + client.push_operations([], device_id="desktop", now=1e9) + client.push_operations([], device_id="laptop", now=1e9) + assert len(server.puts) == puts + + +def test_retired_peer_requires_full_sync(server): + client = _client() + entry = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + client.push_operations([_op(entry)], device_id="laptop") + client.push_operations([], device_id="desktop") + stale_edit = entry.edited({"combo": "made while away"}, "desktop") + + client.retire_peer("desktop") + client.push_operations([_op(entry.removed("laptop"))], device_id="laptop") + # With the desktop retired the laptop alone confirms, and the tombstone is collected. + collected = client.push_operations([], device_id="laptop") + assert collected.bucket.sections["hotkeys"] == {} + + before = server.revision + with pytest.raises(FullResyncRequired): + client.push_operations([_op(stale_edit)], device_id="desktop") + assert server.revision == before, "a retired device must not write" + + adopted = client.full_resync(device_id="desktop") + assert adopted.values("hotkeys") == {}, "the deleted entry did not come back" + assert adopted.peer_states()["desktop"].retired is False + assert client.push_operations([], device_id="desktop").bucket.values("hotkeys") == {} + + +def test_peers_unseen_for_too_long_are_retired_when_asked(server): + client = _client() + client.push_operations([], device_id="desktop", now=0.0) + entry = SyncEntry.create("hk1", {"combo": "a"}, "laptop") + client.push_operations([_op(entry)], device_id="laptop", now=100.0, max_offline_s=50.0) + assert server.body["peers"]["desktop"]["retired"] is True + assert server.body["peers"]["laptop"]["retired"] is False + + +def test_two_devices_pushing_at_once_lose_nothing(server): + client = _client() + mine = SyncEntry.create("mine", {"combo": "m"}, "laptop") + theirs = SyncEntry.create("theirs", {"combo": "t"}, "desktop") + real = server.request + raced = [] + + def racing(method, body=None): + if method == "PUT" and not raced: + raced.append(True) + # The desktop's push lands between the laptop's fetch and its push. + ConfigSyncClient("https://sync.invalid", user_id="alice").push_operations( + [_op(theirs)], device_id="desktop") + return real(method, body) + + server.request = racing + result = client.push_operations([_op(mine)], device_id="laptop") + assert set(result.bucket.values("hotkeys")) == {"mine", "theirs"} + assert result.revision == 2 + + +def test_resending_a_batch_changes_nothing(server): + client = _client() + operations = [_op(SyncEntry.create("hk1", {"combo": "a"}, "laptop"))] + first = client.push_operations(operations, device_id="laptop") + again = client.push_operations(operations, device_id="laptop") + assert first.pushed is True and again.pushed is False + assert again.revision == first.revision == 1 + + +# --- the outbox ------------------------------------------------------------ + +def _outbox(tmp_path, **options): + return SyncOutbox(tmp_path / "outbox.sqlite3", account="alice", + endpoint="https://sync.invalid", **options) + + +def test_restart_retries_outbox(tmp_path, server): + entry = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + server.offline = True + first_run = _outbox(tmp_path, base_delay_s=0.0) + first_run.enqueue(_op(entry)) + report = first_run.drain( + lambda batch: _client().push_operations(batch, device_id="laptop"), max_attempts=2) + assert report.offline is True and report.pending == 1 and report.attempts == 2 + + # The program restarts; the server comes back. + server.offline = False + second_run = _outbox(tmp_path, base_delay_s=0.0) + queued = second_run.pending() + assert [operation.operation_id for operation in queued] == [entry.operation_id] + report = second_run.drain( + lambda batch: _client().push_operations(batch, device_id="laptop")) + assert report.sent == 1 and report.pending == 0 and report.offline is False + assert server.body["sections"]["hotkeys"]["hk1"]["operation_id"] == entry.operation_id + + +def test_an_uncertain_send_is_resent_under_the_same_operation_id(tmp_path, server): + """The push arrived but its reply did not: the resend must not double-apply.""" + entry = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + outbox = _outbox(tmp_path, base_delay_s=0.0) + outbox.enqueue(_op(entry)) + outbox.enqueue(_op(entry)) # queued twice by mistake: still one operation + lost = [] + + def send(batch): + _client().push_operations(batch, device_id="laptop") + if not lost: + lost.append(True) + raise ConfigSyncError("config sync PUT failed: timed out") + + report = outbox.drain(send) + assert report.sent == 1 and report.attempts == 2 + assert server.revision == 1, "the resend found its change already there" + + +def test_backoff_is_bounded_and_the_queue_waits_for_it(tmp_path): + outbox = _outbox(tmp_path, base_delay_s=2.0, max_delay_s=30.0) + assert [outbox.retry_delay(n) for n in (0, 1, 2, 3, 4, 5, 6, 60)] == [ + 0.0, 2.0, 4.0, 8.0, 16.0, 30.0, 30.0, 30.0] + entry = SyncEntry.create("hk1", {"combo": "a"}, "laptop") + outbox.enqueue(_op(entry)) + outbox.record_failure([entry.operation_id], now=100.0, error="offline") + assert outbox.seconds_until_due(100.0) == pytest.approx(2.0) + calls = [] + report = outbox.drain(calls.append, wait=False, clock=lambda: 100.5) + assert calls == [] and report.pending == 1 and report.offline is True + + +def test_a_drain_waiting_to_retry_can_be_cancelled(tmp_path): + outbox = _outbox(tmp_path, base_delay_s=3600.0, max_delay_s=3600.0) + outbox.enqueue(_op(SyncEntry.create("hk1", {"combo": "a"}, "laptop"))) + cancel = threading.Event() + attempts = [] + + def send(batch): + attempts.append(batch) + cancel.set() # cancelled while the hour-long back-off is pending + raise ConfigSyncError("offline") + + report = outbox.drain(send, cancel=cancel, max_attempts=10) + assert report.cancelled is True and len(attempts) == 1 and report.pending == 1 + + +def test_outboxes_are_isolated_by_account_and_endpoint(tmp_path): + path = tmp_path / "outbox.sqlite3" + alice = SyncOutbox(path, account="alice", endpoint="https://one.invalid") + bob = SyncOutbox(path, account="bob", endpoint="https://one.invalid") + elsewhere = SyncOutbox(path, account="alice", endpoint="https://two.invalid") + alice.enqueue(_op(SyncEntry.create("hk1", {"combo": "a"}, "laptop"))) + alice.save_baseline(ConfigBucket(user_id="alice", revision=7)) + assert len(alice.pending()) == 1 + assert bob.pending() == [] and elsewhere.pending() == [] + assert bob.load_baseline() is None and elsewhere.load_baseline() is None + assert SyncOutbox(path, account="alice", + endpoint="https://one.invalid/").load_baseline().revision == 7 + + +def test_a_full_resync_is_not_retried_by_the_outbox(tmp_path): + outbox = _outbox(tmp_path, base_delay_s=0.0) + outbox.enqueue(_op(SyncEntry.create("hk1", {"combo": "a"}, "laptop"))) + + def send(_batch): + raise FullResyncRequired("retired") + + with pytest.raises(FullResyncRequired): + outbox.drain(send) + assert [operation.entry.key for operation in outbox.clear()] == ["hk1"] + assert outbox.pending() == [] diff --git a/test/unit_test/headless/test_signaling_config_routes.py b/test/unit_test/headless/test_signaling_config_routes.py index adf4c525d..9fd377698 100644 --- a/test/unit_test/headless/test_signaling_config_routes.py +++ b/test/unit_test/headless/test_signaling_config_routes.py @@ -38,7 +38,7 @@ def test_a_bucket_round_trips(client): assert client.put("/config/alice", json=_put(bucket), headers=_SECRET).status_code == 200 # The revision is the one the server committed, not the one the client sent. assert client.get("/config/alice", headers=_SECRET).json() == { - **bucket, "revision": 1, "version": 2} + **bucket, "revision": 1, "version": 2, "peers": {}} def test_config_routes_need_the_secret(client): From c96286bf61e7ce302fefa32909d077980b5d6dc5 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:50:25 +0800 Subject: [PATCH 10/44] Let a list use the package it loads when the gate allows that package Every command name is validated before the first action runs, and AC_add_package_to_executor only registers _ when it runs, so a list that loaded an allowed package and then used it was refused for an unknown command whatever the gate said. Validation now leaves such a name to run time, but only after a load command that names the package literally, earlier in the walk, and only if the gate would let it through; every other unknown name is rejected up front as before. The repository's sample keyword1.json still loaded time, which the gate now refuses by default: it is regenerated from the current project template. --- AutoControl/keyword/keyword1.json | 30 +-- .../keyword_and_executor_doc.rst | 23 ++ .../keyword_and_executor_doc.rst | 19 ++ je_auto_control/cli.py | 2 +- .../utils/executor/action_executor.py | 13 +- .../utils/executor/action_schema.py | 91 ++++++- .../package_manager/package_manager_class.py | 10 + .../headless/test_package_gate_same_list.py | 247 ++++++++++++++++++ 8 files changed, 391 insertions(+), 44 deletions(-) create mode 100644 test/unit_test/headless/test_package_gate_same_list.py diff --git a/AutoControl/keyword/keyword1.json b/AutoControl/keyword/keyword1.json index d50d68df7..51a454110 100644 --- a/AutoControl/keyword/keyword1.json +++ b/AutoControl/keyword/keyword1.json @@ -86,39 +86,15 @@ [ "AC_write", { - "write_string": "now i will add time module and sleep 3 sec" + "write_string": "now i will sleep 3 sec" } ], [ - "AC_add_package_to_executor", - [ - "time" - ] - ], - [ - "time_sleep", - [ - 3 - ] - ], - [ - "AC_type_keyboard", + "AC_sleep", { - "keycode": "return" + "seconds": 3 } ], - [ - "AC_write", - { - "write_string": "also you can use builtin function on executor like print" - } - ], - [ - "print", - [ - "Google Bye World" - ] - ], [ "AC_type_keyboard", { diff --git a/docs/source/Eng/doc/keyword_and_executor/keyword_and_executor_doc.rst b/docs/source/Eng/doc/keyword_and_executor/keyword_and_executor_doc.rst index b7c0660e0..dc3de8b62 100644 --- a/docs/source/Eng/doc/keyword_and_executor/keyword_and_executor_doc.rst +++ b/docs/source/Eng/doc/keyword_and_executor/keyword_and_executor_doc.rst @@ -126,6 +126,29 @@ For example, ``time.sleep`` becomes ``time_sleep``: ["time_sleep", {"secs": 2}] ] +An action file may load a package and use it in the same list, provided the +gate allows that package: + +.. code-block:: json + + [ + ["AC_add_package_to_executor", ["time"]], + ["time_sleep", [2]] + ] + +Every command name is checked before the first action runs, and +``time_sleep`` does not exist until the load has run, so this list used to be +refused for an unknown command whatever the gate said. The check now leaves a +name to run time when all of these hold: it starts with ``_``; an +``AC_add_package_to_executor`` earlier in the file names that package +literally (``["time"]`` or ``{"package": "time"}``, not a ``${variable}``); +and the gate would allow the package. Anything else -- a package that is not +allowed, a name used before its load, a typo in an ``AC_*`` name -- is still +rejected before anything runs. A deferred name that turns out not to exist +(``time_slep``) fails its own action with ``Unknown action`` when the run +reaches it. ``je_auto_control validate`` and the REST pre-check follow the +same rule. + To inspect the current executor command dictionary: .. code-block:: python diff --git a/docs/source/Zh/doc/keyword_and_executor/keyword_and_executor_doc.rst b/docs/source/Zh/doc/keyword_and_executor/keyword_and_executor_doc.rst index ac87b7f88..e90bff394 100644 --- a/docs/source/Zh/doc/keyword_and_executor/keyword_and_executor_doc.rst +++ b/docs/source/Zh/doc/keyword_and_executor/keyword_and_executor_doc.rst @@ -122,6 +122,25 @@ JSON 陣列（關鍵字），由執行者解析並執行。 ["time_sleep", {"secs": 2}] ] +只要閘門允許該套件，動作檔可以在同一份清單裡載入套件並使用它： + +.. code-block:: json + + [ + ["AC_add_package_to_executor", ["time"]], + ["time_sleep", [2]] + ] + +所有指令名稱會在第一個動作執行前先檢查，而 ``time_sleep`` 要等載入指令執行後 +才存在，所以這樣的清單以前不論閘門怎麼設定都會以 unknown command 被拒絕。 +現在符合下列全部條件的名稱會留到執行時才檢查：名稱以 ``<套件>_`` 開頭；檔案中 +較前面有一個 ``AC_add_package_to_executor`` 以字面值指名該套件（``["time"]`` +或 ``{"package": "time"}``\ ，不能是 ``${變數}``\ ）；而且閘門會放行該套件。 +其他情況——套件未被允許、名稱用在載入之前、``AC_*`` 名稱打錯——仍然在任何動作 +執行前就被拒絕。留到執行時的名稱若其實不存在（``time_slep``\ ），執行到它時 +該動作會以 ``Unknown action`` 失敗。``je_auto_control validate`` 與 REST 的 +事前檢查遵循同一條規則。 + 查看目前執行者的指令字典： .. code-block:: python diff --git a/je_auto_control/cli.py b/je_auto_control/cli.py index 672b9e4bc..c9d45a2b9 100644 --- a/je_auto_control/cli.py +++ b/je_auto_control/cli.py @@ -106,7 +106,7 @@ def cmd_validate(args: argparse.Namespace) -> int: # Unwrapped and checked like `run` does: {"auto_control": [...]} failed # here and ran there, and [] passed here and failed there. actions = executor._unwrap_action_list(read_action_json(args.script)) - validate_actions(actions, executor.known_commands()) + validate_actions(actions, executor.known_commands(), executor._self_loadable()) sys.stdout.write(f"OK: {len(actions)} action(s)\n") return 0 diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 22dd31e24..c95566b22 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -8133,7 +8133,16 @@ def unknown_commands_in(self, action_list: Union[list, dict]) -> List[str]: failed" *before* the first action moves anything. """ return unknown_command_names(self._unwrap_action_list(action_list), - self.known_commands()) + self.known_commands(), self._self_loadable()) + + def _self_loadable(self) -> Optional[Callable[[str], bool]]: + """The package gate's verdict, on the executor a load command fills. + + ``AC_add_package_to_executor`` registers ``_`` names + on the package manager's executor, so only there may validation leave + such a name to run time; anywhere else it would never appear. + """ + return package_manager.would_allow if package_manager.executor is self else None def _resolve_runtime_args(self, args: Any, command: str = "") -> Any: """Interpolate ``${var}`` placeholders against the current scope. @@ -8235,7 +8244,7 @@ def _execute_list(self, action_list: Union[list, dict], raise_on_error: bool, """The body of :meth:`execute_action`, with strictness already settled.""" action_list = self._unwrap_action_list(action_list) if not _validated: - validate_actions(action_list, self.known_commands()) + validate_actions(action_list, self.known_commands(), self._self_loadable()) execute_record_dict: Dict[str, Any] = {} for action in action_list: diff --git a/je_auto_control/utils/executor/action_schema.py b/je_auto_control/utils/executor/action_schema.py index 72d0fdf8f..1b5a7227c 100644 --- a/je_auto_control/utils/executor/action_schema.py +++ b/je_auto_control/utils/executor/action_schema.py @@ -3,8 +3,19 @@ Validates the outer shape (``[name]`` / ``[name, params]``), that names are in the executor allowlist, and that flow-control nested bodies are themselves valid lists. + +Names a list brings in itself. ``AC_add_package_to_executor`` registers a +package's members as ``_`` commands when it runs, which is +after validation: a list that loaded a package and then used it was rejected +for an unknown command before its first action. A validator given a +``loadable`` predicate (the package gate's verdict, without importing +anything) therefore leaves such a name to run time -- but only a name that +starts with ``_``, only after a load command that names that +package literally, earlier in the walk, and only if the gate would let the +load through. Every other unknown name is rejected up front as before, and +so is a deferred name that turns out not to exist, when its action runs. """ -from typing import Any, Iterable, Iterator, List, Tuple +from typing import Any, Callable, Iterable, Iterator, List, Optional, Tuple from je_auto_control.utils.exception.exceptions import AutoControlActionException @@ -69,28 +80,80 @@ } -def validate_actions(actions: Any, known_commands: Iterable[str]) -> None: - """Validate an action list recursively; raise on the first problem.""" +#: Commands that register a package's members on the executor when they run. +PACKAGE_LOAD_COMMANDS = frozenset({"AC_add_package_to_executor"}) + + +class _SelfLoadedNames: + """Command names the list being walked will register by loading a package.""" + + def __init__(self, loadable: Optional[Callable[[str], bool]]) -> None: + self._loadable = loadable + self._prefixes: List[str] = [] + + def note(self, name: Any, action: list) -> None: + """Remember the package ``action`` loads, if it is a load the gate allows.""" + if self._loadable is None or name not in PACKAGE_LOAD_COMMANDS: + return + package = _literal_package(action) + if package is not None and self._loadable(package): + self._prefixes.append(package + "_") + + def covers(self, name: str) -> bool: + """Whether ``name`` could be a member of a package loaded so far.""" + return any(name.startswith(prefix) and len(name) > len(prefix) + for prefix in self._prefixes) + + +def _literal_package(action: list) -> Optional[str]: + """The package a load command names: ``[name, [pkg]]`` or ``[name, {"package": pkg}]``.""" + if len(action) != 2: + return None + params = action[1] + if isinstance(params, dict): + package = params.get("package") + elif isinstance(params, list) and len(params) == 1: + package = params[0] + else: + return None + return package if isinstance(package, str) else None + + +def validate_actions(actions: Any, known_commands: Iterable[str], + loadable: Optional[Callable[[str], bool]] = None) -> None: + """Validate an action list recursively; raise on the first problem. + + ``loadable`` says whether the package gate would let the list load a + package; with it, names of a package the list itself loads earlier are + left to run time (see the module docstring). Without it every name must + be in ``known_commands``. + """ known = set(known_commands) - for trail, name in _iter_actions(actions, "root"): - if not isinstance(name, str) or name not in known: + self_loaded = _SelfLoadedNames(loadable) + for trail, name, action in _iter_actions(actions, "root"): + if not isinstance(name, str) or (name not in known and not self_loaded.covers(name)): raise AutoControlActionException(f"{trail}: unknown command {name!r}") + self_loaded.note(name, action) def unknown_command_names(actions: Any, - known_commands: Iterable[str]) -> List[str]: + known_commands: Iterable[str], + loadable: Optional[Callable[[str], bool]] = None, + ) -> List[str]: """Return every unrecognised command name in ``actions``, in order. Structural problems still raise, exactly as :func:`validate_actions` does; the only difference is that an unknown *name* is collected instead of ending the walk. A boundary that has to answer a caller — the REST API — reports the whole list, so a client fixes every typo in one round trip - rather than one per request. + rather than one per request. ``loadable`` is as in :func:`validate_actions`. """ known = set(known_commands) + self_loaded = _SelfLoadedNames(loadable) unknown: List[str] = [] - for _trail, name in _iter_actions(actions, "root"): - if isinstance(name, str) and name in known: + for _trail, name, action in _iter_actions(actions, "root"): + if isinstance(name, str) and (name in known or self_loaded.covers(name)): + self_loaded.note(name, action) continue label = name if isinstance(name, str) else repr(name) if label not in unknown: @@ -98,8 +161,8 @@ def unknown_command_names(actions: Any, return unknown -def _iter_actions(actions: Any, trail: str) -> Iterator[Tuple[str, Any]]: - """Yield ``(trail, command_name)`` for every action in the tree, in order. +def _iter_actions(actions: Any, trail: str) -> Iterator[Tuple[str, Any, list]]: + """Yield ``(trail, command_name, action)`` for every action in the tree, in order. Structural problems raise as they are met, but whether the name is in the allowlist is left to the caller. That is what lets "reject the first @@ -120,7 +183,7 @@ def _iter_actions(actions: Any, trail: str) -> Iterator[Tuple[str, Any]]: raise AutoControlActionException( f"{node}: must be [name] or [name, params]" ) - yield node, action[0] + yield node, action[0], action if len(action) == 2 and not isinstance(action[1], (dict, list)): raise AutoControlActionException( f"{node}: params must be dict or list" @@ -129,7 +192,7 @@ def _iter_actions(actions: Any, trail: str) -> Iterator[Tuple[str, Any]]: def _iter_nested_actions(name: Any, action: list, - trail: str) -> Iterator[Tuple[str, Any]]: + trail: str) -> Iterator[Tuple[str, Any, list]]: """Yield the actions held in a flow-control command's nested body keys.""" # ``name`` arrives unvalidated — the collector does not stop on a bad one — # and an unhashable name would blow up the two lookups below. @@ -154,6 +217,6 @@ def _iter_nested_actions(name: Any, action: list, __all__ = [ "BLOCK_REQUIRED_KEYS", - "FLOW_BODY_KEYS", "FLOW_BRANCH_LIST_KEYS", + "FLOW_BODY_KEYS", "FLOW_BRANCH_LIST_KEYS", "PACKAGE_LOAD_COMMANDS", "validate_actions", "unknown_command_names", ] diff --git a/je_auto_control/utils/package_manager/package_manager_class.py b/je_auto_control/utils/package_manager/package_manager_class.py index ffdbbe662..8f3cb9b5b 100644 --- a/je_auto_control/utils/package_manager/package_manager_class.py +++ b/je_auto_control/utils/package_manager/package_manager_class.py @@ -79,6 +79,16 @@ def _is_allowlisted(self, package: str) -> bool: return any(package == allowed or package.startswith(allowed + ".") for allowed in self.allowed_packages) + def would_allow(self, package: object) -> bool: + """ + 閘門是否會放行這個套件（不匯入任何東西） + Whether the gate would let ``package`` be loaded. Nothing is imported: this is + what lets an action list be validated before the load command in it has run. + """ + if not is_package_name(package): + return False + return self.allow_arbitrary_packages or self._is_allowlisted(package) + def _check_allowed(self, package: object) -> None: """Refuse ``package`` before it is imported, unless the gate lets it through.""" if isinstance(package, str) and self._is_allowlisted(package): diff --git a/test/unit_test/headless/test_package_gate_same_list.py b/test/unit_test/headless/test_package_gate_same_list.py new file mode 100644 index 000000000..73387e36b --- /dev/null +++ b/test/unit_test/headless/test_package_gate_same_list.py @@ -0,0 +1,247 @@ +"""A list may load a package and use it, when the gate allows that package. + +``execute_action`` validates every command name before it runs anything, and +``AC_add_package_to_executor`` only registers ``_`` when it +runs -- so a list that loaded an allowed package and then used it was refused +for an unknown command whatever the gate said. Validation now leaves exactly +those names to run time: a name with the prefix of a package that a load +command names literally, earlier in the walk, and that the gate would allow. +Everything else is rejected up front as before. + +``json`` is the package loaded here; only its members and a probe run. +""" +import json +from pathlib import Path + +import pytest + +from je_auto_control.utils.exception.exceptions import AutoControlActionException +from je_auto_control.utils.executor.action_executor import Executor, executor +from je_auto_control.utils.executor.action_schema import ( + unknown_command_names, validate_actions, +) +from je_auto_control.utils.package_manager import package_manager_class +from je_auto_control.utils.package_manager.package_manager_class import PackageManager + +PROBE = "AC_gate_probe" +_LOAD = ["AC_add_package_to_executor", ["json"]] +_USE = ["json_dumps", [[1, 2]]] +_REPO = Path(__file__).resolve().parents[3] + + +@pytest.fixture(autouse=True) +def _no_inherited_allowlist(monkeypatch): + monkeypatch.delenv("JE_AUTOCONTROL_ALLOWED_PACKAGES", raising=False) + + +@pytest.fixture +def gate(): + """The shared gate, closed and empty; it and the command table are restored.""" + manager = package_manager_class.package_manager + saved_gate = (manager.allow_arbitrary_packages, set(manager.allowed_packages)) + saved_commands = dict(executor.event_dict) + manager.allow_arbitrary_packages, manager.allowed_packages = False, set() + for name in [name for name in executor.event_dict if name.startswith("json_")]: + del executor.event_dict[name] + yield manager + manager.allow_arbitrary_packages, manager.allowed_packages = saved_gate[0], saved_gate[1] + executor.event_dict.clear() + executor.event_dict.update(saved_commands) + + +@pytest.fixture +def ran(gate): + """A probe command; the list records every call it receives.""" + calls = [] + executor.event_dict[PROBE] = lambda value=None: calls.append(value) or value + return calls + + +def _values(record): + return list(record.values()) + + +# --- the defect ------------------------------------------------------------- + +def test_a_list_loads_an_allowed_package_and_uses_it(gate): + gate.allow_packages("json") + record = executor.execute_action([_LOAD, _USE]) + assert _values(record)[1] == "[1, 2]" + + +def test_the_keyword_form_of_the_load_command_counts_too(gate): + gate.allow_packages("json") + record = executor.execute_action( + [["AC_add_package_to_executor", {"package": "json"}], _USE]) + assert _values(record)[1] == "[1, 2]" + + +def test_an_open_gate_defers_any_package_the_list_loads(gate): + gate.set_allow_arbitrary_packages(True) + record = executor.execute_action([_LOAD, _USE]) + assert _values(record)[1] == "[1, 2]" + + +def test_a_submodule_of_an_allowed_package_is_deferred(gate): + gate.allow_packages("json") + record = executor.execute_action([ + ["AC_add_package_to_executor", ["json.decoder"]], + ["json.decoder_JSONDecoder"], + ]) + assert "JSONDecoder object" in str(_values(record)[1]) + + +def test_the_name_may_be_used_inside_a_nested_body(gate, ran): + gate.allow_packages("json") + record = executor.execute_action([ + _LOAD, + ["AC_loop", {"times": 2, "body": [_USE, [PROBE, {"value": "x"}]]}], + ]) + assert ran == ["x", "x"] + assert "Unknown" not in str(record) + + +def test_a_load_inside_a_body_covers_what_follows_it(gate): + gate.allow_packages("json") + record = executor.execute_action([ + ["AC_loop", {"times": 1, "body": [_LOAD]}], + _USE, + ]) + assert _values(record)[1] == "[1, 2]" + + +# --- what is still rejected before anything runs ---------------------------- + +def _refused(actions, ran, name): + with pytest.raises(AutoControlActionException, match=f"unknown command '{name}'"): + executor.execute_action([[PROBE, {"value": "first"}], *actions]) + assert ran == [] # nothing ran, not even the first action + assert "json_dumps" not in executor.event_dict # ...and nothing was imported + + +def test_a_package_the_gate_refuses_is_not_deferred(ran): + _refused([_LOAD, _USE], ran, "json_dumps") + + +def test_a_name_used_before_its_load_is_rejected(gate, ran): + gate.allow_packages("json") + _refused([_USE, _LOAD], ran, "json_dumps") + + +def test_a_name_of_another_package_is_rejected(gate, ran): + gate.allow_packages("json") + _refused([_LOAD, ["os_getcwd"]], ran, "os_getcwd") + + +def test_the_bare_prefix_is_rejected(gate, ran): + gate.allow_packages("json") + _refused([_LOAD, ["json_"]], ran, "json_") + + +def test_an_unknown_name_elsewhere_in_the_list_is_still_rejected(gate, ran): + gate.allow_packages("json") + _refused([_LOAD, _USE, ["AC_no_such_command"]], ran, "AC_no_such_command") + + +@pytest.mark.parametrize("params", [ + ["${package}"], ["json", "extra"], [], {"name": "json"}, {"package": 7}, [["json"]], +], ids=["placeholder", "two-args", "no-args", "wrong-key", "not-a-string", "nested"]) +def test_a_load_that_does_not_name_a_package_literally_defers_nothing(gate, ran, params): + gate.set_allow_arbitrary_packages(True) + _refused([["AC_add_package_to_executor", params], _USE], ran, "json_dumps") + + +def test_a_private_executor_never_receives_the_names_so_they_are_rejected(gate): + gate.allow_packages("json") + private = Executor() + with pytest.raises(AutoControlActionException, match="unknown command 'json_dumps'"): + private.execute_action([_LOAD, _USE]) + + +def test_the_callback_executor_load_command_defers_nothing(gate, ran): + gate.allow_packages("json") + _refused([["AC_add_package_to_callback_executor", ["json"]], _USE], ran, "json_dumps") + + +# --- a deferred name is checked when its action runs ------------------------ + +def test_a_deferred_name_that_does_not_exist_fails_its_own_action(gate, ran): + from je_auto_control.utils.executor.action_executor import ( + recorded_failures, reset_recorded_failures, + ) + gate.allow_packages("json") + reset_recorded_failures() + record = executor.execute_action([ + _LOAD, ["json_no_such_member"], [PROBE, {"value": "after"}]]) + assert "Unknown action: json_no_such_member" in str(_values(record)[1]) + assert recorded_failures() == 1 + assert ran == ["after"] + + +def test_a_deferred_name_fails_loudly_under_raise_on_error(gate): + gate.allow_packages("json") + with pytest.raises(AutoControlActionException, match="Unknown action: json_no_such_member"): + executor.execute_action([_LOAD, ["json_no_such_member"]], raise_on_error=True) + + +# --- the other validators agree --------------------------------------------- + +def test_unknown_commands_in_follows_the_same_rule(gate): + assert executor.unknown_commands_in([_LOAD, _USE]) == ["json_dumps"] + gate.allow_packages("json") + assert executor.unknown_commands_in([_LOAD, _USE, ["nope"], ["os_getcwd"]]) == [ + "nope", "os_getcwd"] + assert executor.unknown_commands_in([_USE, _LOAD]) == ["json_dumps"] + + +def test_cli_validate_and_run_agree(gate, tmp_path, capsys): + from je_auto_control.cli import main + script = tmp_path / "script.json" + script.write_text(json.dumps([_LOAD, _USE]), encoding="utf-8") + assert main(["validate", str(script)]) == 1 + assert main(["run", str(script)]) == 1 + assert "unknown command 'json_dumps'" in capsys.readouterr().err + assert main(["run", str(script), "--allow-package", "json"]) == 0 + assert '"[1, 2]"' in capsys.readouterr().out + assert main(["validate", str(script)]) == 0 # the gate is still open in this process + + +def test_the_validators_are_unchanged_without_a_gate(): + known = {"AC_add_package_to_executor"} + with pytest.raises(AutoControlActionException, match="unknown command 'json_dumps'"): + validate_actions([_LOAD, _USE], known) + assert unknown_command_names([_LOAD, _USE], known) == ["json_dumps"] + validate_actions([_LOAD, _USE], known, loadable=lambda package: package == "json") + assert unknown_command_names( + [_LOAD, _USE], known, loadable=lambda package: False) == ["json_dumps"] + + +def test_would_allow_imports_nothing(): + from unittest.mock import patch + manager = PackageManager() + manager.allowed_packages = {"json"} + with patch.object(package_manager_class.importlib, "import_module") as importer: + assert manager.would_allow("json") is True + assert manager.would_allow("json.decoder") is True + assert manager.would_allow("jsonx") is False + assert manager.would_allow("os") is False + for not_a_name in ("", "os; rm", "${package}", 3, None, ["json"]): + assert manager.would_allow(not_a_name) is False + manager.set_allow_arbitrary_packages(True) + assert manager.would_allow("os") is True + assert manager.would_allow("os; rm") is False + importer.assert_not_called() + + +# --- the sample project in the repository root ------------------------------ + +@pytest.mark.parametrize("name", ["keyword1.json", "keyword2.json"]) +def test_the_repository_sample_runs_without_opening_the_gate(name): + """The checked-in sample is what ``--create_project`` writes: no package load.""" + from je_auto_control.utils.project.template import template_keyword + actions = json.loads((_REPO / "AutoControl" / "keyword" / name).read_text(encoding="utf-8")) + validate_actions(actions, executor.known_commands()) + assert not [action for action in actions if action[0].startswith("AC_add_package")] + template = {"keyword1.json": template_keyword.template_keyword_1, + "keyword2.json": template_keyword.template_keyword_2}[name] + assert actions == json.loads(json.dumps(template)) From f2c8fd6a3f01f968bafbd9aa81e9106262b7c7d6 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:51:29 +0800 Subject: [PATCH 11/44] Measure self-healing locator versions on the same labelled frames and stop a located point from reading as a verified action A heal log could say what the locator did but not whether it was right: a hit at the wrong place counted the same as a recovery, an unlabelled frame could not be told from a correct one, and screen_region reached only the VLM path, so the template match answered from outside the region the caller asked for. evaluate_locators scores strategy versions against one captured frame and one region per sample, reports every rate with its numerator and denominator plus p50/p95 latency, and keeps unknown, false positive and error apart from correct. HealEvent gains optional version / context / verification fields and still loads older lines. A new template image is a candidate that has to be previewed against labelled frames before it replaces anything, and can be reverted. benchmarks/self_healing holds the fixed synthetic regression set. --- benchmarks/self_healing/dataset.json | 93 ++++ benchmarks/self_healing/run.py | 71 +++ benchmarks/self_healing/synthetic.py | 106 ++++ .../Eng/doc/new_features/v2_features_doc.rst | 142 +++++ .../Zh/doc/new_features/v2_features_doc.rst | 128 +++++ je_auto_control/__init__.py | 13 + je_auto_control/actions.pyi | 29 + .../gui/language_wrapper/english.py | 28 + .../gui/language_wrapper/japanese.py | 28 + .../language_wrapper/simplified_chinese.py | 28 + .../language_wrapper/traditional_chinese.py | 28 + .../gui/script_builder/command_schema.py | 45 ++ je_auto_control/gui/self_healing_tab.py | 187 ++++++- .../utils/executor/action_executor.py | 100 +++- .../utils/heal_analytics/heal_analytics.py | 18 + .../utils/mcp_server/tools/_factories.py | 81 +++ .../mcp_server/tools/_handlers_locators.py | 76 ++- .../utils/self_healing/__init__.py | 45 +- .../utils/self_healing/eval_strategies.py | 241 +++++++++ .../utils/self_healing/evaluation.py | 510 ++++++++++++++++++ .../utils/self_healing/heal_log.py | 45 +- je_auto_control/utils/self_healing/locator.py | 222 ++++++-- .../utils/self_healing/template_revision.py | 351 ++++++++++++ .../test_self_healing_eval_surfaces.py | 248 +++++++++ .../headless/test_self_healing_evaluation.py | 344 ++++++++++++ .../test_self_healing_template_revision.py | 331 ++++++++++++ 26 files changed, 3429 insertions(+), 109 deletions(-) create mode 100644 benchmarks/self_healing/dataset.json create mode 100644 benchmarks/self_healing/run.py create mode 100644 benchmarks/self_healing/synthetic.py create mode 100644 je_auto_control/utils/self_healing/eval_strategies.py create mode 100644 je_auto_control/utils/self_healing/evaluation.py create mode 100644 je_auto_control/utils/self_healing/template_revision.py create mode 100644 test/unit_test/headless/test_self_healing_eval_surfaces.py create mode 100644 test/unit_test/headless/test_self_healing_evaluation.py create mode 100644 test/unit_test/headless/test_self_healing_template_revision.py diff --git a/benchmarks/self_healing/dataset.json b/benchmarks/self_healing/dataset.json new file mode 100644 index 000000000..6209e65f4 --- /dev/null +++ b/benchmarks/self_healing/dataset.json @@ -0,0 +1,93 @@ +{ + "schema_version": 1, + "seed": 20261002, + "template_size": [56, 32], + "samples": [ + { + "id": "plain", + "description": "target on the primary monitor at 100% scale", + "frame_size": [320, 200], + "targets": [{"at": [100, 60]}], + "expected_target": 0 + }, + { + "id": "scaled-150", + "description": "the same screen captured at 150% display scale", + "frame_size": [480, 300], + "scale": 1.5, + "targets": [{"at": [100, 60]}], + "expected_target": 0 + }, + { + "id": "negative-origin", + "description": "target on a monitor left of and above the primary", + "frame_size": [320, 200], + "origin": [-1920, -300], + "targets": [{"at": [40, 120]}], + "expected_target": 0 + }, + { + "id": "negative-origin-scaled-125", + "description": "negative-origin monitor at 125% display scale", + "frame_size": [400, 250], + "origin": [-2560, -200], + "scale": 1.25, + "targets": [{"at": [200, 40]}], + "expected_target": 0 + }, + { + "id": "region-second-of-two", + "description": "two identical targets; the region selects the second", + "frame_size": [320, 200], + "targets": [{"at": [20, 20]}, {"at": [220, 140]}], + "region": [160, 100, 320, 200], + "expected_target": 1 + }, + { + "id": "region-negative-origin", + "description": "region and target both in negative screen coordinates", + "frame_size": [320, 200], + "origin": [-320, -200], + "targets": [{"at": [30, 30]}, {"at": [200, 150]}], + "region": [-320, -200, -160, -100], + "expected_target": 0 + }, + { + "id": "failing-restyled", + "description": "the control was redesigned; the old template cannot find it", + "frame_size": [320, 200], + "targets": [{"at": [100, 60], "style": "restyled"}], + "expected_target": 0 + }, + { + "id": "failing-absent", + "description": "the target is not on screen; the right answer is a miss", + "frame_size": [320, 200], + "targets": [], + "expect_miss": true + }, + { + "id": "failing-decoy-only", + "description": "only a look-alike neighbour is on screen; the right answer is a miss", + "frame_size": [320, 200], + "targets": [{"at": [100, 60], "style": "decoy"}], + "expect_miss": true + }, + { + "id": "unlabelled", + "description": "a frame nobody labelled: reported as unknown, never as correct", + "frame_size": [320, 200], + "targets": [{"at": [60, 90]}] + } + ], + "versions": { + "v1-exact": {"strategy": "template", "threshold": 0.9}, + "v2-multiscale": {"strategy": "template", "threshold": 0.9, "scales": [1.0, 1.25, 1.5, 2.0]}, + "v3-loose": {"strategy": "template", "threshold": 0.5, "scales": [1.0, 1.25, 1.5, 2.0]} + }, + "thresholds": { + "v1-exact": {"min_correct": 4, "max_false_positive": 0, "max_error": 0}, + "v2-multiscale": {"min_correct": 6, "min_accuracy": 0.88, "max_false_positive": 0, "max_error": 0}, + "v3-loose": {"min_correct": 6, "max_false_positive": 1, "max_error": 0} + } +} diff --git a/benchmarks/self_healing/run.py b/benchmarks/self_healing/run.py new file mode 100644 index 000000000..371e48751 --- /dev/null +++ b/benchmarks/self_healing/run.py @@ -0,0 +1,71 @@ +"""Compare self-healing locator versions on the fixed synthetic dataset. + + python benchmarks/self_healing/run.py # text report + python benchmarks/self_healing/run.py --json out.json # also write JSON + python benchmarks/self_healing/run.py --check # exit 1 on a threshold miss + +The frames are drawn in memory from ``dataset.json`` (see ``synthetic.py``); +no screen is captured and no pointer or key event is sent. The dataset covers +a plain hit, 125% / 150% display scale, a negative-origin monitor, a region +that has to pick the second of two identical targets, a redesigned control, an +absent target, a look-alike neighbour and one unlabelled frame. + +Counts are deterministic. Latency is whatever this machine measured and is +reported, never gated. +""" +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +_HERE = Path(__file__).resolve().parent +_REPO = _HERE.parent.parent +for _entry in (str(_HERE), str(_REPO)): + if _entry not in sys.path: + sys.path.insert(0, _entry) + +import synthetic # noqa: E402 # reason: needs the sys.path entry added above + +from je_auto_control.utils.self_healing.eval_strategies import ( # noqa: E402 # reason: as above + build_strategy, comparison_payload, +) +from je_auto_control.utils.self_healing.evaluation import ( # noqa: E402 # reason: as above + HealingComparison, evaluate_locators, format_comparison, +) + + +def run() -> HealingComparison: + """Evaluate every version in ``dataset.json`` on the same drawn frames.""" + descriptor = synthetic.load_descriptor() + samples = synthetic.build_samples(descriptor) + versions = {name: build_strategy(config) + for name, config in descriptor["versions"].items()} + return evaluate_locators(samples, versions) + + +def report() -> Dict[str, Any]: + """The JSON-safe report: comparison, thresholds, violations, ``passed``.""" + return comparison_payload(run(), synthetic.load_descriptor()["thresholds"]) + + +def main(argv: Optional[List[str]] = None) -> int: + """Command-line entry; returns the process exit code.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--json", metavar="PATH", help="write the report as JSON") + parser.add_argument("--check", action="store_true", + help="exit 1 when a version misses its thresholds") + args = parser.parse_args(argv) + comparison = run() + payload = comparison_payload(comparison, synthetic.load_descriptor()["thresholds"]) + print(format_comparison(comparison)) + for violation in payload["violations"]: + print(f"THRESHOLD {violation}") + if args.json: + Path(args.json).write_text(json.dumps(payload, indent=2, sort_keys=True), + encoding="utf-8") + return 1 if args.check and payload["violations"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/self_healing/synthetic.py b/benchmarks/self_healing/synthetic.py new file mode 100644 index 000000000..4f2e6ba02 --- /dev/null +++ b/benchmarks/self_healing/synthetic.py @@ -0,0 +1,106 @@ +"""Build the fixed self-healing evaluation samples from ``dataset.json``. + +The frames are drawn here, from the descriptor, instead of being stored as +images: the descriptor is small enough to review in a diff, and a frame can be +regenerated bit for bit on any machine. Nothing in this module reads the +screen. + +Every position in the descriptor is in *logical* units relative to the frame's +top-left corner. A sample's ``scale`` is frame pixels per logical unit, so a +target at ``[100, 60]`` on a ``scale: 1.5`` sample is drawn at frame pixel +``(150, 90)`` and 1.5 times the template's size — what a HiDPI capture of the +same screen looks like. ``origin`` is the screen coordinate of the frame's +top-left corner and may be negative (a monitor left of or above the primary). +""" +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import cv2 +import numpy as np + +from je_auto_control.utils.self_healing.evaluation import EvaluationSample + +DATASET_PATH = Path(__file__).with_name("dataset.json") + +STYLE_NORMAL = "normal" +STYLE_DECOY = "decoy" +STYLE_RESTYLED = "restyled" + + +def draw_element(width: int, height: int, style: str = STYLE_NORMAL) -> np.ndarray: + """The target control as a ``height`` x ``width`` grayscale image. + + ``decoy`` is a neighbouring control that shares the frame and the label + bar but not the icon; ``restyled`` is the same control after a redesign + (inverted, different layout), which no version of the old template finds. + """ + ramp = np.linspace(70, 190, width, dtype=np.float32) + image = np.tile(ramp, (height, 1)).astype(np.uint8) + cv2.rectangle(image, (0, 0), (width - 1, height - 1), 20, 2) + if style == STYLE_RESTYLED: + image = 255 - image + cv2.line(image, (4, 4), (width - 5, height - 5), 0, 3) + return image + cv2.rectangle(image, (width // 2, height // 3), + (width - 7, height // 3 + 4), 240, -1) + if style == STYLE_NORMAL: + cv2.circle(image, (height // 2, height // 2), height // 4, 250, -1) + cv2.rectangle(image, (width // 2, 2 * height // 3), + (width - 14, 2 * height // 3 + 3), 30, -1) + else: + cv2.rectangle(image, (6, height // 3), (height // 2 + 4, 2 * height // 3), 40, -1) + return image + + +def _background(rng: np.random.Generator, width: int, height: int) -> np.ndarray: + return rng.integers(105, 135, size=(height, width), dtype=np.uint8) + + +def _paste(frame: np.ndarray, element: np.ndarray, left: int, top: int) -> None: + height, width = element.shape[:2] + frame[top:top + height, left:left + width] = element + + +def _build_sample(entry: Dict[str, Any], template: np.ndarray, + rng: np.random.Generator) -> EvaluationSample: + scale = float(entry.get("scale", 1.0)) + origin = tuple(entry.get("origin", (0, 0))) + frame_width, frame_height = entry["frame_size"] + frame = _background(rng, frame_width, frame_height) + base_height, base_width = template.shape[:2] + boxes: List[Tuple[int, int, int, int]] = [] + for target in entry.get("targets", ()): + at_x, at_y = target["at"] + element = draw_element(base_width, base_height, target.get("style", STYLE_NORMAL)) + if scale != 1.0: + # INTER_AREA / INTER_CUBIC on purpose: the strategy under test + # resizes with INTER_LINEAR, so a scaled match is close, not exact. + element = cv2.resize(element, None, fx=scale, fy=scale, + interpolation=cv2.INTER_CUBIC) + _paste(frame, element, int(round(at_x * scale)), int(round(at_y * scale))) + boxes.append((origin[0] + at_x, origin[1] + at_y, + origin[0] + at_x + base_width, origin[1] + at_y + base_height)) + expected: Optional[int] = entry.get("expected_target") + frame.setflags(write=False) + return EvaluationSample( + sample_id=entry["id"], frame=frame, + expected_box=None if expected is None else boxes[expected], + origin=origin, scale=scale, region=entry.get("region"), + expect_miss=bool(entry.get("expect_miss", False)), + template=template, description=entry.get("description")) + + +def load_descriptor(path: Path = DATASET_PATH) -> Dict[str, Any]: + """The dataset descriptor as parsed JSON.""" + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def build_samples(descriptor: Optional[Dict[str, Any]] = None) -> List[EvaluationSample]: + """Every sample of the fixed dataset, drawn deterministically.""" + data = load_descriptor() if descriptor is None else descriptor + width, height = data["template_size"] + template = draw_element(width, height) + template.setflags(write=False) + rng = np.random.default_rng(int(data["seed"])) + return [_build_sample(entry, template, rng) for entry in data["samples"]] diff --git a/docs/source/Eng/doc/new_features/v2_features_doc.rst b/docs/source/Eng/doc/new_features/v2_features_doc.rst index edeec5565..febb13b1f 100644 --- a/docs/source/Eng/doc/new_features/v2_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v2_features_doc.rst @@ -32,6 +32,148 @@ locators can be tuned over time:: Executor: ``AC_self_heal_locate / _click / _log_list / _log_clear``. MCP: ``ac_self_heal_*``. GUI: **Self-Healing** tab. +``screen_region`` is ``[x1, y1, x2, y2]`` in screen coordinates and confines +**both** strategies; it used to reach only the VLM, so the template match +could answer from outside the region. + +**Located is not verified.** ``HealOutcome.found`` says a strategy returned a +point. Whether the click did what it was for is a separate field, +``action_verified``, filled from a check the caller passes — ``None`` when no +check ran, never inferred from the hit:: + + outcome = self_heal_click( + template_path="submit.png", + description="the green Submit button", + verify=lambda result: dialog_is_open(), + ) + outcome.found # a point was returned + outcome.action_verified # True / False from the check, None without one + +Heal-log lines gained optional fields (``schema_version``, ``run_id``, +``step_id``, ``locator_id``, ``locator_version``, ``backend``, ``model``, +``screen_region``, ``image_ms``, ``vlm_ms``, ``action``, ``action_verified``). +Lines written before they existed still load, with those fields ``None``. +Stamp the ids from the caller's side:: + + from je_auto_control import heal_context + + with heal_context(run_id="nightly-42", locator_id="submit", locator_version="v2"): + self_heal_click(template_path="submit.png") + +From a JSON action or MCP, pass the same keys as ``context`` on +``AC_self_heal_locate`` / ``AC_self_heal_click``. A JSON step cannot carry a +check, so its ``action_verified`` stays ``None``. ``AC_heal_stats`` reports +``action_verification`` (``actions`` / ``verified`` / ``failed`` / +``unchecked``) apart from ``healed``, which only counts returned points. + +Measuring locator versions +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``evaluate_locators`` runs every strategy version over the same labelled +frames — one captured frame, one region, one origin and scale per sample, the +same request object for every version — and scores the answers:: + + from je_auto_control import ( + EvaluationSample, evaluate_locators, template_match_strategy, + ) + + samples = [ + EvaluationSample("submit", frame, expected_box=(100, 60, 156, 92), + template="submit.png"), + EvaluationSample("left-monitor-150", hidpi_frame, + expected_box=(-1880, -180, -1824, -148), + origin=(-1920, -300), scale=1.5, template="submit.png"), + EvaluationSample("dialog-closed", other_frame, expect_miss=True, + template="submit.png"), + ] + comparison = evaluate_locators(samples, { + "v1": template_match_strategy(0.9), + "v2": template_match_strategy(0.9, scales=(1.0, 1.25, 1.5, 2.0)), + }) + report = comparison.report("v2") + report.accuracy # Ratio(numerator, denominator); .value is None over 0 + report.recovery_rate # targets v1 failed on that v2 hit correctly + report.p50_ms, report.p95_ms + comparison.failures("v2") + +How a result is counted: + +* a hit inside ``expected_box`` is ``correct``; a hit anywhere else, or on a + sample marked ``expect_miss``, is a ``false_positive`` — located, and not a + recovery; +* a sample with neither ``expected_box`` nor ``expect_miss`` is ``unknown``. + It counts in the hit rate and in no rate that claims correctness; +* a strategy that raises is ``error``, including on a sample that expects a + miss; +* a strategy that modifies the frame is refused with + ``HealingEvaluationError``: the next version would be measured on a + different image. + +Coordinates are screen coordinates. ``origin`` is the screen position of the +frame's top-left pixel (negative on a monitor left of or above the primary) +and ``scale`` is frame pixels per screen unit. + +A dataset can also be a JSON file with frame images beside it, which is what +the executor command, the MCP tool and the GUI evaluate:: + + {"schema_version": 1, + "samples": [{"id": "submit", "frame": "frames/submit.png", + "template": "submit.png", + "expected_box": [100, 60, 156, 92], + "origin": [0, 0], "scale": 1.0, "region": null}], + "versions": {"v1": {"strategy": "template", "threshold": 0.9}, + "v2": {"strategy": "template", "threshold": 0.9, + "scales": [1.0, 1.5]}}, + "thresholds": {"v2": {"min_correct": 1, "max_false_positive": 0}}} + + from je_auto_control import evaluate_healing_dataset + payload = evaluate_healing_dataset("dataset.json") + payload["passed"], payload["violations"] + +Image paths are relative to the dataset file and may not leave its directory. +Only the ``template`` strategy can be named from JSON; evaluate a VLM by +passing ``evaluate_locators`` a callable that wraps it. + +``benchmarks/self_healing/run.py`` is the fixed regression set: ten frames +drawn in memory (plain, 125% / 150% scale, negative-origin monitor, a region +that must pick the second of two identical targets, a redesigned control, an +absent target, a look-alike neighbour, one unlabelled frame) and three +versions. It captures no screen:: + + python benchmarks/self_healing/run.py --check --json report.json + +Candidate template revisions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A healed point is never written over a template. A new image is a candidate +that has to be previewed before it replaces anything:: + + from je_auto_control import ( + propose_template_revision, preview_template_revision, + accept_template_revision, revert_template_revision, + ) + + revision = propose_template_revision("submit.png", "submit_new.png") + preview = preview_template_revision(revision.revision_id, + dataset_path="dataset.json") + preview["revision"]["validated"] + accept_template_revision(revision.revision_id) # backup kept + revert_template_revision(revision.revision_id) # backup restored + +``propose`` and ``preview`` never touch the template. With a dataset (or +``samples=``), preview runs the current and the candidate template over the +same frames; the candidate is ``validated`` only when it is correct at least +once, at least as often as the current template, and raises no false positive +or error. ``accept`` refuses an unvalidated candidate unless +``allow_unvalidated=True``, and both ``accept`` and ``revert`` refuse when the +template file changed underneath the revision. Revisions live under +``~/.je_auto_control/template_revisions``. + +Executor: ``AC_self_heal_evaluate``, ``AC_self_heal_revision_propose / +_preview / _accept / _revert / _list``. MCP: ``ac_self_heal_evaluate``, +``ac_self_heal_revision_*``. GUI: **Self-Healing** tab → Actions menu +(*Evaluate dataset*, *Propose / Preview / Accept / Revert revision*). + Anchor-based locator -------------------- diff --git a/docs/source/Zh/doc/new_features/v2_features_doc.rst b/docs/source/Zh/doc/new_features/v2_features_doc.rst index df7709db9..054b66307 100644 --- a/docs/source/Zh/doc/new_features/v2_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v2_features_doc.rst @@ -30,6 +30,134 @@ Executor：``AC_self_heal_locate / _click / _log_list / _log_clear``。 MCP：``ac_self_heal_*``。GUI：**Self-Healing** 分頁。 +``screen_region`` 是螢幕座標的 ``[x1, y1, x2, y2]``，**兩種**策略都受它限制； +以前只有 VLM 收到它，樣板比對可能在區域之外回報命中。 + +**找到不等於驗證過。** ``HealOutcome.found`` 只表示某個策略回傳了一個點。 +點擊有沒有達到目的是另一個欄位 ``action_verified``，由呼叫端傳入的檢查填入—— +沒有檢查時是 ``None``，絕不從命中推論:: + + outcome = self_heal_click( + template_path="submit.png", + description="綠色的 Submit 按鈕", + verify=lambda result: dialog_is_open(), + ) + outcome.found # 回傳了一個點 + outcome.action_verified # 檢查結果 True / False，沒有檢查則為 None + +自愈記錄新增選用欄位（``schema_version``、``run_id``、``step_id``、``locator_id``、 +``locator_version``、``backend``、``model``、``screen_region``、``image_ms``、 +``vlm_ms``、``action``、``action_verified``）。這些欄位出現之前寫入的記錄仍可讀取， +缺少的欄位為 ``None``。識別資訊由呼叫端標註:: + + from je_auto_control import heal_context + + with heal_context(run_id="nightly-42", locator_id="submit", locator_version="v2"): + self_heal_click(template_path="submit.png") + +從 JSON 動作或 MCP 呼叫時，把同樣的鍵放在 ``AC_self_heal_locate`` / +``AC_self_heal_click`` 的 ``context`` 參數。JSON 步驟無法攜帶檢查函式，所以它的 +``action_verified`` 會維持 ``None``。``AC_heal_stats`` 另外回報 +``action_verification``（``actions`` / ``verified`` / ``failed`` / ``unchecked``）， +與只計算「有回傳座標」的 ``healed`` 分開。 + +量測定位器版本 +~~~~~~~~~~~~~~ + +``evaluate_locators`` 讓每個策略版本跑同一批已標註的畫面——每個樣本只有一張 +擷取畫面、一個區域、一組原點與縮放，所有版本收到同一個 request 物件——再替答案計分:: + + from je_auto_control import ( + EvaluationSample, evaluate_locators, template_match_strategy, + ) + + samples = [ + EvaluationSample("submit", frame, expected_box=(100, 60, 156, 92), + template="submit.png"), + EvaluationSample("left-monitor-150", hidpi_frame, + expected_box=(-1880, -180, -1824, -148), + origin=(-1920, -300), scale=1.5, template="submit.png"), + EvaluationSample("dialog-closed", other_frame, expect_miss=True, + template="submit.png"), + ] + comparison = evaluate_locators(samples, { + "v1": template_match_strategy(0.9), + "v2": template_match_strategy(0.9, scales=(1.0, 1.25, 1.5, 2.0)), + }) + report = comparison.report("v2") + report.accuracy # Ratio(分子, 分母)；分母為 0 時 .value 是 None + report.recovery_rate # v1 失敗而 v2 正確命中的目標 + report.p50_ms, report.p95_ms + comparison.failures("v2") + +計分方式： + +* 命中點落在 ``expected_box`` 內才是 ``correct``；落在別處，或樣本標為 + ``expect_miss`` 卻命中，都是 ``false_positive``——有找到，但不算恢復； +* 既沒有 ``expected_box`` 也沒有 ``expect_miss`` 的樣本是 ``unknown``， + 只計入命中率，不計入任何宣稱正確的比率； +* 策略拋出例外記為 ``error``，即使該樣本預期 miss； +* 策略若修改了畫面，會以 ``HealingEvaluationError`` 拒絕：下一個版本量到的 + 就不是同一張圖了。 + +座標一律是螢幕座標。``origin`` 是畫面左上角像素的螢幕位置（在主螢幕左方或上方的 +螢幕為負值），``scale`` 是每一個螢幕單位對應的畫面像素數。 + +資料集也可以是一個 JSON 檔，畫面影像放在它旁邊；Executor 指令、MCP 工具與 GUI +評估的就是這種檔案:: + + {"schema_version": 1, + "samples": [{"id": "submit", "frame": "frames/submit.png", + "template": "submit.png", + "expected_box": [100, 60, 156, 92], + "origin": [0, 0], "scale": 1.0, "region": null}], + "versions": {"v1": {"strategy": "template", "threshold": 0.9}, + "v2": {"strategy": "template", "threshold": 0.9, + "scales": [1.0, 1.5]}}, + "thresholds": {"v2": {"min_correct": 1, "max_false_positive": 0}}} + + from je_auto_control import evaluate_healing_dataset + payload = evaluate_healing_dataset("dataset.json") + payload["passed"], payload["violations"] + +影像路徑相對於資料集檔案，且不得離開它所在的目錄。JSON 裡只能指名 ``template`` +策略；要評估 VLM，請把包裝它的 callable 傳給 ``evaluate_locators``。 + +``benchmarks/self_healing/run.py`` 是固定的回歸資料集：十張在記憶體中繪製的畫面 +（一般、125%／150% 縮放、負原點螢幕、必須選中兩個相同目標中第二個的區域、 +改版後的控制項、目標不存在、相似的鄰近控制項、一張未標註畫面）與三個版本。 +它不擷取螢幕:: + + python benchmarks/self_healing/run.py --check --json report.json + +候選樣板修訂 +~~~~~~~~~~~~ + +自愈得到的點絕不直接覆寫樣板。新影像只是候選，必須先預覽才能取代任何東西:: + + from je_auto_control import ( + propose_template_revision, preview_template_revision, + accept_template_revision, revert_template_revision, + ) + + revision = propose_template_revision("submit.png", "submit_new.png") + preview = preview_template_revision(revision.revision_id, + dataset_path="dataset.json") + preview["revision"]["validated"] + accept_template_revision(revision.revision_id) # 保留備份 + revert_template_revision(revision.revision_id) # 還原備份 + +``propose`` 與 ``preview`` 都不會動到樣板。給了資料集（或 ``samples=``）時， +preview 會讓現行樣板與候選樣板跑同一批畫面；候選必須至少正確一次、正確次數不少於 +現行樣板、且沒有 false positive 或 error，才會標為 ``validated``。``accept`` 會拒絕 +未驗證的候選，除非傳入 ``allow_unvalidated=True``；樣板檔在修訂之後被別人改過時， +``accept`` 與 ``revert`` 都會拒絕。修訂存放在 ``~/.je_auto_control/template_revisions``。 + +Executor：``AC_self_heal_evaluate``、``AC_self_heal_revision_propose / +_preview / _accept / _revert / _list``。MCP：``ac_self_heal_evaluate``、 +``ac_self_heal_revision_*``。GUI：**Self-Healing** 分頁 → Actions 選單 +（*評估資料集*、*提出／預覽／接受／回復修訂*）。 + 錨點定位器 ---------- diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 475c40ca2..a143ac40d 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -199,6 +199,13 @@ from je_auto_control.utils.self_healing import ( HealEvent, HealEventLog, HealOutcome, SelfHealError, default_heal_log, self_heal_click, self_heal_locate, + # measurement: labelled evaluation of strategy versions, template revisions + EvaluationSample, HealingComparison, HealingEvaluationError, + TemplateRevision, TemplateRevisionError, TemplateRevisionStore, + accept_template_revision, evaluate_healing_dataset, evaluate_locators, + heal_context, list_template_revisions, preview_template_revision, + propose_template_revision, revert_template_revision, + template_match_strategy, ) # Screenshot PII redaction (blur regions before VLM upload / audit log). from je_auto_control.utils.redaction import ( @@ -1950,6 +1957,12 @@ def start_autocontrol_gui(*args, **kwargs): # Self-healing locator (image → VLM fallback) "HealEvent", "HealEventLog", "HealOutcome", "SelfHealError", "default_heal_log", "self_heal_click", "self_heal_locate", + "EvaluationSample", "HealingComparison", "HealingEvaluationError", + "TemplateRevision", "TemplateRevisionError", "TemplateRevisionStore", + "accept_template_revision", "evaluate_healing_dataset", "evaluate_locators", + "heal_context", "list_template_revisions", "preview_template_revision", + "propose_template_revision", "revert_template_revision", + "template_match_strategy", # Screenshot redaction (PII blur) "POLICY_MODERATE", "POLICY_OFF", "POLICY_STRICT", "RedactionEngine", "RedactionPolicy", "RedactionResult", diff --git a/je_auto_control/actions.pyi b/je_auto_control/actions.pyi index 35e4e39ee..f6c1c25eb 100644 --- a/je_auto_control/actions.pyi +++ b/je_auto_control/actions.pyi @@ -2580,9 +2580,13 @@ def AC_self_heal_click( screen_region: List[int] | None = ..., model: str | None = ..., raise_on_miss: bool = ..., + context: Dict[str, str] | None = ..., ) -> Dict[str, Any]: """Executor adapter: locate with self-heal, then click.""" +def AC_self_heal_evaluate(dataset_path: str, versions: Dict[str, Any] | None = ...) -> Dict[str, Any]: + """Executor adapter: score strategy versions on a labelled dataset file.""" + def AC_self_heal_locate( template_path: str | None = ..., description: str | None = ..., @@ -2590,6 +2594,7 @@ def AC_self_heal_locate( screen_region: List[int] | None = ..., model: str | None = ..., raise_on_miss: bool = ..., + context: Dict[str, str] | None = ..., ) -> Dict[str, Any]: """Executor adapter: template-first locate with VLM fallback.""" @@ -2599,6 +2604,30 @@ def AC_self_heal_log_clear() -> Dict[str, Any]: def AC_self_heal_log_list(limit: int = ...) -> List[Dict[str, Any]]: """Executor adapter: return the recent self-healing events.""" +def AC_self_heal_revision_accept(revision_id: str, allow_unvalidated: bool = ...) -> Dict[str, Any]: + """Executor adapter: replace the template with a validated candidate.""" + +def AC_self_heal_revision_list() -> List[Dict[str, Any]]: + """Executor adapter: every stored template revision, oldest first.""" + +def AC_self_heal_revision_preview( + revision_id: str, + dataset_path: str | None = ..., + detect_threshold: float = ..., +) -> Dict[str, Any]: + """Executor adapter: current vs candidate, validated when given a dataset.""" + +def AC_self_heal_revision_propose( + template_path: str, + candidate_path: str, + source: str = ..., + note: str | None = ..., +) -> Dict[str, Any]: + """Executor adapter: store a candidate template; the template is untouched.""" + +def AC_self_heal_revision_revert(revision_id: str) -> Dict[str, Any]: + """Executor adapter: restore the template an accepted revision replaced.""" + def AC_send_email(message: Any, smtp: Any) -> Dict[str, Any]: """Adapter: send an email via SMTP (message/smtp config dicts).""" diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 6bc8d1ebe..f95526da4 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -1016,6 +1016,34 @@ "self_heal_col_template_path": "Template", "self_heal_col_description": "Description", "self_heal_col_duration_ms": "ms", + "self_heal_col_locator_version": "Version", + "self_heal_col_action_verified": "Verified", + "self_heal_verified_yes": "verified", + "self_heal_verified_no": "failed check", + "self_heal_verified_unknown": "not checked", + "self_heal_measure_title": "Measure versions and revise templates", + "self_heal_dataset_label": "Dataset:", + "self_heal_dataset_placeholder": "labelled dataset JSON", + "self_heal_candidate_label": "Candidate image:", + "self_heal_candidate_placeholder": "new template image to propose", + "self_heal_revision_label": "Revision id:", + "self_heal_revision_placeholder": "filled in by Propose", + "self_heal_report_placeholder": "Evaluation and revision results appear here", + "self_heal_browse_dataset": "Choose dataset…", + "self_heal_browse_candidate": "Choose candidate image…", + "self_heal_evaluate": "Evaluate dataset", + "self_heal_rev_propose": "Propose template revision", + "self_heal_rev_preview": "Preview revision", + "self_heal_rev_accept": "Accept revision", + "self_heal_rev_revert": "Revert revision", + "self_heal_rev_list": "List revisions", + "self_heal_done": "Done", + "self_heal_dataset_required": "Provide a dataset JSON path", + "self_heal_revision_required": "Provide a revision id", + "self_heal_propose_required": "Provide the template and a candidate image", + "self_heal_eval_passed": "Evaluation finished — thresholds met", + "self_heal_eval_failed": "Evaluation finished — thresholds NOT met", + "self_heal_accept_unvalidated": "This candidate has not been validated against labelled frames. Accept it anyway?", # WebRunner Tab "web_available": "WebRunner is installed.", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 0740bfc51..481827dbb 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -905,6 +905,34 @@ "self_heal_col_template_path": "テンプレート", "self_heal_col_description": "説明", "self_heal_col_duration_ms": "ミリ秒", + "self_heal_col_locator_version": "バージョン", + "self_heal_col_action_verified": "検証", + "self_heal_verified_yes": "検証済み", + "self_heal_verified_no": "検証失敗", + "self_heal_verified_unknown": "未確認", + "self_heal_measure_title": "バージョンの測定とテンプレートの改訂", + "self_heal_dataset_label": "データセット:", + "self_heal_dataset_placeholder": "ラベル付きデータセット JSON", + "self_heal_candidate_label": "候補画像:", + "self_heal_candidate_placeholder": "提案する新しいテンプレート画像", + "self_heal_revision_label": "改訂 ID:", + "self_heal_revision_placeholder": "提案すると自動入力されます", + "self_heal_report_placeholder": "評価と改訂の結果がここに表示されます", + "self_heal_browse_dataset": "データセットを選択…", + "self_heal_browse_candidate": "候補画像を選択…", + "self_heal_evaluate": "データセットを評価", + "self_heal_rev_propose": "テンプレート改訂を提案", + "self_heal_rev_preview": "改訂をプレビュー", + "self_heal_rev_accept": "改訂を承認", + "self_heal_rev_revert": "改訂を取り消す", + "self_heal_rev_list": "改訂を一覧表示", + "self_heal_done": "完了", + "self_heal_dataset_required": "データセット JSON のパスを入力してください", + "self_heal_revision_required": "改訂 ID を入力してください", + "self_heal_propose_required": "テンプレートと候補画像を入力してください", + "self_heal_eval_passed": "評価完了 — しきい値を満たしています", + "self_heal_eval_failed": "評価完了 — しきい値を満たしていません", + "self_heal_accept_unvalidated": "この候補はラベル付きフレームで検証されていません。承認しますか？", # WebRunner Tab "web_available": "WebRunner はインストール済みです。", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index d002ab239..f5287ab83 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -888,6 +888,34 @@ "self_heal_col_template_path": "模板", "self_heal_col_description": "描述", "self_heal_col_duration_ms": "毫秒", + "self_heal_col_locator_version": "版本", + "self_heal_col_action_verified": "已验证", + "self_heal_verified_yes": "已验证", + "self_heal_verified_no": "验证失败", + "self_heal_verified_unknown": "未检查", + "self_heal_measure_title": "测量版本与修订模板", + "self_heal_dataset_label": "数据集：", + "self_heal_dataset_placeholder": "已标注的数据集 JSON", + "self_heal_candidate_label": "候选图像：", + "self_heal_candidate_placeholder": "要提出的新模板图像", + "self_heal_revision_label": "修订编号：", + "self_heal_revision_placeholder": "提出后自动填入", + "self_heal_report_placeholder": "评估与修订结果显示于此", + "self_heal_browse_dataset": "选择数据集…", + "self_heal_browse_candidate": "选择候选图像…", + "self_heal_evaluate": "评估数据集", + "self_heal_rev_propose": "提出模板修订", + "self_heal_rev_preview": "预览修订", + "self_heal_rev_accept": "接受修订", + "self_heal_rev_revert": "回退修订", + "self_heal_rev_list": "列出修订", + "self_heal_done": "完成", + "self_heal_dataset_required": "请提供数据集 JSON 路径", + "self_heal_revision_required": "请提供修订编号", + "self_heal_propose_required": "请提供模板与候选图像", + "self_heal_eval_passed": "评估完成 — 达到阈值", + "self_heal_eval_failed": "评估完成 — 未达阈值", + "self_heal_accept_unvalidated": "此候选尚未用已标注的画面验证，仍要接受吗？", # WebRunner Tab "web_available": "WebRunner 已安装。", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 046af99cd..aebb3a06c 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -889,6 +889,34 @@ "self_heal_col_template_path": "樣板", "self_heal_col_description": "描述", "self_heal_col_duration_ms": "毫秒", + "self_heal_col_locator_version": "版本", + "self_heal_col_action_verified": "已驗證", + "self_heal_verified_yes": "已驗證", + "self_heal_verified_no": "驗證失敗", + "self_heal_verified_unknown": "未檢查", + "self_heal_measure_title": "量測版本與修訂樣板", + "self_heal_dataset_label": "資料集：", + "self_heal_dataset_placeholder": "已標註的資料集 JSON", + "self_heal_candidate_label": "候選影像：", + "self_heal_candidate_placeholder": "要提出的新樣板影像", + "self_heal_revision_label": "修訂編號：", + "self_heal_revision_placeholder": "提出後自動填入", + "self_heal_report_placeholder": "評估與修訂結果顯示於此", + "self_heal_browse_dataset": "選擇資料集…", + "self_heal_browse_candidate": "選擇候選影像…", + "self_heal_evaluate": "評估資料集", + "self_heal_rev_propose": "提出樣板修訂", + "self_heal_rev_preview": "預覽修訂", + "self_heal_rev_accept": "接受修訂", + "self_heal_rev_revert": "回復修訂", + "self_heal_rev_list": "列出修訂", + "self_heal_done": "完成", + "self_heal_dataset_required": "請提供資料集 JSON 路徑", + "self_heal_revision_required": "請提供修訂編號", + "self_heal_propose_required": "請提供樣板與候選影像", + "self_heal_eval_passed": "評估完成 — 達到門檻", + "self_heal_eval_failed": "評估完成 — 未達門檻", + "self_heal_accept_unvalidated": "此候選尚未用已標註的畫面驗證，仍要接受嗎？", # WebRunner Tab "web_available": "WebRunner 已安裝。", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index a9638555d..af3cfc3d4 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2994,6 +2994,51 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: description="Aggregate the self-heal log (heal rate, brittle " "locators).", )) + specs.append(CommandSpec( + "AC_self_heal_evaluate", "Testing", "Self-Heal: Evaluate Dataset", + fields=(FieldSpec("dataset_path", FieldType.FILE_PATH, + placeholder="dataset.json"),), + description="Score locator versions on the same labelled frames " + "(accuracy, false positives, recovery, p50/p95).", + )) + specs.append(CommandSpec( + "AC_self_heal_revision_propose", "Testing", "Self-Heal: Propose Template Revision", + fields=( + FieldSpec("template_path", FieldType.FILE_PATH), + FieldSpec("candidate_path", FieldType.FILE_PATH), + FieldSpec("source", FieldType.STRING, optional=True, default="manual"), + FieldSpec("note", FieldType.STRING, optional=True), + ), + description="Store a candidate template; the live template is not changed.", + )) + specs.append(CommandSpec( + "AC_self_heal_revision_preview", "Testing", "Self-Heal: Preview Template Revision", + fields=( + FieldSpec("revision_id", FieldType.STRING), + FieldSpec("dataset_path", FieldType.FILE_PATH, optional=True, + placeholder="dataset.json"), + FieldSpec("detect_threshold", FieldType.FLOAT, optional=True, + default=0.9, min_value=0.0, max_value=1.0), + ), + description="Compare candidate and current template; a dataset validates it.", + )) + specs.append(CommandSpec( + "AC_self_heal_revision_accept", "Testing", "Self-Heal: Accept Template Revision", + fields=( + FieldSpec("revision_id", FieldType.STRING), + FieldSpec("allow_unvalidated", FieldType.BOOL, optional=True, default=False), + ), + description="Replace the template with a validated candidate (backup kept).", + )) + specs.append(CommandSpec( + "AC_self_heal_revision_revert", "Testing", "Self-Heal: Revert Template Revision", + fields=(FieldSpec("revision_id", FieldType.STRING),), + description="Restore the template an accepted revision replaced.", + )) + specs.append(CommandSpec( + "AC_self_heal_revision_list", "Testing", "Self-Heal: List Template Revisions", + description="List stored template revisions and their status.", + )) specs.append(CommandSpec( "AC_failure_signature", "Testing", "Failure Signature", fields=( diff --git a/je_auto_control/gui/self_healing_tab.py b/je_auto_control/gui/self_healing_tab.py index 66ed19d6a..e58a72950 100644 --- a/je_auto_control/gui/self_healing_tab.py +++ b/je_auto_control/gui/self_healing_tab.py @@ -1,14 +1,18 @@ """Self-Healing Locator tab. Fire a template-first / VLM-fallback locate from the GUI and browse the -audit log of every healing attempt the runtime has performed. +audit log of every healing attempt the runtime has performed. The second +group measures locator versions against a labelled dataset and walks a +candidate template revision through propose / preview / accept / revert; every +one of those is a call into ``utils.self_healing`` and nothing more. """ -from typing import Optional +import json +from typing import Callable, Optional, Sequence from PySide6.QtCore import Qt from PySide6.QtWidgets import ( QCheckBox, QDoubleSpinBox, QFileDialog, QFormLayout, QGroupBox, - QLabel, QLineEdit, QMessageBox, + QLabel, QLineEdit, QMessageBox, QPlainTextEdit, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, ) @@ -17,13 +21,20 @@ language_wrapper, ) from je_auto_control.utils.self_healing import ( - HealOutcome, default_heal_log, self_heal_click, self_heal_locate, + HealOutcome, accept_template_revision, default_heal_log, + evaluate_healing_dataset, list_template_revisions, + preview_template_revision, propose_template_revision, + revert_template_revision, self_heal_click, self_heal_locate, ) from je_auto_control.utils.exception.exceptions import AutoControlException _COLUMNS = ("timestamp", "method", "coordinates", - "template_path", "description", "duration_ms") + "template_path", "description", "duration_ms", + "locator_version", "action_verified") + +_SLOT_ERRORS = (AutoControlException, OSError, ValueError, RuntimeError) +_IMAGE_FILTER = "Images (*.png *.jpg *.bmp);;All (*)" def _t(key: str) -> str: @@ -45,6 +56,11 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._click_check = QCheckBox() self._status = QLabel() self._table = QTableWidget(0, len(_COLUMNS)) + self._dataset_input = QLineEdit() + self._candidate_input = QLineEdit() + self._revision_input = QLineEdit() + self._report_view = QPlainTextEdit() + self._report_view.setReadOnly(True) self._build_layout() def retranslate(self) -> None: @@ -59,6 +75,8 @@ def _build_layout(self) -> None: root = QVBoxLayout(self) root.addWidget(self._build_form_group()) root.addWidget(self._table, stretch=1) + root.addWidget(self._build_measure_group()) + root.addWidget(self._report_view, stretch=1) root.addWidget(self._status) self._apply_translations() self.refresh_log() @@ -73,6 +91,15 @@ def _build_form_group(self) -> QGroupBox: self._group_box = group return group + def _build_measure_group(self) -> QGroupBox: + group = QGroupBox() + form = QFormLayout(group) + form.addRow(QLabel(), self._dataset_input) + form.addRow(QLabel(), self._candidate_input) + form.addRow(QLabel(), self._revision_input) + self._measure_box = group + return group + def menu_actions(self) -> list: """Expose tab commands to the window-level Actions menu.""" return [ @@ -81,6 +108,14 @@ def menu_actions(self) -> list: ("self_heal_click_btn", self._on_click), ("self_heal_refresh", self.refresh_log), ("self_heal_clear", self._on_clear_log), + ("self_heal_browse_dataset", self._on_browse_dataset), + ("self_heal_evaluate", self._on_evaluate), + ("self_heal_browse_candidate", self._on_browse_candidate), + ("self_heal_rev_propose", self._on_propose_revision), + ("self_heal_rev_preview", self._on_preview_revision), + ("self_heal_rev_accept", self._on_accept_revision), + ("self_heal_rev_revert", self._on_revert_revision), + ("self_heal_rev_list", self._on_list_revisions), ] # --- translation ----------------------------------------------- @@ -90,16 +125,19 @@ def _apply_translations(self) -> None: self._template_input.setPlaceholderText(_t("self_heal_template_placeholder")) self._description_input.setPlaceholderText(_t("self_heal_desc_placeholder")) self._click_check.setText(_t("self_heal_click_check")) - layout = self._group_box.layout() - if isinstance(layout, QFormLayout): - labels = ( - "self_heal_template_label", "self_heal_desc_label", - "self_heal_threshold_label", "", - ) - for row, key in enumerate(labels): - item = layout.itemAt(row, QFormLayout.LabelRole) - if item is not None and isinstance(item.widget(), QLabel): - item.widget().setText(_t(key) if key else "") + _set_form_labels(self._group_box, ( + "self_heal_template_label", "self_heal_desc_label", + "self_heal_threshold_label", "", + )) + self._measure_box.setTitle(_t("self_heal_measure_title")) + self._dataset_input.setPlaceholderText(_t("self_heal_dataset_placeholder")) + self._candidate_input.setPlaceholderText(_t("self_heal_candidate_placeholder")) + self._revision_input.setPlaceholderText(_t("self_heal_revision_placeholder")) + self._report_view.setPlaceholderText(_t("self_heal_report_placeholder")) + _set_form_labels(self._measure_box, ( + "self_heal_dataset_label", "self_heal_candidate_label", + "self_heal_revision_label", + )) headers = [_t(f"self_heal_col_{name}") for name in _COLUMNS] self._table.setHorizontalHeaderLabels(headers) @@ -107,12 +145,105 @@ def _apply_translations(self) -> None: def _on_browse_template(self) -> None: path, _selected = QFileDialog.getOpenFileName( - self, _t("self_heal_browse"), "", - "Images (*.png *.jpg *.bmp);;All (*)", + self, _t("self_heal_browse"), "", _IMAGE_FILTER, ) if path: self._template_input.setText(path) + def _on_browse_dataset(self) -> None: + path, _selected = QFileDialog.getOpenFileName( + self, _t("self_heal_browse_dataset"), "", "JSON (*.json);;All (*)", + ) + if path: + self._dataset_input.setText(path) + + def _on_browse_candidate(self) -> None: + path, _selected = QFileDialog.getOpenFileName( + self, _t("self_heal_browse_candidate"), "", _IMAGE_FILTER, + ) + if path: + self._candidate_input.setText(path) + + # --- measurement and template revisions ------------------------- + + def _show(self, call: Callable[[], object]) -> Optional[object]: + """Run a headless call; show its JSON result, or its error in the status.""" + try: + result = call() + except _SLOT_ERRORS as error: + self._status.setText(f"{_t('self_heal_error')}: {error}") + return None + self._report_view.setPlainText( + json.dumps(result, ensure_ascii=False, indent=2, default=str)) + self._status.setText(_t("self_heal_done")) + return result + + def _revision_id(self) -> Optional[str]: + revision_id = self._revision_input.text().strip() + if not revision_id: + self._status.setText(_t("self_heal_revision_required")) + return None + return revision_id + + def _on_evaluate(self) -> None: + dataset = self._dataset_input.text().strip() + if not dataset: + self._status.setText(_t("self_heal_dataset_required")) + return + result = self._show(lambda: evaluate_healing_dataset(dataset)) + if isinstance(result, dict): + self._status.setText(_t( + "self_heal_eval_passed" if result.get("passed") else "self_heal_eval_failed")) + + def _on_propose_revision(self) -> None: + template = self._template_input.text().strip() + candidate = self._candidate_input.text().strip() + if not template or not candidate: + self._status.setText(_t("self_heal_propose_required")) + return + result = self._show( + lambda: propose_template_revision(template, candidate).to_dict()) + if isinstance(result, dict): + self._revision_input.setText(str(result["revision_id"])) + + def _on_preview_revision(self) -> None: + revision_id = self._revision_id() + if revision_id is None: + return + dataset = self._dataset_input.text().strip() or None + threshold = float(self._threshold.value()) + self._show(lambda: preview_template_revision( + revision_id, dataset_path=dataset, detect_threshold=threshold)) + + def _on_accept_revision(self) -> None: + revision_id = self._revision_id() + if revision_id is None: + return + self._show(lambda: self._accept(revision_id)) + + def _accept(self, revision_id: str) -> dict: + """Accept a validated revision; an unvalidated one only after a prompt.""" + pending = {item.revision_id: item for item in list_template_revisions()} + revision = pending.get(revision_id) + allow = False + if revision is not None and not revision.validated: + reply = QMessageBox.question( + self, _t("self_heal_rev_accept"), _t("self_heal_accept_unvalidated"), + ) + if reply != QMessageBox.Yes: + return {"accepted": False, "revision_id": revision_id} + allow = True + return accept_template_revision(revision_id, allow_unvalidated=allow).to_dict() + + def _on_revert_revision(self) -> None: + revision_id = self._revision_id() + if revision_id is None: + return + self._show(lambda: revert_template_revision(revision_id).to_dict()) + + def _on_list_revisions(self) -> None: + self._show(lambda: [item.to_dict() for item in list_template_revisions()]) + def _collect_inputs(self): template = self._template_input.text().strip() or None description = self._description_input.text().strip() or None @@ -178,6 +309,8 @@ def refresh_log(self) -> None: _format_coordinates(event.coordinates), event.template_path or "", event.description or "", f"{event.duration_ms:.1f}", + event.locator_version or "", + _format_verified(event.action, event.action_verified), ) for col, text in enumerate(values): item = QTableWidgetItem(str(text)) @@ -192,4 +325,24 @@ def _format_coordinates(coords) -> str: return f"({coords[0]}, {coords[1]})" +def _format_verified(action: Optional[str], verified: Optional[bool]) -> str: + """Empty for a bare locate; otherwise whether the action was checked.""" + if action is None: + return "" + if verified is None: + return _t("self_heal_verified_unknown") + return _t("self_heal_verified_yes" if verified else "self_heal_verified_no") + + +def _set_form_labels(group: QGroupBox, keys: Sequence[str]) -> None: + """Translate the label column of ``group``'s form, row by row.""" + layout = group.layout() + if not isinstance(layout, QFormLayout): + return + for row, key in enumerate(keys): + item = layout.itemAt(row, QFormLayout.LabelRole) + if item is not None and isinstance(item.widget(), QLabel): + item.widget().setText(_t(key) if key else "") + + __all__ = ["SelfHealingTab"] diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cc475cf85..97910f5a3 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -167,14 +167,21 @@ def _self_heal_locate(template_path: Optional[str] = None, detect_threshold: float = 0.9, screen_region: Optional[List[int]] = None, model: Optional[str] = None, - raise_on_miss: bool = False) -> Dict[str, Any]: - """Executor adapter: template-first locate with VLM fallback.""" - outcome = _self_heal_locate_impl( - template_path=template_path, description=description, - detect_threshold=float(detect_threshold), - screen_region=screen_region, model=model, - raise_on_miss=_as_bool(raise_on_miss), - ) + raise_on_miss: bool = False, + context: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """Executor adapter: template-first locate with VLM fallback. + + ``context`` stamps ``run_id`` / ``step_id`` / ``locator_id`` / + ``locator_version`` / ``backend`` on the logged event. + """ + from je_auto_control.utils.self_healing import heal_context + with heal_context(**(context or {})): + outcome = _self_heal_locate_impl( + template_path=template_path, description=description, + detect_threshold=float(detect_threshold), + screen_region=screen_region, model=model, + raise_on_miss=_as_bool(raise_on_miss), + ) return outcome.to_dict() @@ -184,15 +191,22 @@ def _self_heal_click(template_path: Optional[str] = None, detect_threshold: float = 0.9, screen_region: Optional[List[int]] = None, model: Optional[str] = None, - raise_on_miss: bool = False) -> Dict[str, Any]: - """Executor adapter: locate with self-heal, then click.""" - outcome = _self_heal_click_impl( - template_path=template_path, description=description, - mouse_keycode=mouse_keycode, - detect_threshold=float(detect_threshold), - screen_region=screen_region, model=model, - raise_on_miss=_as_bool(raise_on_miss), - ) + raise_on_miss: bool = False, + context: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """Executor adapter: locate with self-heal, then click. + + The result's ``action_verified`` stays ``None``: a JSON step cannot carry + the check, so the click is recorded as located and unverified. + """ + from je_auto_control.utils.self_healing import heal_context + with heal_context(**(context or {})): + outcome = _self_heal_click_impl( + template_path=template_path, description=description, + mouse_keycode=mouse_keycode, + detect_threshold=float(detect_threshold), + screen_region=screen_region, model=model, + raise_on_miss=_as_bool(raise_on_miss), + ) return outcome.to_dict() @@ -207,6 +221,52 @@ def _self_heal_log_clear() -> Dict[str, Any]: return {"cleared": True, "path": str(default_heal_log.path)} +def _self_heal_evaluate(dataset_path: str, + versions: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Executor adapter: score strategy versions on a labelled dataset file.""" + from je_auto_control.utils.self_healing import evaluate_healing_dataset + return evaluate_healing_dataset(dataset_path, versions=versions) + + +def _self_heal_revision_propose(template_path: str, candidate_path: str, + source: str = "manual", + note: Optional[str] = None) -> Dict[str, Any]: + """Executor adapter: store a candidate template; the template is untouched.""" + from je_auto_control.utils.self_healing import propose_template_revision + return propose_template_revision( + template_path, candidate_path, source=source, note=note).to_dict() + + +def _self_heal_revision_preview(revision_id: str, + dataset_path: Optional[str] = None, + detect_threshold: float = 0.9) -> Dict[str, Any]: + """Executor adapter: current vs candidate, validated when given a dataset.""" + from je_auto_control.utils.self_healing import preview_template_revision + return preview_template_revision( + revision_id, dataset_path=dataset_path or None, + detect_threshold=float(detect_threshold)) + + +def _self_heal_revision_accept(revision_id: str, + allow_unvalidated: bool = False) -> Dict[str, Any]: + """Executor adapter: replace the template with a validated candidate.""" + from je_auto_control.utils.self_healing import accept_template_revision + return accept_template_revision( + revision_id, allow_unvalidated=_as_bool(allow_unvalidated)).to_dict() + + +def _self_heal_revision_revert(revision_id: str) -> Dict[str, Any]: + """Executor adapter: restore the template an accepted revision replaced.""" + from je_auto_control.utils.self_healing import revert_template_revision + return revert_template_revision(revision_id).to_dict() + + +def _self_heal_revision_list() -> List[Dict[str, Any]]: + """Executor adapter: every stored template revision, oldest first.""" + from je_auto_control.utils.self_healing import list_template_revisions + return [revision.to_dict() for revision in list_template_revisions()] + + def _run_dag(definition: Dict[str, Any], max_parallel: int = 4) -> Dict[str, Any]: """Executor adapter: run a cross-host DAG definition.""" @@ -7815,6 +7875,12 @@ def __init__(self): "AC_self_heal_click": _self_heal_click, "AC_self_heal_log_list": _self_heal_log_list, "AC_self_heal_log_clear": _self_heal_log_clear, + "AC_self_heal_evaluate": _self_heal_evaluate, + "AC_self_heal_revision_propose": _self_heal_revision_propose, + "AC_self_heal_revision_preview": _self_heal_revision_preview, + "AC_self_heal_revision_accept": _self_heal_revision_accept, + "AC_self_heal_revision_revert": _self_heal_revision_revert, + "AC_self_heal_revision_list": _self_heal_revision_list, # Assertion DSL (verify screen state; raise on mismatch) "AC_assert_text": _assert_text, diff --git a/je_auto_control/utils/heal_analytics/heal_analytics.py b/je_auto_control/utils/heal_analytics/heal_analytics.py index 71e81d1d8..eea2c15e4 100644 --- a/je_auto_control/utils/heal_analytics/heal_analytics.py +++ b/je_auto_control/utils/heal_analytics/heal_analytics.py @@ -38,12 +38,29 @@ def _accumulate(events: List[Any]) -> Tuple[Dict[str, int], Dict[str, int], return by_method, brittle, durations, fallbacks +def _verification(events: List[Any]) -> Dict[str, int]: + """Count the events that acted on a hit, by what a post-action check said.""" + acted = [event for event in events if _attr(event, "action")] + verdicts = [_attr(event, "action_verified") for event in acted] + return { + "actions": len(acted), + "verified": sum(1 for verdict in verdicts if verdict is True), + "failed": sum(1 for verdict in verdicts if verdict is False), + "unchecked": sum(1 for verdict in verdicts if verdict is None), + } + + def heal_stats(events: List[Any]) -> Dict[str, Any]: """Aggregate self-heal events into a metrics dict. Each event is a :class:`HealEvent` or dict with ``method`` / ``coordinates`` / ``duration_ms`` / ``image_error`` / ``template_path`` / ``description``. + + ``healed`` / ``heal_rate`` count events where a strategy returned a + point. Whether the action that followed worked is reported apart, in + ``action_verification`` — ``verified`` over ``actions``, with the + ``unchecked`` ones never folded into either side. """ events = list(events) total = len(events) @@ -60,6 +77,7 @@ def heal_stats(events: List[Any]) -> Dict[str, Any]: if durations else 0.0), "top_brittle": [{"locator": key, "fallbacks": count} for key, count in top_brittle], + "action_verification": _verification(events), } diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fecbc25ae..277867aa3 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -2250,6 +2250,8 @@ def self_healing_tools() -> List[MCPTool]: "items": {"type": "integer"}}, "model": {"type": "string"}, "raise_on_miss": {"type": "boolean"}, + "context": {"type": "object", + "additionalProperties": {"type": "string"}}, }), handler=h_loc.self_heal_locate, annotations=READ_ONLY, @@ -2269,6 +2271,8 @@ def self_healing_tools() -> List[MCPTool]: "items": {"type": "integer"}}, "model": {"type": "string"}, "raise_on_miss": {"type": "boolean"}, + "context": {"type": "object", + "additionalProperties": {"type": "string"}}, }), handler=h_loc.self_heal_click, annotations=DESTRUCTIVE, @@ -2290,6 +2294,83 @@ def self_healing_tools() -> List[MCPTool]: handler=h_loc.self_heal_log_clear, annotations=DESTRUCTIVE, ), + MCPTool( + name="ac_self_heal_evaluate", + description=("Score locator strategy versions on a labelled " + "dataset JSON file: every version runs on the same " + "stored frames and regions; no screen is captured. " + "Returns per-version counts and rates with " + "numerator/denominator (accuracy, false positives, " + "recovery vs the baseline), p50/p95 latency, the " + "failing samples, and threshold violations. " + "Unlabelled samples are 'unknown', never correct."), + input_schema=schema({ + "dataset_path": {"type": "string", "format": "path"}, + "versions": {"type": "object"}, + }, required=["dataset_path"]), + handler=h_loc.self_heal_evaluate, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_self_heal_revision_propose", + description=("Store an image as a candidate replacement for a " + "template. The live template is not changed; the " + "candidate stays pending until previewed and " + "accepted."), + input_schema=schema({ + "template_path": {"type": "string", "format": "path"}, + "candidate_path": {"type": "string", "format": "path"}, + "source": {"type": "string"}, + "note": {"type": "string"}, + }, required=["template_path", "candidate_path"]), + handler=h_loc.self_heal_revision_propose, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_self_heal_revision_preview", + description=("Compare a candidate template revision with the " + "current template. With dataset_path both are run " + "over the same labelled frames and the candidate is " + "marked validated only if it is correct at least as " + "often and raises no false positive."), + input_schema=schema({ + "revision_id": {"type": "string"}, + "dataset_path": {"type": "string", "format": "path"}, + "detect_threshold": {"type": "number"}, + }, required=["revision_id"]), + handler=h_loc.self_heal_revision_preview, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_self_heal_revision_accept", + description=("Replace the live template file with a candidate " + "revision, keeping a backup. Refuses a candidate " + "that was not validated unless allow_unvalidated " + "is true."), + input_schema=schema({ + "revision_id": {"type": "string"}, + "allow_unvalidated": {"type": "boolean"}, + }, required=["revision_id"]), + handler=h_loc.self_heal_revision_accept, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_self_heal_revision_revert", + description=("Restore the template file an accepted revision " + "replaced."), + input_schema=schema({ + "revision_id": {"type": "string"}, + }, required=["revision_id"]), + handler=h_loc.self_heal_revision_revert, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_self_heal_revision_list", + description="List stored template revisions (pending / accepted / reverted).", + input_schema=schema({}), + handler=h_loc.self_heal_revision_list, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_locators.py b/je_auto_control/utils/mcp_server/tools/_handlers_locators.py index 5a322a459..70ec731b5 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_locators.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_locators.py @@ -279,14 +279,17 @@ def self_heal_locate(template_path: Optional[str] = None, detect_threshold: float = 0.9, screen_region: Optional[List[int]] = None, model: Optional[str] = None, - raise_on_miss: bool = False) -> Dict[str, Any]: + raise_on_miss: bool = False, + context: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + from je_auto_control.utils.self_healing import heal_context from je_auto_control.utils.self_healing import self_heal_locate as _impl - return _impl( - template_path=template_path, description=description, - detect_threshold=float(detect_threshold), - screen_region=screen_region, model=model, - raise_on_miss=bool(raise_on_miss), - ).to_dict() + with heal_context(**(context or {})): + return _impl( + template_path=template_path, description=description, + detect_threshold=float(detect_threshold), + screen_region=screen_region, model=model, + raise_on_miss=bool(raise_on_miss), + ).to_dict() def self_heal_click(template_path: Optional[str] = None, @@ -295,15 +298,18 @@ def self_heal_click(template_path: Optional[str] = None, detect_threshold: float = 0.9, screen_region: Optional[List[int]] = None, model: Optional[str] = None, - raise_on_miss: bool = False) -> Dict[str, Any]: + raise_on_miss: bool = False, + context: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + from je_auto_control.utils.self_healing import heal_context from je_auto_control.utils.self_healing import self_heal_click as _impl - return _impl( - template_path=template_path, description=description, - mouse_keycode=mouse_keycode, - detect_threshold=float(detect_threshold), - screen_region=screen_region, model=model, - raise_on_miss=bool(raise_on_miss), - ).to_dict() + with heal_context(**(context or {})): + return _impl( + template_path=template_path, description=description, + mouse_keycode=mouse_keycode, + detect_threshold=float(detect_threshold), + screen_region=screen_region, model=model, + raise_on_miss=bool(raise_on_miss), + ).to_dict() def self_heal_log_list(limit: int = 50) -> List[Dict[str, Any]]: @@ -318,6 +324,46 @@ def self_heal_log_clear() -> Dict[str, Any]: return {"cleared": True, "path": str(default_heal_log.path)} +def self_heal_evaluate(dataset_path: str, + versions: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + from je_auto_control.utils.self_healing import evaluate_healing_dataset + return evaluate_healing_dataset(dataset_path, versions=versions) + + +def self_heal_revision_propose(template_path: str, candidate_path: str, + source: str = "manual", + note: Optional[str] = None) -> Dict[str, Any]: + from je_auto_control.utils.self_healing import propose_template_revision + return propose_template_revision( + template_path, candidate_path, source=source, note=note).to_dict() + + +def self_heal_revision_preview(revision_id: str, + dataset_path: Optional[str] = None, + detect_threshold: float = 0.9) -> Dict[str, Any]: + from je_auto_control.utils.self_healing import preview_template_revision + return preview_template_revision( + revision_id, dataset_path=dataset_path or None, + detect_threshold=float(detect_threshold)) + + +def self_heal_revision_accept(revision_id: str, + allow_unvalidated: bool = False) -> Dict[str, Any]: + from je_auto_control.utils.self_healing import accept_template_revision + return accept_template_revision( + revision_id, allow_unvalidated=bool(allow_unvalidated)).to_dict() + + +def self_heal_revision_revert(revision_id: str) -> Dict[str, Any]: + from je_auto_control.utils.self_healing import revert_template_revision + return revert_template_revision(revision_id).to_dict() + + +def self_heal_revision_list() -> List[Dict[str, Any]]: + from je_auto_control.utils.self_healing import list_template_revisions + return [revision.to_dict() for revision in list_template_revisions()] + + def a11y_dump(app_name: Optional[str] = None, max_results: int = 500) -> Dict[str, Any]: from je_auto_control.utils.accessibility import dump_accessibility_tree diff --git a/je_auto_control/utils/self_healing/__init__.py b/je_auto_control/utils/self_healing/__init__.py index d699cae08..21b0358c7 100644 --- a/je_auto_control/utils/self_healing/__init__.py +++ b/je_auto_control/utils/self_healing/__init__.py @@ -4,23 +4,56 @@ * :func:`self_heal_locate` — return resolved screen coordinates or a miss outcome without raising; -* :func:`self_heal_click` — same, then click the resolved point; +* :func:`self_heal_click` — same, then click the resolved point, and record + whether a caller-supplied check verified the click; * :class:`HealOutcome` — structured result both helpers return; * :data:`default_heal_log` — singleton JSON-lines log every heal - attempt is appended to (override per-call via ``log=`` argument). + attempt is appended to (override per-call via ``log=`` argument); +* :func:`heal_context` — stamp run / step / locator ids and a locator + version on the events logged inside a block; +* :func:`evaluate_locators` / :func:`evaluate_healing_dataset` — score strategy + versions against the same labelled frames (:mod:`evaluation`); +* :func:`propose_template_revision` and friends — candidate template + revisions with preview, accept and revert (:mod:`template_revision`). """ +from je_auto_control.utils.self_healing.eval_strategies import ( + EvaluationDataset, build_strategy, evaluate_healing_dataset, + load_evaluation_dataset, template_match_strategy, +) +from je_auto_control.utils.self_healing.evaluation import ( + EvaluationSample, HealingComparison, HealingEvaluationError, + LocateRequest, LocatorStrategy, Ratio, SampleResult, VersionReport, + check_thresholds, evaluate_locators, format_comparison, +) from je_auto_control.utils.self_healing.heal_log import ( - HealEvent, HealEventLog, default_heal_log, + HEAL_EVENT_SCHEMA_VERSION, HealEvent, HealEventLog, default_heal_log, ) from je_auto_control.utils.self_healing.locator import ( HealOutcome, SelfHealError, METHOD_IMAGE, METHOD_MISS, METHOD_VLM, - self_heal_click, self_heal_locate, + heal_context, self_heal_click, self_heal_locate, +) +from je_auto_control.utils.self_healing.template_revision import ( + TemplateRevision, TemplateRevisionError, TemplateRevisionStore, + accept_template_revision, default_template_revisions, + list_template_revisions, preview_template_revision, + propose_template_revision, revert_template_revision, ) __all__ = [ - "HealEvent", "HealEventLog", "HealOutcome", "SelfHealError", + "EvaluationDataset", "EvaluationSample", "HEAL_EVENT_SCHEMA_VERSION", + "HealEvent", "HealEventLog", "HealOutcome", "HealingComparison", + "HealingEvaluationError", "LocateRequest", "LocatorStrategy", "METHOD_IMAGE", "METHOD_MISS", "METHOD_VLM", - "default_heal_log", "self_heal_click", "self_heal_locate", + "Ratio", "SampleResult", "SelfHealError", + "TemplateRevision", "TemplateRevisionError", "TemplateRevisionStore", + "VersionReport", + "accept_template_revision", "build_strategy", "check_thresholds", + "default_heal_log", "default_template_revisions", "evaluate_healing_dataset", + "evaluate_locators", "format_comparison", "heal_context", + "list_template_revisions", "load_evaluation_dataset", + "preview_template_revision", "propose_template_revision", + "revert_template_revision", "self_heal_click", "self_heal_locate", + "template_match_strategy", ] diff --git a/je_auto_control/utils/self_healing/eval_strategies.py b/je_auto_control/utils/self_healing/eval_strategies.py new file mode 100644 index 000000000..661a0b76b --- /dev/null +++ b/je_auto_control/utils/self_healing/eval_strategies.py @@ -0,0 +1,241 @@ +"""Built-in strategy versions and the file-based dataset for heal evaluation. + +:mod:`evaluation` scores any callable; this module supplies the ones that can +be named from JSON, so the executor, the MCP tool and the GUI produce the same +report from the same dataset file: + +* :func:`template_match_strategy` — OpenCV template match on the sample's + frame, confined to the request's region, optionally trying the template at + several scales. Coordinates go through :meth:`LocateRequest.to_screen`, so a + hit on a HiDPI or negative-origin frame comes back as a screen point; +* :func:`load_evaluation_dataset` / :func:`evaluate_healing_dataset` — a JSON file of + frame images, labels, version configs and thresholds. + +Nothing here captures the screen or calls a model: a VLM version is evaluated +by passing :func:`evaluation.evaluate_locators` a callable that wraps one. +""" +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union + +from je_auto_control.utils.self_healing.evaluation import ( + EvaluationSample, HealingComparison, HealingEvaluationError, LocateRequest, + LocatorStrategy, Point, check_thresholds, evaluate_locators, +) + +#: A pixel-identical match scores 0.99999905 in float32 (see template_detection). +_SCORE_EPSILON = 1e-5 + +DATASET_SCHEMA_VERSION = 1 +STRATEGY_TEMPLATE = "template" + +PathLike = Union[str, "os.PathLike[str]"] + + +def _gray(image: Any) -> Any: + """``image`` (path or array; a 3-D array is RGB) as a 2-D ``uint8`` array.""" + import cv2 + import numpy as np + if isinstance(image, (str, os.PathLike)): + from je_auto_control.utils.cv2_utils.image_file import read_image + return read_image(os.fspath(image), cv2.IMREAD_GRAYSCALE) + array = np.asarray(image) + if array.ndim == 3 and array.shape[2] == 1: + array = array[:, :, 0] + if array.ndim == 3: + array = cv2.cvtColor(array, cv2.COLOR_RGB2GRAY) + if array.ndim != 2: + raise ValueError(f"expected a 2-D or 3-D image, got shape {array.shape}") + return array + + +def _best_match(area: Any, template: Any, scales: Sequence[float], + ) -> Tuple[float, Tuple[int, int], Tuple[int, int]]: + """Best ``(score, (left, top), (width, height))`` over ``scales``; score -1 if none fits.""" + import cv2 + best: Tuple[float, Tuple[int, int], Tuple[int, int]] = (-1.0, (0, 0), (0, 0)) + for scale in scales: + scaled = template if scale == 1.0 else cv2.resize( + template, None, fx=scale, fy=scale, interpolation=cv2.INTER_LINEAR) + height, width = scaled.shape[:2] + if height < 1 or width < 1 or height > area.shape[0] or width > area.shape[1]: + continue + scores = cv2.matchTemplate(area, scaled, cv2.TM_CCOEFF_NORMED) + _low, high, _low_at, at = cv2.minMaxLoc(scores) + if high > best[0]: + best = (float(high), (int(at[0]), int(at[1])), (int(width), int(height))) + return best + + +def template_match_strategy(threshold: float = 0.9, + scales: Sequence[float] = (1.0,), + template: Any = None) -> LocatorStrategy: + """A strategy version that template-matches inside the request's frame. + + ``scales`` are factors applied to the template before matching; add the + display scales the template may meet to make a version tolerant of them. + ``template`` overrides the sample's own, which is how two revisions of one + template are compared on the same frames. + """ + limit = float(threshold) + if not 0.0 <= limit <= 1.0: + raise HealingEvaluationError(f"threshold must be between 0 and 1, got {threshold!r}") + factors = tuple(float(scale) for scale in scales) + if not factors or any(scale <= 0 for scale in factors): + raise HealingEvaluationError(f"scales must be positive numbers, got {scales!r}") + effective = min(limit, 1.0 - _SCORE_EPSILON) + + def locate(request: LocateRequest) -> Optional[Point]: + source = template if template is not None else request.template + if source is None: + raise ValueError(f"sample {request.sample_id!r} has no template") + frame = _gray(request.frame) + box = request.frame_box(frame.shape[1], frame.shape[0]) + if box is None: + return None + left, top, right, bottom = box + import cv2 + try: + score, (at_x, at_y), (width, height) = _best_match( + frame[top:bottom, left:right], _gray(source), factors) + except cv2.error as error: + # A pixel type OpenCV cannot match is this sample's error, in a + # type the evaluation records instead of one that ends the run. + raise ValueError(f"cannot match template: {error}") from error + if score < effective: + return None + return request.to_screen(left + at_x + width / 2.0, top + at_y + height / 2.0) + + return locate + + +def build_strategy(config: Mapping[str, Any]) -> LocatorStrategy: + """Build a strategy from ``{"strategy": "template", "threshold", "scales"}``.""" + if not isinstance(config, Mapping): + raise HealingEvaluationError(f"a version must be an object, got {config!r}") + kind = config.get("strategy", STRATEGY_TEMPLATE) + if kind != STRATEGY_TEMPLATE: + raise HealingEvaluationError( + f"unknown strategy {kind!r}; built-in strategies: [{STRATEGY_TEMPLATE!r}]") + unknown = sorted(set(config) - {"strategy", "threshold", "scales"}) + if unknown: + raise HealingEvaluationError(f"unknown strategy option(s) {unknown}") + return template_match_strategy(config.get("threshold", 0.9), config.get("scales", (1.0,))) + + +@dataclass(frozen=True) +class EvaluationDataset: + """A loaded dataset file: samples, version configs and thresholds.""" + + path: Path + samples: Tuple[EvaluationSample, ...] + versions: Mapping[str, Mapping[str, Any]] + thresholds: Mapping[str, Mapping[str, float]] + + +def _inside_root(root: Path, relative: Any, what: str) -> Path: + """``relative`` resolved under ``root``; a path that leaves it is refused.""" + if not isinstance(relative, str) or not relative: + raise HealingEvaluationError(f"{what} must be a relative file path, got {relative!r}") + resolved = Path(os.path.realpath(root / relative)) + if os.path.commonpath([str(root), str(resolved)]) != str(root): + raise HealingEvaluationError(f"{what} {relative!r} is outside the dataset directory") + if not resolved.is_file(): + raise HealingEvaluationError(f"{what} {relative!r} does not exist") + return resolved + + +def _sample_from(entry: Any, root: Path) -> EvaluationSample: + if not isinstance(entry, dict) or not isinstance(entry.get("id"), str): + raise HealingEvaluationError(f"a sample must be an object with a string id, got {entry!r}") + sample_id = entry["id"] + template = entry.get("template") + try: + frame = _gray(str(_inside_root(root, entry.get("frame"), f"{sample_id}: frame"))) + except ValueError as error: + raise HealingEvaluationError(f"{sample_id}: {error}") from error + return EvaluationSample( + sample_id=sample_id, frame=frame, + expected_box=entry.get("expected_box"), + origin=entry.get("origin", (0, 0)), scale=entry.get("scale", 1.0), + region=entry.get("region"), expect_miss=bool(entry.get("expect_miss", False)), + template=(None if template is None + else str(_inside_root(root, template, f"{sample_id}: template"))), + description=entry.get("description")) + + +def load_evaluation_dataset(path: PathLike) -> EvaluationDataset: + """Load a dataset JSON file; image paths are relative to its directory. + + :: + + {"schema_version": 1, + "samples": [{"id": "ok", "frame": "frames/ok.png", "template": "t.png", + "expected_box": [10, 20, 50, 40], "origin": [0, 0], + "scale": 1.0, "region": null, "expect_miss": false}], + "versions": {"v1": {"strategy": "template", "threshold": 0.9}}, + "thresholds": {"v1": {"min_correct": 1, "max_false_positive": 0}}} + + A sample with neither ``expected_box`` nor ``expect_miss`` is unlabelled. + """ + file_path = Path(os.path.realpath(os.fspath(path))) + try: + data = json.loads(file_path.read_text(encoding="utf-8-sig")) + except (OSError, ValueError) as error: + raise HealingEvaluationError(f"cannot read dataset {file_path}: {error}") from error + if not isinstance(data, dict) or not isinstance(data.get("samples"), list): + raise HealingEvaluationError(f"dataset {file_path} needs a 'samples' list") + if data.get("schema_version", DATASET_SCHEMA_VERSION) != DATASET_SCHEMA_VERSION: + raise HealingEvaluationError( + f"dataset schema_version {data.get('schema_version')!r} is not " + f"{DATASET_SCHEMA_VERSION}") + versions = data.get("versions") or {} + thresholds = data.get("thresholds") or {} + if not isinstance(versions, dict) or not isinstance(thresholds, dict): + raise HealingEvaluationError("'versions' and 'thresholds' must be objects") + samples = tuple(_sample_from(entry, file_path.parent) for entry in data["samples"]) + return EvaluationDataset(path=file_path, samples=samples, + versions=versions, thresholds=thresholds) + + +def comparison_payload(comparison: HealingComparison, + thresholds: Mapping[str, Mapping[str, float]], + ) -> Dict[str, Any]: + """``comparison.to_dict()`` plus the threshold verdict, as every surface returns it.""" + violations: List[str] = check_thresholds(comparison, thresholds) + payload = comparison.to_dict() + payload["thresholds"] = {name: dict(limits) for name, limits in thresholds.items()} + payload["violations"] = violations + payload["passed"] = not violations + return payload + + +def evaluate_healing_dataset(path: PathLike, + versions: Optional[Mapping[str, Mapping[str, Any]]] = None, + ) -> Dict[str, Any]: + """Evaluate a dataset file and return the JSON-safe report. + + ``versions`` replaces the file's own version configs when given. The + report is :meth:`HealingComparison.to_dict` plus ``dataset``, + ``thresholds``, ``violations`` and ``passed``. + """ + dataset = load_evaluation_dataset(path) + configs = dataset.versions if versions is None else versions + if not configs: + raise HealingEvaluationError(f"dataset {dataset.path} names no versions to compare") + strategies = {name: build_strategy(config) for name, config in configs.items()} + comparison = evaluate_locators(dataset.samples, strategies) + payload = comparison_payload(comparison, dataset.thresholds) + payload["dataset"] = str(dataset.path) + return payload + + +__all__ = [ + "DATASET_SCHEMA_VERSION", "EvaluationDataset", "STRATEGY_TEMPLATE", + "build_strategy", "comparison_payload", "evaluate_healing_dataset", + "load_evaluation_dataset", "template_match_strategy", +] diff --git a/je_auto_control/utils/self_healing/evaluation.py b/je_auto_control/utils/self_healing/evaluation.py new file mode 100644 index 000000000..4d476c9eb --- /dev/null +++ b/je_auto_control/utils/self_healing/evaluation.py @@ -0,0 +1,510 @@ +"""Measure locator strategy versions against the same labelled frames. + +The heal log answers "what did the locator do"; it cannot answer "was that +right", because a strategy that returns a confident wrong point looks exactly +like one that healed. This module compares strategy versions on samples that +carry the answer: + +* every version is called with the **same** :class:`LocateRequest` — one + captured frame, one region, one origin and scale — so a difference between + two reports is a difference between two strategies and never between two + screenshots. A strategy that edits the frame in place is refused, since the + next version would be measured on something else; +* a hit is ``correct`` only when it lands inside the sample's ``expected_box``. + A hit anywhere else, or on a sample that expects a miss, is a + ``false_positive`` — it is located, and it is not a recovery; +* a sample with no label is ``unknown``. It counts in the hit rate, where no + label is needed, and in no rate that claims correctness; +* every rate is a :class:`Ratio` carrying its numerator and denominator, and a + rate over nothing has no value rather than a perfect one. + +All coordinates are screen coordinates. ``origin`` is the screen position of +the frame's top-left pixel (negative on a monitor left of or above the +primary) and ``scale`` is frame pixels per screen unit, so a frame pixel +``(fx, fy)`` is the screen point ``(origin_x + fx / scale, origin_y + fy / +scale)``. :meth:`LocateRequest.to_screen` and :meth:`LocateRequest.frame_box` +do that arithmetic for a strategy. + +Qt-free and capture-free: nothing here reads the screen. +""" +from __future__ import annotations + +import hashlib +import math +from dataclasses import dataclass +from time import perf_counter +from typing import ( + Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, +) + +from je_auto_control.utils.exception.exceptions import AutoControlException + +Box = Tuple[int, int, int, int] +Point = Tuple[int, int] + +OUTCOME_CORRECT = "correct" +OUTCOME_FALSE_POSITIVE = "false_positive" +OUTCOME_MISS = "miss" +OUTCOME_TRUE_NEGATIVE = "true_negative" +OUTCOME_UNKNOWN = "unknown" +OUTCOME_ERROR = "error" + +#: Outcomes that are a failure of the strategy on a labelled sample. +FAILURE_OUTCOMES = (OUTCOME_FALSE_POSITIVE, OUTCOME_MISS, OUTCOME_ERROR) + +_STRATEGY_ERRORS = (AutoControlException, OSError, RuntimeError, ValueError, + TypeError, ArithmeticError, LookupError) + + +class HealingEvaluationError(AutoControlException, ValueError): + """An evaluation was set up so that its result could not be trusted.""" + + +def _as_box(value: Optional[Sequence[int]], name: str) -> Optional[Box]: + if value is None: + return None + try: + x1, y1, x2, y2 = (int(item) for item in value) + except (TypeError, ValueError) as error: + raise HealingEvaluationError( + f"{name} must be [x1, y1, x2, y2]; got {value!r}") from error + if x2 <= x1 or y2 <= y1: + raise HealingEvaluationError( + f"{name} must have x2 > x1 and y2 > y1; got {value!r}") + return x1, y1, x2, y2 + + +def _as_point(value: Sequence[int], name: str) -> Point: + try: + x, y = (int(item) for item in value) + except (TypeError, ValueError) as error: + raise HealingEvaluationError(f"{name} must be [x, y]; got {value!r}") from error + return x, y + + +def frame_hash(frame: Any) -> Optional[str]: + """SHA-256 of ``frame``'s pixels and shape; ``None`` for an unhashable frame.""" + if isinstance(frame, (bytes, bytearray, memoryview)): + return hashlib.sha256(bytes(frame)).hexdigest() + to_bytes = getattr(frame, "tobytes", None) + if to_bytes is None: + return None + digest = hashlib.sha256(repr(getattr(frame, "shape", None)).encode("ascii")) + digest.update(to_bytes()) + return digest.hexdigest() + + +@dataclass(frozen=True) +class EvaluationSample: + """One captured frame and what a correct locator returns for it. + + ``expected_box`` and ``region`` are ``(x1, y1, x2, y2)`` in screen + coordinates. A sample with neither ``expected_box`` nor ``expect_miss`` is + unlabelled and is reported as ``unknown``. + """ + + sample_id: str + frame: Any + expected_box: Optional[Box] = None + origin: Point = (0, 0) + scale: float = 1.0 + region: Optional[Box] = None + expect_miss: bool = False + template: Any = None + description: Optional[str] = None + + def __post_init__(self) -> None: + object.__setattr__(self, "expected_box", _as_box(self.expected_box, "expected_box")) + object.__setattr__(self, "region", _as_box(self.region, "region")) + object.__setattr__(self, "origin", _as_point(self.origin, "origin")) + try: + scale = float(self.scale) + except (TypeError, ValueError) as error: + raise HealingEvaluationError(f"scale must be a number; got {self.scale!r}") from error + if not math.isfinite(scale) or scale <= 0: + raise HealingEvaluationError(f"scale must be positive; got {self.scale!r}") + object.__setattr__(self, "scale", scale) + if self.expect_miss and self.expected_box is not None: + raise HealingEvaluationError( + f"sample {self.sample_id!r}: expect_miss and expected_box contradict") + + @property + def labelled(self) -> bool: + """Whether the sample says what the right answer is.""" + return self.expect_miss or self.expected_box is not None + + +@dataclass(frozen=True) +class LocateRequest: + """What every strategy version receives for one sample — the same object.""" + + sample_id: str + frame: Any + origin: Point + scale: float + region: Optional[Box] + template: Any + description: Optional[str] + frame_hash: Optional[str] + + def to_screen(self, frame_x: float, frame_y: float) -> Point: + """A frame pixel as the screen point a click would be sent to.""" + return (self.origin[0] + int(math.floor(frame_x / self.scale)), + self.origin[1] + int(math.floor(frame_y / self.scale))) + + def frame_box(self, width: int, height: int) -> Optional[Box]: + """``region`` in frame pixels, clipped to a ``width`` x ``height`` frame. + + The whole frame when there is no region; ``None`` when the region does + not overlap the frame, which a strategy must treat as nothing to search. + """ + if self.region is None: + return 0, 0, int(width), int(height) + x1, y1, x2, y2 = self.region + left = max(0, int(math.floor((x1 - self.origin[0]) * self.scale))) + top = max(0, int(math.floor((y1 - self.origin[1]) * self.scale))) + right = min(int(width), int(math.ceil((x2 - self.origin[0]) * self.scale))) + bottom = min(int(height), int(math.ceil((y2 - self.origin[1]) * self.scale))) + if right <= left or bottom <= top: + return None + return left, top, right, bottom + + +#: A strategy version: screen ``(x, y)`` for a hit, ``None`` for a miss. +LocatorStrategy = Callable[[LocateRequest], Optional[Sequence[int]]] + + +@dataclass(frozen=True) +class Ratio: + """A rate that keeps the counts it was computed from.""" + + numerator: int + denominator: int + + @property + def value(self) -> Optional[float]: + """The rate, or ``None`` when there was nothing to divide by.""" + if self.denominator == 0: + return None + return self.numerator / self.denominator + + def to_dict(self) -> Dict[str, Any]: + """``{numerator, denominator, value}``; ``value`` is ``None`` over zero.""" + value = self.value + return {"numerator": self.numerator, "denominator": self.denominator, + "value": None if value is None else round(value, 4)} + + def __str__(self) -> str: + value = self.value + shown = "n/a" if value is None else f"{value * 100:.1f}%" + return f"{self.numerator}/{self.denominator} ({shown})" + + +@dataclass(frozen=True) +class SampleResult: + """What one strategy version did with one sample.""" + + sample_id: str + version: str + outcome: str + located: bool + coordinates: Optional[Point] + latency_ms: float + frame_hash: Optional[str] + error: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + """JSON-safe row.""" + return { + "sample_id": self.sample_id, "version": self.version, + "outcome": self.outcome, "located": self.located, + "coordinates": None if self.coordinates is None else list(self.coordinates), + "latency_ms": self.latency_ms, "frame_hash": self.frame_hash, + "error": self.error, + } + + +@dataclass(frozen=True) +class VersionReport: + """Counts for one strategy version; the six outcomes add up to ``total``.""" + + version: str + total: int + located: int + correct: int + false_positive: int + miss: int + true_negative: int + unknown: int + error: int + recovered: int + recoverable: int + p50_ms: Optional[float] + p95_ms: Optional[float] + + @property + def labelled(self) -> int: + """Samples that say what the right answer is.""" + return self.total - self.unknown + + @property + def hit_rate(self) -> Ratio: + """Samples a point was returned for, right or wrong, over all samples.""" + return Ratio(self.located, self.total) + + @property + def accuracy(self) -> Ratio: + """Correct hits plus correct rejections, over labelled samples.""" + return Ratio(self.correct + self.true_negative, self.labelled) + + @property + def false_positive_rate(self) -> Ratio: + """Hits in the wrong place or on an expected miss, over labelled samples.""" + return Ratio(self.false_positive, self.labelled) + + @property + def recovery_rate(self) -> Ratio: + """Targets the baseline failed on that this version hit correctly.""" + return Ratio(self.recovered, self.recoverable) + + def to_dict(self) -> Dict[str, Any]: + """JSON-safe report; every rate carries numerator and denominator.""" + return { + "version": self.version, "total": self.total, + "labelled": self.labelled, "located": self.located, + "correct": self.correct, "false_positive": self.false_positive, + "miss": self.miss, "true_negative": self.true_negative, + "unknown": self.unknown, "error": self.error, + "hit_rate": self.hit_rate.to_dict(), + "accuracy": self.accuracy.to_dict(), + "false_positive_rate": self.false_positive_rate.to_dict(), + "recovery_rate": self.recovery_rate.to_dict(), + "p50_ms": self.p50_ms, "p95_ms": self.p95_ms, + } + + +@dataclass(frozen=True) +class HealingComparison: + """Reports for every version over one set of samples.""" + + baseline: str + reports: Mapping[str, VersionReport] + results: Tuple[SampleResult, ...] + + def report(self, version: str) -> VersionReport: + """The report for ``version``; unknown names raise.""" + try: + return self.reports[version] + except KeyError: + raise HealingEvaluationError( + f"no such version {version!r}; have {list(self.reports)}") from None + + def failures(self, version: str) -> List[SampleResult]: + """Labelled samples ``version`` got wrong, in sample order.""" + self.report(version) + return [row for row in self.results + if row.version == version and row.outcome in FAILURE_OUTCOMES] + + def to_dict(self) -> Dict[str, Any]: + """The whole comparison as JSON-safe data.""" + return { + "baseline": self.baseline, + "versions": {name: report.to_dict() for name, report in self.reports.items()}, + "results": [row.to_dict() for row in self.results], + } + + +def evaluate_locators(samples: Sequence[EvaluationSample], + versions: Mapping[str, LocatorStrategy], + *, baseline: Optional[str] = None, + clock: Callable[[], float] = perf_counter, + ) -> HealingComparison: + """Run every strategy in ``versions`` over every sample and score it. + + ``baseline`` names the version recovery is measured against; it defaults + to the first one. ``clock`` returns seconds and exists so a test can make + the latency columns deterministic. + """ + names = list(versions) + if not names: + raise HealingEvaluationError("evaluate_locators needs at least one version") + base = names[0] if baseline is None else baseline + if base not in versions: + raise HealingEvaluationError(f"baseline {base!r} is not one of {names}") + _reject_duplicate_ids(samples) + rows: List[SampleResult] = [] + for sample in samples: + request = LocateRequest( + sample_id=sample.sample_id, frame=sample.frame, origin=sample.origin, + scale=sample.scale, region=sample.region, template=sample.template, + description=sample.description, frame_hash=frame_hash(sample.frame)) + for name in names: + rows.append(_run_one(sample, request, name, versions[name], clock)) + reports = _build_reports(samples, names, base, rows) + return HealingComparison(baseline=base, reports=reports, results=tuple(rows)) + + +def _reject_duplicate_ids(samples: Sequence[EvaluationSample]) -> None: + seen = set() + for sample in samples: + if sample.sample_id in seen: + raise HealingEvaluationError(f"duplicate sample id {sample.sample_id!r}") + seen.add(sample.sample_id) + + +def _run_one(sample: EvaluationSample, request: LocateRequest, version: str, + strategy: LocatorStrategy, clock: Callable[[], float]) -> SampleResult: + started = clock() + point: Optional[Point] = None + error: Optional[str] = None + try: + raw = strategy(request) + point = None if raw is None else _as_point(raw, f"{version} result") + except _STRATEGY_ERRORS as exc: + error = repr(exc) + latency = round((clock() - started) * 1000.0, 3) + if frame_hash(sample.frame) != request.frame_hash: + raise HealingEvaluationError( + f"version {version!r} modified the frame of sample {sample.sample_id!r}; " + "later versions would be measured on a different image") + return SampleResult( + sample_id=sample.sample_id, version=version, + outcome=_classify(sample, point, error), located=point is not None, + coordinates=point, latency_ms=latency, frame_hash=request.frame_hash, + error=error) + + +def _classify(sample: EvaluationSample, point: Optional[Point], + error: Optional[str]) -> str: + if not sample.labelled: + return OUTCOME_UNKNOWN + if error is not None: + return OUTCOME_ERROR + if point is None: + return OUTCOME_TRUE_NEGATIVE if sample.expect_miss else OUTCOME_MISS + if sample.expected_box is not None and _inside(point, sample.expected_box): + return OUTCOME_CORRECT + return OUTCOME_FALSE_POSITIVE + + +def _inside(point: Point, box: Box) -> bool: + return box[0] <= point[0] < box[2] and box[1] <= point[1] < box[3] + + +def _build_reports(samples: Sequence[EvaluationSample], names: Sequence[str], + baseline: str, rows: Sequence[SampleResult], + ) -> Dict[str, VersionReport]: + by_version: Dict[str, Dict[str, SampleResult]] = {name: {} for name in names} + for row in rows: + by_version[row.version][row.sample_id] = row + # Recoverable: a target that exists and that the baseline did not hit + # correctly. An expected miss is never in this set. + recoverable = [sample.sample_id for sample in samples + if sample.expected_box is not None + and by_version[baseline][sample.sample_id].outcome != OUTCOME_CORRECT] + reports: Dict[str, VersionReport] = {} + for name in names: + own = by_version[name] + recovered = sum(1 for sample_id in recoverable + if own[sample_id].outcome == OUTCOME_CORRECT) + reports[name] = _report(name, list(own.values()), recovered, len(recoverable)) + return reports + + +def _report(version: str, rows: Sequence[SampleResult], recovered: int, + recoverable: int) -> VersionReport: + counts = {outcome: 0 for outcome in ( + OUTCOME_CORRECT, OUTCOME_FALSE_POSITIVE, OUTCOME_MISS, + OUTCOME_TRUE_NEGATIVE, OUTCOME_UNKNOWN, OUTCOME_ERROR)} + for row in rows: + counts[row.outcome] += 1 + latencies = sorted(row.latency_ms for row in rows) + return VersionReport( + version=version, total=len(rows), + located=sum(1 for row in rows if row.located), + correct=counts[OUTCOME_CORRECT], + false_positive=counts[OUTCOME_FALSE_POSITIVE], + miss=counts[OUTCOME_MISS], true_negative=counts[OUTCOME_TRUE_NEGATIVE], + unknown=counts[OUTCOME_UNKNOWN], error=counts[OUTCOME_ERROR], + recovered=recovered, recoverable=recoverable, + p50_ms=_percentile(latencies, 50), p95_ms=_percentile(latencies, 95)) + + +def _percentile(ordered: Sequence[float], percent: int) -> Optional[float]: + """Nearest-rank percentile of an ascending list; ``None`` when it is empty.""" + if not ordered: + return None + rank = max(1, math.ceil(percent / 100.0 * len(ordered))) + return ordered[rank - 1] + + +_THRESHOLD_CHECKS: Dict[str, Callable[[VersionReport], Optional[float]]] = { + "min_correct": lambda report: report.correct, + "min_accuracy": lambda report: report.accuracy.value, + "max_false_positive": lambda report: report.false_positive, + "max_error": lambda report: report.error, + "max_p95_ms": lambda report: report.p95_ms, +} + + +def check_thresholds(comparison: HealingComparison, + thresholds: Mapping[str, Mapping[str, float]]) -> List[str]: + """Return one line per threshold ``comparison`` does not meet. + + ``thresholds`` maps a version to ``min_correct`` / ``min_accuracy`` / + ``max_false_positive`` / ``max_error`` / ``max_p95_ms``. A version or key + that does not exist is a violation too: a gate that silently checks + nothing is worse than none. A rate with no value fails its ``min_``. + """ + violations: List[str] = [] + for version, limits in thresholds.items(): + report = comparison.reports.get(version) + if report is None: + violations.append(f"{version}: no such version in the comparison") + continue + for key, limit in limits.items(): + problem = _threshold_problem(report, key, limit) + if problem: + violations.append(f"{version}: {problem}") + return violations + + +def _threshold_problem(report: VersionReport, key: str, limit: float) -> Optional[str]: + reader = _THRESHOLD_CHECKS.get(key) + if reader is None: + return f"unknown threshold {key!r}" + measured = reader(report) + if key.startswith("min_"): + failed = measured is None or measured < limit + else: + failed = measured is not None and measured > limit + return f"{key}={limit} not met (measured {measured})" if failed else None + + +def format_comparison(comparison: HealingComparison) -> str: + """Plain-text table of the comparison, followed by each version's failures.""" + lines = [f"baseline: {comparison.baseline}"] + for name, report in comparison.reports.items(): + lines.append( + f"[{name}] samples={report.total} labelled={report.labelled} " + f"unknown={report.unknown} error={report.error}") + lines.append(f" located {report.hit_rate}") + lines.append(f" accuracy {report.accuracy}") + lines.append(f" false pos. {report.false_positive_rate}") + lines.append(f" recovery {report.recovery_rate}") + lines.append(f" latency ms p50={report.p50_ms} p95={report.p95_ms}") + for row in comparison.failures(name): + where = "" if row.coordinates is None else f" at {list(row.coordinates)}" + reason = "" if row.error is None else f" ({row.error})" + lines.append(f" ! {row.sample_id}: {row.outcome}{where}{reason}") + return "\n".join(lines) + + +__all__ = [ + "Box", "EvaluationSample", "FAILURE_OUTCOMES", "HealingComparison", + "HealingEvaluationError", "LocateRequest", "LocatorStrategy", + "OUTCOME_CORRECT", "OUTCOME_ERROR", "OUTCOME_FALSE_POSITIVE", + "OUTCOME_MISS", "OUTCOME_TRUE_NEGATIVE", "OUTCOME_UNKNOWN", + "Point", "Ratio", "SampleResult", "VersionReport", + "check_thresholds", "evaluate_locators", "format_comparison", "frame_hash", +] diff --git a/je_auto_control/utils/self_healing/heal_log.py b/je_auto_control/utils/self_healing/heal_log.py index 98e3fcde7..52ca47764 100644 --- a/je_auto_control/utils/self_healing/heal_log.py +++ b/je_auto_control/utils/self_healing/heal_log.py @@ -1,9 +1,15 @@ -"""Append-only JSON-lines log of self-healing locator events.""" +"""Append-only JSON-lines log of self-healing locator events. + +A line written before the version / context fields existed has none of them +and still loads: every field added since is optional. A line written by a +newer build may carry fields this one does not know; they are dropped rather +than costing the reader the whole event. +""" from __future__ import annotations import json import threading -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, fields from pathlib import Path from typing import Any, Dict, List, Optional @@ -22,6 +28,25 @@ class HealEvent: description: Optional[str] = None image_error: Optional[str] = None vlm_error: Optional[str] = None + # --- added with schema_version 2; absent from older lines ------------- + #: ``None`` on a line written before the field existed. + schema_version: Optional[int] = None + run_id: Optional[str] = None + step_id: Optional[str] = None + locator_id: Optional[str] = None + locator_version: Optional[str] = None + backend: Optional[str] = None + model: Optional[str] = None + #: ``[x1, y1, x2, y2]`` both strategies were confined to, if any. + screen_region: Optional[List[int]] = None + image_ms: Optional[float] = None + vlm_ms: Optional[float] = None + #: What was done with the hit (``"click"``); ``None`` for a bare locate. + action: Optional[str] = None + #: ``True`` / ``False`` once a caller-supplied check ran after the action; + #: ``None`` when nothing verified it. A located element is not a verified + #: action, so this is never inferred from ``method``. + action_verified: Optional[bool] = None def to_dict(self) -> Dict[str, Any]: """Return a plain-dict snapshot safe for JSON / network transport.""" @@ -100,13 +125,25 @@ def _parse_line(raw: str) -> Optional[HealEvent]: payload = json.loads(text) except ValueError: return None + if not isinstance(payload, dict): + return None + known = {name: payload[name] for name in _FIELD_NAMES if name in payload} try: - return HealEvent(**payload) + return HealEvent(**known) except TypeError: return None +#: Schema written by this build. 1 is the original eight-field line, which +#: carried no ``schema_version`` key at all. +HEAL_EVENT_SCHEMA_VERSION = 2 + +_FIELD_NAMES = frozenset(spec.name for spec in fields(HealEvent)) + + default_heal_log = HealEventLog() -__all__ = ["HealEvent", "HealEventLog", "default_heal_log"] +__all__ = [ + "HEAL_EVENT_SCHEMA_VERSION", "HealEvent", "HealEventLog", "default_heal_log", +] diff --git a/je_auto_control/utils/self_healing/locator.py b/je_auto_control/utils/self_healing/locator.py index 98253553b..b83f9a229 100644 --- a/je_auto_control/utils/self_healing/locator.py +++ b/je_auto_control/utils/self_healing/locator.py @@ -9,30 +9,66 @@ * records every attempt (hit / heal / miss) to a JSON-lines log so flaky locators can be audited and tuned over time; * never raises on a miss by default — returns a :class:`HealOutcome` - the caller can branch on. + the caller can branch on; +* keeps "the locator found something" (``found``) apart from "the action + that followed was checked" (``action_verified``): a VLM that answers with + a confident wrong point is a hit by the first measure and nothing by the + second, and counting it as a recovery is how a healing rate lies. + +``screen_region`` is ``[x1, y1, x2, y2]`` in screen coordinates and confines +**both** strategies. It used to reach only the VLM, so a template match could +"find" the element outside the region the caller asked to search. The wrapper is platform-agnostic and Qt-free; the GUI panel and MCP tool are thin shells over it. """ from __future__ import annotations -from dataclasses import asdict, dataclass +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import asdict, dataclass, replace from datetime import datetime, timezone from time import monotonic -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple from je_auto_control.utils.exception.exceptions import ( AutoControlException, ImageNotFoundException, ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.self_healing.heal_log import ( - HealEvent, HealEventLog, default_heal_log, + HEAL_EVENT_SCHEMA_VERSION, HealEvent, HealEventLog, default_heal_log, ) METHOD_IMAGE = "image" METHOD_VLM = "vlm" METHOD_MISS = "miss" +ACTION_CLICK = "click" + +#: Keys :func:`heal_context` accepts; each is a string column of the heal log. +CONTEXT_KEYS = ("run_id", "step_id", "locator_id", "locator_version", "backend") + +_context: ContextVar[Mapping[str, str]] = ContextVar("self_heal_context") + + +@contextmanager +def heal_context(**values: Optional[str]) -> Iterator[None]: + """Stamp every heal event logged inside the block with ``values``. + + Accepts the :data:`CONTEXT_KEYS` (``run_id``, ``step_id``, ``locator_id``, + ``locator_version``, ``backend``); ``None`` values are ignored and an + unknown key raises ``ValueError``. Nested blocks layer over outer ones. + """ + unknown = sorted(set(values) - set(CONTEXT_KEYS)) + if unknown: + raise ValueError(f"unknown heal context key(s) {unknown}; expected {list(CONTEXT_KEYS)}") + merged = dict(_context.get({})) + merged.update({key: str(value) for key, value in values.items() if value is not None}) + token = _context.set(merged) + try: + yield + finally: + _context.reset(token) class SelfHealError(AutoControlException, RuntimeError): @@ -53,6 +89,14 @@ class HealOutcome: image_error: Optional[str] = None vlm_error: Optional[str] = None duration_ms: float = 0.0 + screen_region: Optional[Tuple[int, int, int, int]] = None + model: Optional[str] = None + image_ms: Optional[float] = None + vlm_ms: Optional[float] = None + #: ``"click"`` once the hit was acted on; ``None`` for a bare locate. + action: Optional[str] = None + #: Result of the caller's post-action check; ``None`` when none ran. + action_verified: Optional[bool] = None def to_dict(self) -> Dict[str, Any]: """JSON-safe dict (tuple → list) for executor / MCP responses.""" @@ -60,6 +104,8 @@ def to_dict(self) -> Dict[str, Any]: if self.coordinates is not None: data["coordinates"] = [int(self.coordinates[0]), int(self.coordinates[1])] + if self.screen_region is not None: + data["screen_region"] = [int(value) for value in self.screen_region] return data @@ -75,44 +121,13 @@ def self_heal_locate(template_path: Optional[str] = None, At least one of ``template_path`` / ``description`` must be given. Provide both for full self-healing — the VLM path only runs when - the template match fails or returns no candidates. + the template match fails or returns no candidates. ``screen_region`` + (``[x1, y1, x2, y2]``, screen coordinates) confines both strategies. """ - if not template_path and not description: - raise ValueError( - "self_heal_locate requires template_path or description", - ) - started = monotonic() - coords, image_error = _try_image(template_path, detect_threshold) - if coords is not None: - return _finish( - HealOutcome(found=True, coordinates=coords, method=METHOD_IMAGE, - description=description, template_path=template_path, - duration_ms=_ms_since(started)), - log, - ) - coords, vlm_error = _try_vlm(description, screen_region, model) - if coords is not None: - autocontrol_logger.warning( - f"self_heal: image miss ({image_error}); VLM healed → {coords}", - ) - return _finish( - HealOutcome(found=True, coordinates=coords, method=METHOD_VLM, - description=description, template_path=template_path, - image_error=image_error, - duration_ms=_ms_since(started)), - log, - ) - outcome = HealOutcome( - found=False, coordinates=None, method=METHOD_MISS, - description=description, template_path=template_path, - image_error=image_error, vlm_error=vlm_error, - duration_ms=_ms_since(started), - ) + outcome = _locate(template_path, description, detect_threshold, + _checked_region(screen_region), model) _finish(outcome, log) - if raise_on_miss: - raise SelfHealError( - f"self_heal_locate failed: image={image_error!r} vlm={vlm_error!r}", - ) + _raise_on_miss(outcome, raise_on_miss) return outcome @@ -124,18 +139,110 @@ def self_heal_click(template_path: Optional[str] = None, model: Optional[str] = None, log: Optional[HealEventLog] = None, raise_on_miss: bool = False, + verify: Optional[Callable[[HealOutcome], bool]] = None, ) -> HealOutcome: - """``self_heal_locate`` + a click at the resolved coordinates.""" - outcome = self_heal_locate( - template_path=template_path, description=description, - detect_threshold=detect_threshold, screen_region=screen_region, - model=model, log=log, raise_on_miss=raise_on_miss, - ) - if outcome.found and outcome.coordinates is not None: - _click_at(outcome.coordinates, mouse_keycode) + """``self_heal_locate`` + a click at the resolved coordinates. + + ``verify`` is called with the outcome after the click and its truth value + is recorded as ``action_verified``; without it the field stays ``None``. + ``found`` only ever says that a strategy returned a point. + """ + outcome = _locate(template_path, description, detect_threshold, + _checked_region(screen_region), model) + if not outcome.found or outcome.coordinates is None: + _finish(outcome, log) + _raise_on_miss(outcome, raise_on_miss) + return outcome + target = outcome.coordinates + outcome = replace(outcome, action=ACTION_CLICK) + try: + _click_at(target, mouse_keycode) + if verify is not None: + outcome = replace(outcome, action_verified=bool(verify(outcome))) + finally: + # Logged whether or not the click or the check raised: the attempt + # happened, and an unverified one must not read as a verified one. + _finish(outcome, log) return outcome +def _locate(template_path: Optional[str], description: Optional[str], + detect_threshold: float, region: Optional[Tuple[int, int, int, int]], + model: Optional[str]) -> HealOutcome: + """Run image then VLM once; never logs, never raises on a miss.""" + if not template_path and not description: + raise ValueError( + "self_heal_locate requires template_path or description", + ) + started = monotonic() + base = HealOutcome(found=False, coordinates=None, method=METHOD_MISS, + description=description, template_path=template_path, + screen_region=region, model=model) + coords, image_error = _try_image_in_region(template_path, detect_threshold, region) + image_ms = _ms_since(started) + if coords is not None: + return replace(base, found=True, coordinates=coords, method=METHOD_IMAGE, + image_ms=image_ms, duration_ms=_ms_since(started)) + vlm_started = monotonic() + coords, vlm_error = _try_vlm(description, None if region is None else list(region), model) + timed = replace(base, image_error=image_error, image_ms=image_ms, + vlm_ms=_ms_since(vlm_started) if description else None) + if coords is not None: + autocontrol_logger.warning( + f"self_heal: image miss ({image_error}); VLM healed → {coords}", + ) + return replace(timed, found=True, coordinates=coords, method=METHOD_VLM, + duration_ms=_ms_since(started)) + return replace(timed, vlm_error=vlm_error, duration_ms=_ms_since(started)) + + +def _raise_on_miss(outcome: HealOutcome, raise_on_miss: bool) -> None: + if raise_on_miss and not outcome.found: + raise SelfHealError( + f"self_heal_locate failed: image={outcome.image_error!r} vlm={outcome.vlm_error!r}", + ) + + +def _checked_region(screen_region: Optional[Sequence[int]], + ) -> Optional[Tuple[int, int, int, int]]: + """``[x1, y1, x2, y2]`` as four ints with a positive area, or ``None``.""" + if screen_region is None: + return None + try: + x1, y1, x2, y2 = (int(value) for value in screen_region) + except (TypeError, ValueError) as error: + raise ValueError( + f"screen_region must be [x1, y1, x2, y2]; got {screen_region!r}") from error + if x2 <= x1 or y2 <= y1: + raise ValueError( + f"screen_region must have x2 > x1 and y2 > y1; got {screen_region!r}") + return x1, y1, x2, y2 + + +def _try_image_in_region(template_path: Optional[str], detect_threshold: float, + region: Optional[Tuple[int, int, int, int]], + ) -> Tuple[Optional[Tuple[int, int]], Optional[str]]: + """Template match confined to ``region``; the whole screen when it is ``None``.""" + if region is None or not template_path: + return _try_image(template_path, detect_threshold) + x1, y1, x2, y2 = region + try: + from je_auto_control.utils.cv2_utils import template_detection + # The matcher takes (x, y, width, height); the heal API, like the VLM + # path, takes two corners. + result = template_detection.find_image( + template_path, float(detect_threshold), + screen_region=(x1, y1, x2 - x1, y2 - y1)) + except ImageNotFoundException as exc: + return None, str(exc) + except (AutoControlException, OSError, RuntimeError, ValueError, TypeError) as exc: + return None, repr(exc) + if not result[0]: + return None, f"template not found in region {list(region)}" + left, top, right, bottom = result[1] + return (int((left + right) // 2), int((top + bottom) // 2)), None + + def _try_image(template_path: Optional[str], detect_threshold: float, ) -> Tuple[Optional[Tuple[int, int]], Optional[str]]: @@ -203,6 +310,7 @@ def _as_event(outcome: HealOutcome) -> HealEvent: [int(outcome.coordinates[0]), int(outcome.coordinates[1])] if outcome.coordinates is not None else None ) + context = _context.get({}) return HealEvent( timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"), method=outcome.method, @@ -212,10 +320,24 @@ def _as_event(outcome: HealOutcome) -> HealEvent: description=outcome.description, image_error=outcome.image_error, vlm_error=outcome.vlm_error, + schema_version=HEAL_EVENT_SCHEMA_VERSION, + run_id=context.get("run_id"), + step_id=context.get("step_id"), + locator_id=context.get("locator_id"), + locator_version=context.get("locator_version"), + backend=context.get("backend"), + model=outcome.model, + screen_region=(None if outcome.screen_region is None + else [int(value) for value in outcome.screen_region]), + image_ms=outcome.image_ms, + vlm_ms=outcome.vlm_ms, + action=outcome.action, + action_verified=outcome.action_verified, ) __all__ = [ - "HealOutcome", "METHOD_IMAGE", "METHOD_MISS", "METHOD_VLM", - "SelfHealError", "self_heal_click", "self_heal_locate", + "ACTION_CLICK", "CONTEXT_KEYS", "HealOutcome", + "METHOD_IMAGE", "METHOD_MISS", "METHOD_VLM", + "SelfHealError", "heal_context", "self_heal_click", "self_heal_locate", ] diff --git a/je_auto_control/utils/self_healing/template_revision.py b/je_auto_control/utils/self_healing/template_revision.py new file mode 100644 index 000000000..2fab4535a --- /dev/null +++ b/je_auto_control/utils/self_healing/template_revision.py @@ -0,0 +1,351 @@ +"""Candidate template revisions: propose, preview, accept, revert. + +A locator that "healed" hands back a point, and the tempting next step is to +crop the screen there and overwrite the template. That turns one unverified +answer into every later answer. Here a new template image is only ever a +**candidate**: + +1. :meth:`TemplateRevisionStore.propose` copies the candidate into the store. + The live template is not touched. +2. :meth:`~TemplateRevisionStore.preview` shows both files side by side and, + given a labelled dataset, runs the current and the candidate template over + the *same* frames with :func:`evaluation.evaluate_locators`. A candidate + that hits at least one target correctly, raises no false positive and is + correct at least as often as the current template is marked ``validated``. +3. :meth:`~TemplateRevisionStore.accept` replaces the live template — after + backing it up — and refuses an unvalidated candidate unless the caller + says ``allow_unvalidated=True``. +4. :meth:`~TemplateRevisionStore.revert` puts the backup back. + +Accept and revert both check the live file's hash first: a template that +changed underneath a revision is reported, never silently overwritten. +""" +from __future__ import annotations + +import hashlib +import os +import re +import threading +import uuid +from dataclasses import asdict, dataclass, fields, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Union + +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.json_store.json_store import ( + atomic_write_bytes, read_json_dict, write_json_dict, +) +from je_auto_control.utils.self_healing.evaluation import ( + EvaluationSample, VersionReport, evaluate_locators, +) + +STATUS_PENDING = "pending" +STATUS_ACCEPTED = "accepted" +STATUS_REVERTED = "reverted" + +VERSION_CURRENT = "current" +VERSION_CANDIDATE = "candidate" + +_ID_PATTERN = re.compile(r"^[0-9a-f]{12}$") +_INDEX_NAME = "index.json" + +PathLike = Union[str, "os.PathLike[str]"] + + +class TemplateRevisionError(AutoControlException, RuntimeError): + """A template revision could not be proposed, accepted or reverted.""" + + +@dataclass(frozen=True) +class TemplateRevision: + """One candidate replacement for a template image.""" + + revision_id: str + template_path: str + candidate_file: str + status: str + created_at: str + #: Hash of the live template when the candidate was proposed. + previous_sha256: str + candidate_sha256: str + source: str = "manual" + note: Optional[str] = None + validated: bool = False + #: Counts from the last preview that ran a dataset, or ``None``. + validation: Optional[Dict[str, Any]] = None + backup_file: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + """JSON-safe snapshot.""" + return asdict(self) + + +def default_revision_root() -> Path: + """``~/.je_auto_control/template_revisions``, resolved at call time.""" + return Path.home() / ".je_auto_control" / "template_revisions" + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _existing_file(path: PathLike, what: str) -> Path: + resolved = Path(os.path.realpath(os.fspath(path))) + if not resolved.is_file(): + raise TemplateRevisionError(f"{what} is not a file: {path!r}") + return resolved + + +def _file_facts(path: Path) -> Dict[str, Any]: + """Size, hash and — when the file decodes as an image — pixel dimensions.""" + facts: Dict[str, Any] = {"path": str(path), "bytes": path.stat().st_size, + "sha256": _sha256(path), "width": None, "height": None} + try: + import cv2 + from je_auto_control.utils.cv2_utils.image_file import read_image + image = read_image(str(path), cv2.IMREAD_UNCHANGED) + except (ImportError, ValueError): + return facts + facts["height"], facts["width"] = int(image.shape[0]), int(image.shape[1]) + return facts + + +def _is_validated(current: VersionReport, candidate: VersionReport) -> bool: + return (candidate.correct >= 1 and candidate.false_positive == 0 + and candidate.error == 0 and candidate.correct >= current.correct) + + +class TemplateRevisionStore: + """Thread-safe store of candidate template revisions under one directory.""" + + def __init__(self, root: Optional[PathLike] = None) -> None: + # As HealEventLog: only an explicit root is kept, the default follows HOME. + self._explicit_root: Optional[Path] = None if root is None else Path(root) + self._lock = threading.Lock() + + @property + def root(self) -> Path: + """Directory holding the index, the candidates and the backups.""" + return self._explicit_root or default_revision_root() + + # --- index ----------------------------------------------------------- + + def _load(self) -> Dict[str, TemplateRevision]: + known = {spec.name for spec in fields(TemplateRevision)} + revisions: Dict[str, TemplateRevision] = {} + for raw in read_json_dict(self.root / _INDEX_NAME).get("revisions", []): + if not isinstance(raw, dict): + continue + try: + revision = TemplateRevision(**{k: v for k, v in raw.items() if k in known}) + except TypeError: + continue + revisions[revision.revision_id] = revision + return revisions + + def _save(self, revisions: Mapping[str, TemplateRevision]) -> None: + write_json_dict(self.root / _INDEX_NAME, + {"revisions": [item.to_dict() for item in revisions.values()]}) + + def _require(self, revisions: Mapping[str, TemplateRevision], + revision_id: str) -> TemplateRevision: + if not isinstance(revision_id, str) or not _ID_PATTERN.match(revision_id): + raise TemplateRevisionError(f"not a revision id: {revision_id!r}") + revision = revisions.get(revision_id) + if revision is None: + raise TemplateRevisionError(f"no such template revision: {revision_id}") + return revision + + def _put(self, revision: TemplateRevision) -> TemplateRevision: + """Write ``revision`` into the index under the lock the caller holds.""" + revisions = self._load() + revisions[revision.revision_id] = revision + self._save(revisions) + return revision + + # --- public API ------------------------------------------------------ + + def list_revisions(self) -> List[TemplateRevision]: + """Every revision, oldest first.""" + with self._lock: + return list(self._load().values()) + + def get(self, revision_id: str) -> TemplateRevision: + """The revision ``revision_id`` names; unknown ids raise.""" + with self._lock: + return self._require(self._load(), revision_id) + + def propose(self, template_path: PathLike, candidate_path: PathLike, *, + source: str = "manual", note: Optional[str] = None) -> TemplateRevision: + """Store ``candidate_path`` as a pending replacement for ``template_path``. + + The live template is left exactly as it is. + """ + template = _existing_file(template_path, "template") + candidate = _existing_file(candidate_path, "candidate") + previous, proposed = _sha256(template), _sha256(candidate) + if previous == proposed: + raise TemplateRevisionError("the candidate is identical to the current template") + revision_id = uuid.uuid4().hex[:12] + stored = self.root / revision_id / f"candidate{candidate.suffix.lower()}" + with self._lock: + stored.parent.mkdir(parents=True, exist_ok=True) + atomic_write_bytes(stored, candidate.read_bytes()) + return self._put(TemplateRevision( + revision_id=revision_id, template_path=str(template), + candidate_file=str(stored), status=STATUS_PENDING, + created_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + previous_sha256=previous, candidate_sha256=proposed, + source=str(source), note=note)) + + def preview(self, revision_id: str, *, + samples: Optional[Sequence[EvaluationSample]] = None, + dataset_path: Optional[PathLike] = None, + detect_threshold: float = 0.9) -> Dict[str, Any]: + """Describe what accepting would change; validate it when given frames. + + With ``samples`` or ``dataset_path`` the current and the candidate + template are run over the same frames and the result decides the + revision's ``validated`` flag. Without either, nothing is validated + and the flag keeps its value. Nothing is written to the template. + """ + revision = self.get(revision_id) + payload: Dict[str, Any] = { + "revision": revision.to_dict(), + "current": _file_facts(_existing_file(revision.template_path, "template")), + "candidate": _file_facts(_existing_file(revision.candidate_file, "candidate")), + "comparison": None, + } + frames = _preview_samples(samples, dataset_path) + if frames is None: + return payload + from je_auto_control.utils.self_healing.eval_strategies import ( + template_match_strategy, + ) + comparison = evaluate_locators(frames, { + VERSION_CURRENT: template_match_strategy( + detect_threshold, template=revision.template_path), + VERSION_CANDIDATE: template_match_strategy( + detect_threshold, template=revision.candidate_file), + }) + current, candidate = (comparison.report(VERSION_CURRENT), + comparison.report(VERSION_CANDIDATE)) + validated = _is_validated(current, candidate) + with self._lock: + latest = self._require(self._load(), revision_id) + updated = self._put(replace(latest, validated=validated, validation={ + "detect_threshold": float(detect_threshold), + VERSION_CURRENT: current.to_dict(), + VERSION_CANDIDATE: candidate.to_dict(), + })) + payload["revision"] = updated.to_dict() + payload["comparison"] = comparison.to_dict() + return payload + + def accept(self, revision_id: str, *, + allow_unvalidated: bool = False) -> TemplateRevision: + """Replace the live template with the candidate, keeping a backup.""" + with self._lock: + revision = self._require(self._load(), revision_id) + if revision.status == STATUS_ACCEPTED: + raise TemplateRevisionError(f"revision {revision_id} is already accepted") + if not revision.validated and not allow_unvalidated: + raise TemplateRevisionError( + f"revision {revision_id} has not been validated; preview it against " + "labelled frames, or pass allow_unvalidated=True to accept it anyway") + template = _existing_file(revision.template_path, "template") + candidate = _existing_file(revision.candidate_file, "candidate") + if _sha256(template) != revision.previous_sha256: + raise TemplateRevisionError( + f"{template} changed since revision {revision_id} was proposed; " + "propose a new revision against the current file") + backup = candidate.parent / f"backup{template.suffix.lower()}" + atomic_write_bytes(backup, template.read_bytes()) + atomic_write_bytes(template, candidate.read_bytes()) + return self._put(replace(revision, status=STATUS_ACCEPTED, + backup_file=str(backup))) + + def revert(self, revision_id: str) -> TemplateRevision: + """Restore the template an accepted revision replaced.""" + with self._lock: + revision = self._require(self._load(), revision_id) + if revision.status != STATUS_ACCEPTED or revision.backup_file is None: + raise TemplateRevisionError( + f"revision {revision_id} is {revision.status}, not accepted") + template = _existing_file(revision.template_path, "template") + backup = _existing_file(revision.backup_file, "backup") + if _sha256(template) != revision.candidate_sha256: + raise TemplateRevisionError( + f"{template} changed since revision {revision_id} was accepted; " + "reverting would discard that change") + atomic_write_bytes(template, backup.read_bytes()) + return self._put(replace(revision, status=STATUS_REVERTED)) + + +def _preview_samples(samples: Optional[Sequence[EvaluationSample]], + dataset_path: Optional[PathLike], + ) -> Optional[Sequence[EvaluationSample]]: + if samples is not None and dataset_path is not None: + raise TemplateRevisionError("pass samples or dataset_path, not both") + if dataset_path is None: + return samples + from je_auto_control.utils.self_healing.eval_strategies import ( + load_evaluation_dataset, + ) + return load_evaluation_dataset(dataset_path).samples + + +default_template_revisions = TemplateRevisionStore() + + +def propose_template_revision(template_path: PathLike, candidate_path: PathLike, *, + source: str = "manual", note: Optional[str] = None, + store: Optional[TemplateRevisionStore] = None, + ) -> TemplateRevision: + """Store a candidate replacement for a template; the template is not changed.""" + return (store or default_template_revisions).propose( + template_path, candidate_path, source=source, note=note) + + +def preview_template_revision(revision_id: str, *, + samples: Optional[Sequence[EvaluationSample]] = None, + dataset_path: Optional[PathLike] = None, + detect_threshold: float = 0.9, + store: Optional[TemplateRevisionStore] = None, + ) -> Dict[str, Any]: + """Show current vs candidate and, given labelled frames, validate the candidate.""" + return (store or default_template_revisions).preview( + revision_id, samples=samples, dataset_path=dataset_path, + detect_threshold=detect_threshold) + + +def accept_template_revision(revision_id: str, *, allow_unvalidated: bool = False, + store: Optional[TemplateRevisionStore] = None, + ) -> TemplateRevision: + """Replace the live template with a validated candidate, keeping a backup.""" + return (store or default_template_revisions).accept( + revision_id, allow_unvalidated=allow_unvalidated) + + +def revert_template_revision(revision_id: str, *, + store: Optional[TemplateRevisionStore] = None, + ) -> TemplateRevision: + """Put back the template an accepted revision replaced.""" + return (store or default_template_revisions).revert(revision_id) + + +def list_template_revisions(store: Optional[TemplateRevisionStore] = None, + ) -> List[TemplateRevision]: + """Every stored revision, oldest first.""" + return (store or default_template_revisions).list_revisions() + + +__all__ = [ + "STATUS_ACCEPTED", "STATUS_PENDING", "STATUS_REVERTED", + "TemplateRevision", "TemplateRevisionError", "TemplateRevisionStore", + "accept_template_revision", "default_revision_root", + "default_template_revisions", "list_template_revisions", + "preview_template_revision", "propose_template_revision", + "revert_template_revision", +] diff --git a/test/unit_test/headless/test_self_healing_eval_surfaces.py b/test/unit_test/headless/test_self_healing_eval_surfaces.py new file mode 100644 index 000000000..a35a26312 --- /dev/null +++ b/test/unit_test/headless/test_self_healing_eval_surfaces.py @@ -0,0 +1,248 @@ +"""The evaluation and template-revision features on every delivery surface. + +Facade, ``AC_*`` commands, MCP tools, Script Builder schema and the GUI tab +must all reach the same headless code and return the same report. +""" +import json + +import numpy as np +import pytest + +import je_auto_control as ac +from je_auto_control.utils.cv2_utils.image_file import write_image +from je_auto_control.utils.executor.action_executor import executor +from je_auto_control.utils.self_healing import template_revision as revision_mod + +_COMMANDS = ( + "AC_self_heal_evaluate", "AC_self_heal_revision_propose", + "AC_self_heal_revision_preview", "AC_self_heal_revision_accept", + "AC_self_heal_revision_revert", "AC_self_heal_revision_list", +) +_FACADE = ( + "EvaluationSample", "HealingComparison", "HealingEvaluationError", + "TemplateRevision", "TemplateRevisionError", "TemplateRevisionStore", + "accept_template_revision", "evaluate_healing_dataset", "evaluate_locators", + "heal_context", "list_template_revisions", "preview_template_revision", + "propose_template_revision", "revert_template_revision", + "template_match_strategy", +) + + +def _noise(seed: int) -> np.ndarray: + return np.random.default_rng(seed).integers(0, 255, (20, 30), dtype=np.uint8) + + +@pytest.fixture +def workspace(tmp_path, monkeypatch): + """A template, a candidate, a labelled dataset and an isolated revision store.""" + monkeypatch.setattr(revision_mod, "default_revision_root", lambda: tmp_path / "store") + old, new = _noise(1), _noise(2) + frame = np.full((100, 160), 128, dtype=np.uint8) + frame[30:50, 60:90] = new + write_image(tmp_path / "template.png", old) + write_image(tmp_path / "candidate.png", new) + write_image(tmp_path / "frame.png", frame) + (tmp_path / "dataset.json").write_text(json.dumps({ + "samples": [{"id": "new-look", "frame": "frame.png", "template": "template.png", + "expected_box": [60, 30, 90, 50]}], + "versions": {"old": {"threshold": 0.9}}, + "thresholds": {"old": {"min_correct": 1}}, + }), encoding="utf-8") + return tmp_path + + +def _mcp(name: str): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + return {tool.name: tool for tool in build_default_tool_registry()}[name] + + +def _counts(payload): + """The report without its latency columns, which differ run to run.""" + return {name: {key: value for key, value in report.items() if not key.endswith("_ms")} + for name, report in payload["versions"].items()} + + +def test_facade_exports_the_names(): + for name in _FACADE: + assert name in ac.__all__ and hasattr(ac, name), name + + +def test_commands_are_registered_and_in_the_script_builder(): + from je_auto_control.gui.script_builder.command_schema import _build_specs + known = set(executor.known_commands()) + in_builder = {spec.command for spec in _build_specs()} + for command in _COMMANDS: + assert command in known, command + assert command in in_builder, command + + +def test_executor_and_mcp_return_the_same_evaluation(workspace): + dataset = str(workspace / "dataset.json") + from_executor = executor.event_dict["AC_self_heal_evaluate"](dataset_path=dataset) + from_mcp = _mcp("ac_self_heal_evaluate").handler(dataset_path=dataset) + from_api = ac.evaluate_healing_dataset(dataset) + for payload in (from_executor, from_mcp, from_api): + assert payload["versions"]["old"]["miss"] == 1 + assert payload["violations"] and payload["passed"] is False + assert _counts(from_executor) == _counts(from_mcp) == _counts(from_api) + + +def test_evaluate_runs_from_a_json_action_list(workspace): + record = executor.execute_action( + [["AC_self_heal_evaluate", {"dataset_path": str(workspace / "dataset.json")}]]) + payload = next(iter(record.values())) + assert payload["versions"]["old"]["total"] == 1 + + +def test_revision_lifecycle_through_the_executor(workspace): + events = executor.event_dict + template = workspace / "template.png" + before = template.read_bytes() + proposed = events["AC_self_heal_revision_propose"]( + template_path=str(template), candidate_path=str(workspace / "candidate.png")) + revision_id = proposed["revision_id"] + assert template.read_bytes() == before + with pytest.raises(ac.TemplateRevisionError): + events["AC_self_heal_revision_accept"](revision_id=revision_id) + preview = events["AC_self_heal_revision_preview"]( + revision_id=revision_id, dataset_path=str(workspace / "dataset.json")) + assert preview["revision"]["validated"] is True + assert events["AC_self_heal_revision_accept"](revision_id=revision_id)["status"] == "accepted" + assert template.read_bytes() == (workspace / "candidate.png").read_bytes() + assert events["AC_self_heal_revision_revert"](revision_id=revision_id)["status"] == "reverted" + assert template.read_bytes() == before + assert [item["revision_id"] for item in events["AC_self_heal_revision_list"]()] == [revision_id] + + +def test_revision_lifecycle_through_mcp(workspace): + template = workspace / "template.png" + before = template.read_bytes() + revision_id = _mcp("ac_self_heal_revision_propose").handler( + template_path=str(template), + candidate_path=str(workspace / "candidate.png"))["revision_id"] + preview = _mcp("ac_self_heal_revision_preview").handler(revision_id=revision_id) + assert preview["comparison"] is None and preview["revision"]["validated"] is False + accepted = _mcp("ac_self_heal_revision_accept").handler( + revision_id=revision_id, allow_unvalidated=True) + assert accepted["status"] == "accepted" + _mcp("ac_self_heal_revision_revert").handler(revision_id=revision_id) + assert template.read_bytes() == before + assert len(_mcp("ac_self_heal_revision_list").handler()) == 1 + assert _mcp("ac_self_heal_evaluate").annotations.read_only is True + assert _mcp("ac_self_heal_revision_accept").annotations.destructive is True + + +def test_context_reaches_the_log_from_executor_and_mcp(tmp_path, monkeypatch): + from je_auto_control.utils.executor import action_executor + from je_auto_control.utils.self_healing import HealEventLog + from je_auto_control.utils.self_healing import locator as locator_mod + log = HealEventLog(tmp_path / "events.jsonl") + monkeypatch.setattr(locator_mod, "default_heal_log", log) + monkeypatch.setattr(locator_mod, "_try_image", lambda path, threshold: ((3, 4), None)) + monkeypatch.setattr(locator_mod, "_click_at", lambda coords, keycode: None) + result = action_executor.executor.event_dict["AC_self_heal_click"]( + template_path="t.png", context={"locator_version": "v2", "run_id": "r1"}) + assert result["found"] is True and result["action"] == "click" + assert result["action_verified"] is None + _mcp("ac_self_heal_locate").handler( + template_path="t.png", context={"locator_version": "v3"}) + first, second = log.list_events() + assert (first.locator_version, first.run_id) == ("v2", "r1") + assert second.locator_version == "v3" and second.run_id is None + with pytest.raises(ValueError): + _mcp("ac_self_heal_locate").handler(template_path="t.png", context={"x": "y"}) + + +# --- GUI: a thin shell over the same calls -------------------------------- + + +@pytest.fixture +def tab(workspace): + pytest.importorskip("PySide6") + from PySide6.QtWidgets import QApplication + app = QApplication.instance() or QApplication([]) + from je_auto_control.gui.self_healing_tab import SelfHealingTab + widget = SelfHealingTab() + yield widget + widget.deleteLater() + app.processEvents() + + +def test_tab_commands_are_menu_actions_with_translations(tab): + from je_auto_control.gui.language_wrapper import ( + english, japanese, simplified_chinese, traditional_chinese, + ) + actions = dict(tab.menu_actions()) + wanted = {"self_heal_evaluate", "self_heal_rev_propose", "self_heal_rev_preview", + "self_heal_rev_accept", "self_heal_rev_revert", "self_heal_rev_list", + "self_heal_browse_dataset", "self_heal_browse_candidate"} + assert wanted <= set(actions) and all(callable(actions[key]) for key in wanted) + for module in (english, japanese, simplified_chinese, traditional_chinese): + catalogue = next(value for value in vars(module).values() + if isinstance(value, dict) and "self_heal_refresh" in value) + missing = [key for key in wanted | {"self_heal_col_action_verified"} + if key not in catalogue] + assert missing == [], (module.__name__, missing) + + +def test_tab_evaluates_and_walks_a_revision(tab, workspace, monkeypatch): + from je_auto_control.gui import self_healing_tab as module + actions = dict(tab.menu_actions()) + template = workspace / "template.png" + before = template.read_bytes() + + actions["self_heal_evaluate"]() # no dataset typed yet + assert tab._report_view.toPlainText() == "" + tab._dataset_input.setText(str(workspace / "dataset.json")) + actions["self_heal_evaluate"]() + shown = json.loads(tab._report_view.toPlainText()) + assert shown["versions"]["old"]["miss"] == 1 and shown["passed"] is False + + tab._template_input.setText(str(template)) + tab._candidate_input.setText(str(workspace / "candidate.png")) + actions["self_heal_rev_propose"]() + revision_id = tab._revision_input.text() + assert len(revision_id) == 12 and template.read_bytes() == before + + # Not validated yet: the tab asks, and "No" leaves the template alone. + monkeypatch.setattr(module.QMessageBox, "question", + lambda *_args, **_kwargs: module.QMessageBox.No) + actions["self_heal_rev_accept"]() + assert template.read_bytes() == before + + actions["self_heal_rev_preview"]() + assert json.loads(tab._report_view.toPlainText())["revision"]["validated"] is True + + def no_prompt(*_args, **_kwargs): + raise AssertionError("a validated revision must not prompt") + + monkeypatch.setattr(module.QMessageBox, "question", no_prompt) + actions["self_heal_rev_accept"]() + assert template.read_bytes() == (workspace / "candidate.png").read_bytes() + actions["self_heal_rev_revert"]() + assert template.read_bytes() == before + actions["self_heal_rev_list"]() + assert json.loads(tab._report_view.toPlainText())[0]["status"] == "reverted" + + +def test_tab_reports_errors_in_the_status(tab): + tab._revision_input.setText("not-an-id") + dict(tab.menu_actions())["self_heal_rev_revert"]() + assert "not a revision id" in tab._status.text() + tab._revision_input.setText("") + dict(tab.menu_actions())["self_heal_rev_preview"]() + assert tab._status.text() + + +def test_log_table_shows_version_and_verification(tab, tmp_path, monkeypatch): + from je_auto_control.gui import self_healing_tab as module + from je_auto_control.utils.self_healing import HealEvent, HealEventLog + log = HealEventLog(tmp_path / "events.jsonl") + log.append(HealEvent("t1", "vlm", [1, 2], 3.0, locator_version="v2", + action="click", action_verified=False)) + log.append(HealEvent("t2", "image", [1, 2], 3.0)) + monkeypatch.setattr(module, "default_heal_log", log) + tab.refresh_log() + assert tab._table.item(0, 6).text() == "v2" + assert tab._table.item(0, 7).text() != "" + assert tab._table.item(1, 7).text() == "" diff --git a/test/unit_test/headless/test_self_healing_evaluation.py b/test/unit_test/headless/test_self_healing_evaluation.py new file mode 100644 index 000000000..408fec50f --- /dev/null +++ b/test/unit_test/headless/test_self_healing_evaluation.py @@ -0,0 +1,344 @@ +"""Self-healing measurement: labelled evaluation, heal-log schema, verified actions. + +Everything here runs on arrays built in the test and on fakes for the two +locator strategies; nothing captures the screen or moves the pointer. +""" +import json + +import numpy as np +import pytest + +from je_auto_control.utils.self_healing import ( + HealEvent, HealEventLog, self_heal_click, self_heal_locate, +) +from je_auto_control.utils.self_healing import locator as locator_mod +from je_auto_control.utils.self_healing.evaluation import ( + EvaluationSample, HealingEvaluationError, Ratio, check_thresholds, + evaluate_locators, format_comparison, +) +from je_auto_control.utils.self_healing.heal_log import HEAL_EVENT_SCHEMA_VERSION +from je_auto_control.utils.self_healing.locator import heal_context + + +def _frame(seed: int = 0) -> np.ndarray: + return np.random.default_rng(seed).integers(0, 255, (40, 60), dtype=np.uint8) + + +def _always(point): + return lambda request: point + + +def _never(request): + return None + + +# --- evaluate_locators --------------------------------------------------- + + +def test_unlabelled_is_unknown(): + samples = [ + EvaluationSample("labelled", _frame(1), expected_box=(0, 0, 10, 10)), + EvaluationSample("unlabelled", _frame(2)), + ] + report = evaluate_locators(samples, {"v1": _always((5, 5))}).report("v1") + assert report.unknown == 1 + # The unlabelled hit is counted as located, and as nothing else. + assert report.located == 2 + assert report.correct == 1 + assert report.accuracy == Ratio(1, 1) + assert report.hit_rate == Ratio(2, 2) + + +def test_false_positive_is_not_recovery(): + samples = [ + EvaluationSample("a", _frame(1), expected_box=(0, 0, 10, 10)), + EvaluationSample("b", _frame(2), expected_box=(20, 20, 30, 30)), + ] + # v2 "finds" both, but the second hit is nowhere near the target. + comparison = evaluate_locators( + samples, {"v1": _never, "v2": _always((5, 5))}) + report = comparison.report("v2") + assert report.correct == 1 and report.false_positive == 1 + assert report.located == 2 + assert report.recovery_rate == Ratio(1, 2) + assert comparison.report("v1").miss == 2 + assert [row.sample_id for row in comparison.failures("v2")] == ["b"] + + +def test_hit_on_an_expected_miss_is_a_false_positive(): + samples = [EvaluationSample("gone", _frame(), expect_miss=True)] + comparison = evaluate_locators( + samples, {"strict": _never, "loose": _always((1, 1))}) + assert comparison.report("strict").true_negative == 1 + assert comparison.report("strict").accuracy == Ratio(1, 1) + assert comparison.report("loose").false_positive == 1 + assert comparison.report("loose").accuracy == Ratio(0, 1) + # A sample that expects a miss can never be "recovered". + assert comparison.report("loose").recovery_rate == Ratio(0, 0) + assert comparison.report("loose").recovery_rate.value is None + + +def test_same_frame_versions_are_comparable(): + seen = {} + + def record(name): + def strategy(request): + seen[name] = request + return None + return strategy + + frame = _frame(7) + sample = EvaluationSample("s", frame, expected_box=(0, 0, 5, 5), + origin=(-1920, -200), scale=1.5) + comparison = evaluate_locators([sample], {"a": record("a"), "b": record("b")}) + assert seen["a"].frame is frame and seen["b"].frame is frame + assert seen["a"] is seen["b"] + assert seen["a"].origin == (-1920, -200) and seen["a"].scale == 1.5 + hashes = {row.frame_hash for row in comparison.results} + assert len(hashes) == 1 and None not in hashes + + +def test_region_passed_to_both_strategies(): + regions = {} + + def record(name): + def strategy(request): + regions[name] = request.region + return None + return strategy + + sample = EvaluationSample("s", _frame(), expected_box=(12, 12, 20, 20), + region=(10, 10, 50, 30)) + evaluate_locators([sample], {"image": record("image"), "vlm": record("vlm")}) + image_region, vlm_region = regions["image"], regions["vlm"] + assert image_region == vlm_region + assert image_region == (10, 10, 50, 30) + + +def test_a_strategy_that_edits_the_frame_is_refused(): + def vandal(request): + request.frame[0, 0] ^= 0xFF + return None + + sample = EvaluationSample("s", _frame(), expected_box=(0, 0, 5, 5)) + with pytest.raises(HealingEvaluationError, match="modified the frame"): + evaluate_locators([sample], {"vandal": vandal, "later": _never}) + + +def test_negative_origin_box_is_judged_in_screen_coordinates(): + sample = EvaluationSample("left-monitor", _frame(), + expected_box=(-1900, -180, -1880, -160), + origin=(-1920, -200)) + comparison = evaluate_locators( + [sample], {"right": _always((-1890, -170)), "frame-local": _always((30, 30))}) + assert comparison.report("right").correct == 1 + assert comparison.report("frame-local").false_positive == 1 + + +def test_strategy_error_is_its_own_bucket(): + def broken(request): + raise RuntimeError("backend down") + + samples = [EvaluationSample("s", _frame(), expect_miss=True)] + report = evaluate_locators(samples, {"broken": broken}).report("broken") + # Raising on a sample that expects a miss is not a correct rejection. + assert report.error == 1 and report.true_negative == 0 + assert report.accuracy == Ratio(0, 1) + + +def test_latency_percentiles_and_counts_are_reported(): + ticks = iter(range(0, 1000)) + samples = [EvaluationSample(f"s{i}", _frame(i), expected_box=(0, 0, 9, 9)) + for i in range(4)] + comparison = evaluate_locators( + samples, {"v1": _always((1, 1))}, clock=lambda: next(ticks) / 1000.0) + report = comparison.report("v1") + assert report.p50_ms == pytest.approx(1.0) + assert report.p95_ms == pytest.approx(1.0) + payload = json.loads(json.dumps(comparison.to_dict())) + version = payload["versions"]["v1"] + assert version["accuracy"] == {"numerator": 4, "denominator": 4, "value": 1.0} + assert version["false_positive_rate"]["denominator"] == 4 + assert payload["baseline"] == "v1" + assert "4/4" in format_comparison(comparison) + + +def test_empty_ratio_has_no_value_rather_than_a_perfect_one(): + report = evaluate_locators( + [EvaluationSample("u", _frame())], {"v1": _never}).report("v1") + assert report.accuracy.to_dict() == { + "numerator": 0, "denominator": 0, "value": None} + + +def test_invalid_inputs_are_rejected(): + sample = EvaluationSample("s", _frame(), expected_box=(0, 0, 5, 5)) + with pytest.raises(HealingEvaluationError): + evaluate_locators([sample], {}) + with pytest.raises(HealingEvaluationError, match="duplicate"): + evaluate_locators([sample, sample], {"v1": _never}) + with pytest.raises(HealingEvaluationError, match="expect_miss"): + EvaluationSample("x", _frame(), expected_box=(0, 0, 5, 5), expect_miss=True) + with pytest.raises(HealingEvaluationError, match="scale"): + EvaluationSample("x", _frame(), scale=0) + + +def test_thresholds_report_each_violation(): + samples = [EvaluationSample("a", _frame(), expected_box=(0, 0, 10, 10)), + EvaluationSample("b", _frame(1), expect_miss=True)] + comparison = evaluate_locators(samples, {"v1": _always((5, 5))}) + assert check_thresholds(comparison, {"v1": {"min_correct": 1}}) == [] + violations = check_thresholds( + comparison, {"v1": {"max_false_positive": 0, "min_accuracy": 0.9}, + "missing": {"min_correct": 1}}) + assert len(violations) == 3 + + +# --- the live locator: one region for both strategies ---------------------- + + +@pytest.fixture +def temp_log(tmp_path): + return HealEventLog(path=tmp_path / "events.jsonl") + + +def test_live_locate_gives_the_region_to_image_and_vlm(monkeypatch, temp_log): + seen = {} + + def fake_find_image(image, detect_threshold=1.0, draw_image=False, + all_screens=True, screen_region=None): + seen["image"] = screen_region + return [False, []] + + def fake_vlm(description, screen_region, model): + seen["vlm"] = screen_region + return None, "no match" + + from je_auto_control.utils.cv2_utils import template_detection + monkeypatch.setattr(template_detection, "find_image", fake_find_image) + monkeypatch.setattr(locator_mod, "_try_vlm", fake_vlm) + outcome = self_heal_locate(template_path="t.png", description="the button", + screen_region=[-100, 20, 300, 220], log=temp_log) + assert outcome.found is False + # Two corners for the VLM, (x, y, width, height) for the matcher: one box. + assert seen["vlm"] == [-100, 20, 300, 220] + assert seen["image"] == (-100, 20, 400, 200) + assert temp_log.list_events()[0].screen_region == [-100, 20, 300, 220] + + +def test_live_locate_rejects_a_region_without_area(temp_log): + with pytest.raises(ValueError): + self_heal_locate(template_path="t.png", screen_region=[10, 10, 10, 50], + log=temp_log) + + +# --- located vs verified ------------------------------------------------- + + +def _patch_hit(monkeypatch, clicks): + monkeypatch.setattr(locator_mod, "_try_image", lambda path, threshold: ((7, 9), None)) + monkeypatch.setattr(locator_mod, "_click_at", + lambda coords, keycode: clicks.append(coords)) + + +def test_click_without_a_check_is_located_but_not_verified(monkeypatch, temp_log): + clicks = [] + _patch_hit(monkeypatch, clicks) + outcome = self_heal_click(template_path="t.png", log=temp_log) + assert outcome.found is True and outcome.action == "click" + assert outcome.action_verified is None + event = temp_log.list_events()[0] + assert event.action == "click" and event.action_verified is None + assert clicks == [(7, 9)] + + +@pytest.mark.parametrize("verdict", [True, False]) +def test_click_records_what_the_check_said(monkeypatch, temp_log, verdict): + _patch_hit(monkeypatch, []) + outcome = self_heal_click(template_path="t.png", log=temp_log, + verify=lambda result: verdict) + assert outcome.found is True + assert outcome.action_verified is verdict + events = temp_log.list_events() + assert len(events) == 1 and events[0].action_verified is verdict + + +def test_a_failed_click_is_still_logged_and_never_verified(monkeypatch, temp_log): + monkeypatch.setattr(locator_mod, "_try_image", lambda path, threshold: ((1, 2), None)) + + def boom(coords, keycode): + raise OSError("pointer unavailable") + + monkeypatch.setattr(locator_mod, "_click_at", boom) + with pytest.raises(OSError): + self_heal_click(template_path="t.png", log=temp_log, verify=lambda result: True) + event = temp_log.list_events()[0] + assert event.action == "click" and event.action_verified is None + + +def test_bare_locate_has_no_action(monkeypatch, temp_log): + _patch_hit(monkeypatch, []) + outcome = self_heal_locate(template_path="t.png", log=temp_log) + assert outcome.action is None and outcome.action_verified is None + + +# --- heal log schema ----------------------------------------------------- + + +def test_old_format_lines_still_load(tmp_path): + path = tmp_path / "events.jsonl" + old = {"timestamp": "2026-05-01T00:00:00+00:00", "method": "image", + "coordinates": [1, 2], "duration_ms": 3.5, "template_path": "a.png", + "description": None, "image_error": None, "vlm_error": None} + newer = dict(old, schema_version=99, locator_version="v9", + a_field_from_the_future={"x": 1}) + path.write_text(json.dumps(old) + "\n" + json.dumps(newer) + "\n[1, 2]\n", + encoding="utf-8") + events = HealEventLog(path=path).list_events() + assert len(events) == 2 + assert events[0].schema_version is None and events[0].locator_version is None + assert events[0].coordinates == [1, 2] + assert events[1].locator_version == "v9" + + +def test_context_is_stamped_on_events(monkeypatch, temp_log): + _patch_hit(monkeypatch, []) + with heal_context(run_id="run-1", locator_id="submit", locator_version="v2"): + self_heal_locate(template_path="t.png", model="m-1", log=temp_log) + self_heal_locate(template_path="t.png", log=temp_log) + inside, outside = temp_log.list_events() + assert (inside.run_id, inside.locator_id, inside.locator_version) == ( + "run-1", "submit", "v2") + assert inside.model == "m-1" + assert inside.schema_version == HEAL_EVENT_SCHEMA_VERSION + assert inside.image_ms is not None + assert outside.run_id is None and outside.locator_version is None + + +def test_unknown_context_key_is_rejected(): + with pytest.raises(ValueError): + with heal_context(colour="blue"): + pass + + +def test_event_round_trips_through_the_log(temp_log): + event = HealEvent(timestamp="t", method="vlm", coordinates=[-5, 6], + duration_ms=1.0, locator_version="v2", + screen_region=[-10, 0, 10, 20], action="click", + action_verified=False) + temp_log.append(event) + assert temp_log.list_events() == [event] + + +def test_heal_stats_report_verification_apart_from_hits(): + from je_auto_control.utils.heal_analytics import heal_stats + events = [ + HealEvent("t", "image", [1, 1], 1.0), + HealEvent("t", "vlm", [1, 1], 1.0, action="click", action_verified=True), + HealEvent("t", "vlm", [1, 1], 1.0, action="click", action_verified=False), + HealEvent("t", "vlm", [1, 1], 1.0, action="click"), + {"method": "image", "coordinates": [1, 1], "duration_ms": 1.0}, + ] + stats = heal_stats(events) + assert stats["healed"] == 5 + assert stats["action_verification"] == { + "actions": 3, "verified": 1, "failed": 1, "unchecked": 1} diff --git a/test/unit_test/headless/test_self_healing_template_revision.py b/test/unit_test/headless/test_self_healing_template_revision.py new file mode 100644 index 000000000..f73c9cacb --- /dev/null +++ b/test/unit_test/headless/test_self_healing_template_revision.py @@ -0,0 +1,331 @@ +"""Candidate template revisions, the dataset file, and the fixed benchmark. + +Frames and templates are arrays written to ``tmp_path``; the screen is never +captured and the template store lives under ``tmp_path`` too. +""" +import importlib.util +import json +import sys +from pathlib import Path + +import numpy as np +import pytest + +from je_auto_control.utils.cv2_utils.image_file import write_image +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.self_healing import template_revision as revision_mod +from je_auto_control.utils.self_healing.eval_strategies import ( + build_strategy, evaluate_healing_dataset, load_evaluation_dataset, + template_match_strategy, +) +from je_auto_control.utils.self_healing.evaluation import ( + EvaluationSample, HealingEvaluationError, evaluate_locators, +) +from je_auto_control.utils.self_healing.template_revision import ( + STATUS_ACCEPTED, STATUS_PENDING, STATUS_REVERTED, TemplateRevisionError, + TemplateRevisionStore, +) + +_REPO = Path(__file__).resolve().parents[3] + + +def _patch(seed: int, width: int = 30, height: int = 20) -> np.ndarray: + return np.random.default_rng(seed).integers(0, 255, (height, width), dtype=np.uint8) + + +def _frame(*placed) -> np.ndarray: + """A flat frame with each ``(patch, left, top)`` pasted in.""" + frame = np.full((120, 200), 128, dtype=np.uint8) + for patch, left, top in placed: + frame[top:top + patch.shape[0], left:left + patch.shape[1]] = patch + return frame + + +OLD, NEW, OTHER = _patch(1), _patch(2), _patch(3) + + +@pytest.fixture +def files(tmp_path): + """The live template (old look), candidates, and a labelled dataset.""" + paths = {"template": tmp_path / "live" / "button.png", + "new": tmp_path / "new.png", "other": tmp_path / "other.png", + "dataset": tmp_path / "data" / "dataset.json"} + for path in (paths["template"], paths["dataset"]): + path.parent.mkdir(parents=True) + write_image(paths["template"], OLD) + write_image(paths["new"], NEW) + write_image(paths["other"], OTHER) + data_dir = paths["dataset"].parent + write_image(data_dir / "new_look.png", _frame((NEW, 50, 40))) + write_image(data_dir / "other_only.png", _frame((OTHER, 10, 10))) + write_image(data_dir / "t.png", OLD) + paths["dataset"].write_text(json.dumps({ + "schema_version": 1, + "samples": [ + {"id": "new-look", "frame": "new_look.png", "template": "t.png", + "expected_box": [50, 40, 80, 60]}, + {"id": "absent", "frame": "other_only.png", "template": "t.png", + "expect_miss": True}, + ], + "versions": {"v1": {"strategy": "template", "threshold": 0.9}}, + "thresholds": {"v1": {"min_correct": 1}}, + }), encoding="utf-8") + return paths + + +@pytest.fixture +def store(tmp_path): + return TemplateRevisionStore(tmp_path / "revisions") + + +# --- propose / preview / accept / revert --------------------------------- + + +def test_propose_leaves_the_template_alone(store, files): + before = files["template"].read_bytes() + revision = store.propose(files["template"], files["new"], source="vlm", note="redesign") + assert revision.status == STATUS_PENDING and revision.validated is False + assert files["template"].read_bytes() == before + assert Path(revision.candidate_file).read_bytes() == files["new"].read_bytes() + assert [item.revision_id for item in store.list_revisions()] == [revision.revision_id] + assert store.get(revision.revision_id).source == "vlm" + + +def test_unvalidated_candidate_is_not_accepted(store, files): + before = files["template"].read_bytes() + revision = store.propose(files["template"], files["new"]) + preview = store.preview(revision.revision_id) + assert preview["comparison"] is None + assert preview["current"]["width"] == 30 and preview["candidate"]["height"] == 20 + assert preview["current"]["sha256"] != preview["candidate"]["sha256"] + with pytest.raises(TemplateRevisionError, match="not been validated"): + store.accept(revision.revision_id) + assert files["template"].read_bytes() == before + + +def test_preview_with_a_dataset_validates_then_accept_and_revert(store, files): + before = files["template"].read_bytes() + revision = store.propose(files["template"], files["new"]) + preview = store.preview(revision.revision_id, dataset_path=files["dataset"]) + versions = preview["comparison"]["versions"] + assert versions["current"]["correct"] == 0 and versions["current"]["miss"] == 1 + assert versions["candidate"]["correct"] == 1 + assert versions["candidate"]["false_positive"] == 0 + assert preview["revision"]["validated"] is True + assert files["template"].read_bytes() == before, "preview must not write the template" + + accepted = store.accept(revision.revision_id) + assert accepted.status == STATUS_ACCEPTED + assert files["template"].read_bytes() == files["new"].read_bytes() + assert Path(accepted.backup_file).read_bytes() == before + with pytest.raises(TemplateRevisionError, match="already accepted"): + store.accept(revision.revision_id) + + reverted = store.revert(revision.revision_id) + assert reverted.status == STATUS_REVERTED + assert files["template"].read_bytes() == before + with pytest.raises(TemplateRevisionError, match="not accepted"): + store.revert(revision.revision_id) + + +def test_a_candidate_that_hits_the_wrong_thing_is_not_validated(store, files): + # OTHER is on the frame that expects a miss: located, and wrong. + revision = store.propose(files["template"], files["other"]) + preview = store.preview(revision.revision_id, dataset_path=files["dataset"]) + candidate = preview["comparison"]["versions"]["candidate"] + assert candidate["false_positive"] == 1 and candidate["located"] == 1 + assert preview["revision"]["validated"] is False + with pytest.raises(TemplateRevisionError): + store.accept(revision.revision_id) + + +def test_preview_takes_samples_built_in_code(store, files): + revision = store.propose(files["template"], files["new"]) + samples = [EvaluationSample("s", _frame((NEW, 5, 5)), expected_box=(5, 5, 35, 25))] + preview = store.preview(revision.revision_id, samples=samples) + assert preview["revision"]["validated"] is True + with pytest.raises(TemplateRevisionError): + store.preview(revision.revision_id, samples=samples, dataset_path=files["dataset"]) + + +def test_allow_unvalidated_is_an_explicit_opt_in(store, files): + revision = store.propose(files["template"], files["new"]) + accepted = store.accept(revision.revision_id, allow_unvalidated=True) + assert accepted.status == STATUS_ACCEPTED and accepted.validated is False + + +def test_a_template_changed_underneath_is_never_overwritten(store, files): + revision = store.propose(files["template"], files["new"]) + write_image(files["template"], _patch(9)) + edited = files["template"].read_bytes() + with pytest.raises(TemplateRevisionError, match="changed since"): + store.accept(revision.revision_id, allow_unvalidated=True) + assert files["template"].read_bytes() == edited + + second = store.propose(files["template"], files["new"]) + store.accept(second.revision_id, allow_unvalidated=True) + write_image(files["template"], _patch(10)) + with pytest.raises(TemplateRevisionError, match="changed since"): + store.revert(second.revision_id) + + +def test_bad_inputs_raise_the_framework_error(store, files, tmp_path): + with pytest.raises(TemplateRevisionError): + store.propose(tmp_path / "missing.png", files["new"]) + with pytest.raises(TemplateRevisionError, match="identical"): + store.propose(files["template"], files["template"]) + for bad in ("../../etc", "", "zzzzzzzzzzzz", "0123456789ab"): + with pytest.raises(TemplateRevisionError): + store.get(bad) + assert issubclass(TemplateRevisionError, AutoControlException) + + +def test_module_functions_use_the_default_store(monkeypatch, files, tmp_path): + monkeypatch.setattr(revision_mod, "default_revision_root", lambda: tmp_path / "home-store") + revision = revision_mod.propose_template_revision(files["template"], files["new"]) + assert (tmp_path / "home-store" / "index.json").is_file() + assert revision_mod.list_template_revisions()[0].revision_id == revision.revision_id + revision_mod.preview_template_revision(revision.revision_id, dataset_path=files["dataset"]) + revision_mod.accept_template_revision(revision.revision_id) + assert revision_mod.revert_template_revision(revision.revision_id).status == STATUS_REVERTED + + +# --- template strategy and dataset file ---------------------------------- + + +def test_template_strategy_honours_region_origin_and_scale(): + import cv2 + twice = cv2.resize(NEW, None, fx=2.0, fy=2.0, interpolation=cv2.INTER_NEAREST) + frame = np.full((240, 400), 128, dtype=np.uint8) + frame[20:60, 20:80] = twice # first copy, screen (-990, -490) + frame[160:200, 300:360] = twice # second copy, screen (-850, -420) + sample = EvaluationSample( + "s", frame, expected_box=(-850, -420, -820, -400), origin=(-1000, -500), + scale=2.0, region=(-900, -450, -800, -380), template=NEW) + comparison = evaluate_locators([sample], { + "exact": template_match_strategy(0.9), + "scaled": template_match_strategy(0.9, scales=(1.0, 2.0)), + }) + assert comparison.report("exact").miss == 1 + assert comparison.report("scaled").correct == 1 + assert comparison.results[1].coordinates == (-835, -410) + assert comparison.report("scaled").recovery_rate.numerator == 1 + + +def test_region_outside_the_frame_is_a_miss_not_an_error(): + sample = EvaluationSample("s", _frame((NEW, 5, 5)), expected_box=(5, 5, 35, 25), + region=(5000, 5000, 5100, 5100), template=NEW) + report = evaluate_locators([sample], {"v": template_match_strategy()}).report("v") + assert report.miss == 1 and report.error == 0 + + +def test_missing_template_is_an_error_outcome(): + sample = EvaluationSample("s", _frame(), expect_miss=True) + report = evaluate_locators([sample], {"v": template_match_strategy()}).report("v") + assert report.error == 1 + + +def test_dataset_file_round_trip(files): + dataset = load_evaluation_dataset(files["dataset"]) + assert [sample.sample_id for sample in dataset.samples] == ["new-look", "absent"] + payload = evaluate_healing_dataset(files["dataset"]) + # The old template misses the redesigned control and rejects the other frame. + assert payload["versions"]["v1"]["miss"] == 1 + assert payload["versions"]["v1"]["true_negative"] == 1 + assert payload["passed"] is False and len(payload["violations"]) == 1 + json.dumps(payload) + + +def test_dataset_paths_cannot_leave_the_dataset_directory(files): + data = json.loads(files["dataset"].read_text(encoding="utf-8")) + data["samples"][0]["frame"] = "../new.png" + files["dataset"].write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(HealingEvaluationError, match="outside the dataset directory"): + load_evaluation_dataset(files["dataset"]) + + +@pytest.mark.parametrize("config", [ + {"strategy": "telepathy"}, {"threshold": 2}, {"scales": [0]}, + {"strategy": "template", "colour": "blue"}, +]) +def test_unknown_strategy_configs_are_rejected(config): + with pytest.raises(HealingEvaluationError): + build_strategy(config) + + +def test_unreadable_dataset_is_the_framework_error(tmp_path): + bad = tmp_path / "bad.json" + bad.write_text("{not json", encoding="utf-8") + with pytest.raises(HealingEvaluationError): + evaluate_healing_dataset(bad) + with pytest.raises(HealingEvaluationError): + evaluate_healing_dataset(tmp_path / "missing.json") + + +# --- the fixed benchmark ------------------------------------------------- + + +@pytest.fixture +def benchmark(monkeypatch): + """``benchmarks/self_healing/run.py`` loaded with every capture path armed to fail.""" + from je_auto_control.utils.monitor_layout import logical_frame + from je_auto_control.wrapper import auto_control_mouse + + def forbidden(*_args, **_kwargs): + raise AssertionError("the benchmark touched the real screen or pointer") + + monkeypatch.setattr(logical_frame, "grab_logical", forbidden) + monkeypatch.setattr(auto_control_mouse, "click_mouse", forbidden) + monkeypatch.setattr(auto_control_mouse, "set_mouse_position", forbidden) + saved_path = list(sys.path) + spec = importlib.util.spec_from_file_location( + "self_healing_benchmark_run", _REPO / "benchmarks" / "self_healing" / "run.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + yield module + sys.path[:] = saved_path + sys.modules.pop("synthetic", None) + + +def test_benchmark_counts_are_fixed(benchmark): + payload = benchmark.report() + versions = payload["versions"] + assert payload["baseline"] == "v1-exact" and payload["passed"] is True + exact, multi, loose = (versions[name] for name in ( + "v1-exact", "v2-multiscale", "v3-loose")) + assert all(report["total"] == 10 and report["unknown"] == 1 + for report in (exact, multi, loose)) + assert (exact["correct"], exact["miss"], exact["true_negative"]) == (4, 3, 2) + assert multi["accuracy"] == {"numerator": 8, "denominator": 9, "value": 0.8889} + assert multi["recovery_rate"]["numerator"] == 2 + assert multi["recovery_rate"]["denominator"] == 3 + # The loose version locates more and is right less: that is the point. + assert loose["located"] > multi["located"] + assert loose["false_positive"] == 1 + assert loose["accuracy"]["numerator"] < multi["accuracy"]["numerator"] + assert exact["p50_ms"] is not None and exact["p95_ms"] >= exact["p50_ms"] + + +def test_benchmark_covers_failing_scaled_and_negative_cases(benchmark): + samples = {sample.sample_id: sample for sample in benchmark.synthetic.build_samples()} + assert any(sample.scale != 1.0 for sample in samples.values()) + assert any(sample.origin[0] < 0 and sample.origin[1] < 0 for sample in samples.values()) + assert any(sample.region and sample.region[0] < 0 for sample in samples.values()) + assert any(sample.expect_miss for sample in samples.values()) + assert any(not sample.labelled for sample in samples.values()) + by_sample = {(row["sample_id"], row["version"]): row["outcome"] + for row in benchmark.report()["results"]} + assert by_sample[("failing-restyled", "v3-loose")] == "miss" + assert by_sample[("failing-decoy-only", "v3-loose")] == "false_positive" + assert by_sample[("region-negative-origin", "v1-exact")] == "correct" + assert by_sample[("negative-origin-scaled-125", "v2-multiscale")] == "correct" + assert by_sample[("unlabelled", "v1-exact")] == "unknown" + + +def test_benchmark_is_deterministic_and_writes_json(benchmark, tmp_path, capsys): + first, second = (benchmark.synthetic.build_samples() for _ in range(2)) + assert all(np.array_equal(a.frame, b.frame) for a, b in zip(first, second)) + out = tmp_path / "report.json" + assert benchmark.main(["--check", "--json", str(out)]) == 0 + assert json.loads(out.read_text(encoding="utf-8"))["passed"] is True + assert "v2-multiscale" in capsys.readouterr().out From 0d0f3bdcb915b59e7d9c1a34751822217ec89fa3 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:51:34 +0800 Subject: [PATCH 12/44] Stop mobile input from reporting success for text and taps the device never got adb input text carries printable ASCII only and exits 0 for anything else, so Unicode text was dropped while the step passed; and a pixel found in an iOS screenshot was tapped as if it were a point. Android text now goes through the ADBKeyBoard IME or uiautomator2 when input text cannot carry it and raises when neither exists. DeviceSession.capture() returns a DeviceFrame that is turned upright and carries the pixel-to-point mapping; template, OCR, VLM and self-heal locating search that frame instead of the desktop and answer in device points. Long-press, drag and pinch join tap and swipe as gestures. --- docs/source/Eng/doc/mobile/mobile_doc.rst | 105 ++++++ docs/source/Zh/doc/mobile/mobile_doc.rst | 97 +++++ je_auto_control/__init__.py | 5 +- je_auto_control/android/__init__.py | 10 +- je_auto_control/android/screen.py | 57 +++ je_auto_control/android/session.py | 36 +- je_auto_control/ios/__init__.py | 8 +- je_auto_control/ios/input.py | 38 +- je_auto_control/ios/screen.py | 59 +++- je_auto_control/ios/session.py | 31 +- .../utils/executor/action_executor.py | 5 +- je_auto_control/utils/self_healing/locator.py | 22 +- je_auto_control/wrapper/device_frame.py | 234 +++++++++++++ .../headless/test_mobile_input_capture.py | 330 ++++++++++++++++++ 14 files changed, 1011 insertions(+), 26 deletions(-) create mode 100644 je_auto_control/android/screen.py create mode 100644 je_auto_control/wrapper/device_frame.py create mode 100644 test/unit_test/headless/test_mobile_input_capture.py diff --git a/docs/source/Eng/doc/mobile/mobile_doc.rst b/docs/source/Eng/doc/mobile/mobile_doc.rst index 25d241b1e..98747fc13 100644 --- a/docs/source/Eng/doc/mobile/mobile_doc.rst +++ b/docs/source/Eng/doc/mobile/mobile_doc.rst @@ -130,3 +130,108 @@ Every error derives from ``DeviceError`` (an ``AutoControlException`` and a ``AdbError`` and the other pre-existing names are still raised and still caught by existing ``except`` clauses; they now share this base. + +Text +==== + +``session.type_text(text)`` delivers the text or raises; it never reports +success for text the device did not receive. + +**Android.** ``adb shell input text`` carries printable ASCII only. It drops +or mangles everything else, turns every ``%s`` into a space, and exits 0 +either way. So: + +1. Printable ASCII without a literal ``%s`` goes through ``input text``. +2. Anything else goes to the `ADBKeyBoard + `_ IME when it is the device's + selected input method (``adb shell ime set + com.android.adbkeyboard/.AdbIME``). +3. Otherwise it goes to ``uiautomator2`` when that is installed on the host. +4. With neither, ``AdbUnsupportedError`` (a ``DeviceUnsupportedError``) is + raised and nothing is sent. + +``AC_android_text`` follows the same route. ``AdbClient.text()`` itself now +refuses text ``input text`` cannot deliver instead of sending it. + +.. note:: + + The ADBKeyBoard broadcast returns the same result whether or not an IME + consumed it, so delivery is inferred from the IME being selected, not + confirmed by the device. + +**iOS.** WebDriverAgent's key endpoint carries Unicode; the text is passed +through unchanged. + +Gestures +======== + +``session.perform(gesture)`` takes one of five frozen dataclasses, all in +device input coordinates: + +.. list-table:: + :header-rows: 1 + :widths: 22 39 39 + + * - Gesture + - Android + - iOS + * - ``Tap(x, y)`` + - ``input tap`` + - WDA tap + * - ``LongPress(x, y, duration_s)`` + - ``input swipe`` that does not move + - WDA touch-and-hold + * - ``Swipe(x1, y1, x2, y2, duration_s)`` + - ``input swipe`` + - WDA drag + * - ``Drag(x1, y1, x2, y2, hold_s, duration_s)`` + - ``input draganddrop``; falls back to ``uiautomator2`` on a device + whose ``input`` lacks it + - WDA drag with ``hold_s`` as the press duration + * - ``Pinch(x, y, scale, duration_s, span)`` + - ``uiautomator2`` two-pointer gesture; ``adb shell input`` has one + pointer, so without it this raises ``DeviceUnsupportedError`` + - Pinch on the frontmost application. WDA pinches an element, not a + coordinate, so ``x`` / ``y`` / ``span`` are ignored. + +Frames and coordinates +====================== + +``session.capture()`` returns a :class:`DeviceFrame`: the screenshot, upright, +together with the size of the input coordinate space. + +* On **iOS** WebDriverAgent takes *points* and returns *pixels* (three per + point on most iPhones). ``frame.pixel_to_point(x, y)`` converts; tapping a + screenshot pixel as if it were a point lands in the wrong place. +* On **Android** ``screencap`` and ``input tap`` share one coordinate space + and the mapping is the identity. This is checked against ``wm size`` and the + display rotation rather than assumed. +* A screenshot that comes back in the panel's natural orientation while the + display is rotated is turned upright before anything is located in it. Which + way to turn it (270 degrees for ``landscape_left``, 90 for + ``landscape_right``) follows the platforms' documented conventions and has + not been checked on a device. An upside-down display cannot be told from the + image shape and is taken as already upright. + +Locating +-------- + +A frame is searched directly — the host's desktop is never captured — and +every answer is in device points, ready for a gesture:: + + from je_auto_control import Tap, self_heal_locate + + frame = session.capture() + session.perform(Tap(*frame.locate_image("login_button.png"))) + + point = frame.locate_text("Sign in") # OCR, or None + point = frame.locate_description("the blue button") # VLM, or None + + outcome = self_heal_locate(template_path="login_button.png", + description="the login button", frame=frame) + if outcome.found: + session.perform(Tap(*outcome.coordinates)) + +``self_heal_locate(..., frame=frame)`` runs the same template-then-VLM +fallback and writes the same heal log; ``screen_region`` does not apply to a +frame. ``self_heal_click`` still clicks the desktop mouse and takes no frame. diff --git a/docs/source/Zh/doc/mobile/mobile_doc.rst b/docs/source/Zh/doc/mobile/mobile_doc.rst index 528a02348..5f15f2a93 100644 --- a/docs/source/Zh/doc/mobile/mobile_doc.rst +++ b/docs/source/Zh/doc/mobile/mobile_doc.rst @@ -118,3 +118,100 @@ iOS 只請求 WDA 的 ``/status``\ 。 ``AdbError`` 等既有名稱照樣丟出，既有的 ``except`` 也照樣接得到； 它們現在共用這個基底。 + +文字 +==== + +``session.type_text(text)`` 要嘛把文字送到，要嘛丟例外；裝置沒收到的文字 +絕不會回報成功。 + +**Android。** ``adb shell input text`` 只能送可列印的 ASCII。其他字元會被丟掉 +或弄亂，每個 ``%s`` 會變成空白，而且不管怎樣結束碼都是 0。所以： + +1. 不含字面 ``%s`` 的可列印 ASCII 走 ``input text``\ 。 +2. 其他文字在裝置選用 `ADBKeyBoard `_ + 輸入法時交給它（``adb shell ime set com.android.adbkeyboard/.AdbIME``\ ）。 +3. 否則在主機裝有 ``uiautomator2`` 時交給它。 +4. 兩者都沒有就丟 ``AdbUnsupportedError``\ （屬於 ``DeviceUnsupportedError``\ ）， + 什麼都不送。 + +``AC_android_text`` 走同一條路。\ ``AdbClient.text()`` 本身現在也會拒絕 +``input text`` 送不了的文字，而不是照送。 + +.. note:: + + ADBKeyBoard 的 broadcast 不論有沒有輸入法接收，回傳結果都一樣，所以「送達」 + 是從輸入法已被選用推斷的，並非由裝置確認。 + +**iOS。** WebDriverAgent 的按鍵端點可以送 Unicode；文字原樣傳遞。 + +手勢 +==== + +``session.perform(gesture)`` 接受五種 frozen dataclass 之一，全部使用裝置的 +輸入座標： + +.. list-table:: + :header-rows: 1 + :widths: 22 39 39 + + * - 手勢 + - Android + - iOS + * - ``Tap(x, y)`` + - ``input tap`` + - WDA tap + * - ``LongPress(x, y, duration_s)`` + - 不移動的 ``input swipe`` + - WDA touch-and-hold + * - ``Swipe(x1, y1, x2, y2, duration_s)`` + - ``input swipe`` + - WDA drag + * - ``Drag(x1, y1, x2, y2, hold_s, duration_s)`` + - ``input draganddrop``\ ；裝置的 ``input`` 沒有這個子指令時退回 + ``uiautomator2`` + - WDA drag，\ ``hold_s`` 是按住的時間 + * - ``Pinch(x, y, scale, duration_s, span)`` + - ``uiautomator2`` 的雙指手勢；\ ``adb shell input`` 只有一個觸控點， + 沒裝的話丟 ``DeviceUnsupportedError`` + - 對最前景的 App 做 pinch。WDA 的 pinch 對象是元素而不是座標，所以 + ``x`` / ``y`` / ``span`` 會被忽略。 + +畫面與座標 +========== + +``session.capture()`` 回傳 :class:`DeviceFrame`\ ：轉正後的截圖，加上輸入座標 +空間的大小。 + +* **iOS** 的 WebDriverAgent 收的是 *point*\ 、回的是 *pixel*\ （多數 iPhone 是 + 一個 point 三個 pixel）。\ ``frame.pixel_to_point(x, y)`` 負責換算；把截圖的 + pixel 當成 point 去點會點錯位置。 +* **Android** 的 ``screencap`` 與 ``input tap`` 共用同一個座標空間，換算是恆等 + 的。這一點是對照 ``wm size`` 與顯示旋轉檢查出來的，不是假設。 +* 顯示已旋轉、截圖卻以面板原生方向回來時，會先轉正再做任何定位。要往哪邊轉 + （\ ``landscape_left`` 轉 270 度、\ ``landscape_right`` 轉 90 度）依據的是兩個 + 平台文件記載的慣例，沒有在實機上確認過。上下顛倒的顯示無法從影像形狀判斷， + 視為已經是正的。 + +定位 +---- + +定位直接在 frame 上做——不會去截主機的桌面——而且答案一律是裝置的 point， +可以直接交給手勢:: + + from je_auto_control import Tap, self_heal_locate + + frame = session.capture() + session.perform(Tap(*frame.locate_image("login_button.png"))) + + point = frame.locate_text("Sign in") # OCR，找不到回 None + point = frame.locate_description("the blue button") # VLM，找不到回 None + + outcome = self_heal_locate(template_path="login_button.png", + description="the login button", frame=frame) + if outcome.found: + session.perform(Tap(*outcome.coordinates)) + +``self_heal_locate(..., frame=frame)`` 跑的是同一套「先樣板、後 VLM」的退路， +寫的也是同一份 heal log；\ ``screen_region`` 不適用於 frame。 +``self_heal_click`` 仍然是點桌面的滑鼠，不接受 frame。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 6e4502f7d..91421adda 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1042,8 +1042,10 @@ from je_auto_control.wrapper.device_context import ( DeviceCancelledError, DeviceCapability, DeviceClosedError, DeviceContext, DeviceError, DevicePermissionError, DeviceSession, DeviceTimeoutError, - DeviceUnavailableError, DeviceUnsupportedError, open_device, use_device, + DeviceUnavailableError, DeviceUnsupportedError, Drag, LongPress, Pinch, + Swipe, Tap, open_device, use_device, ) +from je_auto_control.wrapper.device_frame import DeviceFrame # Media assertions (audio activity, video motion) from je_auto_control.utils.media_assert import ( MediaAssertionResult, assert_audio_activity, assert_video_changes, @@ -1944,6 +1946,7 @@ def start_autocontrol_gui(*args, **kwargs): "DeviceCancelledError", "DeviceCapability", "DeviceClosedError", "DeviceContext", "DeviceError", "DevicePermissionError", "DeviceSession", "DeviceTimeoutError", "DeviceUnavailableError", "DeviceUnsupportedError", "open_device", "use_device", + "DeviceFrame", "Drag", "LongPress", "Pinch", "Swipe", "Tap", # Media assertions "MediaAssertionResult", "assert_audio_activity", "assert_video_changes", "measure_audio_rms", "video_segment_motion", diff --git a/je_auto_control/android/__init__.py b/je_auto_control/android/__init__.py index e36216f42..4007fc6d2 100644 --- a/je_auto_control/android/__init__.py +++ b/je_auto_control/android/__init__.py @@ -33,7 +33,7 @@ """ from je_auto_control.android.adb_client import ( AdbClient, AdbDeviceMissingError, AdbError, AdbNotAvailable, - AdbTimeoutError, AdbUnauthorizedError, AndroidDevice, + AdbTimeoutError, AdbUnauthorizedError, AdbUnsupportedError, AndroidDevice, ) from je_auto_control.android.client import ( UIAutomatorDevice, UIAutomatorUnavailableError, @@ -42,11 +42,15 @@ from je_auto_control.android.find import ( ElementNotFoundError, click_element, dump_hierarchy, find_element, ) +from je_auto_control.android.input import ADB_KEYBOARD_IME, type_text +from je_auto_control.android.screen import capture_frame from je_auto_control.android.session import AndroidSession __all__ = [ - "AdbClient", "AdbDeviceMissingError", "AdbError", "AdbNotAvailable", - "AdbTimeoutError", "AdbUnauthorizedError", "AndroidDevice", "AndroidSession", + "ADB_KEYBOARD_IME", "AdbClient", "AdbDeviceMissingError", "AdbError", + "AdbNotAvailable", "AdbTimeoutError", "AdbUnauthorizedError", + "AdbUnsupportedError", "AndroidDevice", "AndroidSession", "capture_frame", + "type_text", "ElementNotFoundError", "UIAutomatorDevice", "UIAutomatorUnavailableError", "click_element", "default_ui_device", "dump_hierarchy", diff --git a/je_auto_control/android/screen.py b/je_auto_control/android/screen.py new file mode 100644 index 000000000..9acf6a0af --- /dev/null +++ b/je_auto_control/android/screen.py @@ -0,0 +1,57 @@ +"""Android screen geometry and capture as a :class:`DeviceFrame`. + +``input tap`` takes coordinates in the display's *current* orientation, and on +current Android ``screencap`` returns the image in that same orientation, so a +located pixel is the tap coordinate. That is checked rather than assumed: the +display size and rotation are read from the device, and a screenshot that came +back in the panel's natural orientation instead is turned upright before +anything is located in it. +""" +from __future__ import annotations + +import io +import re +from typing import Any, Optional, Tuple + +from je_auto_control.android.adb_client import AdbError +from je_auto_control.wrapper.device_frame import ORIENTATIONS, DeviceFrame + +_SIZE = re.compile(r"(Override|Physical) size:\s*(\d+)x(\d+)") +_SURFACE_ORIENTATION = re.compile(r"SurfaceOrientation:\s*(\d)") + + +def display_size(adb: Any) -> Optional[Tuple[int, int]]: + """The display size in its natural orientation, honouring a ``wm size`` override.""" + sizes = {kind: (int(width), int(height)) + for kind, width, height in _SIZE.findall(adb.shell("wm size"))} + return sizes.get("Override") or sizes.get("Physical") + + +def display_rotation(adb: Any) -> int: + """Quarter turns the display is rotated from natural (``Surface.ROTATION_*``), 0 if unknown.""" + found = _SURFACE_ORIENTATION.search(adb.shell("dumpsys input")) + return int(found.group(1)) % 4 if found else 0 + + +def capture_frame(adb: Any, device_id: str = "") -> DeviceFrame: + """Capture the screen with the geometry needed to tap what is found in it.""" + from PIL import Image + data = adb.screencap_png() + try: + image = Image.open(io.BytesIO(data)) + image.load() + except (OSError, ValueError) as error: + raise AdbError(f"adb screencap did not return an image: {error}") from error + rotation = display_rotation(adb) + natural = display_size(adb) + if natural is None: + # No geometry to check against: the screenshot is the coordinate space. + return DeviceFrame(image=image, point_size=image.size, platform="android", + device_id=device_id) + point_size = (natural[1], natural[0]) if rotation % 2 else natural + return DeviceFrame.from_capture( + image, point_size, orientation=ORIENTATIONS[rotation], + platform="android", device_id=device_id) + + +__all__ = ["capture_frame", "display_rotation", "display_size"] diff --git a/je_auto_control/android/session.py b/je_auto_control/android/session.py index ef3150ddf..133857bf8 100644 --- a/je_auto_control/android/session.py +++ b/je_auto_control/android/session.py @@ -2,15 +2,19 @@ from __future__ import annotations from importlib.util import find_spec -from typing import Any, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple +from je_auto_control.android import input as android_input from je_auto_control.android.adb_client import AdbClient, AdbError from je_auto_control.android.client import UIAutomatorDevice from je_auto_control.android.input import ADB_KEYBOARD_IME, current_input_method +from je_auto_control.android.screen import capture_frame from je_auto_control.wrapper.device_context import ( CAPABILITY_NAMES, STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, STATE_NEEDS_PERMISSION, - DeviceCapability, DeviceContext, DeviceError, DeviceSession, bound_session, + DeviceCapability, DeviceContext, DeviceError, DeviceSession, Drag, LongPress, + Pinch, Swipe, Tap, bound_session, ) +from je_auto_control.wrapper.device_frame import DeviceFrame #: Capabilities plain ``adb`` provides once the device is authorised. _ADB_CAPABILITIES = ("input", "screenshot", "app_lifecycle", "install", "files", "recording") @@ -116,6 +120,34 @@ def _unicode_capability(self) -> DeviceCapability: "uiautomator2 on the host or the ADBKeyBoard IME selected on the device", alternative="ASCII text works without either") + def _ui_or_none(self) -> Optional[UIAutomatorDevice]: + """The uiautomator2 wrapper when that path can be used, else ``None``.""" + return self.ui_device if self.has_ui_automator else None + + def capture(self) -> DeviceFrame: + """The current screen, upright, in the coordinates ``input tap`` takes.""" + return self.invoke("capture", lambda: capture_frame(self.adb, self.device_id)) + + def type_text(self, text: str) -> None: + """Type ``text`` through a path that can carry it, or raise (see ``android.input``).""" + self.invoke("type_text", lambda: android_input.type_text( + self.adb, text, ui_device=self._ui_or_none())) + + def press_key(self, key: str) -> None: + """Send a keycode (``KEYCODE_HOME``, ``BACK``, or a number).""" + self.invoke("press_key", lambda: self.adb.key_event(key)) + + def _gesture_handlers(self) -> Dict[type, Callable[[Any], None]]: + adb = self.adb + ui_device = self._ui_or_none() + return { + Tap: lambda g: android_input.tap(adb, g), + LongPress: lambda g: android_input.long_press(adb, g), + Swipe: lambda g: android_input.swipe(adb, g), + Drag: lambda g: android_input.drag(adb, g, ui_device=ui_device), + Pinch: lambda g: android_input.pinch(adb, g, ui_device=ui_device), + } + def _release(self) -> None: self._adb = None if not self._ui_injected: diff --git a/je_auto_control/ios/__init__.py b/je_auto_control/ios/__init__.py index 008e21344..f996af60a 100644 --- a/je_auto_control/ios/__init__.py +++ b/je_auto_control/ios/__init__.py @@ -22,16 +22,16 @@ ElementNotFoundError, click_element, dump_source, find_element, ) from je_auto_control.ios.input import ( - long_press, press_key, swipe, tap, type_text, + drag, long_press, pinch, press_key, swipe, tap, type_text, ) -from je_auto_control.ios.screen import screen_size, screenshot +from je_auto_control.ios.screen import capture_frame, screen_size, screenshot from je_auto_control.ios.session import IOSSession __all__ = [ "ElementNotFoundError", "IOSDevice", "IOSSession", "IOSUnavailableError", - "click_element", "default_ios_device", "dump_source", - "find_element", "long_press", "press_key", + "capture_frame", "click_element", "default_ios_device", "drag", "dump_source", + "find_element", "long_press", "pinch", "press_key", "reset_default_ios_device", "screen_size", "screenshot", "swipe", "tap", "type_text", ] diff --git a/je_auto_control/ios/input.py b/je_auto_control/ios/input.py index a73e6d34a..efc8b8e95 100644 --- a/je_auto_control/ios/input.py +++ b/je_auto_control/ios/input.py @@ -1,4 +1,9 @@ -"""iOS touch + key primitives via WebDriverAgent.""" +"""iOS touch + key primitives via WebDriverAgent. + +Every coordinate here is in **points**, WebDriverAgent's input space — not in +screenshot pixels, which are larger by the Retina scale. +:meth:`DeviceFrame.pixel_to_point` converts. +""" from __future__ import annotations from typing import Optional @@ -8,7 +13,7 @@ @translate_device_errors def tap(x: int, y: int, *, device: Optional[IOSDevice] = None) -> None: - """Single-tap absolute pixel coordinates.""" + """Single-tap at ``(x, y)`` in points.""" handle = (device or default_ios_device()).handle handle.tap(int(x), int(y)) @@ -48,4 +53,31 @@ def press_key(name: str, *, device: Optional[IOSDevice] = None) -> None: handle.press(name) -__all__ = ["long_press", "press_key", "swipe", "tap", "type_text"] +@translate_device_errors +def drag(x1: int, y1: int, x2: int, y2: int, hold_s: float = 0.5, + *, device: Optional[IOSDevice] = None) -> None: + """Press at ``(x1, y1)``, hold ``hold_s`` so the item lifts, then move to ``(x2, y2)``. + + WebDriverAgent's drag takes the press duration, not the travel time. + """ + handle = (device or default_ios_device()).handle + handle.swipe(int(x1), int(y1), int(x2), int(y2), float(hold_s)) + + +@translate_device_errors +def pinch(scale: float, duration_s: float = 0.5, + *, device: Optional[IOSDevice] = None) -> None: + """Pinch the frontmost application: ``scale`` above 1 zooms in, below 1 zooms out. + + WebDriverAgent pinches an element, not a coordinate, so the gesture is + centred on the application rather than on a chosen point. + """ + if scale <= 0 or scale == 1: + raise ValueError("pinch scale must be positive and not 1") + handle = (device or default_ios_device()).handle + # XCTest wants scale per second, negative when the fingers close. + velocity = (float(scale) - 1.0) / max(float(duration_s), 0.05) + handle(className="XCUIElementTypeApplication").pinch(float(scale), velocity) + + +__all__ = ["drag", "long_press", "pinch", "press_key", "swipe", "tap", "type_text"] diff --git a/je_auto_control/ios/screen.py b/je_auto_control/ios/screen.py index fdee48749..f4dd311e1 100644 --- a/je_auto_control/ios/screen.py +++ b/je_auto_control/ios/screen.py @@ -1,15 +1,34 @@ """Screen capture + sizing for the attached iOS device.""" from __future__ import annotations +import io from pathlib import Path -from typing import Optional, Tuple +from typing import Any, Optional, Tuple -from je_auto_control.ios.client import IOSDevice, default_ios_device, translate_device_errors +from je_auto_control.ios.client import ( + IOSDevice, IOSUnavailableError, default_ios_device, translate_device_errors, +) +from je_auto_control.wrapper.device_frame import ( + ORIENTATION_LANDSCAPE_LEFT, ORIENTATION_LANDSCAPE_RIGHT, ORIENTATION_PORTRAIT, + ORIENTATION_PORTRAIT_UPSIDE_DOWN, DeviceFrame, +) + +#: WebDriverAgent's orientation names. +_ORIENTATIONS = { + "PORTRAIT": ORIENTATION_PORTRAIT, + "LANDSCAPE": ORIENTATION_LANDSCAPE_LEFT, + "UIA_DEVICE_ORIENTATION_LANDSCAPERIGHT": ORIENTATION_LANDSCAPE_RIGHT, + "UIA_DEVICE_ORIENTATION_PORTRAIT_UPSIDEDOWN": ORIENTATION_PORTRAIT_UPSIDE_DOWN, +} @translate_device_errors def screen_size(*, device: Optional[IOSDevice] = None) -> Tuple[int, int]: - """Return the device's current pixel size as ``(width, height)``.""" + """Return the screen size in **points**, in the current orientation, as ``(width, height)``. + + Points are WebDriverAgent's input space; a screenshot is larger by the + Retina scale. + """ handle = (device or default_ios_device()).handle size = handle.window_size() if isinstance(size, dict): @@ -31,4 +50,36 @@ def screenshot(file_path: Optional[str] = None, return str(target) -__all__ = ["screen_size", "screenshot"] +def _as_image(shot: Any) -> Any: + """A WDA screenshot (PIL image, or PNG bytes) as a loaded PIL image.""" + if hasattr(shot, "size") and hasattr(shot, "save"): + return shot + from PIL import Image + try: + image = Image.open(io.BytesIO(bytes(shot))) + image.load() + except (OSError, ValueError, TypeError) as error: + raise IOSUnavailableError( + f"WebDriverAgent did not return a screenshot: {error}") from error + return image + + +@translate_device_errors +def capture_frame(*, device: Optional[IOSDevice] = None) -> DeviceFrame: + """Capture the screen with the point size and orientation needed to tap what is found. + + WebDriverAgent takes points and returns pixels, and some builds return a + landscape screen in the panel's portrait buffer; the frame carries the + mapping and is turned upright. + """ + target = device or default_ios_device() + handle = target.handle + image = _as_image(handle.screenshot()) + orientation = _ORIENTATIONS.get(str(getattr(handle, "orientation", "PORTRAIT")), + ORIENTATION_PORTRAIT) + return DeviceFrame.from_capture( + image, screen_size(device=target), orientation=orientation, + platform="ios", device_id=target.url) + + +__all__ = ["capture_frame", "screen_size", "screenshot"] diff --git a/je_auto_control/ios/session.py b/je_auto_control/ios/session.py index 544dfbec8..2ab86a012 100644 --- a/je_auto_control/ios/session.py +++ b/je_auto_control/ios/session.py @@ -1,13 +1,17 @@ """One iOS device as a :class:`DeviceSession`: its own WebDriverAgent client.""" from __future__ import annotations -from typing import Any, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple +from je_auto_control.ios import input as ios_input from je_auto_control.ios.client import IOSDevice, translate_device_errors +from je_auto_control.ios.screen import capture_frame from je_auto_control.wrapper.device_context import ( CAPABILITY_NAMES, STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, - DeviceCapability, DeviceContext, DeviceError, DeviceSession, bound_session, + DeviceCapability, DeviceContext, DeviceError, DeviceSession, Drag, LongPress, + Pinch, Swipe, Tap, bound_session, ) +from je_auto_control.wrapper.device_frame import DeviceFrame #: Capabilities WebDriverAgent itself provides. _WDA_CAPABILITIES = ("input", "unicode_text", "multi_touch", "screenshot", "ui_tree", @@ -74,6 +78,29 @@ def _blocker(self) -> Optional[Tuple[str, str]]: return STATE_NEEDS_DEPENDENCY, str(error) return None + def capture(self) -> DeviceFrame: + """The current screen, upright, with its pixel-to-point mapping.""" + return self.invoke("capture", lambda: capture_frame(device=self.device)) + + def type_text(self, text: str) -> None: + """Type ``text`` into the focused field (WebDriverAgent carries Unicode).""" + self.invoke("type_text", lambda: ios_input.type_text(text, device=self.device)) + + def press_key(self, key: str) -> None: + """Press a hardware key (``"home"``, ``"volumeUp"``, ``"volumeDown"``).""" + self.invoke("press_key", lambda: ios_input.press_key(key, device=self.device)) + + def _gesture_handlers(self) -> Dict[type, Callable[[Any], None]]: + device = self.device + return { + Tap: lambda g: ios_input.tap(g.x, g.y, device=device), + LongPress: lambda g: ios_input.long_press(g.x, g.y, g.duration_s, device=device), + Swipe: lambda g: ios_input.swipe(g.x1, g.y1, g.x2, g.y2, g.duration_s, + device=device), + Drag: lambda g: ios_input.drag(g.x1, g.y1, g.x2, g.y2, g.hold_s, device=device), + Pinch: lambda g: ios_input.pinch(g.scale, g.duration_s, device=device), + } + def _adapter_capability(self, name: str) -> DeviceCapability: return DeviceCapability(name, STATE_NEEDS_DEPENDENCY, _NEEDS_ADAPTER) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 79a2358a6..45c2c7d52 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -1711,8 +1711,9 @@ def _ac_android_key(key: str, def _ac_android_text(text: str, serial: Optional[str] = None, adb_path: Optional[str] = None) -> None: - """Type a string via ``input text``.""" - _android_client(serial, adb_path).text(text) + """Type a string: ``input text`` for ASCII, a Unicode-capable path otherwise.""" + from je_auto_control.android.input import type_text + type_text(_android_client(serial, adb_path), text) def _ac_android_screenshot(file_path: str, diff --git a/je_auto_control/utils/self_healing/locator.py b/je_auto_control/utils/self_healing/locator.py index 98253553b..e5b3d9ec9 100644 --- a/je_auto_control/utils/self_healing/locator.py +++ b/je_auto_control/utils/self_healing/locator.py @@ -70,19 +70,25 @@ def self_heal_locate(template_path: Optional[str] = None, model: Optional[str] = None, log: Optional[HealEventLog] = None, raise_on_miss: bool = False, + frame: Optional[Any] = None, ) -> HealOutcome: """Locate an element by template; fall back to VLM on miss. At least one of ``template_path`` / ``description`` must be given. Provide both for full self-healing — the VLM path only runs when the template match fails or returns no candidates. + + ``frame`` (a ``DeviceFrame`` from ``DeviceSession.capture()``) is searched + instead of the desktop; the coordinates are then device points. """ if not template_path and not description: raise ValueError( "self_heal_locate requires template_path or description", ) started = monotonic() - coords, image_error = _try_image(template_path, detect_threshold) + # The two-argument call when there is no frame: callers replace these strategies. + coords, image_error = (_try_image(template_path, detect_threshold) if frame is None + else _try_image(template_path, detect_threshold, frame)) if coords is not None: return _finish( HealOutcome(found=True, coordinates=coords, method=METHOD_IMAGE, @@ -90,7 +96,8 @@ def self_heal_locate(template_path: Optional[str] = None, duration_ms=_ms_since(started)), log, ) - coords, vlm_error = _try_vlm(description, screen_region, model) + coords, vlm_error = (_try_vlm(description, screen_region, model) if frame is None + else _try_vlm(description, screen_region, model, frame)) if coords is not None: autocontrol_logger.warning( f"self_heal: image miss ({image_error}); VLM healed → {coords}", @@ -138,10 +145,14 @@ def self_heal_click(template_path: Optional[str] = None, def _try_image(template_path: Optional[str], detect_threshold: float, + frame: Optional[Any] = None, ) -> Tuple[Optional[Tuple[int, int]], Optional[str]]: if not template_path: return None, "no template_path supplied" try: + if frame is not None: + cx, cy = frame.locate_image(template_path, float(detect_threshold)) + return (int(cx), int(cy)), None from je_auto_control.wrapper.auto_control_image import ( locate_image_center, ) @@ -158,14 +169,15 @@ def _try_image(template_path: Optional[str], def _try_vlm(description: Optional[str], screen_region: Optional[List[int]], model: Optional[str], + frame: Optional[Any] = None, ) -> Tuple[Optional[Tuple[int, int]], Optional[str]]: if not description: return None, "no description supplied" try: from je_auto_control.utils.vision.vlm_api import locate_by_description - coords = locate_by_description( - description, screen_region=screen_region, model=model, - ) + coords = (frame.locate_description(description, model=model) if frame is not None + else locate_by_description( + description, screen_region=screen_region, model=model)) # AutoControlException: a failed screenshot (AutoControlScreenException) # escaped the self-heal and left no heal-log entry. except (AutoControlException, OSError, RuntimeError, ValueError, TypeError) as exc: diff --git a/je_auto_control/wrapper/device_frame.py b/je_auto_control/wrapper/device_frame.py new file mode 100644 index 000000000..597ba5321 --- /dev/null +++ b/je_auto_control/wrapper/device_frame.py @@ -0,0 +1,234 @@ +"""A captured device screen that knows how its pixels map to input coordinates. + +A locator finds things in *pixels of the screenshot*; a tap is sent in the +device's *input coordinates*. The two differ on iOS, where WebDriverAgent +takes points and returns Retina pixels, and on any device whose screenshot +comes back in the panel's natural orientation while the display is rotated. +Tapping a located pixel as if it were a point lands in the wrong place, and +nothing reports it. + +A :class:`DeviceFrame` carries the mapping with the image. Template matching, +OCR and the VLM locator search the frame — never the host's desktop — and +answer in device points, so the result can go straight into a gesture. +""" +from __future__ import annotations + +import io +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional, Tuple, Union + +from je_auto_control.utils.exception.exceptions import ImageNotFoundException +from je_auto_control.wrapper.device_context import DeviceError + +ORIENTATION_PORTRAIT = "portrait" +ORIENTATION_LANDSCAPE_LEFT = "landscape_left" +ORIENTATION_PORTRAIT_UPSIDE_DOWN = "portrait_upside_down" +ORIENTATION_LANDSCAPE_RIGHT = "landscape_right" +ORIENTATIONS = (ORIENTATION_PORTRAIT, ORIENTATION_LANDSCAPE_LEFT, + ORIENTATION_PORTRAIT_UPSIDE_DOWN, ORIENTATION_LANDSCAPE_RIGHT) + +#: TM_CCOEFF_NORMED is float32: an identical match scores just under 1.0. +_SCORE_EPSILON = 1e-5 + +Point = Tuple[int, int] + + +def _is_landscape(size: Tuple[int, int]) -> bool: + return size[0] > size[1] + + +def upright_turn(image_size: Tuple[int, int], point_size: Tuple[int, int], + orientation: str) -> int: + """Clockwise degrees a captured image must be turned to match the display. + + ``0`` when the image already has the display's shape. A landscape display + with a portrait-shaped screenshot means the backend returned the panel's + natural buffer; the orientation then says which way to turn it. An + upside-down display cannot be told from the shape and is taken as already + upright. + """ + if _is_landscape(image_size) == _is_landscape(point_size) or image_size[0] == image_size[1]: + return 0 + if orientation == ORIENTATION_LANDSCAPE_RIGHT: + return 90 + return 270 + + +@dataclass(frozen=True) +class DeviceFrame: + """One screenshot of a device, upright, with its pixel-to-point mapping. + + ``image`` is a PIL image in the display's current orientation. + ``point_size`` is the size of the input coordinate space in that same + orientation: equal to the pixel size on Android, smaller by the Retina + scale on iOS. + """ + + image: Any + point_size: Tuple[int, int] + orientation: str = ORIENTATION_PORTRAIT + platform: str = "" + device_id: str = "" + + def __post_init__(self) -> None: + if min(self.point_size) <= 0 or min(self.pixel_size) <= 0: + raise DeviceError( + f"a device frame needs positive sizes, got pixels {self.pixel_size} " + f"and points {self.point_size}") + + @classmethod + def from_capture(cls, image: Any, point_size: Tuple[int, int], *, + orientation: str = ORIENTATION_PORTRAIT, platform: str = "", + device_id: str = "") -> "DeviceFrame": + """Build a frame from a raw screenshot, turning it upright when needed.""" + size = (int(point_size[0]), int(point_size[1])) + turn = upright_turn(image.size, size, orientation) + if turn: + # PIL turns counter-clockwise for positive angles. + image = image.rotate(-turn, expand=True) + return cls(image=image, point_size=size, orientation=orientation, + platform=platform, device_id=device_id) + + @property + def pixel_size(self) -> Tuple[int, int]: + """``(width, height)`` of the image in pixels.""" + return int(self.image.size[0]), int(self.image.size[1]) + + @property + def scale(self) -> float: + """Pixels per input point along the width.""" + return self.pixel_size[0] / float(self.point_size[0]) + + def pixel_to_point(self, x: float, y: float) -> Point: + """The input coordinate under the centre of image pixel ``(x, y)``.""" + width, height = self.pixel_size + if not (0 <= x < width and 0 <= y < height): + raise DeviceError(f"pixel ({x}, {y}) is outside the {width}x{height} frame") + return (int((x + 0.5) * self.point_size[0] / width), + int((y + 0.5) * self.point_size[1] / height)) + + def point_to_pixel(self, x: float, y: float) -> Point: + """The image pixel under input coordinate ``(x, y)``.""" + width, height = self.pixel_size + return (min(width - 1, max(0, int(x * width / self.point_size[0]))), + min(height - 1, max(0, int(y * height / self.point_size[1])))) + + def png(self) -> bytes: + """The frame as PNG bytes.""" + buffer = io.BytesIO() + self.image.save(buffer, format="PNG") + return buffer.getvalue() + + def save(self, file_path: Union[str, "os.PathLike[str]"]) -> str: + """Write the frame as a PNG; returns the path.""" + target = Path(file_path) + target.parent.mkdir(parents=True, exist_ok=True) + self.image.save(str(target), format="PNG") + return str(target) + + def to_dict(self) -> dict: + """Geometry only (no pixels), for executor / MCP results.""" + return {"platform": self.platform, "device_id": self.device_id, + "orientation": self.orientation, + "pixel_size": list(self.pixel_size), "point_size": list(self.point_size), + "scale": round(self.scale, 4)} + + # --- locating: every method searches this frame and answers in points --- + + def locate_image(self, template: Any, detect_threshold: float = 0.9) -> Point: + """Centre of the best template match, in device points. + + Raises :class:`ImageNotFoundException` when nothing reaches the threshold. + """ + threshold = float(detect_threshold) + if not 0.0 <= threshold <= 1.0: + raise ImageNotFoundException( + f"detect_threshold must be between 0 and 1, got {detect_threshold!r}") + from je_auto_control.utils.cv2_utils.optional import require_cv2 + cv2 = require_cv2() + needle = _gray_template(cv2, template) + haystack = _gray(cv2, self.image) + if needle.shape[0] > haystack.shape[0] or needle.shape[1] > haystack.shape[1]: + raise ImageNotFoundException("template is larger than the device frame") + try: + scores = cv2.matchTemplate(haystack, needle, cv2.TM_CCOEFF_NORMED) + except cv2.error as error: + raise ImageNotFoundException(f"cannot match template image: {error}") from error + _low, best, _low_at, (left, top) = cv2.minMaxLoc(scores) + if best < min(threshold, 1.0 - _SCORE_EPSILON): + raise ImageNotFoundException( + f"template not found in the device frame (best score {best:.3f})") + return self.pixel_to_point(left + needle.shape[1] // 2, top + needle.shape[0] // 2) + + def locate_text(self, target: str, *, lang: str = "eng", min_confidence: float = 60.0, + case_sensitive: bool = False, backend: Any = None) -> Optional[Point]: + """Centre of the first OCR match for ``target``, in device points, or ``None``.""" + from je_auto_control.utils.ocr.backends import get_backend + from je_auto_control.utils.ocr.text_span import find_spans + engine = get_backend(backend) if backend is None or isinstance(backend, str) else backend + matches = engine.image_to_matches(self.image, lang, float(min_confidence)) + spans = find_spans(matches, target, case_sensitive=case_sensitive) + if not spans: + return None + span = spans[0] + left = min(box.x for box in span) + top = min(box.y for box in span) + right = max(box.x + box.width for box in span) + bottom = max(box.y + box.height for box in span) + return self.pixel_to_point((left + right) // 2, (top + bottom) // 2) + + def locate_description(self, description: str, *, model: Optional[str] = None, + backend: Any = None) -> Optional[Point]: + """Where a vision-language model says ``description`` is, in device points.""" + if not description or not description.strip(): + raise ValueError("description must be a non-empty string") + from je_auto_control.utils.vision.backends import get_backend + from je_auto_control.utils.vision.backends.base import VLMNotAvailableError + bound = backend if backend is not None else get_backend() + if not bound.available: + raise VLMNotAvailableError( + "no VLM backend configured; set ANTHROPIC_API_KEY or " + "OPENAI_API_KEY and install the matching SDK") + reply = bound.locate(self.png(), description, model=model) + if reply is None: + return None + width, height = self.pixel_size + if not (0 <= reply[0] < width and 0 <= reply[1] < height): + # Off the image the model was shown: a misread, not a location. + return None + return self.pixel_to_point(reply[0], reply[1]) + + +def _gray(cv2: Any, image: Any) -> Any: + """``image`` (PIL image or array) as a 2-D ``uint8`` grayscale array.""" + import numpy as np + array = np.asarray(image) + if array.ndim == 3 and array.shape[2] == 4: + array = array[:, :, :3] + if array.ndim == 3: + array = cv2.cvtColor(array, cv2.COLOR_RGB2GRAY) + if array.ndim != 2: + raise ImageNotFoundException(f"cannot search an image of shape {array.shape}") + return array.astype(np.uint8, copy=False) + + +def _gray_template(cv2: Any, template: Any) -> Any: + """Load ``template`` (path, PIL image or array) as a grayscale array.""" + try: + if isinstance(template, (str, os.PathLike)): + if not os.path.isfile(template): + raise ImageNotFoundException(f"template image not found: {template}") + from je_auto_control.utils.cv2_utils.image_file import read_image + return read_image(os.fspath(template), cv2.IMREAD_GRAYSCALE) + return _gray(cv2, template) + except (ValueError, TypeError, cv2.error) as error: + raise ImageNotFoundException(f"cannot read template image: {template!r}") from error + + +__all__ = [ + "DeviceFrame", "ORIENTATIONS", "ORIENTATION_LANDSCAPE_LEFT", + "ORIENTATION_LANDSCAPE_RIGHT", "ORIENTATION_PORTRAIT", + "ORIENTATION_PORTRAIT_UPSIDE_DOWN", "upright_turn", +] diff --git a/test/unit_test/headless/test_mobile_input_capture.py b/test/unit_test/headless/test_mobile_input_capture.py new file mode 100644 index 000000000..84c7a4da4 --- /dev/null +++ b/test/unit_test/headless/test_mobile_input_capture.py @@ -0,0 +1,330 @@ +"""Mobile text, gestures, frames and locating, against the fake backends. + +No device is involved: these prove which commands are sent and how frame +pixels map to input coordinates, not that a phone reacts to them. +""" +import io + +import pytest +from PIL import Image + +from headless._mobile_doubles import ( + ADB_KEYBOARD_IME, FakeAdbHost, FakeU2, FakeWda, png_bytes, +) +from je_auto_control.android import AdbClient, UIAutomatorDevice +from je_auto_control.ios import IOSDevice +from je_auto_control.utils.exception.exceptions import ImageNotFoundException +from je_auto_control.utils.ocr.ocr_engine import TextMatch +from je_auto_control.utils.self_healing.locator import self_heal_locate +from je_auto_control.wrapper.device_context import ( + DeviceContext, DeviceError, DeviceUnsupportedError, Drag, LongPress, Pinch, + Swipe, Tap, open_device, +) +from je_auto_control.wrapper.device_frame import DeviceFrame + +UNICODE_TEXT = "測試 café 🙂" + + +@pytest.fixture +def adb_host(monkeypatch): + """A fake ADB host with one attached device, ``phone``.""" + host = FakeAdbHost() + host.add("phone") + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", host.run) + from je_auto_control.utils.executor import action_executor + monkeypatch.setattr(action_executor, "_android_client_cache", {}) + return host + + +@pytest.fixture +def no_uiautomator(monkeypatch): + """Pretend uiautomator2 is not installed, whatever the host has.""" + monkeypatch.setattr("je_auto_control.android.input.find_spec", lambda _name: None) + monkeypatch.setattr("je_auto_control.android.session.find_spec", lambda _name: None) + + +def _android(host, **transports): + return open_device(DeviceContext("android", "phone", adb_path="fake-adb"), **transports) + + +def _ios(handle): + return open_device(DeviceContext("ios", "http://wda.test:8100"), + device=IOSDevice(url="http://wda.test:8100", handle=handle)) + + +# --- text -------------------------------------------------------------- + +def test_unicode_round_trip(adb_host, no_uiautomator): + phone = adb_host.devices["phone"] + phone.ime = ADB_KEYBOARD_IME + _android(adb_host).type_text(UNICODE_TEXT) + received_text = phone.typed[-1] + assert received_text == '測試 café 🙂' + # It did not go through the path that would have dropped it. + assert not any(command.startswith("input text") for command in phone.shell_commands) + + +def test_ascii_still_uses_input_text(adb_host, no_uiautomator): + phone = adb_host.devices["phone"] + _android(adb_host).type_text("hello world") + assert phone.typed == ["hello world"] + assert "input text hello%sworld" in phone.shell_commands + + +def test_unicode_goes_to_uiautomator_without_the_ime(adb_host): + handle = FakeU2() + _android(adb_host, ui_device=UIAutomatorDevice(handle=handle)).type_text(UNICODE_TEXT) + assert handle.typed == [UNICODE_TEXT] + assert adb_host.devices["phone"].typed == [] + + +@pytest.mark.parametrize("text", [UNICODE_TEXT, "100%s done"]) +def test_text_no_path_can_carry_is_refused_not_sent(adb_host, no_uiautomator, text): + phone = adb_host.devices["phone"] + with pytest.raises(DeviceUnsupportedError) as caught: + _android(adb_host).type_text(text) + assert caught.value.alternative + assert phone.input_calls == [] + assert phone.typed == [] + # The low-level method refuses it too, instead of reporting success. + with pytest.raises(DeviceUnsupportedError): + AdbClient(adb_path="fake-adb", default_serial="phone").text(text) + assert phone.input_calls == [] + + +def test_executor_text_command_carries_unicode(adb_host, no_uiautomator): + from je_auto_control.utils.executor.action_executor import executor + phone = adb_host.devices["phone"] + phone.ime = ADB_KEYBOARD_IME + executor.event_dict["AC_android_text"](UNICODE_TEXT, serial="phone", adb_path="fake-adb") + assert phone.typed == [UNICODE_TEXT] + + +def test_ios_text_is_sent_whole(): + handle = FakeWda() + _ios(handle).type_text(UNICODE_TEXT) + assert handle.typed == [UNICODE_TEXT] + + +# --- gestures ---------------------------------------------------------- + +def test_long_press_drag_pinch(adb_host): + phone = adb_host.devices["phone"] + handle = FakeU2() + session = _android(adb_host, ui_device=UIAutomatorDevice(handle=handle)) + session.perform(Tap(5, 6)) + session.perform(LongPress(10, 20, duration_s=1.5)) + session.perform(Swipe(1, 2, 3, 4, duration_s=0.2)) + session.perform(Drag(10, 20, 300, 400, hold_s=0.5, duration_s=0.5)) + session.perform(Pinch(500, 800, scale=2.0, duration_s=0.5, span=400)) + assert phone.input_calls == [ + "input tap 5 6", + "input swipe 10 20 10 20 1500", + "input swipe 1 2 3 4 200", + "input draganddrop 10 20 300 400 1000", + ] + # Zooming in: the fingers start close and end `span` apart, about the centre. + assert handle.calls == [{"op": "gesture", "start1": (400, 800), "start2": (600, 800), + "end1": (300, 800), "end2": (700, 800), "steps": 100}] + + +def test_pinch_out_closes_the_fingers(adb_host): + handle = FakeU2() + session = _android(adb_host, ui_device=UIAutomatorDevice(handle=handle)) + session.perform(Pinch(500, 800, scale=0.5, span=400)) + assert handle.calls[0]["start1"] == (300, 800) + assert handle.calls[0]["end1"] == (400, 800) + + +def test_drag_falls_back_when_input_lacks_draganddrop(adb_host, no_uiautomator): + phone = adb_host.devices["phone"] + phone.has_draganddrop = False + handle = FakeU2() + _android(adb_host, ui_device=UIAutomatorDevice(handle=handle)).perform( + Drag(1, 2, 3, 4, hold_s=0.25, duration_s=0.25)) + assert handle.calls == [{"op": "drag", "sx": 1, "sy": 2, "ex": 3, "ey": 4, + "duration": 0.5}] + # With no fallback, the usage text `input` printed is an error, not a success. + with pytest.raises(DeviceUnsupportedError): + _android(adb_host).perform(Drag(1, 2, 3, 4)) + + +def test_pinch_without_multi_touch_says_why(adb_host, no_uiautomator): + with pytest.raises(DeviceUnsupportedError) as caught: + _android(adb_host).perform(Pinch(1, 2, scale=2.0)) + assert "uiautomator2" in caught.value.alternative + assert adb_host.devices["phone"].input_calls == [] + + +def test_ios_gestures(): + handle = FakeWda() + session = _ios(handle) + session.perform(LongPress(10, 20, duration_s=1.5)) + session.perform(Drag(1, 2, 3, 4, hold_s=0.75)) + session.perform(Pinch(0, 0, scale=0.5, duration_s=0.5)) + session.press_key("home") + assert handle.input_calls == [ + {"op": "tap_hold", "x": 10, "y": 20, "duration": 1.5}, + {"op": "swipe", "x1": 1, "y1": 2, "x2": 3, "y2": 4, "duration": 0.75}, + {"op": "pinch", "scale": 0.5, "velocity": -1.0}, + {"op": "press", "name": "home"}, + ] + + +def test_invalid_gestures_are_rejected(): + with pytest.raises(DeviceError): + Pinch(1, 2, scale=1) + with pytest.raises(DeviceError): + LongPress(1, 2, duration_s=-1) + + +# --- frames ------------------------------------------------------------ + +def _upright_with_patch(size, patch): + return Image.open(io.BytesIO(png_bytes(size[0], size[1], patch=patch))) + + +def _as_png(image): + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +def test_rotated_frame_maps_to_device_points(): + # An iPhone held in landscape: 844x390 points, 3 pixels per point. This WDA + # build returns the panel's portrait buffer, so the screenshot is 1170x2532. + upright = _upright_with_patch((2532, 1170), (600, 810, 90, 60)) + template = upright.crop((600, 810, 690, 870)) + raw = upright.rotate(-90, expand=True) + assert raw.size == (1170, 2532) + handle = FakeWda(points=(390, 844), scale=3, orientation="LANDSCAPE") + handle.screen = _as_png(raw) + session = _ios(handle) + frame = session.capture() + assert frame.pixel_size == (2532, 1170) + assert frame.point_size == (844, 390) + assert frame.orientation == "landscape_left" + # Patch centre (645, 840) px upright -> (215, 280) points. + expected_device_point = (215, 280) + click_point = frame.locate_image(template) + session.perform(Tap(*click_point)) + assert click_point == expected_device_point + assert handle.input_calls == [{"op": "tap", "x": 215, "y": 280}] + + +def test_retina_pixels_are_not_tapped_as_points(): + handle = FakeWda(points=(390, 844), scale=3) + frame = _ios(handle).capture() + assert frame.scale == 3 + assert frame.pixel_to_point(600, 1500) == (200, 500) + assert frame.point_to_pixel(200, 500) == (600, 1500) + with pytest.raises(DeviceError): + frame.pixel_to_point(1170, 0) + + +def test_android_frame_follows_the_display_rotation(adb_host): + phone = adb_host.devices["phone"] + phone.rotation = 1 + frame = _android(adb_host).capture() + # screencap already comes back in landscape: pixels are the tap coordinates. + assert frame.pixel_size == (1920, 1080) + assert frame.point_size == (1920, 1080) + assert frame.orientation == "landscape_left" + assert frame.pixel_to_point(1500, 900) == (1500, 900) + # A device that returns the natural portrait buffer instead is turned upright. + upright = _upright_with_patch((1920, 1080), (300, 200, 90, 60)) + phone.screen = _as_png(upright.rotate(-90, expand=True)) + turned = _android(adb_host).capture() + assert turned.pixel_size == (1920, 1080) + assert turned.locate_image(upright.crop((300, 200, 390, 260))) == (345, 230) + # Reading the geometry sent no input. + assert phone.input_calls == [] + + +def test_template_absent_from_the_frame_is_not_found(adb_host): + frame = _android(adb_host).capture() + template = _upright_with_patch((90, 60), (0, 0, 90, 60)) + with pytest.raises(ImageNotFoundException): + frame.locate_image(template) + + +# --- the locating pipeline never looks at the host's desktop ------------ + +class _FakeOcr: + name = "fake" + available = True + + def image_to_matches(self, image, lang, min_confidence): + self.seen = image.size + return [TextMatch("Sign", 600, 300, 60, 30, 99.0), + TextMatch("in", 666, 300, 30, 30, 99.0)] + + +class _FakeVlm: + name = "fake" + available = True + + def __init__(self, reply): + self.reply = reply + self.images = [] + + def locate(self, image_bytes, description, model=None, image_mime="image/png"): + self.images.append(Image.open(io.BytesIO(image_bytes)).size) + return self.reply + + +def test_mobile_pipeline_uses_device_frame(monkeypatch, tmp_path): + desktop = [] + + def touched(name): + def record(*_args, **_kwargs): + desktop.append(name) + raise AssertionError(f"the desktop was used: {name}") + return record + + for target in ( + "je_auto_control.utils.monitor_layout.logical_frame.grab_logical", + "je_auto_control.utils.ocr.ocr_engine.grab_logical", + "je_auto_control.utils.cv2_utils.template_detection.grab_logical", + "je_auto_control.utils.vision.vlm_api._capture_screenshot_bytes", + "je_auto_control.wrapper.auto_control_mouse.click_mouse", + "je_auto_control.wrapper.auto_control_mouse.set_mouse_position"): + monkeypatch.setattr(target, touched(target)) + vlm = _FakeVlm((900, 600)) + monkeypatch.setattr("je_auto_control.utils.vision.backends.get_backend", lambda: vlm) + + handle = FakeWda(points=(390, 844), scale=3) + handle.screen = png_bytes(1170, 2532, patch=(300, 900, 90, 60)) + session = _ios(handle) + frame = session.capture() + template = tmp_path / "button.png" + frame.image.crop((300, 900, 390, 960)).save(template) + missing = tmp_path / "absent.png" + Image.open(io.BytesIO(png_bytes(40, 40, color=(200, 30, 30), + patch=(5, 5, 30, 30)))).save(missing) + + ocr = _FakeOcr() + assert frame.locate_image(str(template)) == (115, 310) + assert frame.locate_text("Sign in", backend=ocr) == (216, 105) + assert ocr.seen == (1170, 2532) + assert frame.locate_description("the blue button") == (300, 200) + assert vlm.images == [(1170, 2532)] + + from je_auto_control.utils.self_healing.heal_log import HealEventLog + log = HealEventLog(str(tmp_path / "heal.jsonl")) + hit = self_heal_locate(template_path=str(template), frame=frame, log=log) + healed = self_heal_locate(template_path=str(missing), description="the blue button", + frame=frame, log=log) + assert (hit.method, hit.coordinates) == ("image", (115, 310)) + assert (healed.method, healed.coordinates) == ("vlm", (300, 200)) + session.perform(Tap(*healed.coordinates)) + + desktop_capture_calls = len(desktop) + assert desktop_capture_calls == 0 + assert handle.input_calls == [{"op": "tap", "x": 300, "y": 200}] + + +def test_vlm_reply_off_the_frame_is_not_a_location(): + frame = DeviceFrame(image=Image.new("RGB", (300, 600)), point_size=(100, 200)) + assert frame.locate_description("x", backend=_FakeVlm((300, 10))) is None + assert frame.locate_description("x", backend=_FakeVlm(None)) is None From a855815406bb3817dbc2f00ce05e02a23eafd017 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:51:56 +0800 Subject: [PATCH 13/44] Add an opt-in action journal and candidate scripts built from it Executed actions had no traceable record: the run record is keyed by str(action) and says nothing about nesting, order across AC_parallel branches, or a step that never finished. The journal writes a start and an end line per action at the executor boundary, masks secrets before the line exists, and costs one global read per action when off. A journal run can be turned back into a reviewable script without evaluating anything from the log: recorded blocks keep their control flow, what cannot be rebuilt is emitted as a labelled observed path, masked secrets become loud placeholders, and the product is parsed, name-checked and dry-run only. --- .../Eng/doc/new_features/v5_features_doc.rst | 82 ++++ .../Zh/doc/new_features/v5_features_doc.rst | 69 +++ je_auto_control/__init__.py | 13 + je_auto_control/cli.py | 55 ++- je_auto_control/gui/_journal_import.py | 71 ++++ .../gui/language_wrapper/english.py | 13 + .../gui/language_wrapper/japanese.py | 13 + .../language_wrapper/simplified_chinese.py | 13 + .../language_wrapper/traditional_chinese.py | 13 + je_auto_control/gui/recording_editor_tab.py | 13 + je_auto_control/gui/run_history_tab.py | 58 ++- .../gui/script_builder/builder_tab.py | 21 +- .../gui/script_builder/command_schema.py | 50 +++ .../utils/action_journal/__init__.py | 21 + .../utils/action_journal/events.py | 161 +++++++ .../utils/action_journal/recorder.py | 281 +++++++++++++ .../utils/action_journal/sanitize.py | 114 +++++ je_auto_control/utils/action_journal/store.py | 141 +++++++ je_auto_control/utils/codegen/__init__.py | 11 +- .../utils/codegen/journal_import.py | 392 ++++++++++++++++++ .../utils/executor/action_executor.py | 59 ++- .../utils/executor/flow_control.py | 5 +- .../utils/mcp_server/tools/_factories.py | 78 ++++ .../utils/mcp_server/tools/_handlers_qa.py | 14 + .../utils/mcp_server/tools/_handlers_runs.py | 29 ++ .../unit_test/headless/test_action_journal.py | 327 +++++++++++++++ .../headless/test_action_journal_gui.py | 160 +++++++ .../headless/test_codegen_from_journal.py | 307 ++++++++++++++ 28 files changed, 2571 insertions(+), 13 deletions(-) create mode 100644 je_auto_control/gui/_journal_import.py create mode 100644 je_auto_control/utils/action_journal/__init__.py create mode 100644 je_auto_control/utils/action_journal/events.py create mode 100644 je_auto_control/utils/action_journal/recorder.py create mode 100644 je_auto_control/utils/action_journal/sanitize.py create mode 100644 je_auto_control/utils/action_journal/store.py create mode 100644 je_auto_control/utils/codegen/journal_import.py create mode 100644 test/unit_test/headless/test_action_journal.py create mode 100644 test/unit_test/headless/test_action_journal_gui.py create mode 100644 test/unit_test/headless/test_codegen_from_journal.py diff --git a/docs/source/Eng/doc/new_features/v5_features_doc.rst b/docs/source/Eng/doc/new_features/v5_features_doc.rst index c142cbbb3..49a835228 100644 --- a/docs/source/Eng/doc/new_features/v5_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v5_features_doc.rst @@ -59,6 +59,88 @@ fails at the first failed action. An action list the executor would refuse Executor command: ``AC_generate_code``. CLI: ``je_auto_control codegen``. +Action journal and candidate scripts +==================================== + +The action journal is an opt-in, append-only JSON-lines record of every action +the executor runs -- whichever entry point started it (Python, CLI, GUI, REST, +socket, MCP):: + + from je_auto_control import ( + start_action_journal, stop_action_journal, read_events, + generate_candidate_from_log, + ) + + run_id = start_action_journal("journal.jsonl")["run_id"] + execute_action(actions) + stop_action_journal() + + for event in read_events("journal.jsonl", run_id=run_id): + print(event.sequence, event.command, event.status, event.parent_id) + +With no journal started, dispatch pays one global read per action and allocates +nothing. Each event (``ActionEvent``, ``schema_version`` 1) carries: + +* ``run_id`` / ``step_id`` / ``parent_id`` / ``sequence`` -- a step nested in a + block names the block as its parent; an ``AC_parallel`` branch keeps the + block as parent and its ``branch`` index, and ``sequence`` is the order the + steps started in (also the order of the lines in the file); +* ``command`` and ``params`` -- the arguments *as written*: ``${var}`` and + ``${secrets.NAME}`` references stay references. Secrets are masked **before** + the line is written (the same rules as the executor's log, plus tuples), and + a value JSON cannot hold is replaced by ``{"$unserialisable": ""}``; +* ``unreplayable`` -- for each such path (``params.password``, + ``params.body[0][1].token``) the reason it cannot be replayed; +* ``status`` -- ``ok``, ``error`` or ``incomplete``. A step is written when it + starts and again when it ends, so a step whose end never reached the file + (the process died, ``Ctrl+C``) reads back as ``incomplete``, never as a + success; +* ``outcome`` -- the returned value by type and size only (numbers and booleans + by value); returned text is never stored, and an outcome is never an input. + +Everything recorded between start and stop belongs to one ``run_id`` (pass +``run_id=`` to choose it). If the journal file cannot be written, journalling +stops and the automation continues; ``action_journal_status()`` reports the +error. Steps a runner hands to a thread pool other than ``AC_parallel`` (the +DAG runner, a device matrix) are recorded without a parent. + +``generate_candidate_from_log(path, run_id=..., target="pytest", +style="actions")`` turns one run into a ``CandidateScript`` -- ``code``, +``actions``, ``manifest``, ``warnings`` and ``observed_path_only``: + +* a top-level step is emitted as it was written, so a recorded ``AC_loop`` / + ``AC_if_*`` / ``AC_parallel`` keeps its control flow; +* where a block cannot be rebuilt (the journal started inside it, its + arguments could not be stored, it calls a macro the run did not define) the + steps that actually ran beneath it are emitted instead, marked ``observed`` + in the manifest, and ``observed_path_only`` is true -- the candidate replays + the path that run took, and no branch that did not run is invented. Inside + an observed ``AC_retry`` only the last attempt is kept (the manifest counts + the attempts); +* a masked secret becomes a ``${journal_redacted_N_M}`` reference, which fails + as an unknown variable until you replace it; +* failed and unfinished steps are kept (they are the script's input) and + listed in the warnings; +* the manifest records, per step, the journal line it came from, and the + checks that were run: the source is parsed, command names are looked up and + the list goes through the executor's dry run. Nothing read from the log is + executed or evaluated. + +Executor commands: ``AC_journal_start`` / ``AC_journal_stop`` / +``AC_journal_status`` / ``AC_journal_read`` / ``AC_journal_runs`` and +``AC_generate_code_from_journal``. MCP tools: ``ac_journal_start`` / +``ac_journal_stop`` / ``ac_journal_status`` / ``ac_journal_read`` / +``ac_journal_runs`` and ``ac_generate_code_from_log``. CLI:: + + je_auto_control codegen --from-log journal.jsonl --run-id RUN \ + --target pytest -o test_flow.py --manifest test_flow.manifest.json + +``--run-id`` may be omitted when the journal holds one run; ``--style`` +defaults to ``actions`` here (``calls`` for an action file). GUI: Run History +tab → Actions menu (start / stop the journal, save a candidate script); +Recording Editor → *Import journal run…*; Script Builder → *Import journal*. + + HTTP / API ========== diff --git a/docs/source/Zh/doc/new_features/v5_features_doc.rst b/docs/source/Zh/doc/new_features/v5_features_doc.rst index 965142cc9..f4be0193a 100644 --- a/docs/source/Zh/doc/new_features/v5_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v5_features_doc.rst @@ -53,6 +53,75 @@ 執行器指令:``AC_generate_code``。CLI:``je_auto_control codegen``。 +動作日誌與候選腳本 +================== + +動作日誌是選用、僅附加的 JSON-lines 紀錄,記下執行器跑過的每一個動作——不論是從 +Python、CLI、GUI、REST、socket 還是 MCP 進來的:: + + from je_auto_control import ( + start_action_journal, stop_action_journal, read_events, + generate_candidate_from_log, + ) + + run_id = start_action_journal("journal.jsonl")["run_id"] + execute_action(actions) + stop_action_journal() + + for event in read_events("journal.jsonl", run_id=run_id): + print(event.sequence, event.command, event.status, event.parent_id) + +沒有啟動日誌時,每個動作只多一次全域變數讀取,不配置任何物件。每筆事件 +(``ActionEvent``,``schema_version`` 為 1)包含: + +* ``run_id`` / ``step_id`` / ``parent_id`` / ``sequence``——巢狀在區塊內的步驟以該區塊 + 為 parent;``AC_parallel`` 的分支保留區塊為 parent 並帶 ``branch`` 編號;``sequence`` + 是步驟開始的順序(也是檔案中各行的順序); +* ``command`` 與 ``params``——參數依 *原樣* 記錄:``${var}`` 與 ``${secrets.NAME}`` 參照 + 維持參照。秘密在該行 **寫入前** 就已遮罩(規則同執行器的 log,另含 tuple),JSON 無法 + 表示的值改記為 ``{"$unserialisable": "<型別>"}``; +* ``unreplayable``——上述每個路徑(``params.password``、``params.body[0][1].token``) + 無法重播的原因; +* ``status``——``ok``、``error`` 或 ``incomplete``。步驟在開始時寫一行、結束時再寫一行, + 所以結束行沒寫進檔案的步驟(行程死掉、``Ctrl+C``)讀回來是 ``incomplete``,不會看起來 + 像成功; +* ``outcome``——只記回傳值的型別與大小(數字與布林才記值);回傳的文字一律不存,而且 + outcome 永遠不會被當成輸入。 + +start 到 stop 之間記下的一切屬於同一個 ``run_id``(可用 ``run_id=`` 指定)。日誌檔寫不 +進去時,日誌會停止而自動化繼續執行;``action_journal_status()`` 會回報錯誤。除了 +``AC_parallel`` 以外,交給執行緒池執行的步驟(DAG runner、device matrix)記錄時沒有 +parent。 + +``generate_candidate_from_log(path, run_id=..., target="pytest", style="actions")`` +把一次執行轉成 ``CandidateScript``——``code``、``actions``、``manifest``、``warnings`` +與 ``observed_path_only``: + +* 頂層步驟依原樣輸出,所以記錄到的 ``AC_loop`` / ``AC_if_*`` / ``AC_parallel`` 保留其 + 控制流程; +* 區塊無法還原時(日誌是在區塊內才啟動、參數無法儲存、呼叫了這次執行沒定義的巨集), + 改輸出實際在它底下跑過的步驟,manifest 中標為 ``observed``,且 ``observed_path_only`` + 為 true——候選腳本重播的是那次執行走過的路徑,不會憑空產生沒執行的分支。已觀察的 + ``AC_retry`` 只保留最後一次嘗試(manifest 會記嘗試次數); +* 被遮罩的秘密會變成 ``${journal_redacted_N_M}`` 參照,在你替換掉之前會以未知變數 + 失敗; +* 失敗與未完成的步驟會保留(它們是腳本的輸入),並列在 warnings; +* manifest 逐步記錄來源的日誌行號,以及做過的檢查:原始碼可解析、指令名稱已查表、 + 動作清單通過執行器的 dry run。日誌中的任何內容都不會被執行或 eval。 + +執行器指令:``AC_journal_start`` / ``AC_journal_stop`` / ``AC_journal_status`` / +``AC_journal_read`` / ``AC_journal_runs`` 與 ``AC_generate_code_from_journal``。 +MCP 工具:``ac_journal_start`` / ``ac_journal_stop`` / ``ac_journal_status`` / +``ac_journal_read`` / ``ac_journal_runs`` 與 ``ac_generate_code_from_log``。CLI:: + + je_auto_control codegen --from-log journal.jsonl --run-id RUN \ + --target pytest -o test_flow.py --manifest test_flow.manifest.json + +日誌只有一次執行時可省略 ``--run-id``;這裡 ``--style`` 預設為 ``actions``(動作檔則是 +``calls``)。GUI:Run History 分頁 → Actions 選單(啟動/停止日誌、儲存候選腳本); +Recording Editor →「匯入日誌的執行…」;Script Builder →「匯入日誌」。 + + HTTP / API ========== diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 475c40ca2..81f8f69e3 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1214,6 +1214,15 @@ generate_code, generate_code_file, ) +# action journal (opt-in JSONL record of executed actions) + candidate scripts +from je_auto_control.utils.action_journal import ( + ActionEvent, ActionJournal, ActionJournalError, JournalFormatError, + action_journal_status, list_journal_runs, read_events, + start_action_journal, stop_action_journal, +) +from je_auto_control.utils.codegen.journal_import import ( + CandidateScript, JournalImportError, generate_candidate_from_log, +) # HTTP/API request action (dependency-free, stdlib urllib) from je_auto_control.utils.http_client.http_client import http_request # Ad-hoc read-only SQL query against SQLite @@ -1324,6 +1333,10 @@ def start_autocontrol_gui(*args, **kwargs): "execute_action", "execute_files", "executor", "execute_action_with_vars", "record_to_json", "generate_code", "generate_code_file", "http_request", "query_sqlite", + "ActionEvent", "ActionJournal", "ActionJournalError", "JournalFormatError", + "action_journal_status", "list_journal_runs", "read_events", + "start_action_journal", "stop_action_journal", + "CandidateScript", "JournalImportError", "generate_candidate_from_log", "send_email", "assert_pdf_text", "extract_pdf_text", "pdf_metadata", "pdf_page_count", "add_command_to_executor", "test_record_instance", "pil_screenshot", diff --git a/je_auto_control/cli.py b/je_auto_control/cli.py index ed6e99c65..1ee7f7180 100644 --- a/je_auto_control/cli.py +++ b/je_auto_control/cli.py @@ -12,6 +12,8 @@ je_auto_control fmt script.json [--check] je_auto_control record out.json [--duration 5] je_auto_control codegen script.json [--target pytest] [-o test_flow.py] + je_auto_control codegen --from-log journal.jsonl [--run-id ID] + [--manifest candidate.json] [-o test_flow.py] je_auto_control failure-bundle failure.zip [--error "message"] je_auto_control version je_auto_control list-jobs @@ -164,24 +166,55 @@ def _set_on_enter(stop_event: threading.Event) -> None: def cmd_codegen(args: argparse.Namespace) -> int: - """Generate pytest/python/robot source from an action file.""" + """Generate pytest/python/robot source from an action file or a journal.""" from je_auto_control.utils.codegen.codegen import ( generate_code, generate_code_file, ) from je_auto_control.utils.json.json_file import read_action_json + if bool(args.script) == bool(args.from_log): + raise ValueError("codegen needs an action file or --from-log JOURNAL (not both)") + if args.from_log: + return _codegen_from_log(args) + style = args.style or "calls" if args.output: generate_code_file(args.script, args.output, target=args.target, - name=args.name, style=args.style, + name=args.name, style=style, failure_bundle=args.failure_bundle) sys.stderr.write(f"Wrote {args.target} code to {args.output}\n") else: code = generate_code(read_action_json(args.script), target=args.target, - name=args.name, style=args.style, + name=args.name, style=style, failure_bundle=args.failure_bundle) sys.stdout.write(code) return 0 +def _codegen_from_log(args: argparse.Namespace) -> int: + """Write a candidate script built from one run of an action journal.""" + from je_auto_control.utils.codegen.journal_import import ( + generate_candidate_from_log, only_run_id, write_candidate, + ) + if args.failure_bundle: + raise ValueError("--failure-bundle does not apply to --from-log") + if args.manifest and not args.output: + raise ValueError("--manifest needs -o/--output") + run_id = args.run_id or only_run_id(args.from_log) + candidate = generate_candidate_from_log( + args.from_log, run_id=run_id, target=args.target, + style=args.style or "actions") + for warning in candidate.warnings: + sys.stderr.write(f"warning: {warning}\n") + if candidate.observed_path_only: + sys.stderr.write("warning: observed path only -- the candidate replays what " + "this run did, not the script's control flow\n") + if args.output: + write_candidate(candidate, args.output, args.manifest) + sys.stderr.write(f"Wrote {args.target} candidate to {args.output}\n") + else: + sys.stdout.write(candidate.code) + return 0 + + def cmd_version(_: argparse.Namespace) -> int: """Print the installed ``je_auto_control`` version.""" from importlib.metadata import PackageNotFoundError, version @@ -298,12 +331,20 @@ def build_parser() -> argparse.ArgumentParser: p_record.set_defaults(func=cmd_record) p_codegen = sub.add_parser( - "codegen", help="Generate test code from an action file") - p_codegen.add_argument("script") + "codegen", help="Generate test code from an action file or a journal") + p_codegen.add_argument("script", nargs="?") + p_codegen.add_argument( + "--from-log", metavar="JOURNAL", + help="Build a candidate from an action journal (.jsonl) instead of a script") + p_codegen.add_argument( + "--run-id", help="Journal run to use (optional when the journal holds one run)") + p_codegen.add_argument( + "--manifest", metavar="PATH", + help="With --from-log and -o: also write the provenance manifest (JSON)") p_codegen.add_argument("--target", choices=("pytest", "python", "robot"), default="pytest") - p_codegen.add_argument("--style", choices=("calls", "actions"), - default="calls") + p_codegen.add_argument("--style", choices=("calls", "actions"), default=None, + help="Default: calls for a script, actions for --from-log") p_codegen.add_argument("--name", default="recorded_flow") p_codegen.add_argument("-o", "--output", help="Write to file instead of stdout") p_codegen.add_argument( diff --git a/je_auto_control/gui/_journal_import.py b/je_auto_control/gui/_journal_import.py new file mode 100644 index 000000000..02119cee8 --- /dev/null +++ b/je_auto_control/gui/_journal_import.py @@ -0,0 +1,71 @@ +"""Shared dialog flow: pick an action journal run and build its candidate script. + +The tabs that open a candidate (Run History, Recording Editor, Script Builder) +only differ in what they do with it; choosing the file and the run, building +the candidate and showing its warnings is the same everywhere. All logic stays +in ``utils/codegen/journal_import`` -- nothing here is unreachable headlessly. +""" +from typing import Optional + +from PySide6.QtWidgets import QFileDialog, QInputDialog, QMessageBox, QWidget + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) +from je_auto_control.utils.action_journal.store import list_journal_runs +from je_auto_control.utils.codegen.journal_import import ( + CandidateScript, generate_candidate_from_log, +) +from je_auto_control.utils.exception.exceptions import AutoControlException + +JOURNAL_FILTER = "Journal (*.jsonl);;All files (*)" + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +def _pick_run(parent: QWidget, path: str) -> Optional[str]: + """The run to import: the only one, or the one the user chooses.""" + runs = [run["run_id"] for run in list_journal_runs(path)] + if not runs: + QMessageBox.information(parent, _t("jr_dialog_open"), _t("jr_no_runs")) + return None + if len(runs) == 1: + return runs[0] + # Newest first: the run just recorded is the one usually wanted. + run_id, accepted = QInputDialog.getItem( + parent, _t("jr_dialog_open"), _t("jr_pick_run"), runs[::-1], 0, False) + return run_id if accepted else None + + +def candidate_summary(candidate: CandidateScript) -> str: + """The candidate's caveats as text: the observed-path notice, then warnings.""" + lines = [_t("jr_observed_only")] if candidate.observed_path_only else [] + lines.extend(candidate.warnings) + return "\n".join(lines) + + +def pick_journal_candidate(parent: QWidget, target: str = "pytest" + ) -> Optional[CandidateScript]: + """Ask for a journal file and run, build the candidate, show what to check. + + Returns ``None`` when the user cancels or the journal cannot be used (the + reason is shown). The candidate is built and dry-run only, never executed. + """ + path, _ = QFileDialog.getOpenFileName( + parent, _t("jr_dialog_open"), "", JOURNAL_FILTER) + if not path: + return None + try: + run_id = _pick_run(parent, path) + if run_id is None: + return None + candidate = generate_candidate_from_log(path, run_id=run_id, target=target) + except (AutoControlException, OSError, ValueError) as error: + QMessageBox.warning(parent, _t("jr_dialog_open"), str(error)) + return None + summary = candidate_summary(candidate) + if summary: + QMessageBox.information(parent, _t("jr_dialog_open"), summary) + return candidate diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 6bc8d1ebe..ab0dc13fb 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -853,6 +853,19 @@ "rh_col_artifact": "Artifact", "rh_open_artifact": "Open artifact", "rh_no_artifact": "Selected run has no artifact.", + "rh_journal_start": "Start action journal…", + "rh_journal_stop": "Stop action journal", + "rh_journal_candidate": "Candidate script from journal…", + "rh_journal_off": "Action journal: off", + "rh_journal_on": "Action journal: run {run}, {n} step(s) → {path}", + "re_import_journal": "Import journal run…", + "sb_import_journal": "Import journal", + "jr_dialog_open": "Open action journal", + "jr_dialog_start": "Journal file to append to", + "jr_dialog_save_code": "Save candidate script", + "jr_pick_run": "Run", + "jr_no_runs": "This journal holds no runs.", + "jr_observed_only": "Observed path only: this candidate replays what the run did, not the script's control flow.", "rh_artifact_missing": "Artifact file no longer exists.", "rh_timeline_heading": "Timeline (oldest left → newest right)", "rh_preview_heading": "Preview", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 0740bfc51..7fd069f61 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -742,6 +742,19 @@ "rh_col_artifact": "スクリーンショット", "rh_open_artifact": "スクリーンショットを開く", "rh_no_artifact": "選択した実行にスクリーンショットはありません。", + "rh_journal_start": "アクションジャーナルを開始…", + "rh_journal_stop": "アクションジャーナルを停止", + "rh_journal_candidate": "ジャーナルから候補スクリプトを作成…", + "rh_journal_off": "アクションジャーナル: オフ", + "rh_journal_on": "アクションジャーナル: 実行 {run}、{n} ステップ → {path}", + "re_import_journal": "ジャーナルの実行を取り込む…", + "sb_import_journal": "ジャーナル取込", + "jr_dialog_open": "アクションジャーナルを開く", + "jr_dialog_start": "追記するジャーナルファイル", + "jr_dialog_save_code": "候補スクリプトを保存", + "jr_pick_run": "実行", + "jr_no_runs": "このジャーナルには実行がありません。", + "jr_observed_only": "観測経路のみ: この候補は実行時の動作を再生するもので、スクリプトの制御フローではありません。", "rh_artifact_missing": "スクリーンショットファイルが存在しません。", "rh_timeline_heading": "タイムライン（左:古い → 右:新しい）", "rh_preview_heading": "プレビュー", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index d002ab239..55a4c3a86 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -725,6 +725,19 @@ "rh_col_artifact": "截图", "rh_open_artifact": "打开截图", "rh_no_artifact": "所选记录没有截图。", + "rh_journal_start": "开始动作日志…", + "rh_journal_stop": "停止动作日志", + "rh_journal_candidate": "从日志生成候选脚本…", + "rh_journal_off": "动作日志：未启用", + "rh_journal_on": "动作日志：运行 {run}，{n} 个步骤 → {path}", + "re_import_journal": "导入日志的运行…", + "sb_import_journal": "导入日志", + "jr_dialog_open": "打开动作日志", + "jr_dialog_start": "要追加写入的日志文件", + "jr_dialog_save_code": "保存候选脚本", + "jr_pick_run": "运行", + "jr_no_runs": "这份日志没有任何运行记录。", + "jr_observed_only": "仅为已观察路径：此候选脚本重放的是该次运行实际做的事，不是脚本的控制流程。", "rh_artifact_missing": "截图文件已不存在。", "rh_timeline_heading": "时间轴（左旧右新）", "rh_preview_heading": "预览", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 046af99cd..34caa4fe7 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -726,6 +726,19 @@ "rh_col_artifact": "截圖", "rh_open_artifact": "開啟截圖", "rh_no_artifact": "所選紀錄沒有截圖。", + "rh_journal_start": "開始動作日誌…", + "rh_journal_stop": "停止動作日誌", + "rh_journal_candidate": "從日誌產生候選腳本…", + "rh_journal_off": "動作日誌：未啟用", + "rh_journal_on": "動作日誌：執行 {run}，{n} 個步驟 → {path}", + "re_import_journal": "匯入日誌的執行…", + "sb_import_journal": "匯入日誌", + "jr_dialog_open": "開啟動作日誌", + "jr_dialog_start": "要附加寫入的日誌檔", + "jr_dialog_save_code": "儲存候選腳本", + "jr_pick_run": "執行", + "jr_no_runs": "這份日誌沒有任何執行紀錄。", + "jr_observed_only": "僅為已觀察路徑：此候選腳本重播的是該次執行實際做的事，不是腳本的控制流程。", "rh_artifact_missing": "截圖檔案已不存在。", "rh_timeline_heading": "時間軸（左舊右新）", "rh_preview_heading": "預覽", diff --git a/je_auto_control/gui/recording_editor_tab.py b/je_auto_control/gui/recording_editor_tab.py index 8bce419dd..fc1df8524 100644 --- a/je_auto_control/gui/recording_editor_tab.py +++ b/je_auto_control/gui/recording_editor_tab.py @@ -9,6 +9,7 @@ ) from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui._journal_import import pick_journal_candidate from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( language_wrapper, ) @@ -105,6 +106,7 @@ def menu_actions(self) -> list: return [ ("re_browse", self._browse), ("re_load", self._load), + ("re_import_journal", self._import_journal), ("re_save_as", self._save_as), ("re_export_code", self._export_code), ("re_apply_trim", self._apply_trim), @@ -144,6 +146,17 @@ def _load(self) -> None: self._undo_stack.clear() # a freshly loaded recording starts clean self._refresh() + def _import_journal(self) -> None: + """Open the candidate built from an action journal run for editing.""" + candidate = pick_journal_candidate(self) + if candidate is None: + return + self._path_input.clear() + self._mutate(list(candidate.actions)) + self._status.setText( + f"{len(self._actions)} actions from journal run {candidate.manifest['run_id']}" + f" ({len(candidate.warnings)} warning(s))") + def _save_as(self) -> None: if not self._actions: return diff --git a/je_auto_control/gui/run_history_tab.py b/je_auto_control/gui/run_history_tab.py index 4331892c5..403c292c9 100644 --- a/je_auto_control/gui/run_history_tab.py +++ b/je_auto_control/gui/run_history_tab.py @@ -6,8 +6,8 @@ from PySide6.QtCore import QTimer, Qt, QUrl from PySide6.QtGui import QDesktopServices, QPixmap from PySide6.QtWidgets import ( - QAbstractItemView, QComboBox, QFrame, QHBoxLayout, QHeaderView, QLabel, - QMessageBox, QSplitter, QTableWidget, QTableWidgetItem, + QAbstractItemView, QComboBox, QFileDialog, QFrame, QHBoxLayout, QHeaderView, + QLabel, QMessageBox, QSplitter, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, ) @@ -15,7 +15,14 @@ from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( language_wrapper, ) +from je_auto_control.gui._journal_import import JOURNAL_FILTER, pick_journal_candidate from je_auto_control.gui.run_history_timeline import RunHistoryTimeline +from je_auto_control.utils.action_journal import ( + action_journal_status, default_journal_path, start_action_journal, + stop_action_journal, +) +from je_auto_control.utils.codegen.journal_import import write_candidate +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.run_history.history_store import ( SOURCE_HOTKEY, SOURCE_MANUAL, SOURCE_REST, SOURCE_SCHEDULER, SOURCE_TRIGGER, STATUS_ERROR, STATUS_OK, STATUS_RUNNING, RunRecord, @@ -77,6 +84,8 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: header.setSectionResizeMode(QHeaderView.Interactive) header.setStretchLastSection(True) self._count_label = QLabel() + self._journal_label = QLabel() + self._journal_label.setTextInteractionFlags(Qt.TextSelectableByMouse) self._timeline = RunHistoryTimeline() self._timeline.run_clicked.connect(self._on_timeline_clicked) self._thumb_label = QLabel() @@ -158,6 +167,7 @@ def _build_layout(self) -> None: self._table.cellDoubleClicked.connect(self._on_cell_double_clicked) self._table.itemSelectionChanged.connect(self._on_selection_changed) root.addWidget(self._count_label) + root.addWidget(self._journal_label) def menu_actions(self) -> list: """Expose tab commands to the window-level Actions menu.""" @@ -165,8 +175,51 @@ def menu_actions(self) -> list: ("rh_refresh", self._refresh), ("rh_clear", self._on_clear), ("rh_open_artifact", self._open_selected_artifact), + ("rh_journal_start", self._start_journal), + ("rh_journal_stop", self._stop_journal), + ("rh_journal_candidate", self._export_journal_candidate), ] + def _start_journal(self) -> None: + """Start the action journal, appending to the file the user names.""" + path, _ = QFileDialog.getSaveFileName( + self, _t("jr_dialog_start"), str(default_journal_path()), JOURNAL_FILTER, + options=QFileDialog.Option.DontConfirmOverwrite) # appended, never replaced + if not path: + return + try: + start_action_journal(path) + except (AutoControlException, OSError) as error: + QMessageBox.warning(self, _t("rh_journal_start"), str(error)) + self._refresh_journal_label() + + def _stop_journal(self) -> None: + stop_action_journal() + self._refresh_journal_label() + + def _export_journal_candidate(self) -> None: + """Build a candidate script from a journal run and save it with its manifest.""" + candidate = pick_journal_candidate(self) + if candidate is None: + return + path, _ = QFileDialog.getSaveFileName( + self, _t("jr_dialog_save_code"), "", "Python (*.py)") + if not path: + return + try: + write_candidate(candidate, path, path + ".manifest.json") + except OSError as error: + QMessageBox.warning(self, _t("rh_journal_candidate"), str(error)) + + def _refresh_journal_label(self) -> None: + status = action_journal_status() + if not status["active"]: + self._journal_label.setText(_t("rh_journal_off")) + return + self._journal_label.setText( + _t("rh_journal_on").replace("{run}", str(status["run_id"])) + .replace("{n}", str(status["events"])).replace("{path}", str(status["path"]))) + def _on_clear(self) -> None: reply = QMessageBox.question( self, _t("rh_clear"), _t("rh_confirm_clear"), @@ -191,6 +244,7 @@ def _refresh(self) -> None: ) self._timeline.set_records(runs) self._refresh_preview() + self._refresh_journal_label() def _set_row(self, row: int, record) -> None: status_key = _STATUS_LABEL_KEYS.get(record.status, record.status) diff --git a/je_auto_control/gui/script_builder/builder_tab.py b/je_auto_control/gui/script_builder/builder_tab.py index 72ba3503a..d18688da6 100644 --- a/je_auto_control/gui/script_builder/builder_tab.py +++ b/je_auto_control/gui/script_builder/builder_tab.py @@ -10,6 +10,7 @@ ) from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui._journal_import import candidate_summary, pick_journal_candidate from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( language_wrapper, ) @@ -19,7 +20,7 @@ from je_auto_control.gui.script_builder.step_form_view import StepFormView from je_auto_control.gui.script_builder.step_list_view import StepTreeView from je_auto_control.gui.script_builder.step_model import ( - Step, load_action_file, save_action_file, steps_to_actions, + Step, actions_to_steps, load_action_file, save_action_file, steps_to_actions, ) from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.executor.action_executor import execute_action @@ -78,6 +79,7 @@ def _build_toolbar(self) -> QHBoxLayout: bar.addStretch() for key, handler in ( ("sb_load_json", self._on_load), + ("sb_import_journal", self._on_import_journal), ("sb_save_json", self._on_save), ("sb_run", self._on_run), ): @@ -148,6 +150,23 @@ def _on_load(self) -> None: except (AutoControlException, OSError, ValueError, TypeError) as error: QMessageBox.warning(self, "Error", str(error)) + def _on_import_journal(self) -> None: + """Load the candidate built from an action journal run as steps to review.""" + candidate = pick_journal_candidate(self) + if candidate is None: + return + try: + steps = actions_to_steps(candidate.actions) + except (AutoControlException, ValueError, TypeError) as error: + QMessageBox.warning(self, "Error", str(error)) + return + self._file_extras = None + self._tree.load_steps(steps) + self._form.load_step(None) + self._result.setPlainText("\n".join(( + f"Imported journal run {candidate.manifest['run_id']}: {len(steps)} step(s)", + candidate_summary(candidate)))) + def _on_run(self) -> None: try: actions = steps_to_actions(self._tree.root_steps()) diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index a9638555d..b7105d011 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2994,6 +2994,56 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: description="Aggregate the self-heal log (heal rate, brittle " "locators).", )) + specs.append(CommandSpec( + "AC_journal_start", "Testing", "Start Action Journal", + fields=( + FieldSpec("path", FieldType.FILE_PATH, optional=True, + placeholder="action_journal.jsonl"), + FieldSpec("run_id", FieldType.STRING, optional=True), + FieldSpec("session", FieldType.STRING, optional=True), + ), + description="Append every executed action to a JSON-lines journal " + "(secrets masked before they are written).", + )) + specs.append(CommandSpec( + "AC_journal_stop", "Testing", "Stop Action Journal", + description="Stop the action journal.", + )) + specs.append(CommandSpec( + "AC_journal_status", "Testing", "Action Journal Status", + description="Report whether an action journal is started.", + )) + specs.append(CommandSpec( + "AC_journal_read", "Testing", "Read Action Journal", + fields=( + FieldSpec("path", FieldType.FILE_PATH, + placeholder="action_journal.jsonl"), + FieldSpec("run_id", FieldType.STRING, optional=True), + FieldSpec("limit", FieldType.INT, optional=True, default=0), + ), + description="Read a journal's events (the last 'limit' when positive).", + )) + specs.append(CommandSpec( + "AC_journal_runs", "Testing", "List Journal Runs", + fields=(FieldSpec("path", FieldType.FILE_PATH, + placeholder="action_journal.jsonl"),), + description="Summarise each run a journal file holds.", + )) + specs.append(CommandSpec( + "AC_generate_code_from_journal", "Testing", "Candidate Script from Journal", + fields=( + FieldSpec("path", FieldType.FILE_PATH, + placeholder="action_journal.jsonl"), + FieldSpec("run_id", FieldType.STRING, optional=True), + FieldSpec("target", FieldType.ENUM, optional=True, default="pytest", + choices=("pytest", "python", "robot")), + FieldSpec("style", FieldType.ENUM, optional=True, default="actions", + choices=("actions", "calls")), + FieldSpec("output", FieldType.FILE_PATH, optional=True), + ), + description="Build a reviewable candidate script from one journal " + "run; nothing from the log is executed.", + )) specs.append(CommandSpec( "AC_failure_signature", "Testing", "Failure Signature", fields=( diff --git a/je_auto_control/utils/action_journal/__init__.py b/je_auto_control/utils/action_journal/__init__.py new file mode 100644 index 000000000..9e1919683 --- /dev/null +++ b/je_auto_control/utils/action_journal/__init__.py @@ -0,0 +1,21 @@ +"""Structured, append-only journal of the actions the executor ran.""" +from je_auto_control.utils.action_journal.events import ( + SCHEMA_VERSION, STATUS_ERROR, STATUS_INCOMPLETE, STATUS_OK, ActionEvent, + JournalFormatError, +) +from je_auto_control.utils.action_journal.recorder import ( + ActionJournalError, action_journal_status, start_action_journal, + stop_action_journal, +) +from je_auto_control.utils.action_journal.store import ( + ActionJournal, JournalContents, default_journal_path, list_journal_runs, + load_journal, read_events, +) + +__all__ = [ + "SCHEMA_VERSION", "STATUS_ERROR", "STATUS_INCOMPLETE", "STATUS_OK", + "ActionEvent", "ActionJournal", "ActionJournalError", "JournalContents", + "JournalFormatError", "action_journal_status", "default_journal_path", + "list_journal_runs", "load_journal", "read_events", "start_action_journal", + "stop_action_journal", +] diff --git a/je_auto_control/utils/action_journal/events.py b/je_auto_control/utils/action_journal/events.py new file mode 100644 index 000000000..43407e039 --- /dev/null +++ b/je_auto_control/utils/action_journal/events.py @@ -0,0 +1,161 @@ +"""The record an action journal holds: one executed ``AC_*`` action. + +An :class:`ActionEvent` is what the executor knew at the action boundary -- +the command, its arguments as they were written (placeholders unresolved, +secrets masked), where it sat in the run (``parent_id``, ``sequence``, +``branch``) and how it ended. The journal is JSON lines, so everything here +converts to and from plain JSON; nothing is ever read back with ``eval``. + +Pure standard library; imports no ``PySide6``. +""" +import dataclasses +from dataclasses import dataclass, field +from typing import Any, Dict, Mapping, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlException + +#: The journal line format this module writes and reads. +SCHEMA_VERSION = 1 + +#: The action returned. +STATUS_OK = "ok" +#: The action raised. +STATUS_ERROR = "error" +#: The action started and no end was recorded: the process died, the run was +#: interrupted, or it is still running. Never reported as a success. +STATUS_INCOMPLETE = "incomplete" +STATUSES = frozenset({STATUS_OK, STATUS_ERROR, STATUS_INCOMPLETE}) + +#: ``record`` of a line holding a whole event, written when the action starts. +RECORD_START = "start" +#: ``record`` of a line holding only how a started action ended. +RECORD_END = "end" + +#: What a masked or unserialisable argument is replaced by in ``params``. +MASK = "***" +UNSERIALISABLE_KEY = "$unserialisable" + + +class JournalFormatError(AutoControlException, ValueError): + """A journal line is valid JSON but not a journal record this version reads.""" + + +@dataclass(frozen=True) +class ActionEvent: + """One action the executor ran, as the journal stores it. + + ``params`` is the action's argument value exactly as it was written + (``${var}`` and ``${secrets.NAME}`` references kept as references) after + masking; ``unreplayable`` maps each path in it that cannot be replayed to + the reason. ``outcome`` describes the returned value by type and size and + is never an input to anything. + """ + + run_id: str + step_id: str + sequence: int + command: str + parent_id: Optional[str] = None + params: Any = None + status: str = STATUS_INCOMPLETE + started_at: float = 0.0 + finished_at: Optional[float] = None + error: Optional[str] = None + outcome: Optional[Mapping[str, Any]] = None + unreplayable: Mapping[str, str] = field(default_factory=dict) + branch: Optional[int] = None + thread: Optional[int] = None + session: Optional[str] = None + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> Dict[str, Any]: + """The event as a JSON-ready dict (a whole ``start`` line).""" + data = dataclasses.asdict(self) + data["record"] = RECORD_START + return data + + def end_dict(self) -> Dict[str, Any]: + """Only how the action ended (an ``end`` line for an earlier start).""" + return { + "schema_version": self.schema_version, "record": RECORD_END, + "run_id": self.run_id, "step_id": self.step_id, + "status": self.status, "finished_at": self.finished_at, + "error": self.error, "outcome": self.outcome, + } + + +_REQUIRED = {"run_id": str, "step_id": str, "sequence": int, "command": str} +_OPTIONAL = { + "parent_id": str, "error": str, "branch": int, "thread": int, + "session": str, "outcome": dict, +} + + +def _typed(data: Mapping[str, Any], name: str, kind: type, where: str) -> Any: + value = data.get(name) + # bool is an int: {"sequence": true} is not a sequence number. + if not isinstance(value, kind) or isinstance(value, bool): + raise JournalFormatError( + f"{where}: {name!r} must be {kind.__name__}, got {type(value).__name__}") + return value + + +def _number(data: Mapping[str, Any], name: str, where: str) -> Optional[float]: + value = data.get(name) + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise JournalFormatError(f"{where}: {name!r} must be a number") + return float(value) + + +def check_record(data: Any, where: str) -> str: + """Validate the envelope of one journal line; return its ``record`` kind.""" + if not isinstance(data, dict): + raise JournalFormatError(f"{where}: a journal line must be a JSON object") + version = data.get("schema_version") + if version != SCHEMA_VERSION or isinstance(version, bool): + raise JournalFormatError( + f"{where}: unsupported schema_version {version!r} (this reads {SCHEMA_VERSION})") + record = data.get("record", RECORD_START) + if record not in (RECORD_START, RECORD_END): + raise JournalFormatError(f"{where}: unknown record kind {record!r}") + status = data.get("status", STATUS_INCOMPLETE) + if status not in STATUSES: + raise JournalFormatError(f"{where}: unknown status {status!r}") + for name in ("run_id", "step_id"): + _typed(data, name, str, where) + return record + + +def event_from_dict(data: Mapping[str, Any], where: str = "event") -> ActionEvent: + """Build an :class:`ActionEvent` from a parsed ``start`` line, validating it.""" + check_record(data, where) + values: Dict[str, Any] = { + name: _typed(data, name, kind, where) for name, kind in _REQUIRED.items()} + for name, kind in _OPTIONAL.items(): + if data.get(name) is not None: + values[name] = _typed(data, name, kind, where) + unreplayable = data.get("unreplayable") or {} + if not isinstance(unreplayable, dict) or not all( + isinstance(key, str) and isinstance(reason, str) + for key, reason in unreplayable.items()): + raise JournalFormatError(f"{where}: 'unreplayable' must map paths to reasons") + return ActionEvent( + params=data.get("params"), status=data.get("status", STATUS_INCOMPLETE), + started_at=_number(data, "started_at", where) or 0.0, + finished_at=_number(data, "finished_at", where), + unreplayable=dict(unreplayable), **values) + + +def apply_end(event: ActionEvent, data: Mapping[str, Any], where: str) -> ActionEvent: + """``event`` with the outcome an ``end`` line recorded for it.""" + outcome = data.get("outcome") + if outcome is not None and not isinstance(outcome, dict): + raise JournalFormatError(f"{where}: 'outcome' must be an object") + error = data.get("error") + if error is not None and not isinstance(error, str): + raise JournalFormatError(f"{where}: 'error' must be a string") + return dataclasses.replace( + event, status=data.get("status", STATUS_INCOMPLETE), + finished_at=_number(data, "finished_at", where), error=error, outcome=outcome) diff --git a/je_auto_control/utils/action_journal/recorder.py b/je_auto_control/utils/action_journal/recorder.py new file mode 100644 index 000000000..8b6aabf15 --- /dev/null +++ b/je_auto_control/utils/action_journal/recorder.py @@ -0,0 +1,281 @@ +"""The executor's hook into the action journal, and the switch that turns it on. + +Journalling is opt-in. With no journal started, :func:`step` returns one +shared no-op object -- a global read and a function call per action, nothing +allocated -- so dispatch costs what it did before this module existed. + +Once :func:`start_action_journal` has run, every action the executor runs -- +from the GUI, the CLI, the REST, socket and MCP servers alike, because they +all end in ``Executor._run_one_action`` -- is written when it starts and again +when it ends. The parent of a nested action is whatever action is running on +the same thread; an ``AC_parallel`` branch runs on a new thread, so the block +hands its own step over with :func:`branch_scope`. + +Pure standard library; imports no ``PySide6``. +""" +import dataclasses +import itertools +import threading +import time +import uuid +from pathlib import Path +from types import TracebackType +from typing import Any, Dict, List, Optional, Tuple, Type, Union + +from je_auto_control.utils.action_journal.events import ( + STATUS_ERROR, STATUS_INCOMPLETE, STATUS_OK, ActionEvent, +) +from je_auto_control.utils.action_journal.sanitize import ( + describe_outcome, sanitise_params, +) +from je_auto_control.utils.action_journal.store import ActionJournal +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +_MAX_ERROR_CHARS = 1000 + + +class ActionJournalError(AutoControlException, RuntimeError): + """A journal was started while another one was still running.""" + + +class _Session: + """One started journal: where it writes and the ids it hands out.""" + + def __init__(self, journal: ActionJournal, run_id: str, + session: Optional[str]) -> None: + self.journal = journal + self.run_id = run_id + self.label = session + self.token = uuid.uuid4().hex[:8] + self.started = 0 + self.error: Optional[str] = None + self._counter = itertools.count(1) + self._lock = threading.Lock() + + def start_event(self, event: ActionEvent) -> ActionEvent: + """Number ``event`` and write its start line, in one critical section. + + Numbering and writing together is what makes file order the order + the actions started in, across threads. + """ + with self._lock: + sequence = next(self._counter) + numbered = dataclasses.replace( + event, sequence=sequence, step_id=f"{self.token}-{sequence}") + self.journal.append(numbered) + self.started = sequence + return numbered + + def status(self) -> Dict[str, Any]: + """The session as a JSON-ready dict.""" + return {"path": str(self.journal.path), "run_id": self.run_id, + "session": self.label, "events": self.started, "error": self.error} + + +#: The started journal, or ``None``. Read without a lock on every action. +_ACTIVE: Optional[_Session] = None +#: The session most recently stopped (or failed), for the status report. +_LAST: Optional[_Session] = None +_SWITCH = threading.Lock() +#: Per thread: ``stack`` of running step ids, plus what a branch inherited. +_LOCAL = threading.local() + + +class _NullStep: + """What :func:`step` returns while no journal is started; does nothing.""" + + __slots__ = () + + def __enter__(self) -> "_NullStep": + return self + + def __exit__(self, *_exc: Any) -> None: + """Let any exception through.""" + + def outcome(self, _value: Any) -> None: + """Ignore the action's return value.""" + + +_NULL = _NullStep() + + +def _stack() -> List[str]: + stack = getattr(_LOCAL, "stack", None) + if stack is None: + stack = _LOCAL.stack = [] + return stack + + +def _error_text(error: BaseException) -> str: + # The same masking log lines get: an error message can quote an argument. + from je_auto_control.utils.config_redaction.config_redaction import ( + redact_secret_text, + ) + return redact_secret_text(repr(error)[:_MAX_ERROR_CHARS]) + + +def _control_signals() -> Tuple[type, ...]: + from je_auto_control.utils.executor.flow_control import LoopBreak, LoopContinue + return LoopBreak, LoopContinue + + +class _Step: + """One running action: writes its start on entry and its end on exit.""" + + __slots__ = ("_session", "_action", "_event", "_outcome") + + def __init__(self, session: _Session, action: Any) -> None: + self._session = session + self._action = action + self._event: Optional[ActionEvent] = None + self._outcome: Optional[Dict[str, Any]] = None + + def __enter__(self) -> "_Step": + session, action = self._session, self._action + command = action[0] if action and isinstance(action[0], str) else "" + params, unreplayable = sanitise_params( + command, action[1] if len(action) > 1 else None) + stack = _stack() + event = ActionEvent( + run_id=session.run_id, step_id="", sequence=0, command=command, + parent_id=stack[-1] if stack else getattr(_LOCAL, "parent", None), + params=params, unreplayable=unreplayable, started_at=time.time(), + branch=getattr(_LOCAL, "branch", None), thread=threading.get_ident(), + session=session.label) + try: + self._event = session.start_event(event) + except OSError as error: + _fail(session, error) + return self + stack.append(self._event.step_id) + return self + + def outcome(self, value: Any) -> None: + """Note what the action returned (by type and size only).""" + self._outcome = describe_outcome(value) + + def __exit__(self, exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], _tb: Optional[TracebackType]) -> None: + event = self._event + if event is None: + return + stack = _stack() + if stack and stack[-1] == event.step_id: + stack.pop() + try: + self._session.journal.append_end(self._ended(event, exc)) + except OSError as error: + _fail(self._session, error) + + def _ended(self, event: ActionEvent, exc: Optional[BaseException]) -> ActionEvent: + """``event`` with the status its exit earned.""" + status, error, outcome = STATUS_OK, None, self._outcome + if exc is not None and isinstance(exc, _control_signals()): + # AC_break / AC_continue unwinding through a block is the block + # doing its job, not a failure. + outcome = {"type": "signal", "signal": type(exc).__name__} + elif isinstance(exc, Exception): + status, error = STATUS_ERROR, _error_text(exc) + elif exc is not None: + # KeyboardInterrupt / SystemExit: the action never finished. + status, error = STATUS_INCOMPLETE, _error_text(exc) + return dataclasses.replace( + event, status=status, error=error, outcome=outcome, finished_at=time.time()) + + +def _fail(session: _Session, error: OSError) -> None: + """Stop journalling after a write failed; the automation itself goes on.""" + global _ACTIVE, _LAST + session.error = repr(error) + autocontrol_logger.error( + "action journal %s stopped: %r", session.journal.path, error) + with _SWITCH: + if _ACTIVE is session: + _ACTIVE, _LAST = None, session + + +def step(action: Any) -> Union[_Step, _NullStep]: + """A context manager recording ``action``; a shared no-op when journalling is off.""" + session = _ACTIVE + if session is None: + return _NULL + return _Step(session, action) + + +def current_step() -> Optional[str]: + """The id of the step running on this thread, when a journal is started.""" + if _ACTIVE is None: + return None + stack = getattr(_LOCAL, "stack", None) + return stack[-1] if stack else getattr(_LOCAL, "parent", None) + + +class _BranchScope: + """Binds the parent step and branch index of a worker thread for a block.""" + + __slots__ = ("_parent", "_index") + + def __init__(self, parent: Optional[str], index: Optional[int] = None) -> None: + self._parent = parent + self._index = index + + def __enter__(self) -> "_BranchScope": + _LOCAL.parent = self._parent + _LOCAL.branch = self._index + return self + + def __exit__(self, *_exc: Any) -> None: + _LOCAL.parent = None + _LOCAL.branch = None + + +def branch_scope(parent: Optional[str], index: Optional[int] = None) -> _BranchScope: + """Make ``parent`` the parent of what this (new) thread runs, as branch ``index``.""" + return _BranchScope(parent, index) + + +def start_action_journal(path: Union[str, Path, None] = None, *, + run_id: Optional[str] = None, + session: Optional[str] = None) -> Dict[str, Any]: + """Start writing every executed action to the journal at ``path``. + + ``path`` defaults to ``~/.je_auto_control/action_journal.jsonl``; the file + is appended to, never truncated. Everything recorded until + :func:`stop_action_journal` belongs to ``run_id`` (a fresh one when not + given). ``session`` is a free label -- a device or session name -- stored + on each event. Raises :class:`ActionJournalError` if one is already started. + """ + global _ACTIVE + journal = ActionJournal(Path(path).resolve() if path is not None else None) + journal.path.parent.mkdir(parents=True, exist_ok=True) + started = _Session(journal, str(run_id) if run_id else uuid.uuid4().hex, session) + with _SWITCH: + if _ACTIVE is not None: + raise ActionJournalError( + f"an action journal is already started ({_ACTIVE.journal.path})") + _ACTIVE = started + return started.status() + + +def stop_action_journal() -> Dict[str, Any]: + """Stop journalling; return the stopped session (path, run id, event count). + + Actions still running finish their records. Stopping when nothing is + started is not an error: the result says ``"active": False``. + """ + global _ACTIVE, _LAST + with _SWITCH: + stopped, _ACTIVE = _ACTIVE, None + _LAST = stopped or _LAST + if stopped is None: + return {"active": False} + return {"active": False, **stopped.status()} + + +def action_journal_status() -> Dict[str, Any]: + """Whether a journal is started, and where it writes.""" + session, last = _ACTIVE, _LAST + if session is not None: + return {"active": True, **session.status()} + return {"active": False, **({"last": last.status()} if last is not None else {})} diff --git a/je_auto_control/utils/action_journal/sanitize.py b/je_auto_control/utils/action_journal/sanitize.py new file mode 100644 index 000000000..0a50f9f69 --- /dev/null +++ b/je_auto_control/utils/action_journal/sanitize.py @@ -0,0 +1,114 @@ +"""Make an action's arguments safe to persist, before they are persisted. + +The journal is a file other tools read, so a secret has to be gone *before* +the line is written -- masking a line already on disk leaves the secret in +the file's history. :func:`sanitise_params` reuses the executor's own +redaction (``utils/executor/action_redaction``) and adds what a journal needs +on top of a log line: + +* a value that is only a reference (``${secrets.NAME}``, ``${var}``) is kept: + it names a secret without containing one, and it is what makes the step + replayable; +* every masked value and every value JSON cannot hold is listed with the + reason it cannot be replayed, instead of being silently wrong. + +Pure standard library; imports no ``PySide6``. +""" +import math +from typing import Any, Callable, Dict, Mapping, Tuple + +from je_auto_control.utils.action_journal.events import MASK, UNSERIALISABLE_KEY +from je_auto_control.utils.executor.action_redaction import redact_actions +from je_auto_control.utils.script_vars.interpolate import _PLACEHOLDER + +REASON_SECRET = "secret masked before the journal was written" +_REASON_TYPE = "not JSON-serialisable ({})" +_REASON_FLOAT = "non-finite number ({})" +_SCALARS = (str, int, bool, type(None)) + + +def _plain(value: Any, path: str, notes: Dict[str, str]) -> Any: + """``value`` as JSON-ready data; what JSON cannot hold becomes a marker.""" + if isinstance(value, float): + if math.isfinite(value): + return value + notes[path] = _REASON_FLOAT.format(value) + return {UNSERIALISABLE_KEY: "float"} + if isinstance(value, _SCALARS): + return value + if isinstance(value, Mapping): + return {str(key): _plain(item, f"{path}.{key}", notes) + for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(item, f"{path}[{index}]", notes) + for index, item in enumerate(value)] + # The type name only: a repr can hold a secret and cannot be parsed back. + kind = type(value).__qualname__ + notes[path] = _REASON_TYPE.format(kind) + return {UNSERIALISABLE_KEY: kind} + + +def _is_reference(value: Any) -> bool: + return isinstance(value, str) and _PLACEHOLDER.fullmatch(value) is not None + + +def _settle_masked(plain: Any, path: str, notes: Dict[str, str]) -> Any: + if _is_reference(plain): + return plain + notes[path] = REASON_SECRET + return MASK + + +def _settle(plain: Any, masked: Any, path: str, notes: Dict[str, str]) -> Any: + """Keep references the redaction masked; note everything else it masked.""" + if masked == MASK and plain != MASK: + return _settle_masked(plain, path, notes) + if isinstance(plain, dict) and isinstance(masked, dict): + return {key: _settle(item, masked.get(key, MASK), f"{path}.{key}", notes) + for key, item in plain.items()} + if isinstance(plain, list) and isinstance(masked, list) and len(plain) == len(masked): + return [_settle(item, masked[index], f"{path}[{index}]", notes) + for index, item in enumerate(plain)] + return masked + + +def sanitise_params(command: str, params: Any) -> Tuple[Any, Dict[str, str]]: + """``(safe_params, unreplayable)`` for one action's argument value. + + ``unreplayable`` maps a path such as ``params.body[0][1].password`` to why + the value there is not what the action was given. + """ + notes: Dict[str, str] = {} + plain = _plain(params, "params", notes) + masked = redact_actions([command, plain])[1] + return _settle(plain, masked, "params", notes), notes + + +def replace_unreplayable(params: Any, unreplayable: Mapping[str, str], + substitute: Callable[[str], Any], path: str = "params") -> Any: + """A copy of ``params`` with each unreplayable value swapped for ``substitute(path)``.""" + if path in unreplayable: + return substitute(path) + if isinstance(params, dict): + return {key: replace_unreplayable(item, unreplayable, substitute, f"{path}.{key}") + for key, item in params.items()} + if isinstance(params, list): + return [replace_unreplayable(item, unreplayable, substitute, f"{path}[{index}]") + for index, item in enumerate(params)] + return params + + +def describe_outcome(value: Any) -> Dict[str, Any]: + """What an action returned, by type and size -- never its text. + + A returned string may be a secret read from the vault, the clipboard or + the screen, so only numbers and booleans are stored as values. + """ + kind = type(value).__qualname__ + if value is None or isinstance(value, bool): + return {"type": kind, "value": value} + if isinstance(value, int) or (isinstance(value, float) and math.isfinite(value)): + return {"type": kind, "value": value} + if isinstance(value, (str, bytes, list, tuple, dict, set, frozenset)): + return {"type": kind, "size": len(value)} + return {"type": kind} diff --git a/je_auto_control/utils/action_journal/store.py b/je_auto_control/utils/action_journal/store.py new file mode 100644 index 000000000..97c2865ce --- /dev/null +++ b/je_auto_control/utils/action_journal/store.py @@ -0,0 +1,141 @@ +"""Append-only JSON-lines store for :class:`ActionEvent` records. + +A started action is written at once as a whole ``start`` line with status +``incomplete``; how it ended follows as a short ``end`` line. Reading folds +the two together, so an action whose end never reached the file -- the +process died, the run was interrupted -- reads back as ``incomplete`` instead +of looking like a success. Lines are parsed with ``json`` only. + +Pure standard library; imports no ``PySide6``. +""" +import json +import os +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from je_auto_control.utils.action_journal.events import ( + RECORD_END, STATUS_ERROR, STATUS_INCOMPLETE, STATUS_OK, ActionEvent, + JournalFormatError, apply_end, check_record, event_from_dict, +) +from je_auto_control.utils.json_store.json_store import append_json_line + + +def default_journal_path() -> Path: + """``~/.je_auto_control/action_journal.jsonl``, resolved at call time.""" + return Path.home() / ".je_auto_control" / "action_journal.jsonl" + + +class ActionJournal: + """Thread-safe append-only journal file. Appending creates it.""" + + def __init__(self, path: Union[str, Path, None] = None) -> None: + self._path = Path(path) if path is not None else default_journal_path() + self._lock = threading.Lock() + + @property + def path(self) -> Path: + """The file this journal appends to.""" + return self._path + + def append(self, event: ActionEvent) -> None: + """Append ``event`` as one whole line (the start of an action).""" + self._write(event.to_dict()) + + def append_end(self, event: ActionEvent) -> None: + """Append how the already-written ``event`` ended.""" + self._write(event.end_dict()) + + def _write(self, record: Dict[str, Any]) -> None: + # default=str is never reached for a sanitised event; it only keeps a + # hand-built one from raising half-way through a run. + line = json.dumps(record, ensure_ascii=False, default=str) + with self._lock: + append_json_line(self._path, line) + + def read(self, *, run_id: Optional[str] = None) -> List[ActionEvent]: + """The events of this journal, in the order they started.""" + return read_events(self._path, run_id=run_id) + + +@dataclass(frozen=True) +class JournalContents: + """What a journal file holds: the folded events and what could not be read.""" + + events: Tuple[ActionEvent, ...] + #: ``step_id`` -> 1-based line number of the event's ``start`` line. + lines: Dict[str, int] = field(default_factory=dict) + #: 1-based numbers of lines that are not JSON (a write cut off mid-line). + torn_lines: Tuple[int, ...] = () + + +def _fold(path: Path, run_id: Optional[str]) -> JournalContents: + """Parse every line of ``path`` and fold ``end`` lines into their starts.""" + events: Dict[Tuple[str, str], ActionEvent] = {} + lines: Dict[str, int] = {} + torn: List[int] = [] + with path.open("r", encoding="utf-8", errors="replace") as handle: + for number, raw in enumerate(handle, start=1): + if not raw.strip(): + continue + where = f"{path.name} line {number}" + try: + data = json.loads(raw) + except ValueError: + torn.append(number) + continue + record = check_record(data, where) + if run_id is not None and data["run_id"] != run_id: + continue + key = (data["run_id"], data["step_id"]) + if record == RECORD_END: + # An end whose start is missing has nothing to describe. + if key in events: + events[key] = apply_end(events[key], data, where) + continue + if key in events: + raise JournalFormatError(f"{where}: step {key[1]!r} is recorded twice") + events[key] = event_from_dict(data, where) + lines[key[1]] = number + return JournalContents(tuple(events.values()), lines, tuple(torn)) + + +def load_journal(path: Union[str, Path], *, run_id: Optional[str] = None + ) -> JournalContents: + """Read ``path`` with line numbers and the torn lines it skipped. + + A line that is not JSON is skipped and reported (an append cut off by a + crash is expected in an append-only file); a JSON line that is not a + journal record of this schema raises :class:`JournalFormatError`. + """ + file_path = Path(os.path.realpath(os.fspath(path))) + if not file_path.is_file(): + raise JournalFormatError(f"no journal file at {file_path}") + return _fold(file_path, run_id) + + +def read_events(path: Union[str, Path], *, run_id: Optional[str] = None + ) -> List[ActionEvent]: + """The events in the journal at ``path``, in the order they started. + + ``run_id`` keeps one run's events. An action with no recorded end has + status ``incomplete``. + """ + return list(load_journal(path, run_id=run_id).events) + + +def list_journal_runs(path: Union[str, Path]) -> List[Dict[str, Any]]: + """One summary per run in the journal at ``path``, oldest first.""" + runs: Dict[str, Dict[str, Any]] = {} + for event in load_journal(path).events: + run = runs.setdefault(event.run_id, { + "run_id": event.run_id, "session": event.session, + "started_at": event.started_at, "finished_at": None, "events": 0, + STATUS_OK: 0, STATUS_ERROR: 0, STATUS_INCOMPLETE: 0, + }) + run["events"] += 1 + run[event.status] += 1 + if event.finished_at is not None: + run["finished_at"] = max(run["finished_at"] or 0.0, event.finished_at) + return list(runs.values()) diff --git a/je_auto_control/utils/codegen/__init__.py b/je_auto_control/utils/codegen/__init__.py index 83dd5826b..1bc355a79 100644 --- a/je_auto_control/utils/codegen/__init__.py +++ b/je_auto_control/utils/codegen/__init__.py @@ -4,4 +4,13 @@ generate_code_file, ) -__all__ = ["generate_code", "generate_code_file"] +from je_auto_control.utils.codegen.journal_import import ( + CandidateScript, + JournalImportError, + generate_candidate_from_log, +) + +__all__ = [ + "CandidateScript", "JournalImportError", "generate_candidate_from_log", + "generate_code", "generate_code_file", +] diff --git a/je_auto_control/utils/codegen/journal_import.py b/je_auto_control/utils/codegen/journal_import.py new file mode 100644 index 000000000..c122be6a5 --- /dev/null +++ b/je_auto_control/utils/codegen/journal_import.py @@ -0,0 +1,392 @@ +"""Turn one run of an action journal into a candidate script for review. + +The journal (``utils/action_journal``) holds what the executor ran. This reads +one run of it and rebuilds an action list from the *inputs* it recorded -- +never from outcomes, never by evaluating text -- then renders it through the +existing :func:`~je_auto_control.utils.codegen.codegen.generate_code`. + +What is rebuilt, and what is only observed: + +* A top-level step is emitted as it was written, block body and all: the + journal stored the whole ``AC_loop`` / ``AC_if_*`` / ``AC_parallel``, so its + control flow is recovered, not guessed. +* Where the block itself cannot be rebuilt -- its start is not in the log, its + arguments could not be stored, it calls a macro defined outside the run -- + the steps that actually ran beneath it are emitted in order instead, marked + ``observed``, and the candidate says ``observed_path_only``: it replays the + path this run took, not the script's logic. A branch that did not run is + never invented. +* Inside an observed ``AC_retry`` only the last attempt is kept; the manifest + counts the attempts. +* A masked secret is not replayable. Its place is taken by a + ``${journal_redacted_N_M}`` reference, which fails as an unknown variable + until it is replaced; a ``${secrets.NAME}`` reference in the journal is kept. + +The candidate is checked without touching a device: the source is parsed, the +command names are looked up, and the action list goes through the executor's +dry run. Nothing is executed. + +This module imports no ``PySide6``. +""" +import ast +import json +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union + +from je_auto_control.utils.action_journal.events import ( + SCHEMA_VERSION, STATUS_ERROR, STATUS_INCOMPLETE, STATUS_OK, ActionEvent, + JournalFormatError, +) +from je_auto_control.utils.action_journal.sanitize import ( + REASON_SECRET, replace_unreplayable, +) +from je_auto_control.utils.action_journal.store import ( + JournalContents, list_journal_runs, load_journal, +) +from je_auto_control.utils.codegen.codegen import generate_code +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.json_store.json_store import atomic_write_text +from je_auto_control.utils.executor.action_schema import ( + FLOW_BODY_KEYS, FLOW_BRANCH_LIST_KEYS, unknown_command_names, +) + +MODE_RECORDED = "recorded" +MODE_OBSERVED = "observed" +MODE_DETACHED = "detached" + +_TARGETS = ("pytest", "python", "robot") +_STYLES = ("calls", "actions") +_MACRO_CALL = "AC_call_macro" +_MACRO_DEFINE = "AC_define_macro" +_RETRY = "AC_retry" + + +class JournalImportError(AutoControlException, ValueError): + """The journal run cannot be turned into a candidate script.""" + + +@dataclass(frozen=True) +class CandidateScript: + """A generated script, where each step came from, and what to check. + + ``actions`` is the action list ``code`` replays -- what a recording + editor or the Script Builder opens. ``observed_path_only`` is true when + some control flow could not be rebuilt and the steps that ran stand in + for it. + """ + + code: str + manifest: Dict[str, Any] + warnings: Tuple[str, ...] = () + actions: List[Any] = field(default_factory=list) + observed_path_only: bool = False + + def to_dict(self) -> Dict[str, Any]: + """The candidate as a JSON-ready dict.""" + return {"code": self.code, "manifest": self.manifest, + "warnings": list(self.warnings), "actions": self.actions, + "observed_path_only": self.observed_path_only} + + +def _nesting_commands() -> frozenset: + """Commands whose arguments hold action lists the executor runs.""" + from je_auto_control.utils.executor.action_executor import Executor + return frozenset(FLOW_BODY_KEYS) | frozenset(FLOW_BRANCH_LIST_KEYS) | frozenset( + Executor._DEFERRED_COMMAND_KEYS) | {_MACRO_CALL} + + +class _Run: + """One run's events as a tree, in the order they started.""" + + def __init__(self, contents: JournalContents) -> None: + self.events = contents.events + self.lines = contents.lines + self.children: Dict[str, List[ActionEvent]] = {} + self.roots: List[ActionEvent] = [] + self.orphans: set = set() + seen: set = set() + for event in self.events: + if event.parent_id is None: + self.roots.append(event) + elif event.parent_id in seen: + self.children.setdefault(event.parent_id, []).append(event) + else: + # Its parent started before the journal did (or was lost). + self.roots.append(event) + self.orphans.add(event.step_id) + seen.add(event.step_id) + + def detached(self, nesting: frozenset) -> set: + """Roots that ran on another thread *inside* a nesting root step. + + A runner that hands work to a thread pool (the DAG runner, a device + matrix) records its steps without a parent. Emitting them next to the + block that ran them would run them twice. + """ + hosts = [root for root in self.roots + if root.command in nesting and root.finished_at is not None] + return {root.step_id for root in self.roots for host in hosts + if _inside(root, host)} + + +def _inside(event: ActionEvent, host: ActionEvent) -> bool: + return (event is not host and event.thread != host.thread + and host.started_at <= event.started_at + and event.started_at <= (host.finished_at or 0.0)) + + +def _attempts(parent: ActionEvent, children: Sequence[ActionEvent] + ) -> List[List[ActionEvent]]: + """The children of an ``AC_retry`` step, split into its attempts. + + The body runs strictly, so an attempt ends at its first failed step or + after as many steps as the body holds. + """ + body = parent.params.get("body") if isinstance(parent.params, dict) else None + size = len(body) if isinstance(body, list) and body else len(children) or 1 + attempts: List[List[ActionEvent]] = [[]] + for child in children: + attempts[-1].append(child) + if child.status != STATUS_OK or len(attempts[-1]) >= size: + attempts.append([]) + return [attempt for attempt in attempts if attempt] + + +class _Builder: + """Collects the candidate's actions, manifest steps and warnings.""" + + def __init__(self, run: _Run) -> None: + self.run = run + self.nesting = _nesting_commands() + self.detached = run.detached(self.nesting) + self.actions: List[Any] = [] + self.steps: List[Dict[str, Any]] = [] + self.warnings: List[str] = [] + self.observed = False + self._macros: set = set() + + def build(self) -> None: + """Walk every root step of the run.""" + for root in self.run.roots: + if root.step_id in self.detached: + self._note(root, MODE_DETACHED, emitted=False) + self.warnings.append( + f"{self._where(root)} ran on another thread inside a block of this run " + "with no recorded parent; it is part of that block and is not emitted") + continue + self._emit(root, observed=root.step_id in self.run.orphans) + self._summarise() + + def _where(self, event: ActionEvent) -> str: + return f"line {self.run.lines.get(event.step_id, '?')} ({event.command})" + + def _note(self, event: ActionEvent, mode: str, emitted: bool, + **extra: Any) -> None: + self.steps.append({ + "step_id": event.step_id, "parent_id": event.parent_id, + "line": self.run.lines.get(event.step_id), "sequence": event.sequence, + "command": event.command, "status": event.status, "mode": mode, + "emitted": emitted, "branch": event.branch, + "action_index": len(self.actions) if emitted else None, + "unreplayable": dict(event.unreplayable), **extra}) + + def _unrecoverable(self, event: ActionEvent) -> Optional[str]: + """Why ``event``'s own arguments cannot stand for what ran under it.""" + if event.command not in self.nesting: + return None + if event.command == _MACRO_CALL: + name = event.params.get("name") if isinstance(event.params, dict) else None + return None if name in self._macros else ( + f"macro {name!r} is not defined in this run") + lost = [path for path, reason in event.unreplayable.items() + if reason != REASON_SECRET] + if lost: + return f"{', '.join(sorted(lost))} could not be stored" + return None + + def _emit(self, event: ActionEvent, observed: bool) -> None: + children = self.run.children.get(event.step_id, []) + reason = self._unrecoverable(event) + if reason is not None and children: + self._emit_observed_path(event, children, reason) + return + if reason is not None: + self.warnings.append( + f"{self._where(event)}: {reason}, and no step beneath it was recorded; " + "emitted as written") + if observed: + self.observed = True + self.warnings.append( + f"{self._where(event)} ran inside a block the journal does not hold; " + "emitted as an observed step") + extra = {} + if event.command == _RETRY: + extra["attempts"] = len(_attempts(event, children)) + self._note(event, MODE_OBSERVED if observed else MODE_RECORDED, True, **extra) + self.actions.append(self._action(event)) + self._remember_macro(event) + + def _emit_observed_path(self, event: ActionEvent, children: Sequence[ActionEvent], + reason: str) -> None: + """Stand the steps that ran under ``event`` in for the block itself.""" + self.observed = True + kept = list(children) + extra: Dict[str, Any] = {} + if event.command == _RETRY: + attempts = _attempts(event, children) + kept, extra["attempts"] = attempts[-1], len(attempts) + self.warnings.append( + f"{self._where(event)}: {reason}; its control flow is not rebuilt -- the " + f"{len(kept)} step(s) that ran beneath it are emitted as the observed path") + self._note(event, MODE_OBSERVED, False, **extra) + for child in kept: + self._emit(child, observed=True) + + def _remember_macro(self, event: ActionEvent) -> None: + if event.command == _MACRO_DEFINE and isinstance(event.params, dict): + self._macros.add(event.params.get("name")) + + def _action(self, event: ActionEvent) -> List[Any]: + """The action ``event`` recorded, with unreplayable values made loud.""" + if event.params is None: + return [event.command] + index = len(self.actions) + names: Dict[str, str] = {} + + def substitute(path: str) -> str: + names[path] = f"journal_redacted_{index}_{len(names) + 1}" + return "${" + names[path] + "}" + + params = replace_unreplayable(event.params, event.unreplayable, substitute) + for path, name in names.items(): + self.warnings.append( + f"{self._where(event)}: {path} is not replayable " + f"({event.unreplayable[path]}); the candidate uses ${{{name}}} -- " + "replace it (a ${secrets.NAME} reference for a secret) before running") + return [event.command, params] + + def _summarise(self) -> None: + emitted = [step for step in self.steps if step["emitted"]] + for status, text in ((STATUS_ERROR, "failed"), + (STATUS_INCOMPLETE, "did not finish")): + lines = [str(step["line"]) for step in emitted if step["status"] == status] + if lines: + self.warnings.append( + f"{len(lines)} emitted step(s) {text} in the recorded run " + f"(line {', '.join(lines)}); they are kept because they are the " + "script's input, not its result") + + +def _check_order(contents: JournalContents, warnings: List[str]) -> None: + """Warn when a session's sequence numbers do not rise with the file.""" + last: Dict[str, int] = {} + for event in contents.events: + token = event.step_id.rpartition("-")[0] + if event.sequence <= last.get(token, 0): + warnings.append( + f"line {contents.lines.get(event.step_id)}: sequence {event.sequence} " + "is out of order; steps are taken in file order") + last[token] = max(event.sequence, last.get(token, 0)) + if contents.torn_lines: + warnings.append( + "skipped unreadable line(s) " + + ", ".join(str(number) for number in contents.torn_lines) + + " (a write that was cut off)") + + +def _validate(code: str, actions: List[Any], target: str, + warnings: List[str]) -> Dict[str, Any]: + """Check the product without running it: syntax, command names, dry run.""" + from je_auto_control.utils.executor.action_executor import Executor + checks: Dict[str, Any] = {"ast": None, "unknown_commands": [], "dry_run": None} + if target != "robot": + ast.parse(code) # a SyntaxError here is a codegen defect; let it out + checks["ast"] = True + # A private executor: the dry run never touches the shared one's state. + checker = Executor() + unknown = unknown_command_names(actions, checker.known_commands()) + checks["unknown_commands"] = unknown + if unknown: + warnings.append( + "unknown command(s) " + ", ".join(unknown) + + " -- from a plugin that is not loaded here? The dry run was skipped") + checks["dry_run"] = False + return checks + checks["dry_run"] = len(checker.execute_action(actions, dry_run=True)) == len(actions) + return checks + + +def _script_name(run_id: str) -> str: + return "journal_run_" + re.sub(r"\W+", "_", run_id)[:40] + + +def generate_candidate_from_log(path: Union[str, Path], *, run_id: str, + target: str = "pytest", + style: str = "actions") -> CandidateScript: + """Build a reviewable candidate script from run ``run_id`` of a journal. + + ``target`` is ``pytest`` / ``python`` / ``robot`` and ``style`` ``actions`` + (embed the list, exact) or ``calls``, as for ``generate_code``. Raises + :class:`JournalFormatError` for a file that is not a journal of this + schema and :class:`JournalImportError` when the run holds nothing to + emit. Nothing read from the log is executed. + """ + if target not in _TARGETS: + raise JournalImportError(f"unknown codegen target: {target!r}") + if style not in _STYLES: + raise JournalImportError(f"unknown codegen style: {style!r}") + journal_path = os.path.realpath(os.fspath(path)) + contents = load_journal(journal_path, run_id=run_id) + if not contents.events: + raise JournalImportError(f"run {run_id!r} has no events in {journal_path}") + builder = _Builder(_Run(contents)) + builder.build() + if not builder.actions: + raise JournalImportError(f"run {run_id!r} has no step that can be emitted") + warnings = builder.warnings + _check_order(contents, warnings) + code = generate_code(builder.actions, target=target, + name=_script_name(run_id), style=style) + manifest = { + "schema_version": SCHEMA_VERSION, "journal": journal_path, "run_id": run_id, + "target": target, "style": style, "event_count": len(contents.events), + "action_count": len(builder.actions), + "observed_path_only": builder.observed, + "outcomes_used_as_input": False, "executed": False, + "steps": builder.steps, + "validation": _validate(code, builder.actions, target, warnings), + } + return CandidateScript(code=code, manifest=manifest, warnings=tuple(warnings), + actions=builder.actions, observed_path_only=builder.observed) + + +def only_run_id(path: Union[str, Path]) -> str: + """The id of the journal's single run; raise when there is none or several.""" + runs = [run["run_id"] for run in list_journal_runs(os.path.realpath(os.fspath(path)))] + if len(runs) != 1: + raise JournalImportError( + f"the journal holds {len(runs)} run(s); name one with run_id: " + + (", ".join(runs) or "(none)")) + return runs[0] + + +def write_candidate(candidate: CandidateScript, output: Union[str, Path], + manifest: Optional[Union[str, Path]] = None) -> Mapping[str, str]: + """Write the candidate's code (and its manifest, when a path is given).""" + written = {"output": os.path.realpath(os.fspath(output))} + atomic_write_text(written["output"], candidate.code) + if manifest: + written["manifest"] = os.path.realpath(os.fspath(manifest)) + atomic_write_text(written["manifest"], json.dumps( + {**candidate.manifest, "warnings": list(candidate.warnings)}, + ensure_ascii=False, indent=2)) + return written + + +__all__ = [ + "CandidateScript", "JournalFormatError", "JournalImportError", + "generate_candidate_from_log", "only_run_id", "write_candidate", +] diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cc475cf85..c6c2fa037 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -33,6 +33,7 @@ BLOCK_COMMANDS, LoopBreak, LoopContinue, MacroDepthExceeded, ) from je_auto_control.utils.executor.action_redaction import describe_action, redact_actions +from je_auto_control.utils.action_journal.recorder import step as _journal_step from je_auto_control.utils.executor.mouse_aliases import MOUSE_BUTTON_COMMANDS from je_auto_control.utils.llm.planner import ( plan_actions as llm_plan_actions, @@ -2407,6 +2408,55 @@ def _generate_code(source: Any, output: Optional[str] = None, return generate_code(actions, target=target, name=name, style=style) +def _journal_start(path: Optional[str] = None, run_id: Optional[str] = None, + session: Optional[str] = None) -> Dict[str, Any]: + """Start journalling every executed action to ``path``.""" + from je_auto_control.utils.action_journal.recorder import start_action_journal + return start_action_journal(path or None, run_id=run_id or None, + session=session or None) + + +def _journal_stop() -> Dict[str, Any]: + """Stop the action journal.""" + from je_auto_control.utils.action_journal.recorder import stop_action_journal + return stop_action_journal() + + +def _journal_status() -> Dict[str, Any]: + """Report whether an action journal is started.""" + from je_auto_control.utils.action_journal.recorder import action_journal_status + return action_journal_status() + + +def _journal_read(path: str, run_id: Optional[str] = None, + limit: int = 0) -> List[Dict[str, Any]]: + """Read a journal's events as dicts (the last ``limit`` when positive).""" + from je_auto_control.utils.action_journal.store import read_events + events = read_events(path, run_id=run_id or None) + return [event.to_dict() for event in events[-int(limit):]] + + +def _journal_runs(path: str) -> List[Dict[str, Any]]: + """Summarise each run a journal file holds.""" + from je_auto_control.utils.action_journal.store import list_journal_runs + return list_journal_runs(path) + + +def _generate_code_from_journal(path: str, run_id: Optional[str] = None, + target: str = "pytest", style: str = "actions", + output: Optional[str] = None) -> Dict[str, Any]: + """Build a candidate script from one journal run, optionally writing it.""" + from je_auto_control.utils.codegen.journal_import import ( + generate_candidate_from_log, only_run_id, write_candidate, + ) + candidate = generate_candidate_from_log( + path, run_id=run_id or only_run_id(path), target=target, style=style) + result = candidate.to_dict() + if output: + result.update(write_candidate(candidate, output)) + return result + + def _send_email(message: Any, smtp: Any) -> Dict[str, Any]: """Adapter: send an email via SMTP (message/smtp config dicts).""" from je_auto_control.utils.email_send.email_sender import send_email @@ -7247,6 +7297,12 @@ def __init__(self): "AC_generate_json_report": generate_json_report, "AC_generate_xml_report": generate_xml_report, "AC_generate_code": _generate_code, + "AC_generate_code_from_journal": _generate_code_from_journal, + "AC_journal_start": _journal_start, + "AC_journal_stop": _journal_stop, + "AC_journal_status": _journal_status, + "AC_journal_read": _journal_read, + "AC_journal_runs": _journal_runs, "AC_send_email": _send_email, "AC_assert_pdf_text": _assert_pdf_text, "AC_take_golden": _take_golden, @@ -8292,8 +8348,9 @@ def _run_one_action(self, action: list, record: Dict[str, Any], action_name = action[0] if action and isinstance(action[0], str) else "" started = _time.monotonic() try: - with default_profiler.measure(action_name): + with default_profiler.measure(action_name), _journal_step(action) as step: record[key] = self._execute_event(action) + step.outcome(record[key]) _observe_executor_metrics(action_name, started, error=None) except (LoopBreak, LoopContinue): raise diff --git a/je_auto_control/utils/executor/flow_control.py b/je_auto_control/utils/executor/flow_control.py index 91c0a5233..5f09dfbb7 100644 --- a/je_auto_control/utils/executor/flow_control.py +++ b/je_auto_control/utils/executor/flow_control.py @@ -10,6 +10,7 @@ import time from typing import Any, Callable, Dict, Mapping, Optional, Sequence +from je_auto_control.utils.action_journal.recorder import branch_scope, current_step from je_auto_control.utils.exception.exceptions import ( AutoControlActionException, AutoControlAssertionException, AutoControlException, ImageNotFoundException, @@ -469,6 +470,8 @@ def __init__(self, executor: Any, branches: list) -> None: # The RBAC user is bound to the thread that accepted the request; a # branch thread without it ran privileged commands unchecked. self._caller = current_authorization() + # A branch thread has no running step of its own to be the parent. + self._journal_parent = current_step() self.results: list = [None] * len(branches) self._failures: list = [0] * len(branches) self._errors: Dict[int, str] = {} @@ -498,7 +501,7 @@ def run_branch(self, index: int, branch: Any) -> None: _MACRO_DEPTH.value = self._macro_depth self._module.reset_recorded_failures() try: - with authorization_scope(self._caller): + with authorization_scope(self._caller), branch_scope(self._journal_parent, index): self.results[index] = self._branch_executor().execute_action( branch, raise_on_error=self._strict, _validated=True) self._failures[index] = self._module.recorded_failures() diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fecbc25ae..d6727a982 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -553,6 +553,61 @@ def system_tools() -> List[MCPTool]: handler=h_runs.list_run_history, annotations=READ_ONLY, ), + MCPTool( + name="ac_journal_start", + description=("Start the structured action journal: every action " + "the executor runs from now on (any entry point) is " + "appended to a JSON-lines file with its run/step/" + "parent ids, arguments (secrets masked before they " + "are written) and how it ended. path defaults to " + "~/.je_auto_control/action_journal.jsonl. Returns " + "the path and the run_id."), + input_schema=schema({ + "path": {"type": "string", "format": "path"}, + "run_id": {"type": "string"}, + "session": {"type": "string", + "description": "Free label (device / session) stored on each event."}, + }), + handler=h_runs.journal_start, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_journal_stop", + description="Stop the action journal; returns its path, run_id and event count.", + input_schema=schema({}), + handler=h_runs.journal_stop, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_journal_status", + description="Report whether an action journal is started and where it writes.", + input_schema=schema({}), + handler=h_runs.journal_status, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_journal_read", + description=("Read the events of an action journal file, oldest " + "first. run_id keeps one run; limit keeps the last N. " + "A step with no recorded end has status 'incomplete'."), + input_schema=schema({ + "path": {"type": "string", "format": "path"}, + "run_id": {"type": "string"}, + "limit": {"type": "integer", "minimum": 0}, + }, required=["path"]), + handler=h_runs.journal_read, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_journal_runs", + description=("List the runs an action journal file holds, with " + "their event and ok/error/incomplete counts."), + input_schema=schema({ + "path": {"type": "string", "format": "path"}, + }, required=["path"]), + handler=h_runs.journal_runs, + annotations=READ_ONLY, + ), ] @@ -8697,6 +8752,29 @@ def codegen_tools() -> List[MCPTool]: handler=hq.generate_code, annotations=SIDE_EFFECT_ONLY, ), + MCPTool( + name="ac_generate_code_from_log", + description=("Build a reviewable candidate script from one run of " + "an action journal (see ac_journal_start). Returns " + "code, the action list, a provenance manifest (each " + "step's journal line and whether it was recorded or " + "only observed), warnings and observed_path_only. " + "Nothing from the log is executed: the candidate is " + "parsed, its command names checked and dry-run only. " + "run_id may be omitted when the journal holds one run. " + "Pass 'output' to also write the code."), + input_schema=schema({ + "path": {"type": "string", "format": "path", + "description": "The journal (.jsonl) file."}, + "run_id": {"type": "string"}, + "target": {"type": "string", + "enum": ["pytest", "python", "robot"]}, + "style": {"type": "string", "enum": ["calls", "actions"]}, + "output": {"type": "string", "format": "path"}, + }, required=["path"]), + handler=hq.generate_code_from_log, + annotations=SIDE_EFFECT_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_qa.py b/je_auto_control/utils/mcp_server/tools/_handlers_qa.py index 0456ab3a0..bbb4ea5e4 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_qa.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_qa.py @@ -236,6 +236,20 @@ def generate_code(source: Any, target: str = "pytest", return {"code": _gen(actions, target=target, name=name, style=style)} +def generate_code_from_log(path: str, run_id: Optional[str] = None, + target: str = "pytest", style: str = "actions", + output: Optional[str] = None) -> Dict[str, Any]: + from je_auto_control.utils.codegen.journal_import import ( + generate_candidate_from_log, only_run_id, write_candidate, + ) + candidate = generate_candidate_from_log( + path, run_id=run_id or only_run_id(path), target=target, style=style) + result = candidate.to_dict() + if output: + result.update(write_candidate(candidate, output)) + return result + + # --- Visual regression + state machine ------------------------------------- def take_golden(path: str, region: Optional[List[int]] = None) -> str: diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_runs.py b/je_auto_control/utils/mcp_server/tools/_handlers_runs.py index f2fb7e6bc..e51af5335 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_runs.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_runs.py @@ -45,6 +45,35 @@ def list_run_history(limit: int = 50, } for row in rows] +def journal_start(path: Optional[str] = None, run_id: Optional[str] = None, + session: Optional[str] = None) -> Dict[str, Any]: + from je_auto_control.utils.action_journal.recorder import start_action_journal + return start_action_journal(path or None, run_id=run_id or None, + session=session or None) + + +def journal_stop() -> Dict[str, Any]: + from je_auto_control.utils.action_journal.recorder import stop_action_journal + return stop_action_journal() + + +def journal_status() -> Dict[str, Any]: + from je_auto_control.utils.action_journal.recorder import action_journal_status + return action_journal_status() + + +def journal_read(path: str, run_id: Optional[str] = None, + limit: int = 0) -> List[Dict[str, Any]]: + from je_auto_control.utils.action_journal.store import read_events + events = read_events(path, run_id=run_id or None) + return [event.to_dict() for event in events[-int(limit):]] + + +def journal_runs(path: str) -> List[Dict[str, Any]]: + from je_auto_control.utils.action_journal.store import list_journal_runs + return list_journal_runs(path) + + def record_start() -> str: from je_auto_control.wrapper.auto_control_record import record record() diff --git a/test/unit_test/headless/test_action_journal.py b/test/unit_test/headless/test_action_journal.py new file mode 100644 index 000000000..3b6e0ea4b --- /dev/null +++ b/test/unit_test/headless/test_action_journal.py @@ -0,0 +1,327 @@ +"""The structured action journal: what the executor writes at the action boundary. + +Every test drives a private :class:`Executor` whose commands are fakes, so +nothing here reaches a mouse, a keyboard or a window. +""" +import json +import threading + +import pytest + +import je_auto_control as ac +from je_auto_control.utils.action_journal import recorder, store +from je_auto_control.utils.action_journal.events import ( + SCHEMA_VERSION, STATUS_ERROR, STATUS_INCOMPLETE, STATUS_OK, ActionEvent, + JournalFormatError, +) +from je_auto_control.utils.action_journal.sanitize import REASON_SECRET, sanitise_params +from je_auto_control.utils.action_journal.store import ( + ActionJournal, list_journal_runs, load_journal, read_events, +) +from je_auto_control.utils.executor.action_executor import Executor + +PASSWORD = "hunter2-Zq9!x" + + +@pytest.fixture(autouse=True) +def _journal_off(): + recorder.stop_action_journal() + yield + recorder.stop_action_journal() + + +@pytest.fixture() +def fake_executor(): + """A private executor with fake commands; ``calls`` lists what they received.""" + executor = Executor() + calls = [] + + def fake_step(**kwargs): + calls.append(kwargs) + return len(calls) + + def fake_fail(**kwargs): + raise RuntimeError(f"login refused password={kwargs.get('password')}") + + def fake_interrupt(): + raise KeyboardInterrupt() + + executor.event_dict.update({ + "AC_fake_step": fake_step, "AC_fake_fail": fake_fail, + "AC_fake_interrupt": fake_interrupt, + }) + executor.calls = calls + return executor + + +def _by_command(events, command): + return [event for event in events if event.command == command] + + +def test_redaction_precedes_append(tmp_path, monkeypatch, fake_executor): + path = tmp_path / "journal.jsonl" + written = [] + real_append = store.append_json_line + + def spy(target, line): + written.append(line) + real_append(target, line) + + monkeypatch.setattr(store, "append_json_line", spy) + password = PASSWORD + fake_executor.variables.set("pw", password) + recorder.start_action_journal(path, run_id="run-redact") + fake_executor.execute_action([ + ["AC_fake_step", {"user": "ada", "password": password}], + ["AC_fake_step", {"password": "${pw}"}], + ["AC_loop", {"times": 1, "body": [ + ["AC_fake_step", {"smtp": {"token": password}}]]}], + ["AC_fake_fail", {"password": password}], + ]) + recorder.stop_action_journal() + + # The fakes did receive the real value: masking is the journal's doing. + assert fake_executor.calls[0]["password"] == password + assert fake_executor.calls[1]["password"] == password + journal_text = path.read_text(encoding="utf-8") + assert password not in journal_text + # Not masked afterwards: no line handed to the file ever held it. + assert written and all(password not in line for line in written) + + events = read_events(path) + literal, reference = _by_command(events, "AC_fake_step")[:2] + assert literal.params == {"user": "ada", "password": "***"} + assert literal.unreplayable == {"params.password": REASON_SECRET} + # A reference names the secret without holding it, so it stays replayable. + assert reference.params == {"password": "${pw}"} + assert reference.unreplayable == {} + failed = _by_command(events, "AC_fake_fail")[0] + assert failed.status == STATUS_ERROR and "RuntimeError" in failed.error + + +def test_parallel_parent_and_order(tmp_path, fake_executor): + path = tmp_path / "journal.jsonl" + run_id = recorder.start_action_journal(path)["run_id"] + fake_executor.execute_action([ + ["AC_fake_step", {"n": 0}], + ["AC_parallel", {"branches": [ + [["AC_fake_step", {"n": 10}], ["AC_fake_step", {"n": 11}]], + [["AC_fake_step", {"n": 20}], ["AC_fake_step", {"n": 21}]], + ]}], + ]) + recorder.stop_action_journal() + + events = read_events(path, run_id=run_id) + assert all(event.run_id == run_id for event in events) + assert len(events) == 6 + parallel = _by_command(events, "AC_parallel")[0] + assert parallel.parent_id is None and parallel.status == STATUS_OK + for branch, expected in ((0, [10, 11]), (1, [20, 21])): + steps = [event for event in events if event.branch == branch] + assert [event.params["n"] for event in steps] == expected + assert all(event.parent_id == parallel.step_id for event in steps) + assert all(event.thread != parallel.thread for event in steps) + # File order is start order: sequence rises with the lines. + assert [event.sequence for event in events] == sorted( + event.sequence for event in events) + assert len({event.step_id for event in events}) == 6 + + +def test_incomplete_step_stays_incomplete(tmp_path, fake_executor): + path = tmp_path / "journal.jsonl" + recorder.start_action_journal(path, run_id="run-cut") + with pytest.raises(KeyboardInterrupt): + fake_executor.execute_action([ + ["AC_fake_step", {"n": 1}], + ["AC_loop", {"times": 1, "body": [["AC_fake_interrupt"]]}], + ]) + recorder.stop_action_journal() + events = read_events(path) + unfinished = _by_command(events, "AC_fake_interrupt")[0] + assert unfinished.status == 'incomplete' + assert "KeyboardInterrupt" in unfinished.error + assert _by_command(events, "AC_loop")[0].status == STATUS_INCOMPLETE + assert _by_command(events, "AC_fake_step")[0].status == STATUS_OK + + # A process that died wrote no end line at all. + lines = path.read_text(encoding="utf-8").splitlines() + starts = [line for line in lines if json.loads(line)["record"] == "start"] + path.write_text("\n".join(starts) + "\n", encoding="utf-8") + assert {event.status for event in read_events(path)} == {STATUS_INCOMPLETE} + assert all(event.finished_at is None for event in read_events(path)) + + +def test_off_costs_one_shared_object_and_writes_nothing(tmp_path, fake_executor): + first, second = recorder.step(["AC_fake_step"]), recorder.step(["AC_fake_step"]) + assert first is second is recorder._NULL + assert recorder.current_step() is None + fake_executor.execute_action([["AC_fake_step", {"n": 1}]]) + assert list(tmp_path.iterdir()) == [] + assert recorder.action_journal_status()["active"] is False + + +def test_nested_steps_keep_their_parent(tmp_path, fake_executor): + path = tmp_path / "journal.jsonl" + recorder.start_action_journal(path, session="device-7") + fake_executor.execute_action([ + ["AC_loop", {"times": 2, "body": [ + ["AC_if_var", {"name": "missing", "op": "eq", "value": 1, + "else": [["AC_fake_step", {"n": 1}]]}]]}], + ]) + recorder.stop_action_journal() + events = read_events(path) + loop = _by_command(events, "AC_loop")[0] + branches = _by_command(events, "AC_if_var") + assert [event.parent_id for event in branches] == [loop.step_id] * 2 + steps = _by_command(events, "AC_fake_step") + assert [event.parent_id for event in steps] == [b.step_id for b in branches] + assert all(event.session == "device-7" for event in events) + # The outcome is described, never stored as a value that could be replayed. + assert steps[0].outcome == {"type": "int", "value": 1} + assert loop.params["body"][0][0] == "AC_if_var" + + +def test_outcome_text_and_unserialisable_arguments_are_not_stored(tmp_path, fake_executor): + fake_executor.event_dict["AC_fake_read"] = lambda **_kwargs: PASSWORD + path = tmp_path / "journal.jsonl" + recorder.start_action_journal(path) + fake_executor.execute_action([ + ["AC_fake_read", {"handle": object(), "point": (1, 2), "ratio": float("inf")}]]) + recorder.stop_action_journal() + assert PASSWORD not in path.read_text(encoding="utf-8") + event = read_events(path)[0] + assert event.outcome == {"type": "str", "size": len(PASSWORD)} + assert event.params["point"] == [1, 2] + assert event.params["handle"] == {"$unserialisable": "object"} + assert event.unreplayable["params.handle"] == "not JSON-serialisable (object)" + assert "non-finite" in event.unreplayable["params.ratio"] + + +def test_vault_commands_mask_every_argument_but_keep_references(): + params, notes = sanitise_params("AC_secret_set", {"name": "db", "value": PASSWORD}) + assert params == {"name": "***", "value": "***"} and len(notes) == 2 + params, notes = sanitise_params("AC_write_secret", {"text": "${secrets.db}"}) + assert params == {"text": "${secrets.db}"} and notes == {} + + +def test_stopping_inside_a_script_still_ends_the_running_steps(tmp_path, fake_executor): + path = tmp_path / "journal.jsonl" + fake_executor.execute_action([ + ["AC_journal_start", {"path": str(path), "run_id": "in-script"}], + ["AC_loop", {"times": 1, "body": [ + ["AC_fake_step", {"n": 1}], ["AC_journal_stop"], + ["AC_fake_step", {"n": 2}]]}], + ]) + events = read_events(path) + assert [event.command for event in events] == [ + "AC_loop", "AC_fake_step", "AC_journal_stop"] + assert {event.status for event in events} == {STATUS_OK} + assert {event.run_id for event in events} == {"in-script"} + + +def test_executor_commands_read_and_list_runs(tmp_path, fake_executor): + path = tmp_path / "journal.jsonl" + for run in ("first", "second"): + recorder.start_action_journal(path, run_id=run) + fake_executor.execute_action( + [["AC_fake_step", {"n": 1}], ["AC_fake_fail", {}]]) + assert recorder.action_journal_status()["events"] == 2 + recorder.stop_action_journal() + record = fake_executor.execute_action([ + ["AC_journal_runs", {"path": str(path)}], + ["AC_journal_read", {"path": str(path), "run_id": "second", "limit": 1}], + ["AC_journal_status"], + ]) + runs, tail, status = record.values() + assert [run["run_id"] for run in runs] == ["first", "second"] + assert runs[0]["events"] == 2 and runs[0]["ok"] == 1 and runs[0]["error"] == 1 + assert [event["command"] for event in tail] == ["AC_fake_fail"] + assert status["active"] is False and status["last"]["run_id"] == "second" + assert list_journal_runs(path) == runs + + +def test_starting_twice_is_refused(tmp_path): + recorder.start_action_journal(tmp_path / "a.jsonl") + with pytest.raises(ac.ActionJournalError): + recorder.start_action_journal(tmp_path / "b.jsonl") + assert isinstance(ac.ActionJournalError("x"), ac.AutoControlException) + + +def test_a_failed_write_stops_the_journal_not_the_run(tmp_path, monkeypatch, fake_executor): + recorder.start_action_journal(tmp_path / "journal.jsonl") + + def broken(_target, _line): + raise OSError("disk full") + + monkeypatch.setattr(store, "append_json_line", broken) + record = fake_executor.execute_action( + [["AC_fake_step", {"n": 1}], ["AC_fake_step", {"n": 2}]]) + assert list(record.values()) == [1, 2] + status = recorder.action_journal_status() + assert status["active"] is False and "disk full" in status["last"]["error"] + + +def test_reading_validates_the_schema_and_skips_torn_lines(tmp_path): + path = tmp_path / "journal.jsonl" + journal = ActionJournal(path) + event = ActionEvent(run_id="r", step_id="s-1", sequence=1, command="AC_x") + journal.append(event) + with path.open("a", encoding="utf-8") as handle: + handle.write('{"schema_version": 1, "record": "sta') # a cut-off write + journal.append(ActionEvent(run_id="r", step_id="s-2", sequence=2, command="AC_y")) + contents = load_journal(path) + assert [item.step_id for item in contents.events] == ["s-1", "s-2"] + assert contents.torn_lines == (2,) and contents.lines == {"s-1": 1, "s-2": 3} + assert journal.read(run_id="other") == [] + assert event.schema_version == SCHEMA_VERSION == 1 + + for bad in ({"schema_version": 2, "run_id": "r", "step_id": "s"}, + {"schema_version": 1, "run_id": "r", "step_id": "s-9", + "sequence": 1, "command": 7}, + {"schema_version": 1, "run_id": "r", "step_id": "s-1", + "sequence": 1, "command": "AC_x"}, + {"schema_version": 1, "run_id": "r", "step_id": "s-8", + "sequence": 1, "command": "AC_x", "status": "fine"}, + ["not", "an", "object"]): + broken = tmp_path / "broken.jsonl" + broken.write_text(path.read_text(encoding="utf-8") + json.dumps(bad) + "\n", + encoding="utf-8") + with pytest.raises(JournalFormatError): + read_events(broken) + with pytest.raises(JournalFormatError): + read_events(tmp_path / "absent.jsonl") + + +def test_concurrent_top_level_runs_share_one_ordered_file(tmp_path, fake_executor): + path = tmp_path / "journal.jsonl" + recorder.start_action_journal(path) + + def run(offset): + Executor.execute_action( + fake_executor, [["AC_fake_step", {"n": offset + i}] for i in range(20)]) + + threads = [threading.Thread(target=run, args=(offset,)) for offset in (0, 100, 200)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + recorder.stop_action_journal() + events = read_events(path) + assert len(events) == 60 and {event.status for event in events} == {STATUS_OK} + assert [event.sequence for event in events] == list(range(1, 61)) + + +def test_every_surface_is_wired(): + from je_auto_control.gui.script_builder.command_schema import COMMAND_SPECS + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + commands = {"AC_journal_start", "AC_journal_stop", "AC_journal_status", + "AC_journal_read", "AC_journal_runs", "AC_generate_code_from_journal"} + assert commands <= ac.executor.known_commands() + assert commands <= set(COMMAND_SPECS) + tools = {tool.name for tool in build_default_tool_registry()} + assert {"ac_journal_start", "ac_journal_stop", "ac_journal_status", + "ac_journal_read", "ac_journal_runs", "ac_generate_code_from_log"} <= tools + for name in ("ActionEvent", "ActionJournal", "read_events", "start_action_journal", + "stop_action_journal", "action_journal_status", "list_journal_runs", + "JournalFormatError", "ActionJournalError"): + assert name in ac.__all__ and hasattr(ac, name) diff --git a/test/unit_test/headless/test_action_journal_gui.py b/test/unit_test/headless/test_action_journal_gui.py new file mode 100644 index 000000000..8c7e7cbd9 --- /dev/null +++ b/test/unit_test/headless/test_action_journal_gui.py @@ -0,0 +1,160 @@ +"""GUI affordances for the action journal: thin wrappers over the headless API. + +Every dialog is replaced by a stub, so no window is shown and nothing is run. +""" +import json +import os + +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + +from PySide6.QtWidgets import ( # noqa: E402 + QApplication, QFileDialog, QInputDialog, QMessageBox, +) + +from je_auto_control.gui import _journal_import # noqa: E402 +from je_auto_control.gui.recording_editor_tab import RecordingEditorTab # noqa: E402 +from je_auto_control.gui.run_history_tab import RunHistoryTab # noqa: E402 +from je_auto_control.gui.script_builder.builder_tab import ScriptBuilderTab # noqa: E402 +from je_auto_control.gui.script_builder.step_model import steps_to_actions # noqa: E402 +from je_auto_control.utils.action_journal import recorder # noqa: E402 + + +@pytest.fixture(scope="module") +def qapp(): + app = QApplication.instance() or QApplication([]) + yield app + + +@pytest.fixture(autouse=True) +def _journal_off(): + recorder.stop_action_journal() + yield + recorder.stop_action_journal() + + +@pytest.fixture() +def dialogs(monkeypatch): + """Stub every dialog; ``answers`` holds what each returns, ``shown`` what was said.""" + answers = {"open": "", "save": "", "run": None} + shown = [] + monkeypatch.setattr(QFileDialog, "getOpenFileName", + lambda *_a, **_k: (answers["open"], "")) + monkeypatch.setattr(QFileDialog, "getSaveFileName", + lambda *_a, **_k: (answers["save"], "")) + monkeypatch.setattr( + QInputDialog, "getItem", + lambda _parent, _title, _label, items, *_a, **_k: ( + answers["run"] or items[0], True)) + for name in ("information", "warning"): + monkeypatch.setattr(QMessageBox, name, + lambda _parent, _title, text, *_a, _n=name: shown.append((_n, text))) + answers["shown"] = shown + return answers + + +def _journal(path, *runs): + lines = [] + for run_id, commands in runs: + for index, (command, extra) in enumerate(commands, start=1): + lines.append(json.dumps({ + "schema_version": 1, "record": "start", "run_id": run_id, + "step_id": f"{run_id}-{index}", "sequence": index, "command": command, + "status": "ok", "started_at": float(index), + "finished_at": index + 0.5, **extra})) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return str(path) + + +def test_recording_editor_imports_a_journal_run_and_can_undo(qapp, tmp_path, dialogs): + dialogs["open"] = _journal( + tmp_path / "journal.jsonl", + ("old", [("AC_write", {"params": {"write_string": "old"}})]), + ("new", [("AC_write", {"params": {"write_string": "new"}}), + ("AC_get_mouse_position", {})])) + tab = RecordingEditorTab() + tab._actions = [["AC_a"]] + assert "re_import_journal" in [key for key, _handler in tab.menu_actions()] + tab._import_journal() # the newest run is offered first + assert tab._actions == [["AC_write", {"write_string": "new"}], ["AC_get_mouse_position"]] + assert "journal run new" in tab._status.text() + assert '"new"' in tab._preview.toPlainText() + tab._undo() + assert tab._actions == [["AC_a"]] + tab.deleteLater() + + +def test_observed_path_is_announced_before_the_candidate_opens(qapp, tmp_path, dialogs): + dialogs["open"] = _journal( + tmp_path / "journal.jsonl", + ("run", [("AC_write", {"parent_id": "gone", "params": {"write_string": "x"}})])) + tab = ScriptBuilderTab() + tab._on_import_journal() + assert steps_to_actions(tab._tree.root_steps()) == [["AC_write", {"write_string": "x"}]] + kind, text = dialogs["shown"][0] + assert kind == "information" + assert _journal_import._t("jr_observed_only") in text + assert "Imported journal run run: 1 step(s)" in tab._result.toPlainText() + tab.deleteLater() + + +def test_cancel_and_bad_files_leave_the_tab_alone(qapp, tmp_path, dialogs): + tab = RecordingEditorTab() + tab._actions = [["AC_a"]] + tab._import_journal() # the file dialog was cancelled + assert tab._actions == [["AC_a"]] and dialogs["shown"] == [] + bad = tmp_path / "bad.jsonl" + bad.write_text('{"schema_version": 7, "run_id": "r", "step_id": "s"}\n', encoding="utf-8") + dialogs["open"] = str(bad) + tab._import_journal() + assert tab._actions == [["AC_a"]] + assert dialogs["shown"][0][0] == "warning" and "schema_version" in dialogs["shown"][0][1] + tab.deleteLater() + + +def test_run_history_tab_starts_stops_and_exports(qapp, tmp_path, dialogs): + tab = RunHistoryTab() + tab._timer.stop() + keys = [key for key, _handler in tab.menu_actions()] + assert {"rh_journal_start", "rh_journal_stop", "rh_journal_candidate"} <= set(keys) + assert tab._journal_label.text() == _journal_import._t("rh_journal_off") + + journal = tmp_path / "live.jsonl" + dialogs["save"] = str(journal) + tab._start_journal() + status = recorder.action_journal_status() + assert status["active"] is True and status["path"] == str(journal.resolve()) + assert status["run_id"] in tab._journal_label.text() + tab._start_journal() # a second start is refused, and said so + assert dialogs["shown"][-1][0] == "warning" + tab._stop_journal() + assert recorder.action_journal_status()["active"] is False + assert tab._journal_label.text() == _journal_import._t("rh_journal_off") + + dialogs["open"] = _journal( + tmp_path / "journal.jsonl", + ("run", [("AC_write", {"params": {"write_string": "x"}})])) + output = tmp_path / "test_candidate.py" + dialogs["save"] = str(output) + tab._export_journal_candidate() + assert "def test_journal_run_run" in output.read_text(encoding="utf-8") + manifest = json.loads((tmp_path / "test_candidate.py.manifest.json").read_text( + encoding="utf-8")) + assert manifest["run_id"] == "run" and manifest["executed"] is False + tab.deleteLater() + + +def test_strings_exist_in_every_language(): + from je_auto_control.gui.language_wrapper import ( + english, japanese, simplified_chinese, traditional_chinese, + ) + keys = ("rh_journal_start", "rh_journal_stop", "rh_journal_candidate", + "rh_journal_off", "rh_journal_on", "re_import_journal", "sb_import_journal", + "jr_dialog_open", "jr_dialog_start", "jr_dialog_save_code", "jr_pick_run", + "jr_no_runs", "jr_observed_only") + for module in (english, japanese, simplified_chinese, traditional_chinese): + catalogue = next(value for value in vars(module).values() + if isinstance(value, dict) and "rh_no_artifact" in value) + assert all(catalogue.get(key) for key in keys), module.__name__ diff --git a/test/unit_test/headless/test_codegen_from_journal.py b/test/unit_test/headless/test_codegen_from_journal.py new file mode 100644 index 000000000..685765920 --- /dev/null +++ b/test/unit_test/headless/test_codegen_from_journal.py @@ -0,0 +1,307 @@ +"""Candidate scripts built from an action journal. + +Journals are produced by a private executor running fake commands, or written +by hand as JSON lines; generation itself must never reach a device. +""" +import ast +import json + +import pytest + +import je_auto_control as ac +from je_auto_control import cli +from je_auto_control.utils.action_journal import recorder +from je_auto_control.utils.action_journal.events import JournalFormatError +from je_auto_control.utils.codegen.journal_import import ( + CandidateScript, JournalImportError, generate_candidate_from_log, +) +from je_auto_control.utils.executor.action_executor import Executor + +PASSWORD = "hunter2-Zq9!x" + + +@pytest.fixture(autouse=True) +def _journal_off(): + recorder.stop_action_journal() + yield + recorder.stop_action_journal() + + +def _record(path, run_id, actions, extra=None): + """Journal ``actions`` on a private executor whose device commands are fakes.""" + executor = Executor() + executor.event_dict.update({ + "AC_click_mouse": lambda **_kwargs: None, + "AC_write": lambda **_kwargs: None, + **(extra or {}), + }) + recorder.start_action_journal(path, run_id=run_id) + try: + executor.execute_action(actions) + finally: + recorder.stop_action_journal() + + +def _line(step, sequence, command, **fields): + return json.dumps({ + "schema_version": 1, "record": "start", "run_id": "hand", + "step_id": step, "sequence": sequence, "command": command, + "status": "ok", "started_at": float(sequence), + "finished_at": sequence + 0.5, **fields}) + + +def _write(path, *lines): + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + +@pytest.fixture() +def device_calls(monkeypatch): + """Every action the executor would really dispatch while a test runs.""" + calls = [] + monkeypatch.setattr( + Executor, "_execute_event", lambda _self, action: calls.append(action)) + return calls + + +def test_run_filter_and_provenance(tmp_path): + path = tmp_path / "journal.jsonl" + script = [ + ["AC_click_mouse", {"mouse_keycode": "mouse_left", "x": 10, "y": 20}], + ["AC_loop", {"times": 2, "body": [["AC_write", {"write_string": "hi"}]]}], + ] + _record(path, "other-run", [["AC_write", {"write_string": "noise"}]]) + selected_run = "run-42" + _record(path, selected_run, script) + + candidate = generate_candidate_from_log(path, run_id=selected_run) + assert isinstance(candidate, CandidateScript) + assert candidate.manifest['run_id'] == selected_run + assert candidate.actions == script # the loop is rebuilt, not unrolled + assert candidate.observed_path_only is False + assert "noise" not in candidate.code + steps = candidate.manifest["steps"] + assert [step["command"] for step in steps] == ["AC_click_mouse", "AC_loop"] + assert all(step["mode"] == "recorded" and step["emitted"] for step in steps) + lines = path.read_text(encoding="utf-8").splitlines() + for step in steps: # each step points at the line it was read from + source = json.loads(lines[step["line"] - 1]) + assert source["command"] == step["command"] and source["run_id"] == selected_run + assert candidate.manifest["event_count"] == 4 + assert candidate.manifest["validation"] == { + "ast": True, "unknown_commands": [], "dry_run": True} + assert candidate.manifest["executed"] is False + ast.parse(candidate.code) + assert "raise_on_error=True" in candidate.code + + +def test_observed_branch_is_labelled(tmp_path): + # The journal starts inside a block (the if's own line is missing), and a + # macro is called that this run never defined. + path = _write( + tmp_path / "journal.jsonl", + _line("t-2", 2, "AC_click_mouse", parent_id="t-1", params={"x": 1, "y": 2}), + _line("t-3", 3, "AC_call_macro", params={"name": "login"}), + _line("t-4", 4, "AC_write", parent_id="t-3", params={"write_string": "ada"}), + ) + candidate = generate_candidate_from_log(path, run_id="hand") + assert candidate.observed_path_only is True + assert candidate.manifest["observed_path_only"] is True + assert candidate.actions == [ + ["AC_click_mouse", {"x": 1, "y": 2}], ["AC_write", {"write_string": "ada"}]] + modes = {step["step_id"]: (step["mode"], step["emitted"]) + for step in candidate.manifest["steps"]} + assert modes == {"t-2": ("observed", True), "t-3": ("observed", False), + "t-4": ("observed", True)} + text = " ".join(candidate.warnings) + assert "observed" in text and "'login' is not defined" in text + # Nothing that did not run is invented: no branch, no macro call. + assert "AC_call_macro" not in candidate.code and "AC_if" not in candidate.code + + +def test_a_macro_defined_in_the_run_is_rebuilt_not_observed(tmp_path): + path = tmp_path / "journal.jsonl" + script = [ + ["AC_define_macro", {"name": "greet", "body": [ + ["AC_write", {"write_string": "hi"}]]}], + ["AC_call_macro", {"name": "greet"}], + ] + _record(path, "macro", script) + candidate = generate_candidate_from_log(path, run_id="macro") + assert candidate.actions == script and candidate.observed_path_only is False + + +def test_secret_reference_survives_codegen(tmp_path): + path = tmp_path / "journal.jsonl" + _record(path, "secrets", [ + ["AC_fake_login", {"user": "ada", "password": "${secrets.db_password}"}], + ["AC_fake_login", {"user": "bob", "password": PASSWORD}], + ], extra={"AC_fake_login": lambda **_kwargs: None}) + candidate = generate_candidate_from_log(path, run_id="secrets", target="python") + assert "${secrets.db_password}" in candidate.code + assert candidate.actions[0][1]["password"] == "${secrets.db_password}" + # The literal was masked before it reached the file, so the candidate + # cannot hold it -- and it does not replay the mask as if it were a value. + assert PASSWORD not in candidate.code and PASSWORD not in json.dumps(candidate.manifest) + assert candidate.actions[1][1]["password"] == "${journal_redacted_1_1}" + assert "'***'" not in candidate.code + assert any("params.password is not replayable" in text for text in candidate.warnings) + # AC_fake_login is unknown outside this test: reported, and the dry run skipped. + validation = candidate.manifest["validation"] + assert validation["unknown_commands"] == ["AC_fake_login"] + assert validation["dry_run"] is False + assert any("unknown command(s)" in text for text in candidate.warnings) + + +def test_generation_has_no_device_effect(tmp_path, device_calls, monkeypatch): + path = tmp_path / "journal.jsonl" + _write( + path, + _line("t-1", 1, "AC_click_mouse", params={"mouse_keycode": "mouse_left"}), + _line("t-2", 2, "AC_write", + params={"write_string": "__import__('os').system('calc')"}), + _line("t-3", 3, "AC_shell_command", params={"command": "calc"}), + ) + monkeypatch.setattr("builtins.eval", lambda *_args, **_kwargs: device_calls.append("eval")) + monkeypatch.setattr("builtins.exec", lambda *_args, **_kwargs: device_calls.append("exec")) + for target in ("pytest", "python", "robot"): + for style in ("actions", "calls"): + candidate = generate_candidate_from_log( + path, run_id="hand", target=target, style=style) + assert candidate.manifest["validation"]["dry_run"] is True + assert device_calls == [] + assert recorder.action_journal_status()["active"] is False + + +def test_retry_attempts_are_counted_and_only_the_last_is_observed(tmp_path): + path = tmp_path / "journal.jsonl" + state = {"left": 2} + + def flaky(**_kwargs): + state["left"] -= 1 + if state["left"] >= 0: + raise RuntimeError("not yet") + + retry = ["AC_retry", {"max_attempts": 5, "backoff": 0, "body": [ + ["AC_write", {"write_string": "a"}], ["AC_fake_flaky", {}]]}] + _record(path, "retry", [retry], extra={"AC_fake_flaky": flaky}) + candidate = generate_candidate_from_log(path, run_id="retry") + assert candidate.actions == [retry] + assert candidate.manifest["steps"][0]["attempts"] == 3 + + # The same run with the block's arguments lost: only the final attempt. + lines = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + for line in lines: + if line.get("command") == "AC_retry": + line["unreplayable"] = {"params.body": "not JSON-serialisable (generator)"} + _write(path, *(json.dumps(line) for line in lines)) + observed = generate_candidate_from_log(path, run_id="retry") + assert observed.observed_path_only is True + assert observed.actions == [["AC_write", {"write_string": "a"}], ["AC_fake_flaky", {}]] + assert observed.manifest["steps"][0]["attempts"] == 3 + + +def test_parallel_block_is_emitted_once_with_its_branches(tmp_path): + path = tmp_path / "journal.jsonl" + # A branch runs on a fresh executor that keeps its own stock commands, so + # only a command with a name of its own stays a fake inside one. + script = [["AC_parallel", {"branches": [ + [["AC_fake_type", {"text": "a"}]], [["AC_fake_type", {"text": "b"}]]]}]] + _record(path, "par", script, extra={"AC_fake_type": lambda **_kwargs: None}) + candidate = generate_candidate_from_log(path, run_id="par") + assert candidate.actions == script + assert candidate.manifest["event_count"] == 3 + assert [step["command"] for step in candidate.manifest["steps"]] == ["AC_parallel"] + + +def test_steps_run_by_a_thread_pool_inside_a_block_are_not_emitted_twice(tmp_path): + path = _write( + tmp_path / "journal.jsonl", + _line("t-1", 1, "AC_run_dag", thread=1, finished_at=9.0, + params={"definition": {"nodes": []}}), + _line("t-2", 2, "AC_write", thread=2, params={"write_string": "inside"}), + _line("t-3", 10, "AC_write", thread=2, params={"write_string": "after"}), + ) + candidate = generate_candidate_from_log(path, run_id="hand") + assert [action[0] for action in candidate.actions] == ["AC_run_dag", "AC_write"] + assert candidate.actions[1][1] == {"write_string": "after"} + detached = [step for step in candidate.manifest["steps"] if step["mode"] == "detached"] + assert [step["step_id"] for step in detached] == ["t-2"] + + +def test_failed_and_unfinished_steps_are_flagged_not_dropped(tmp_path): + path = _write( + tmp_path / "journal.jsonl", + _line("t-1", 1, "AC_write", status="error", error="RuntimeError('x')", + params={"write_string": "a"}), + _line("t-2", 2, "AC_write", status="incomplete", finished_at=None, + params={"write_string": "b"}), + "{torn", + ) + candidate = generate_candidate_from_log(path, run_id="hand") + assert len(candidate.actions) == 2 + text = " ".join(candidate.warnings) + assert "1 emitted step(s) failed" in text and "did not finish" in text + assert "skipped unreadable line(s) 3" in text + + +def test_bad_input_is_refused(tmp_path): + path = _write(tmp_path / "journal.jsonl", _line("t-1", 1, "AC_write", params={})) + with pytest.raises(JournalImportError): + generate_candidate_from_log(path, run_id="absent") + with pytest.raises(JournalImportError): + generate_candidate_from_log(path, run_id="hand", target="bash") + with pytest.raises(JournalImportError): + generate_candidate_from_log(path, run_id="hand", style="eval") + _write(path, json.dumps({"schema_version": 99, "run_id": "hand", "step_id": "t"})) + with pytest.raises(JournalFormatError): + generate_candidate_from_log(path, run_id="hand") + assert issubclass(JournalImportError, ac.AutoControlException) + assert ac.generate_candidate_from_log is generate_candidate_from_log + + +def test_cli_from_log(tmp_path, capsys, device_calls): + path = tmp_path / "journal.jsonl" + _write(path, _line("t-1", 1, "AC_write", params={"write_string": "hello"})) + assert cli.main(["codegen", "--from-log", str(path)]) == 0 + out = capsys.readouterr().out + assert "def test_journal_run_hand" in out and "'hello'" in out + + output, manifest = tmp_path / "test_candidate.py", tmp_path / "candidate.json" + assert cli.main(["codegen", "--from-log", str(path), "--run-id", "hand", + "--target", "python", "-o", str(output), + "--manifest", str(manifest)]) == 0 + ast.parse(output.read_text(encoding="utf-8")) + saved = json.loads(manifest.read_text(encoding="utf-8")) + assert saved["run_id"] == "hand" and saved["warnings"] == [] + + with path.open("a", encoding="utf-8") as handle: + handle.write(_line("u-1", 1, "AC_write", params={}).replace('"hand"', '"two"') + "\n") + assert cli.main(["codegen", "--from-log", str(path)]) == 1 + assert "name one with run_id: hand, two" in capsys.readouterr().err + assert cli.main(["codegen"]) == 1 + assert cli.main(["codegen", "x.json", "--from-log", str(path)]) == 1 + assert device_calls == [] + + +def test_cli_script_codegen_keeps_its_default_style(tmp_path, capsys): + script = tmp_path / "script.json" + script.write_text(json.dumps([["AC_get_mouse_position"]]), encoding="utf-8") + assert cli.main(["codegen", str(script)]) == 0 + assert "ac.get_mouse_position()" in capsys.readouterr().out + + +def test_executor_command_and_mcp_handler(tmp_path, device_calls): + from je_auto_control.utils.executor.action_executor import _generate_code_from_journal + from je_auto_control.utils.mcp_server.tools import _handlers_qa + path = _write(tmp_path / "journal.jsonl", + _line("t-1", 1, "AC_write", params={"write_string": "hello"})) + output = tmp_path / "out.py" + result = _generate_code_from_journal(str(path), output=str(output)) + assert result["manifest"]["run_id"] == "hand" and result["observed_path_only"] is False + assert output.read_text(encoding="utf-8") == result["code"] + handled = _handlers_qa.generate_code_from_log(str(path), run_id="hand", target="robot") + assert handled["actions"] == [["AC_write", {"write_string": "hello"}]] + assert handled["manifest"]["validation"]["ast"] is None + json.dumps(handled) + assert device_calls == [] From ba4b693bf7c172d854934a11d50406daa9918a4b Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:54:18 +0800 Subject: [PATCH 14/44] Close the spec block the two merged command groups shared --- je_auto_control/gui/script_builder/command_schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index b62f529bb..134c321e9 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -3038,6 +3038,8 @@ def _add_audit_specs(specs: List[CommandSpec]) -> None: specs.append(CommandSpec( "AC_self_heal_revision_list", "Testing", "Self-Heal: List Template Revisions", description="List stored template revisions and their status.", + )) + specs.append(CommandSpec( "AC_journal_start", "Testing", "Start Action Journal", fields=( FieldSpec("path", FieldType.FILE_PATH, optional=True, From ab42b695e84fc226d54f9f4b4730bea29e19bb16 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:55:29 +0800 Subject: [PATCH 15/44] Name the punctuation keys on Windows by the character they type The Windows key table had no slash, semicolon or bracket names because a fixed alias for an oem_N key is only right on a US keyboard. Eight names now stand for a character and are resolved when they are looked up: VkKeyScanExW against the foreground window's layout, falling back to the US position when the layout has no bare key for the character. Every existing name keeps its code, and the new names never answer a reverse lookup, so recordings are unchanged. --- docs/source/Eng/doc/keyboard/keyboard_doc.rst | 12 + docs/source/Zh/doc/keyboard/keyboard_doc.rst | 10 + .../windows/core/utils/win32_layout_vk.py | 112 +++++++++ je_auto_control/wrapper/_platform_windows.py | 14 +- .../headless/test_key_names_and_aliases.py | 18 +- .../test_windows_layout_key_aliases.py | 224 ++++++++++++++++++ 6 files changed, 385 insertions(+), 5 deletions(-) create mode 100644 je_auto_control/windows/core/utils/win32_layout_vk.py create mode 100644 test/unit_test/headless/test_windows_layout_key_aliases.py diff --git a/docs/source/Eng/doc/keyboard/keyboard_doc.rst b/docs/source/Eng/doc/keyboard/keyboard_doc.rst index 9680ccec6..082ce5cee 100644 --- a/docs/source/Eng/doc/keyboard/keyboard_doc.rst +++ b/docs/source/Eng/doc/keyboard/keyboard_doc.rst @@ -37,6 +37,18 @@ alias: keyboard_key_name(27) # "escape" on Windows keyboard_key_name(0xBB) # "oem_plus" +Eight Windows names stand for a *character* rather than a key position: +``slash`` (``/``), ``backslash`` (``\``), ``semicolon`` (``;``), ``quote`` +(``'``), ``backquote`` (`` ` ``), ``bracketleft`` (``[``), ``bracketright`` +(``]``) and ``equal`` (``=``). Each is resolved when it is looked up, by +asking the keyboard layout of the foreground window which key types that +character (``VkKeyScanExW``), so ``press_keyboard_key("semicolon")`` presses +the right key on a layout where ``;`` is not at the US position. When the +layout has no bare key for the character -- a German keyboard types ``/`` +with Shift+7 -- the name falls back to the key at the US position. The +``oem_1`` ... ``oem_8`` names remain fixed positions, and ``keys_table`` +lists the eight names with their US-position codes. + Press and Release ================= diff --git a/docs/source/Zh/doc/keyboard/keyboard_doc.rst b/docs/source/Zh/doc/keyboard/keyboard_doc.rst index bf37173b3..2e06ea24a 100644 --- a/docs/source/Zh/doc/keyboard/keyboard_doc.rst +++ b/docs/source/Zh/doc/keyboard/keyboard_doc.rst @@ -35,6 +35,16 @@ Windows 的按鍵表也收常見的別名（``ctrl``、``alt``、``enter``、``e keyboard_key_name(27) # Windows 上是 "escape" keyboard_key_name(0xBB) # "oem_plus" +Windows 上有八個名字代表的是\ *字元*\ 而不是按鍵位置：``slash``\ （``/``\ ）、 +``backslash``\ （``\``\ ）、``semicolon``\ （``;``\ ）、``quote``\ （``'``\ ）、 +``backquote``\ （`` ` ``\ ）、``bracketleft``\ （``[``\ ）、``bracketright``\ （``]``\ ） +與 ``equal``\ （``=``\ ）。它們在查表的當下才解析：向前景視窗的鍵盤配置詢問哪個鍵 +打得出該字元（``VkKeyScanExW``\ ），所以在 ``;`` 不在美式位置的配置上， +``press_keyboard_key("semicolon")`` 仍會按到正確的鍵。配置上沒有不加修飾鍵就能 +打出該字元的鍵時——德文鍵盤的 ``/`` 是 Shift+7——這個名字退回美式位置的鍵。 +``oem_1`` … ``oem_8`` 仍然是固定位置；``keys_table`` 列出這八個名字時顯示的是 +美式位置的鍵碼。 + 按下與釋放 ========== diff --git a/je_auto_control/windows/core/utils/win32_layout_vk.py b/je_auto_control/windows/core/utils/win32_layout_vk.py new file mode 100644 index 000000000..7d08f388b --- /dev/null +++ b/je_auto_control/windows/core/utils/win32_layout_vk.py @@ -0,0 +1,112 @@ +"""Key names that mean a character, resolved through the keyboard layout. + +``oem_2`` is the ``/`` key on a US keyboard and something else elsewhere, so a +readable alias that pointed at a fixed ``oem_N`` would lie on most layouts -- +which is why the Windows key table had none. The names here are different: +``slash`` means "the key that types ``/``", and the virtual key is looked up +when the name is, with ``VkKeyScanExW`` on the character against the layout +of the foreground window (where the key press is going). + +A layout that has no bare key for the character -- German types ``/`` with +Shift+7 -- gives the key at the US position, which is also what an unknown +layout, a failed probe and a non-Windows import give. The table therefore +always holds a valid virtual key for every name, and iterating it (the key +list in the GUI, ``AC_get_keyboard_keys_table``) shows those US positions. + +Nothing here sends input, and the module imports no Windows API until a +lookup needs one, so it can be imported -- and tested with a fake ``user32`` +-- on any platform. +""" +from typing import Any, Dict, Iterable, Mapping, Optional, Tuple + +from je_auto_control.utils.keyboard_layout.keyboard_layout import ( + foreground_keyboard_layout, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +#: alias -> (the character the key types, its key name on a US keyboard). +LAYOUT_KEY_ALIASES: Dict[str, Tuple[str, str]] = { + "slash": ("/", "oem_2"), + "backslash": ("\\", "oem_5"), + "semicolon": (";", "oem_1"), + "quote": ("'", "oem_7"), + "backquote": ("`", "oem_3"), + "bracketleft": ("[", "oem_4"), + "bracketright": ("]", "oem_6"), + "equal": ("=", "oem_plus"), +} + +#: ``VkKeyScanExW`` answers -1 when no key of the layout types the character. +_NO_KEY = -1 + + +def _user32() -> Any: + """A private ``user32`` handle, so the prototype set below is ours alone.""" + import ctypes + # getattr: the name exists on Windows only, which is where this is called. + return getattr(ctypes, "WinDLL")("user32") + + +def layout_virtual_key(character: str) -> Optional[int]: + """The key that types ``character`` unmodified on the active layout. + + ``None`` when the layout cannot be probed, has no key for the character, + or types it only with Shift, Ctrl or AltGr held -- the caller then uses + the US position. + """ + layout = foreground_keyboard_layout() + if layout is None: + return None + try: + import ctypes + from ctypes import wintypes + user32 = _user32() + user32.VkKeyScanExW.argtypes = [wintypes.WCHAR, wintypes.HKL] + user32.VkKeyScanExW.restype = ctypes.c_short + scanned = int(user32.VkKeyScanExW(character, layout)) + except (OSError, AttributeError, ValueError, TypeError) as error: + autocontrol_logger.info("layout key lookup for %r failed: %r", character, error) + return None + if scanned == _NO_KEY: + return None + virtual_key, shift_state = scanned & 0xFF, (scanned >> 8) & 0xFF + return virtual_key if shift_state == 0 and virtual_key else None + + +class LayoutKeyTable(dict): + """A key table whose layout-dependent names are resolved at lookup time. + + A plain ``dict`` in every other respect: the layout names are stored with + their US-position codes, so membership, iteration and length behave as + before. Only ``table[name]`` and ``table.get(name)`` ask the layout. + """ + + def __init__(self, table: Mapping[str, int], + aliases: Optional[Mapping[str, Tuple[str, str]]] = None) -> None: + super().__init__(table) + self._characters: Dict[str, str] = {} + for alias, (character, us_name) in (aliases or LAYOUT_KEY_ALIASES).items(): + if alias in self: + continue # an existing name keeps its code and stays static + self._characters[alias] = character + super().__setitem__(alias, super().__getitem__(us_name)) + + def layout_names(self) -> Iterable[str]: + """The names resolved through the layout.""" + return tuple(self._characters) + + def _resolved(self, name: Any) -> Optional[int]: + character = self._characters.get(name) if isinstance(name, str) else None + return layout_virtual_key(character) if character is not None else None + + def __getitem__(self, name: str) -> int: + resolved = self._resolved(name) + return resolved if resolved is not None else super().__getitem__(name) + + def get(self, name: str, default: Any = None) -> Any: + """``dict.get``, asking the layout for the names that depend on it.""" + resolved = self._resolved(name) + return resolved if resolved is not None else super().get(name, default) + + +__all__ = ["LAYOUT_KEY_ALIASES", "LayoutKeyTable", "layout_virtual_key"] diff --git a/je_auto_control/wrapper/_platform_windows.py b/je_auto_control/wrapper/_platform_windows.py index ea89419ce..be577a1e1 100644 --- a/je_auto_control/wrapper/_platform_windows.py +++ b/je_auto_control/wrapper/_platform_windows.py @@ -8,6 +8,9 @@ Win32KeyboardBackend, Win32MouseBackend, ) from je_auto_control.windows.core.utils import win32_keypress_check +from je_auto_control.windows.core.utils.win32_layout_vk import ( + LAYOUT_KEY_ALIASES, LayoutKeyTable, +) from je_auto_control.windows.core.utils.win32_vk import ( WIN32_VK_ACCEPT, WIN32_VK_ADD, WIN32_VK_APPS, WIN32_VK_BACK, WIN32_VK_BROWSER_BACK, WIN32_VK_BROWSER_FAVORITES, @@ -345,12 +348,19 @@ def _build_mouse_keys_table(mouse_module) -> dict: "scrolllock": "scroll", **{f"numpad{digit}": f"num{digit}" for digit in range(10)}, # Microsoft defines these four as the same key on every layout, so the - # readable name does not lie; oem_1 .. oem_8 get no alias for that reason. + # readable name does not lie; oem_1 .. oem_8 get no fixed alias for that + # reason. "plus": "oem_plus", "comma": "oem_comma", "minus": "oem_minus", "period": "oem_period", + # slash, backslash, semicolon, ... name a character, not a position: the + # table asks the active layout for the key that types it when the name is + # looked up, and the oem_N listed here is only its US-position fallback. + **{alias: us_name for alias, (_character, us_name) in LAYOUT_KEY_ALIASES.items()}, } keyboard_keys_table.update( - {alias: keyboard_keys_table[name] for alias, name in keyboard_key_aliases.items()}) + {alias: keyboard_keys_table[name] for alias, name in keyboard_key_aliases.items() + if alias not in LAYOUT_KEY_ALIASES}) +keyboard_keys_table = LayoutKeyTable(keyboard_keys_table) special_mouse_keys_table = None keyboard, mouse = _select_input_backend() diff --git a/test/unit_test/headless/test_key_names_and_aliases.py b/test/unit_test/headless/test_key_names_and_aliases.py index 5b90a98d0..6ac96f286 100644 --- a/test/unit_test/headless/test_key_names_and_aliases.py +++ b/test/unit_test/headless/test_key_names_and_aliases.py @@ -62,9 +62,14 @@ def test_every_alias_resolves_to_its_targets_code(): table = platform_wrapper.keyboard_keys_table aliases = platform_wrapper.keyboard_key_aliases assert len(aliases) >= 40, "the alias map is empty; the checks below would pass vacuously" + # Names resolved through the keyboard layout answer differently per + # machine; test_windows_layout_key_aliases.py covers them with a fake. + layout_names = set(table.layout_names()) for alias, target in aliases.items(): assert target in table, (alias, target) assert target not in aliases, (alias, target) + if alias in layout_names: + continue assert table[alias] == table[target], alias assert auto_control_keyboard._resolve_keycode(alias) == table[target], alias @@ -89,12 +94,19 @@ def test_every_name_is_typable_except_the_legacy_capitals(): @windows_only -def test_layout_dependent_oem_keys_get_no_layout_specific_alias(): - """``oem_1`` is ";" on a US layout only; a name like "semicolon" would lie.""" +def test_layout_dependent_oem_keys_get_no_fixed_alias(): + """``oem_1`` is ";" on a US layout only; a fixed "semicolon" would lie. + + The only aliases that may land on such a key are the ones the table + resolves through the keyboard layout when they are looked up. + """ layout_bound = {0xBA, 0xBF, 0xC0, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE2} table = platform_wrapper.keyboard_keys_table + layout_names = set(table.layout_names()) + stored = dict(table) # the static codes, no layout lookup for alias in platform_wrapper.keyboard_key_aliases: - assert table[alias] not in layout_bound, alias + if alias not in layout_names: + assert stored[alias] not in layout_bound, alias # --- reverse lookup: canonical names only ---------------------------------------- diff --git a/test/unit_test/headless/test_windows_layout_key_aliases.py b/test/unit_test/headless/test_windows_layout_key_aliases.py new file mode 100644 index 000000000..988f3ac9a --- /dev/null +++ b/test/unit_test/headless/test_windows_layout_key_aliases.py @@ -0,0 +1,224 @@ +"""``slash`` and its siblings name a character and follow the keyboard layout. + +The Windows key table had no readable name for the layout-dependent ``oem_N`` +keys because a fixed alias would be right on a US keyboard only. These names +are resolved when they are looked up: ``VkKeyScanExW`` on the character, +against the foreground window's layout, falling back to the US position when +the layout has no bare key for it. + +``user32`` and the layout probe are fakes: no key is sent and the real layout +of the machine running the test does not matter. +""" +import sys + +import pytest + +from je_auto_control.windows.core.utils import win32_layout_vk +from je_auto_control.windows.core.utils.win32_layout_vk import ( + LAYOUT_KEY_ALIASES, LayoutKeyTable, layout_virtual_key, +) + +windows_only = pytest.mark.skipif(not sys.platform.startswith("win"), + reason="the Win32 virtual-key table loads on Windows only") + +#: Microsoft "Virtual-Key Codes": the US positions of the eight characters. +_US = {"slash": 0xBF, "backslash": 0xDC, "semicolon": 0xBA, "quote": 0xDE, + "backquote": 0xC0, "bracketleft": 0xDB, "bracketright": 0xDD, "equal": 0xBB} +_OEM = {"oem_1": 0xBA, "oem_2": 0xBF, "oem_3": 0xC0, "oem_4": 0xDB, "oem_5": 0xDC, + "oem_6": 0xDD, "oem_7": 0xDE, "oem_plus": 0xBB, "oem_minus": 0xBD} +_US_LAYOUT, _OTHER_LAYOUT = 0x04090409, 0x04070407 +_SHIFT = 0x100 + + +class _Function: + """A ctypes function stand-in: accepts prototypes, records its calls.""" + + def __init__(self, answer): + self._answer = answer + self.calls = [] + self.argtypes = None + self.restype = None + + def __call__(self, character, layout): + self.calls.append((character, layout)) + return self._answer(character, layout) + + +class _User32: + """``user32`` with a ``VkKeyScanExW`` that answers from a per-layout map.""" + + def __init__(self, layouts): + self.VkKeyScanExW = _Function( + lambda character, layout: layouts.get(layout, {}).get(character, -1)) + + +#: German-like: "/" is Shift+7, "=" is Shift+0, "\\" needs AltGr, "`" is absent; +#: ";" sits on a bare key at another position, and so do the brackets. +_LAYOUTS = { + _US_LAYOUT: {"/": 0xBF, "\\": 0xDC, ";": 0xBA, "'": 0xDE, "`": 0xC0, + "[": 0xDB, "]": 0xDD, "=": 0xBB}, + _OTHER_LAYOUT: {"/": 0x37 | _SHIFT, "=": 0x30 | _SHIFT, "\\": 0xDB | 0x600, + ";": 0xBC, "'": 0xBF, "[": 0xDE, "]": 0xBA}, +} + + +@pytest.fixture +def layout(monkeypatch): + """Fake ``user32`` and layout probe; set ``layout.active`` to switch layouts.""" + user32 = _User32(_LAYOUTS) + + class _State: + active = _US_LAYOUT + fake = user32 + + monkeypatch.setattr(win32_layout_vk, "_user32", lambda: user32) + monkeypatch.setattr(win32_layout_vk, "foreground_keyboard_layout", lambda: _State.active) + return _State + + +def _table() -> LayoutKeyTable: + return LayoutKeyTable(dict(_OEM)) + + +# --- the lookup ------------------------------------------------------------- + +@pytest.mark.parametrize("name", sorted(_US)) +def test_on_a_us_layout_each_name_is_its_us_key(layout, name): + table = _table() + assert table[name] == _US[name] + assert table.get(name) == _US[name] + + +def test_another_layout_gives_the_key_that_types_the_character(layout): + layout.active = _OTHER_LAYOUT + table = _table() + assert table["semicolon"] == 0xBC # not oem_1 + assert table.get("quote") == 0xBF + assert table["bracketleft"] == 0xDE + assert table["bracketright"] == 0xBA + + +@pytest.mark.parametrize("name", ["slash", "equal", "backslash", "backquote"], + ids=["shift", "shift-digit", "altgr", "absent"]) +def test_a_character_with_no_bare_key_falls_back_to_the_us_position(layout, name): + layout.active = _OTHER_LAYOUT + assert _table()[name] == _US[name] + assert _table().get(name) == _US[name] + + +def test_the_layout_is_asked_at_every_lookup_not_once(layout): + table = _table() + assert table["semicolon"] == 0xBA + layout.active = _OTHER_LAYOUT # the user switches layout mid-run + assert table["semicolon"] == 0xBC + assert layout.fake.VkKeyScanExW.calls == [(";", _US_LAYOUT), (";", _OTHER_LAYOUT)] + + +def test_the_prototype_is_declared_before_the_call(layout): + import ctypes + from ctypes import wintypes + assert layout_virtual_key("/") == 0xBF + function = layout.fake.VkKeyScanExW + assert function.argtypes == [wintypes.WCHAR, wintypes.HKL] + assert function.restype is ctypes.c_short + + +def test_no_layout_means_the_us_position(layout): + layout.active = None # the probe failed, or this is not Windows + assert _table()["slash"] == 0xBF + assert layout.fake.VkKeyScanExW.calls == [] + + +@pytest.mark.parametrize("error", [OSError("no user32"), AttributeError("no WinDLL"), + ValueError("bad argument")]) +def test_a_failing_user32_means_the_us_position(layout, monkeypatch, error): + def broken(): + raise error + + monkeypatch.setattr(win32_layout_vk, "_user32", broken) + assert layout_virtual_key("/") is None + assert _table()["slash"] == 0xBF + + +def test_a_zero_virtual_key_is_not_an_answer(layout, monkeypatch): + monkeypatch.setitem(_LAYOUTS[_US_LAYOUT], "/", 0) + assert layout_virtual_key("/") is None + assert _table()["slash"] == 0xBF + + +# --- the table stays a dict ------------------------------------------------- + +def test_every_other_name_is_untouched_and_never_asks_the_layout(layout): + table = _table() + for name, code in _OEM.items(): + assert table[name] == code + assert table.get(name) == code + assert table.get("no_such_key") is None + assert table.get("no_such_key", 7) == 7 + with pytest.raises(KeyError): + table["no_such_key"] # noqa: B018 # reason: the lookup is the assertion + assert table.get(65) is None # a caller passing a key code, not a name + assert layout.fake.VkKeyScanExW.calls == [] + + +def test_iteration_shows_the_us_positions_without_asking_the_layout(layout): + layout.active = _OTHER_LAYOUT + table = _table() + assert isinstance(table, dict) + assert set(table) == set(_OEM) | set(_US) + assert len(table) == len(_OEM) + len(_US) + assert {name: table.copy()[name] for name in _US} == _US + assert dict(table.items())["semicolon"] == 0xBA + assert "slash" in table + assert sorted(table.layout_names()) == sorted(_US) + assert layout.fake.VkKeyScanExW.calls == [] + + +def test_a_name_the_table_already_has_is_kept_as_it_is(layout): + layout.active = _OTHER_LAYOUT + table = LayoutKeyTable({**_OEM, "semicolon": 0x99}) + assert table["semicolon"] == 0x99 # existing name: its code, no layout lookup + assert "semicolon" not in table.layout_names() + assert table["quote"] == 0xBF # the others still follow the layout + + +def test_the_alias_set_is_the_documented_one(): + assert {name: character for name, (character, _us) in LAYOUT_KEY_ALIASES.items()} == { + "slash": "/", "backslash": "\\", "semicolon": ";", "quote": "'", "backquote": "`", + "bracketleft": "[", "bracketright": "]", "equal": "="} + + +# --- the real Windows table -------------------------------------------------- + +@windows_only +def test_the_platform_table_resolves_the_names_through_the_layout(layout): + from je_auto_control.wrapper import auto_control_keyboard, platform_wrapper + table = platform_wrapper.keyboard_keys_table + assert isinstance(table, LayoutKeyTable) + for name, code in _US.items(): + assert table[name] == code + assert auto_control_keyboard._resolve_keycode(name) == code + layout.active = _OTHER_LAYOUT + assert auto_control_keyboard._resolve_keycode("semicolon") == 0xBC + assert auto_control_keyboard._resolve_keycode("slash") == 0xBF # fallback + assert auto_control_keyboard._resolve_keycode("oem_1") == 0xBA # positions do not move + + +@windows_only +def test_the_platform_aliases_never_answer_a_reverse_lookup(layout): + import je_auto_control as ac + from je_auto_control.wrapper import platform_wrapper + for name, (_character, us_name) in LAYOUT_KEY_ALIASES.items(): + assert platform_wrapper.keyboard_key_aliases[name] == us_name + assert ac.keyboard_key_name(_US[name]) == us_name # what a recorder writes + assert ac.keyboard_key_name(0xBD) == "oem_minus" + + +@windows_only +def test_every_name_that_existed_before_is_still_there(): + from je_auto_control.wrapper import platform_wrapper + table = platform_wrapper.keyboard_keys_table + for name in ("plus", "minus", "comma", "period", "ctrl", "enter", "oem_1", "oem_8", + "oem_102", "a", "A", "f24", "LAUNCH_APP2"): + assert name in table, name + assert dict(table)["plus"] == 0xBB and dict(table)["minus"] == 0xBD From 54702b2ace60af5539d10bdabfafb9e9795c1031 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:56:29 +0800 Subject: [PATCH 16/44] Let an MCP session start with a small tool list and enable what it needs tools/list sends all 680 descriptors, about 327 KB, to every client. Two opt-in tool modes avoid that; the default full mode is unchanged byte for byte. progressive: a session starts with five core tools (search, schema, enable, disable, state). Search returns summaries, never schemas. Enabling is local to one session, sends that session tools/list_changed, and is released when the session ends. tools/list is paged against a snapshot, so a plugin change between two pages cannot mix registries, and a cursor whose snapshot is gone is refused. static: a fixed profile for a client that reads the list once. A 2026-07-28 request gets it in progressive mode too, since that revision has no session. The mode decides what is offered, never what is allowed: every call still passes the role, schema, path-root, rate-limit and confirmation gates, and search, schema and enable answer only from what the caller may call. A tool a plugin removed is forgotten by every session. The connection-scoped state moves out of server.py into its own module to make room; server.py goes from 734 to 643 lines. --- benchmarks/mcp_discovery.py | 130 ++++ .../Eng/doc/ai_agents/ai_agents_doc.rst | 7 + .../Eng/doc/mcp_server/mcp_server_doc.rst | 157 +++++ .../source/Zh/doc/ai_agents/ai_agents_doc.rst | 6 + .../Zh/doc/mcp_server/mcp_server_doc.rst | 135 ++++ je_auto_control/__init__.py | 5 +- je_auto_control/utils/mcp_server/__init__.py | 23 +- je_auto_control/utils/mcp_server/__main__.py | 25 +- .../utils/mcp_server/_connection_state.py | 144 +++++ .../utils/mcp_server/_core_tools.py | 113 ++++ .../utils/mcp_server/disclosure.py | 609 ++++++++++++++++++ je_auto_control/utils/mcp_server/discovery.py | 264 ++++++++ je_auto_control/utils/mcp_server/server.py | 163 ++--- .../utils/mcp_server/tools/__init__.py | 24 +- .../utils/mcp_server/tools/_base.py | 39 +- .../headless/test_mcp_disclosure_policy.py | 329 ++++++++++ .../headless/test_mcp_discovery_index.py | 219 +++++++ .../headless/test_mcp_progressive_sessions.py | 314 +++++++++ 18 files changed, 2563 insertions(+), 143 deletions(-) create mode 100644 benchmarks/mcp_discovery.py create mode 100644 je_auto_control/utils/mcp_server/_connection_state.py create mode 100644 je_auto_control/utils/mcp_server/_core_tools.py create mode 100644 je_auto_control/utils/mcp_server/disclosure.py create mode 100644 je_auto_control/utils/mcp_server/discovery.py create mode 100644 test/unit_test/headless/test_mcp_disclosure_policy.py create mode 100644 test/unit_test/headless/test_mcp_discovery_index.py create mode 100644 test/unit_test/headless/test_mcp_progressive_sessions.py diff --git a/benchmarks/mcp_discovery.py b/benchmarks/mcp_discovery.py new file mode 100644 index 000000000..5b593fe45 --- /dev/null +++ b/benchmarks/mcp_discovery.py @@ -0,0 +1,130 @@ +"""What a new MCP session costs in each tool mode: full against progressive. + +Drives an in-process ``MCPServer`` through ``handle_line``, the same entry +point both transports use, so the figures are the dispatcher's and exclude +pipe and socket time. No tool that touches the desktop is ever called: the +only ``tools/call`` requests are the progressive mode's own search, schema +and enable tools. + +Per mode: + +* ``count`` / ``json_bytes`` -- tools in a new session's ``tools/list`` and the + UTF-8 size of that result, every page added up; +* ``handshake_ms`` -- ``initialize``, ``notifications/initialized`` and the + whole ``tools/list``, on a server that is already built; +* ``search_ms`` / ``schema_ms`` / ``enable_ms`` -- one call of each core tool + (``null`` in full mode, which has none), and ``search_json_bytes``, the size + of one search reply. + +Usage:: + + python benchmarks/mcp_discovery.py # the default registry + python benchmarks/mcp_discovery.py --rounds 50 +""" +import argparse +import json +import statistics +import time + +_QUERY = "screenshot" + + +def _line(method, params=None, msg_id=1): + message = {"jsonrpc": "2.0", "method": method, "params": params or {}} + if msg_id is not None: + message["id"] = msg_id + return json.dumps(message) + + +def _list_all(server): + """``(tool count, result bytes)`` of a session's whole ``tools/list``.""" + count, size, cursor = 0, 0, None + while True: + reply = json.loads(server.handle_line( + _line("tools/list", {"cursor": cursor} if cursor else {}))) + result = reply["result"] + count += len(result["tools"]) + size += len(json.dumps(result, ensure_ascii=False).encode("utf-8")) + cursor = result.get("nextCursor") + if cursor is None: + return count, size + + +def _handshake(server): + server.handle_line(_line("initialize", {"protocolVersion": "2025-11-25", "capabilities": {}})) + server.handle_line(_line("notifications/initialized", msg_id=None)) + return _list_all(server) + + +def _call(server, name, arguments): + reply = json.loads(server.handle_line( + _line("tools/call", {"name": name, "arguments": arguments}))) + result = reply["result"] + if result["isError"]: + raise RuntimeError(f"{name} failed: {result['content'][0]['text']}") + return result["content"][0]["text"] + + +def _median_ms(action, rounds): + samples = [] + for _ in range(rounds): + started = time.perf_counter() + action() + samples.append((time.perf_counter() - started) * 1000) + return round(statistics.median(samples), 3) + + +def _core_figures(server, target, rounds): + """Timings of the progressive mode's own tools against ``server``.""" + def enable_and_disable(): + _call(server, "ac_tools_enable", {"names": [target]}) + _call(server, "ac_tools_disable", {"names": [target]}) + _call(server, "ac_tools_search", {"query": _QUERY}) # builds the index once + return { + "search_ms": _median_ms(lambda: _call(server, "ac_tools_search", {"query": _QUERY}), rounds), + "search_json_bytes": len(_call(server, "ac_tools_search", {"query": _QUERY}).encode("utf-8")), + "schema_ms": _median_ms(lambda: _call(server, "ac_tools_schema", {"name": target}), rounds), + "enable_ms": _median_ms(enable_and_disable, rounds), + } + + +def _measure_mode(mode, tools, rounds): + from je_auto_control.utils.mcp_server.server import MCPServer + server = MCPServer(tools=list(tools), tool_mode=mode) + count, size = _handshake(server) + figures = { + "count": count, "json_bytes": size, + "handshake_ms": _median_ms(lambda: _handshake(server), rounds), + "search_ms": None, "search_json_bytes": None, "schema_ms": None, "enable_ms": None, + } + if mode == "progressive": + figures.update(_core_figures(server, tools[0].name, rounds)) + return figures + + +def measure(tools=None, rounds=20): + """Return the report for ``tools`` (default: the registry a server builds).""" + if tools is None: + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + tools = build_default_tool_registry() + tools = list(tools) + modes = {mode: _measure_mode(mode, tools, rounds) for mode in ("full", "progressive")} + full, progressive = modes["full"], modes["progressive"] + return { + "registry_tools": len(tools), "rounds": rounds, "modes": modes, + "progressive_json_bytes_percent_of_full": round( + 100 * progressive["json_bytes"] / max(1, full["json_bytes"]), 2), + } + + +def main(argv=None): + """Print the report as JSON.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--rounds", type=int, default=20, + help="Samples per timing; the median is reported.") + args = parser.parse_args(argv) + print(json.dumps(measure(rounds=max(1, args.rounds)), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/docs/source/Eng/doc/ai_agents/ai_agents_doc.rst b/docs/source/Eng/doc/ai_agents/ai_agents_doc.rst index 3d050b8f2..12764e388 100644 --- a/docs/source/Eng/doc/ai_agents/ai_agents_doc.rst +++ b/docs/source/Eng/doc/ai_agents/ai_agents_doc.rst @@ -62,6 +62,13 @@ Use read-only mode for inspection-only clients: je_auto_control_mcp --read-only +Start with a small tool list and let the session search and enable the rest +(see "Tool modes" in the MCP server chapter): + +.. code-block:: bash + + je_auto_control_mcp --tool-mode progressive + Recommended agent loop ---------------------- diff --git a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst index 316f691ca..a451d9ca2 100644 --- a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst @@ -209,6 +209,8 @@ exits — useful in CI smoke tests and prompt prep: je_auto_control_mcp --list-resources je_auto_control_mcp --list-prompts je_auto_control_mcp --read-only # serve only the read-only tools + je_auto_control_mcp --tool-mode progressive # a small core; see "Tool modes" + je_auto_control_mcp --list-tools --tool-mode progressive # what a new session sees je_auto_control_mcp --fake-backend # swap in the in-memory backend One ``--list-*`` flag prints its array; several print one object keyed @@ -444,6 +446,161 @@ existing client keeps working unchanged next to a 2026-07-28 one. confirmation included, since the question comes back in the result; an SSE ``POST`` additionally carries the call's progress notifications. +Tool modes: full, progressive, static +===================================== + +By default ``tools/list`` answers with every registered tool. That is the +**full** mode; it is the default and its replies are unchanged. Two other +modes exist for a client that should not carry several hundred schemas it +will never use: + +.. list-table:: + :header-rows: 1 + :widths: 16 84 + + * - Mode + - What ``tools/list`` offers + * - ``full`` + - Every tool, in one reply. The default. + * - ``progressive`` + - Five core tools. A session searches the registry, reads the schema of + what it wants and enables it; from then on the tool is in *that + session's* list and can be called. + * - ``static`` + - A fixed profile and nothing else; nothing can be enabled. For a client + that reads the list once and never again. + +Choose the mode with ``JE_AUTOCONTROL_MCP_TOOL_MODE``, with +``je_auto_control_mcp --tool-mode {full,progressive,static}``, or in code: + +.. code-block:: python + + import je_auto_control as ac + + ac.start_mcp_stdio_server(tool_mode="progressive") + # HTTP: hand the transport a server built in that mode + ac.start_mcp_http_server(mcp=ac.MCPServer(tool_mode="progressive")) + +``AC_start_mcp_server`` takes the same ``tool_mode`` argument, and the +variable reaches every way of starting a server, ``AC_start_mcp_http_server`` +included. A value that is none of the three stops the server from starting +rather than quietly offering everything. + +**The core tools** (progressive mode only): + +.. list-table:: + :header-rows: 1 + :widths: 24 76 + + * - Tool + - What it does + * - ``ac_tools_search`` + - ``query``, ``limit`` (1-50, default 10), optional ``category`` and + ``capability``. Returns rows of ``name`` / ``summary`` / ``category`` / + ``capability`` / ``read_only`` / ``takes_paths`` / ``enabled`` plus + ``total`` and ``truncated`` -- never a schema. + * - ``ac_tools_schema`` + - The full descriptor of one tool, by ``name``. + * - ``ac_tools_enable`` + - ``names``: tool names, or ``category:`` for a whole category. + Replies with ``enabled`` / ``already_enabled`` / ``unavailable`` and + whether ``notifications/tools/list_changed`` was sent. + * - ``ac_tools_disable`` + - Removes tools the session enabled. + * - ``ac_tools_state`` + - The mode, what the session has enabled, the categories and their + sizes, and the limits calls are held to (read-only, path roots, + ``env://`` allowlist, role). + +A category is the name of the factory that builds the tools (``mouse``, +``screen``, ``window`` ...); a plugin's tools are in ``plugin``. + +**Sessions.** The enabled set belongs to one session and to nobody else: +an ``Mcp-Session-Id`` over HTTP (or, for a client that ignores the header, +its connection), and the one implicit session of a stdio server. It is +released when the session is deleted, swept, evicted, when the connection +closes or when the stdio loop ends. Enabling or disabling sends +``notifications/tools/list_changed`` to that session only; a plain HTTP +``POST`` with no standing ``GET`` stream has nowhere to receive it, so the +reply says ``"list_changed_sent": false`` and the client should request +``tools/list`` again. + +**Paging.** In these two modes ``tools/list`` is paged, 100 tools to a page +(``server.disclosure.page_size``): a result with more to come carries +``nextCursor``, to be sent back as ``cursor``. Every page carries the id of +the list it belongs to under ``_meta`` +``io.github.integration-automation/toolSnapshot``. A cursor continues the +list as it was when its first page was requested, even if a plugin was +loaded or removed in between, so no page mixes two registries. A cursor that +is malformed, belongs to another session, or names a list the server no +longer keeps (it remembers the last eight per session) is answered +``-32602``; request ``tools/list`` again without one. The full mode is not +paged and ignores ``cursor``, as before. + +**The static profile.** ``JE_AUTOCONTROL_MCP_TOOL_PROFILE`` is a +comma-separated list of tool names and ``category:`` entries. In +static mode it is the whole list; unset, the profile is the 19 tools behind +the short aliases (``ac_click_mouse``, ``ac_type_text``, ``ac_screenshot`` +...). In progressive mode a configured profile is offered to every session +beside the core, so a client that never refreshes its list still has those +tools. MCP has no client capability that says "I follow +``tools/list_changed``", so the server cannot detect such a client: use the +static mode, or a profile, for one. A 2026-07-28 request is always served +the static profile in progressive mode -- that revision has no session for +an enabled set to live in. + +**What the mode does not change.** It decides what is *offered*, never what +is *allowed*: + +* Every call still goes through the same gates in the same order -- role, + input schema, path roots and ``env://`` allowlist, rate limit, + confirmation -- and is audited as before. +* Search, schema and enable answer from what the caller may call. A tool the + caller's role does not grant, or that read-only mode rules out, is not + found, not described and reported as ``unavailable`` when enabled. +* A call to a tool the session's list does not hold is answered ``-32602`` + and recorded in the audit log as ``denied``. A role refusal keeps its own + code, ``-32003``. +* In read-only mode a mutating tool cannot be enabled, including one a + plugin registers after the server started. +* A tool a plugin removes is gone from every session at once. If the same + name is registered again later, it has to be enabled again. +* The core tools' names cannot be registered over or removed. + +**What it costs.** ``benchmarks/mcp_discovery.py`` measures both modes +in-process through the dispatcher both transports use. On the 680-tool +registry (2026-10-09, one Windows 11 machine, median of 30): + +.. list-table:: + :header-rows: 1 + :widths: 40 30 30 + + * - Figure + - ``full`` + - ``progressive`` + * - Tools in a new session's ``tools/list`` + - 680 + - 5 + * - Size of that result + - 327,378 bytes + - 2,411 bytes (0.74 %) + * - ``initialize`` + ``initialized`` + ``tools/list`` + - 43.7 ms + - 1.0 ms + * - One ``ac_tools_search`` (10 rows, 2,390 bytes) + - -- + - 3.1 ms + * - One ``ac_tools_schema`` + - -- + - 0.3 ms + * - One ``ac_tools_enable`` + ``ac_tools_disable`` + - -- + - 0.9 ms + +.. code-block:: shell + + python benchmarks/mcp_discovery.py --rounds 30 + Read-only / safe mode ===================== diff --git a/docs/source/Zh/doc/ai_agents/ai_agents_doc.rst b/docs/source/Zh/doc/ai_agents/ai_agents_doc.rst index 484023335..707da7d4b 100644 --- a/docs/source/Zh/doc/ai_agents/ai_agents_doc.rst +++ b/docs/source/Zh/doc/ai_agents/ai_agents_doc.rst @@ -60,6 +60,12 @@ MCP 同時提供完整的 ac_* 工具，以及給模型使用的短名稱 alias je_auto_control_mcp --read-only +先只給少量工具，讓 session 自行搜尋並啟用其餘工具（見 MCP 伺服器章節的「工具模式」）： + +.. code-block:: bash + + je_auto_control_mcp --tool-mode progressive + 建議的 Agent Loop ----------------- diff --git a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst index 6118174b4..495eaee2e 100644 --- a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst @@ -192,6 +192,8 @@ CLI 檢視旗標 je_auto_control_mcp --list-resources je_auto_control_mcp --list-prompts je_auto_control_mcp --read-only # 伺服器只提供唯讀工具 + je_auto_control_mcp --tool-mode progressive # 只提供少量核心工具;見「工具模式」 + je_auto_control_mcp --list-tools --tool-mode progressive # 新 session 看到的清單 je_auto_control_mcp --fake-backend # 切換成記憶體版 backend 只給一個 ``--list-*`` 旗標時輸出該陣列;給多個時輸出一個以 ``tools`` / ``resources`` / @@ -382,6 +384,139 @@ client 不用改,可以和 2026-07-28 的 client 並存。 普通的 JSON ``POST`` 就能做所有事,確認也一樣(問題放在結果裡回來);SSE ``POST`` 另外會送出呼叫的進度通知。 +工具模式:full、progressive、static +==================================== + +預設情況下 ``tools/list`` 會回傳所有已註冊的工具,這就是 **full** 模式;它是預設值, +回應內容與以前完全相同。另外兩種模式給不該背著幾百份用不到的 schema 的 client 使用: + +.. list-table:: + :header-rows: 1 + :widths: 16 84 + + * - 模式 + - ``tools/list`` 提供什麼 + * - ``full`` + - 全部工具,一次回完。預設值。 + * - ``progressive`` + - 五個核心工具。Session 先搜尋 registry、讀取想用的工具的 schema、再啟用它; + 之後該工具才出現在 *這個 session* 的清單裡,並且可以呼叫。 + * - ``static`` + - 一份固定的 profile,不能再啟用任何工具。給只讀一次清單、之後不再更新的 client。 + +用 ``JE_AUTOCONTROL_MCP_TOOL_MODE``、``je_auto_control_mcp --tool-mode +{full,progressive,static}`` 或程式碼選擇模式: + +.. code-block:: python + + import je_auto_control as ac + + ac.start_mcp_stdio_server(tool_mode="progressive") + # HTTP:把以該模式建立的伺服器交給傳輸層 + ac.start_mcp_http_server(mcp=ac.MCPServer(tool_mode="progressive")) + +``AC_start_mcp_server`` 接受同一個 ``tool_mode`` 參數;環境變數則對所有啟動方式生效, +包含 ``AC_start_mcp_http_server``。不是這三個值之一時伺服器會拒絕啟動,而不是悄悄地 +提供全部工具。 + +**核心工具** (僅 progressive 模式): + +.. list-table:: + :header-rows: 1 + :widths: 24 76 + + * - 工具 + - 作用 + * - ``ac_tools_search`` + - ``query``、``limit`` (1-50,預設 10),可選的 ``category`` 與 ``capability``。 + 回傳 ``name`` / ``summary`` / ``category`` / ``capability`` / ``read_only`` / + ``takes_paths`` / ``enabled`` 的列,以及 ``total`` 與 ``truncated``——絕不含 schema。 + * - ``ac_tools_schema`` + - 依 ``name`` 取得單一工具的完整描述。 + * - ``ac_tools_enable`` + - ``names``:工具名稱,或用 ``category:<名稱>`` 啟用整個分類。回覆 + ``enabled`` / ``already_enabled`` / ``unavailable``,以及是否送出了 + ``notifications/tools/list_changed``。 + * - ``ac_tools_disable`` + - 移除這個 session 啟用過的工具。 + * - ``ac_tools_state`` + - 目前模式、這個 session 已啟用的工具、各分類及其數量,以及呼叫所受的限制 + (唯讀、路徑根目錄、``env://`` 允許清單、角色)。 + +分類就是建立該組工具的 factory 名稱(``mouse``、``screen``、``window`` ……);plugin +的工具歸在 ``plugin``。 + +**Session。** 已啟用的集合只屬於一個 session,不與任何人共用:HTTP 上是 +``Mcp-Session-Id`` (client 忽略該標頭時則是它的連線),stdio 伺服器則有唯一一個隱含的 +session。Session 被刪除、被清掃、被淘汰、連線關閉或 stdio 迴圈結束時就會釋放。啟用或 +停用只會對該 session 送出 ``notifications/tools/list_changed``;沒有常駐 ``GET`` 串流的 +普通 HTTP ``POST`` 無處可收,所以回覆會是 ``"list_changed_sent": false``,client 應該 +重新請求 ``tools/list``。 + +**分頁。** 在這兩種模式下 ``tools/list`` 會分頁,每頁 100 個工具 +(``server.disclosure.page_size``):還有下一頁的結果會帶 ``nextCursor``,下一次請求 +以 ``cursor`` 送回。每一頁都在 ``_meta`` 的 +``io.github.integration-automation/toolSnapshot`` 帶著所屬清單的 id。Cursor 延續的是 +第一頁被請求當下的那份清單,即使中途有 plugin 載入或移除也一樣,所以不會有一頁混到 +兩份 registry。格式錯誤、屬於別的 session、或指向伺服器已不再保留的清單(每個 session +只記最近八份)的 cursor 會得到 ``-32602``;請不帶 cursor 重新請求 ``tools/list``。 +full 模式不分頁,並且和以前一樣忽略 ``cursor``。 + +**靜態 profile。** ``JE_AUTOCONTROL_MCP_TOOL_PROFILE`` 是以逗號分隔的工具名稱與 +``category:<名稱>`` 項目。在 static 模式下它就是整份清單;未設定時,profile 是短別名 +背後的 19 個工具(``ac_click_mouse``、``ac_type_text``、``ac_screenshot`` ……)。在 +progressive 模式下,有設定的 profile 會和核心工具一起提供給每個 session,因此從不更新 +清單的 client 仍然有這些工具可用。MCP 沒有任何 client capability 能表示「我會處理 +``tools/list_changed``」,所以伺服器無法偵測這種 client:請對它使用 static 模式或 +profile。2026-07-28 的請求在 progressive 模式下一律得到靜態 profile——該版本沒有可以 +存放已啟用集合的 session。 + +**模式不會改變的事。** 它決定的是「提供」什麼,從來不是「允許」什麼: + +* 每一次呼叫仍然依同樣的順序通過同樣的關卡——角色、輸入 schema、路徑根目錄與 + ``env://`` 允許清單、rate limit、確認——稽核紀錄也和以前一樣。 +* 搜尋、schema 與啟用都只依呼叫者可以呼叫的工具回答。角色不允許、或唯讀模式排除的 + 工具找不到、查不到描述,啟用時會被回報為 ``unavailable``。 +* 呼叫 session 清單裡沒有的工具會得到 ``-32602``,並在稽核 log 記為 ``denied``。 + 角色拒絕仍然是它自己的代碼 ``-32003``。 +* 唯讀模式下不能啟用會變更狀態的工具,包含伺服器啟動後才由 plugin 註冊的工具。 +* Plugin 移除的工具會立刻從所有 session 消失。之後若以同名重新註冊,必須重新啟用。 +* 核心工具的名稱不能被覆蓋註冊,也不能被移除。 + +**成本。** ``benchmarks/mcp_discovery.py`` 透過兩種傳輸共用的 dispatcher,在同一個 +process 內量測兩種模式。以 680 個工具的 registry 量測(2026-10-09,一台 Windows 11 +機器,30 次取中位數): + +.. list-table:: + :header-rows: 1 + :widths: 40 30 30 + + * - 項目 + - ``full`` + - ``progressive`` + * - 新 session 的 ``tools/list`` 工具數 + - 680 + - 5 + * - 該結果的大小 + - 327,378 bytes + - 2,411 bytes(0.74 %) + * - ``initialize`` + ``initialized`` + ``tools/list`` + - 43.7 ms + - 1.0 ms + * - 一次 ``ac_tools_search`` (10 列,2,390 bytes) + - -- + - 3.1 ms + * - 一次 ``ac_tools_schema`` + - -- + - 0.3 ms + * - 一次 ``ac_tools_enable`` + ``ac_tools_disable`` + - -- + - 0.9 ms + +.. code-block:: shell + + python benchmarks/mcp_discovery.py --rounds 30 + 唯讀 / 安全模式 =============== diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 475c40ca2..7b4bdfa2f 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1083,7 +1083,8 @@ AuditLogger, HttpMCPServer, MCPContent, MCPPrompt, MCPPromptArgument, MCPResource, MCPServer, MCPTool, MCPToolAnnotations, OperationCancelledError, PromptProvider, RateLimiter, - ResourceProvider, ToolCallContext, build_default_tool_registry, + ResourceProvider, ToolCallContext, ToolDisclosureError, ToolDiscoveryError, + ToolIndex, ToolMode, ToolSummary, ToolView, build_default_tool_registry, default_prompt_provider, default_resource_provider, make_plugin_tool, register_plugin_tools, start_mcp_http_server, start_mcp_stdio_server, @@ -1755,6 +1756,8 @@ def start_autocontrol_gui(*args, **kwargs): "MCPPromptArgument", "MCPResource", "MCPServer", "MCPTool", "MCPToolAnnotations", "OperationCancelledError", "PromptProvider", "RateLimiter", "ResourceProvider", "ToolCallContext", + "ToolDisclosureError", "ToolDiscoveryError", "ToolIndex", "ToolMode", + "ToolSummary", "ToolView", "build_default_tool_registry", "default_prompt_provider", "default_resource_provider", "make_plugin_tool", "register_plugin_tools", diff --git a/je_auto_control/utils/mcp_server/__init__.py b/je_auto_control/utils/mcp_server/__init__.py index 27776f30d..9aaca4a29 100644 --- a/je_auto_control/utils/mcp_server/__init__.py +++ b/je_auto_control/utils/mcp_server/__init__.py @@ -5,6 +5,10 @@ can drive the host machine through AutoControl. The transport is JSON-RPC 2.0 over stdio, implemented with stdlib only — no extra dependencies are required. + +A server offers every tool by default. :mod:`.disclosure` adds the +progressive and static tool modes, and :mod:`.discovery` the search index +those modes are built on. """ from je_auto_control.utils.mcp_server.server import ( MCPServer, start_mcp_stdio_server, @@ -13,6 +17,13 @@ from je_auto_control.utils.mcp_server.context import ( OperationCancelledError, ToolCallContext, ) +from je_auto_control.utils.mcp_server.disclosure import ( + DisclosureResult, ToolDisclosure, ToolDisclosureError, ToolMode, ToolPage, + ToolView, +) +from je_auto_control.utils.mcp_server.discovery import ( + ToolDiscoveryError, ToolIndex, ToolSummary, +) from je_auto_control.utils.mcp_server.fake_backend import ( FakeState, fake_state, install_fake_backend, reset_fake_state, uninstall_fake_backend, @@ -31,16 +42,20 @@ default_resource_provider, ) from je_auto_control.utils.mcp_server.tools import ( - MCPContent, MCPTool, MCPToolAnnotations, build_default_tool_registry, - make_plugin_tool, register_plugin_tools, + MCPContent, MCPTool, MCPToolAnnotations, MCPToolDescriptor, + build_default_tool_registry, make_plugin_tool, register_plugin_tools, ) __all__ = [ - "AuditLogger", "FakeState", "HttpMCPServer", "LiveScreenProvider", + "AuditLogger", "DisclosureResult", "FakeState", "HttpMCPServer", + "LiveScreenProvider", "MCPContent", "MCPLogBridge", "MCPPrompt", "MCPPromptArgument", "MCPResource", "MCPServer", "MCPTool", "MCPToolAnnotations", + "MCPToolDescriptor", "OperationCancelledError", "PluginWatcher", "PromptProvider", - "RateLimiter", "ResourceProvider", "ToolCallContext", + "RateLimiter", "ResourceProvider", "ToolCallContext", "ToolDisclosure", + "ToolDisclosureError", "ToolDiscoveryError", "ToolIndex", "ToolMode", + "ToolPage", "ToolSummary", "ToolView", "build_default_tool_registry", "default_prompt_provider", "default_resource_provider", "fake_state", "install_fake_backend", "make_plugin_tool", diff --git a/je_auto_control/utils/mcp_server/__main__.py b/je_auto_control/utils/mcp_server/__main__.py index a551c67b7..adf6234b8 100644 --- a/je_auto_control/utils/mcp_server/__main__.py +++ b/je_auto_control/utils/mcp_server/__main__.py @@ -2,7 +2,9 @@ Without flags this starts the stdio MCP server. With one of the ``--list-*`` flags it prints the requested catalogue to stdout and -exits — useful for inspection in CI or manual debugging. +exits — useful for inspection in CI or manual debugging. ``--tool-mode`` +chooses how much of the registry ``tools/list`` offers (see +:mod:`.disclosure`); ``--list-tools`` then prints what a new session sees. """ import argparse import json @@ -10,6 +12,7 @@ from typing import Optional from je_auto_control.utils.cli_output import utf8_stdout +from je_auto_control.utils.mcp_server.disclosure import MODE_ENV, ToolMode from je_auto_control.utils.mcp_server.fake_backend import ( install_fake_backend, maybe_install_from_env, ) @@ -17,7 +20,7 @@ from je_auto_control.utils.mcp_server.resources import ( default_resource_provider, ) -from je_auto_control.utils.mcp_server.server import start_mcp_stdio_server +from je_auto_control.utils.mcp_server.server import MCPServer, start_mcp_stdio_server from je_auto_control.utils.mcp_server.tools import ( build_default_tool_registry, ) @@ -44,6 +47,12 @@ def _build_parser() -> argparse.ArgumentParser: "--read-only", action="store_true", help="Restrict tools to those marked readOnlyHint=true.", ) + parser.add_argument( + "--tool-mode", choices=[mode.value for mode in ToolMode], default=None, + help=("How much of the registry tools/list offers: full (every tool, the " + "default), progressive (a small core; a session searches and enables " + f"what it needs) or static (a fixed profile). Default: ${MODE_ENV}."), + ) parser.add_argument( "--fake-backend", action="store_true", help=("Install the in-memory fake backend so tools record but " @@ -64,7 +73,11 @@ def main(argv: Optional[list] = None) -> None: if any(listing_modes): _print_listings(args) return - start_mcp_stdio_server(read_only=True if args.read_only else None) + read_only = True if args.read_only else None + if args.tool_mode is None: + start_mcp_stdio_server(read_only=read_only) + return + start_mcp_stdio_server(read_only=read_only, tool_mode=args.tool_mode) def _print_listings(args: argparse.Namespace) -> None: @@ -76,8 +89,10 @@ def _print_listings(args: argparse.Namespace) -> None: """ listings = {} if args.list_tools: - listings["tools"] = [tool.to_descriptor() for tool in - build_default_tool_registry(read_only=True if args.read_only else None)] + read_only = True if args.read_only else None + server = MCPServer(tools=build_default_tool_registry(read_only=read_only), + tool_mode=args.tool_mode, read_only=read_only) + listings["tools"] = [tool.to_descriptor() for tool in server.disclosure.initial_tools()] if args.list_resources: listings["resources"] = [item.to_descriptor() for item in default_resource_provider().list()] if args.list_prompts: diff --git a/je_auto_control/utils/mcp_server/_connection_state.py b/je_auto_control/utils/mcp_server/_connection_state.py new file mode 100644 index 000000000..8c3f398bc --- /dev/null +++ b/je_auto_control/utils/mcp_server/_connection_state.py @@ -0,0 +1,144 @@ +"""Connection-scoped state of the MCP server: who this thread is serving. + +The stdio transport has exactly one peer, so a server-wide default is right +for it; an HTTP transport has many, and each request runs on its own thread. +Keeping the notifier, the writer, the concurrency switch and the peer's +identity in thread-local storage is what stops one client's progress +notifications from being emitted down another client's socket, and what lets +client capabilities, active calls and the tool view of a progressive session +(:mod:`.disclosure`) be scoped to one peer and released when it goes away. +""" +import contextlib +import threading +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional + + +class ConnectionStateMixin: + """Per-peer state, mixed into :class:`MCPServer`. + + Requires the host to provide ``_local`` (thread-local storage), the + ``_default_*`` slots, ``_client_caps_by_conn`` with ``_caps_lock``, + ``_active_calls`` with ``_calls_lock``, ``_stateless_request``, + ``_end_listeners`` and ``disclosure``. + """ + + if TYPE_CHECKING: + _local: Any + _default_notifier: Optional[Callable[[str, Dict[str, Any]], None]] + _default_writer: Optional[Callable[[str], None]] + _default_concurrent_tools: bool + _default_client_capabilities: Dict[str, Any] + _client_caps_by_conn: Dict[Any, Dict[str, Any]] + _caps_lock: threading.Lock + _active_calls: Dict[Any, Any] + _calls_lock: threading.Lock + disclosure: Any + + @property + def _stateless_request(self) -> Any: + """The 2026-07-28 request being served on this thread, if any.""" + + def _end_listeners(self, which: Callable[[Any], bool], graceful: bool) -> None: + """End every subscription whose connection id satisfies ``which``.""" + + # --- thread-scoped accessors -------------------------------------------- + # + # Each property prefers a value set for the current thread (one HTTP + # request = one thread) and falls back to the server-wide default that + # stdio and set_notifier() use. A plain POST therefore sees *no* notifier + # rather than inheriting whichever SSE connection happens to be open — + # correct, since a plain POST has no stream to deliver notifications on. + + @property + def _notifier(self) -> Optional[Callable[[str, Dict[str, Any]], None]]: + return getattr(self._local, "notifier", None) or self._default_notifier + + @_notifier.setter + def _notifier(self, value) -> None: + self._default_notifier = value + + @property + def _writer(self) -> Optional[Callable[[str], None]]: + return getattr(self._local, "writer", None) or self._default_writer + + @_writer.setter + def _writer(self, value) -> None: + self._default_writer = value + + @property + def _concurrent_tools(self) -> bool: + scoped = getattr(self._local, "concurrent_tools", None) + if scoped is None: + return self._default_concurrent_tools + return scoped + + @_concurrent_tools.setter + def _concurrent_tools(self, value) -> None: + self._default_concurrent_tools = bool(value) + + @property + def _connection_id(self) -> Any: + """Identity of the peer served on this thread (None for stdio).""" + return getattr(self._local, "connection_id", None) + + @property + def _client_capabilities(self) -> Dict[str, Any]: + """Capabilities advertised by the peer served on this thread.""" + if self._stateless_request is not None: + return self._stateless_request.capabilities + conn = self._connection_id + if conn is None: + return self._default_client_capabilities + with self._caps_lock: + return self._client_caps_by_conn.get(conn, {}) + + @_client_capabilities.setter + def _client_capabilities(self, value: Dict[str, Any]) -> None: + conn = self._connection_id + if conn is None: + self._default_client_capabilities = value + return + with self._caps_lock: + self._client_caps_by_conn[conn] = value + + @contextlib.contextmanager + def connection_scope(self, *, notifier=None, writer=None, + concurrent_tools=None, connection_id=None): + """Bind notifier/writer/concurrency/identity to the calling thread only. + + Transports that serve more than one peer must wrap each request in + this. Previously they swapped the attributes on the shared server, so + any concurrent request bound to the wrong peer's socket. ``connection_id`` + scopes active-call slots and client capabilities per peer. + """ + prior = (getattr(self._local, "notifier", None), + getattr(self._local, "writer", None), + getattr(self._local, "concurrent_tools", None), + getattr(self._local, "connection_id", None)) + self._local.notifier = notifier + self._local.writer = writer + self._local.concurrent_tools = concurrent_tools + self._local.connection_id = connection_id + try: + yield self + finally: + (self._local.notifier, self._local.writer, + self._local.concurrent_tools, self._local.connection_id) = prior + + def forget_connection(self, connection_id: Any) -> None: + """Drop per-connection state when a transport connection closes. + + HTTP connections are transient; without this their capabilities and + any stray active-call contexts would accumulate for the server's life. + """ + if connection_id is None: + return + with self._caps_lock: + self._client_caps_by_conn.pop(connection_id, None) + self._end_listeners(lambda conn: conn == connection_id, graceful=False) + self.disclosure.forget(connection_id) + with self._calls_lock: + stale = [key for key in self._active_calls + if isinstance(key, tuple) and key[0] == connection_id] + for key in stale: + self._active_calls.pop(key, None) diff --git a/je_auto_control/utils/mcp_server/_core_tools.py b/je_auto_control/utils/mcp_server/_core_tools.py new file mode 100644 index 000000000..34cd020a2 --- /dev/null +++ b/je_auto_control/utils/mcp_server/_core_tools.py @@ -0,0 +1,113 @@ +"""The five tools a progressive MCP session starts with. + +Search, schema, enable, disable and state: everything a client needs to find +a tool and add it to its own list, and nothing that runs one. They are +ordinary registry entries, so a call to any of them passes the same gates as +every other call, and each answers from the caller's own catalog +(:meth:`ToolDisclosure.catalog`) -- a tool the caller's role or the server's +read-only mode rules out is not found, described or enabled here. + +All five are annotated read-only. They change one session's tool list, not +the machine, and a read-only server or a viewer role has to be able to use +them to reach the read-only tools it is allowed. +""" +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.mcp_server.disclosure import ( + CATEGORY_PREFIX, CORE_TOOL_NAMES, DISABLE_TOOL, ENABLE_TOOL, SCHEMA_TOOL, + SEARCH_TOOL, STATE_TOOL, ToolDisclosure, +) +from je_auto_control.utils.mcp_server.discovery import ( + DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT, checked_limit, +) +from je_auto_control.utils.mcp_server.tools._base import MCPTool, READ_ONLY, schema +from je_auto_control.utils.rbac.authorization import current_authorization +from je_auto_control.utils.rbac.users import Capability + +CORE_CATEGORY = "tool_discovery" +_Spec = Tuple[str, str, Dict[str, Any], Callable[..., Any]] + + +class _CoreHandlers: + """The handlers, bound to one server's :class:`ToolDisclosure`.""" + + def __init__(self, disclosure: ToolDisclosure) -> None: + self._disclosure = disclosure + + def search(self, query: str = "", limit: int = DEFAULT_SEARCH_LIMIT, + category: Optional[str] = None, + capability: Optional[str] = None) -> Dict[str, Any]: + """Summaries of the best matches, each marked with whether it is enabled.""" + bounded = checked_limit(limit) + found = self._disclosure.index().matches(query, category=category, capability=capability) + visible = set(self._disclosure.current_view().visible_names) + return { + "tools": [{**item.to_dict(), "enabled": item.name in visible} + for item in found[:bounded]], + "total": len(found), "truncated": len(found) > bounded, + } + + def schema(self, name: str) -> Dict[str, Any]: + """The full descriptor of one tool.""" + return self._disclosure.index().get_schema(name).to_dict() + + def enable(self, names: Sequence[str]) -> Dict[str, Any]: + """Add tools to the calling session's list.""" + return self._disclosure.current_view().enable(names).to_dict() + + def disable(self, names: Sequence[str]) -> Dict[str, Any]: + """Remove tools the calling session enabled.""" + return self._disclosure.current_view().disable(names).to_dict() + + def state(self) -> Dict[str, Any]: + """The mode, the session's enabled tools and the limits calls are held to.""" + disclosure = self._disclosure + caller = current_authorization() + index = disclosure.index() + return { + "mode": disclosure.mode.value, "read_only": disclosure.read_only, + "role": caller.role if caller is not None else None, + "path_roots": disclosure.argument_policy.path_policy.enabled, + "env_allowlist": disclosure.argument_policy.env_allowlist is not None, + "available": len(index), + "enabled": [name for name in disclosure.current_view().visible_names + if name not in CORE_TOOL_NAMES], + "categories": index.categories(), + } + + +def build_core_tools(disclosure: ToolDisclosure) -> List[MCPTool]: + """The core tools of ``disclosure``'s server, in :data:`CORE_TOOL_NAMES` order.""" + handlers = _CoreHandlers(disclosure) + names = {"type": "array", "items": {"type": "string"}, + "description": f"Tool names, or {CATEGORY_PREFIX} for a whole category."} + specs: List[_Spec] = [ + (SEARCH_TOOL, + "Search the tools this server has. Returns names, one-line summaries, category " + f"and required capability, never schemas. Enable a result with {ENABLE_TOOL}.", + schema({ + "query": {"type": "string", "description": "Words to look for; empty lists all."}, + "limit": {"type": "integer", "minimum": 1, "maximum": MAX_SEARCH_LIMIT, + "default": DEFAULT_SEARCH_LIMIT}, + "category": {"type": "string", "description": f"From {STATE_TOOL}."}, + "capability": {"type": "string", "enum": Capability.all()}, + }), handlers.search), + (SCHEMA_TOOL, "Return the full schema of one tool by name.", + schema({"name": {"type": "string"}}, required=["name"]), handlers.schema), + (ENABLE_TOOL, + "Add tools to this session's tool list so they can be called. The list changes; " + "request tools/list again if no list_changed notification arrives.", + schema({"names": names}, required=["names"]), handlers.enable), + (DISABLE_TOOL, "Remove tools this session enabled from its tool list.", + schema({"names": names}, required=["names"]), handlers.disable), + (STATE_TOOL, + "Report the tool mode, what this session has enabled, the categories that can be " + "searched and the limits (read-only, path roots, role) calls are held to.", + schema({}), handlers.state), + ] + return [MCPTool(name=name, description=description, input_schema=input_schema, + handler=handler, annotations=READ_ONLY, category=CORE_CATEGORY) + for name, description, input_schema, handler in specs] + + +__all__ = ["CORE_CATEGORY", "build_core_tools"] diff --git a/je_auto_control/utils/mcp_server/disclosure.py b/je_auto_control/utils/mcp_server/disclosure.py new file mode 100644 index 000000000..277303772 --- /dev/null +++ b/je_auto_control/utils/mcp_server/disclosure.py @@ -0,0 +1,609 @@ +"""Which tools a session is offered: all of them, or a small core that grows. + +``tools/list`` in the default **full** mode answers with every registered +tool, as it always has. Two other modes exist for clients that should not +carry several hundred schemas they will never use: + +* **progressive** -- a session starts with the core tools of + :mod:`._core_tools` (search, schema, enable, disable, state) and enables + what it needs. The enabled set belongs to one session: an HTTP + ``Mcp-Session-Id`` (or, for a client that ignores the header, its + connection), and the one implicit session of a stdio server. Enabling + sends that session ``notifications/tools/list_changed``. +* **static** -- ``tools/list`` is a fixed profile and nothing can be enabled, + for a client that reads the list once and never again. A 2026-07-28 request + is served this way in progressive mode too: that revision has no session + for an enabled set to live in. + +Disclosure changes what is *offered*, never what is *allowed*. The registry +stays the only source of tools; a view is a set of names filtered through it +on every use, so a tool a plugin removed, the caller's role does not grant +or read-only mode rules out is gone from the view whatever the session +remembers. A call still passes every existing gate in +``MCPServer._prepare_tool_call`` -- role, schema, path roots, rate limit, +confirmation -- and :meth:`ToolDisclosure.require_available` only adds one +more refusal in front of them. + +``tools/list`` is paged in these two modes. A cursor names a snapshot of the +view taken when its first page was requested, so later pages come from that +same list even if a plugin changed the registry in between; a cursor whose +snapshot is no longer kept is refused instead of paging a different list. +""" +import base64 +import binascii +import hashlib +import hmac +import os +import secrets +import threading +from collections import OrderedDict +from dataclasses import dataclass +from enum import Enum +from typing import ( + TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union, +) + +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.mcp_server._authz import visible_tools +from je_auto_control.utils.mcp_server._protocol import _MCPError +from je_auto_control.utils.mcp_server.discovery import ToolIndex, category_of +from je_auto_control.utils.mcp_server.tools import COMMON_TOOL_NAMES, MCPTool +from je_auto_control.utils.mcp_server.tools._base import read_only_env_flag +from je_auto_control.utils.rbac.authorization import current_authorization + +if TYPE_CHECKING: + from je_auto_control.utils.mcp_server.server import MCPServer + +#: ``full`` (default), ``progressive`` or ``static``. +MODE_ENV = "JE_AUTOCONTROL_MCP_TOOL_MODE" +#: Comma-separated tool names and ``category:`` entries. +PROFILE_ENV = "JE_AUTOCONTROL_MCP_TOOL_PROFILE" +#: ``_meta`` key of a paged ``tools/list`` result: the snapshot it is a page of. +SNAPSHOT_META = "io.github.integration-automation/toolSnapshot" +#: The static profile when none is configured: the tools behind the aliases. +DEFAULT_PROFILE: Tuple[str, ...] = COMMON_TOOL_NAMES +DEFAULT_PAGE_SIZE = 100 +#: Entries one enable / disable call may name. +MAX_NAMES_PER_CALL = 100 +CATEGORY_PREFIX = "category:" +_MAX_NAME_CHARS = 200 +#: Views kept at once. HTTP sessions are capped far below this; it bounds a leak. +_MAX_VIEWS = 1024 +_MAX_CACHED_CALLERS = 16 +_LIST_CHANGED = "notifications/tools/list_changed" +_SEND_ERRORS = (OSError, RuntimeError, ValueError) + +SEARCH_TOOL = "ac_tools_search" +SCHEMA_TOOL = "ac_tools_schema" +ENABLE_TOOL = "ac_tools_enable" +DISABLE_TOOL = "ac_tools_disable" +STATE_TOOL = "ac_tools_state" +#: What a progressive session starts with, in the order it is listed. +CORE_TOOL_NAMES: Tuple[str, ...] = ( + SEARCH_TOOL, SCHEMA_TOOL, ENABLE_TOOL, DISABLE_TOOL, STATE_TOOL) + + +class ToolDisclosureError(AutoControlException, ValueError): + """A disclosure request that cannot be honoured as asked.""" + + +class ToolMode(str, Enum): + """How much of the registry ``tools/list`` offers.""" + + FULL = "full" + PROGRESSIVE = "progressive" + STATIC = "static" + + +def resolve_mode(explicit: Union[str, ToolMode, None], + environ: Optional[Mapping[str, str]] = None) -> ToolMode: + """The mode ``explicit`` names, else the one ``JE_AUTOCONTROL_MCP_TOOL_MODE`` does. + + Unset or blank means full. A value that is none of the three raises + :class:`ToolDisclosureError`: a typo must not quietly start a server + that offers every tool. + """ + raw = explicit if explicit is not None else ( + os.environ if environ is None else environ).get(MODE_ENV, "") + if isinstance(raw, ToolMode): + return raw + text = str(raw).strip().lower() + if not text: + return ToolMode.FULL + try: + return ToolMode(text) + except ValueError as error: + known = ", ".join(mode.value for mode in ToolMode) + raise ToolDisclosureError(f"unknown MCP tool mode {raw!r}; expected one of: {known}") from error + + +def profile_from_env(environ: Optional[Mapping[str, str]] = None) -> Optional[Tuple[str, ...]]: + """The entries of ``JE_AUTOCONTROL_MCP_TOOL_PROFILE``; ``None`` when unset.""" + raw = (os.environ if environ is None else environ).get(PROFILE_ENV, "") + entries = tuple(part.strip() for part in raw.split(",") if part.strip()) + return entries or None + + +@dataclass(frozen=True) +class Catalog: + """The tools one caller may use at one registry revision, in registry order.""" + + tools: Mapping[str, MCPTool] + revision: int + #: The RBAC role the tools were filtered for; ``None`` outside RBAC. + caller: Optional[str] = None + + def expand(self, entry: str) -> List[str]: + """The tool names ``entry`` stands for; empty when it names nothing here.""" + if entry.startswith(CATEGORY_PREFIX): + wanted = entry[len(CATEGORY_PREFIX):] + return [name for name, tool in self.tools.items() if category_of(tool) == wanted] + return [entry] if entry in self.tools else [] + + +@dataclass(frozen=True) +class DisclosureResult: + """What one enable or disable call did to a session's view.""" + + enabled: Tuple[str, ...] = () + already_enabled: Tuple[str, ...] = () + disabled: Tuple[str, ...] = () + #: Entries that name nothing this caller may enable (or has enabled). + unavailable: Tuple[str, ...] = () + visible_count: int = 0 + #: Whether ``notifications/tools/list_changed`` went to the session. + list_changed_sent: bool = False + + def to_dict(self) -> Dict[str, Any]: + """Return the JSON shape the enable and disable tools reply with.""" + return { + "enabled": list(self.enabled), "already_enabled": list(self.already_enabled), + "disabled": list(self.disabled), "unavailable": list(self.unavailable), + "visible_count": self.visible_count, + "list_changed_sent": self.list_changed_sent, + } + + +@dataclass(frozen=True) +class ToolPage: + """One page of a session's ``tools/list``.""" + + tools: Tuple[MCPTool, ...] + #: Opaque; ``None`` on the last page. + next_cursor: Optional[str] + #: Identifies the list this is a page of; every page of one list shares it. + snapshot_id: str + + def to_result(self) -> Dict[str, Any]: + """Return the ``tools/list`` result for this page.""" + result: Dict[str, Any] = {"tools": [tool.to_descriptor() for tool in self.tools]} + if self.next_cursor is not None: + result["nextCursor"] = self.next_cursor + result["_meta"] = {SNAPSHOT_META: self.snapshot_id} + return result + + +@dataclass(frozen=True) +class _Snapshot: + tools: Tuple[MCPTool, ...] + caller: Optional[str] + + +def _checked_names(names: Any) -> List[str]: + """``names`` as a de-duplicated list of entries, or :class:`ToolDisclosureError`.""" + if isinstance(names, (str, bytes)) or not isinstance(names, Sequence): + raise ToolDisclosureError("names must be a list of tool names") + if len(names) > MAX_NAMES_PER_CALL: + raise ToolDisclosureError(f"at most {MAX_NAMES_PER_CALL} names per call") + for name in names: + if not isinstance(name, str) or not name or len(name) > _MAX_NAME_CHARS: + raise ToolDisclosureError("every name must be a non-empty string") + return list(dict.fromkeys(names)) + + +class ToolView: + """One session's tools: what is pinned for it plus what it enabled. + + ``catalog`` returns the tools the current caller may use right now. The + view only ever holds names, so everything it reports is the intersection + of those names with that catalog at the moment of asking. ``fixed`` + makes a static profile: its pinned entries are the whole view. + """ + + #: Paged lists remembered per view; the oldest is dropped beyond this. + max_snapshots = 8 + + def __init__(self, catalog: Callable[[], Catalog], *, + pinned: Sequence[str] = (), + page_size: Callable[[], int] = lambda: DEFAULT_PAGE_SIZE, + on_change: Optional[Callable[[], bool]] = None, + fixed: bool = False) -> None: + self._catalog = catalog + self._pinned = tuple(pinned) + self._page_size = page_size + self._on_change = on_change + self._fixed = bool(fixed) + self._enabled: Dict[str, None] = {} + self._revision = 0 + self._snapshots: "OrderedDict[str, _Snapshot]" = OrderedDict() + self._secret = secrets.token_bytes(16) + self._lock = threading.Lock() + + @property + def fixed(self) -> bool: + """Whether this is a static profile, which nothing can be enabled in.""" + return self._fixed + + @property + def visible_names(self) -> Tuple[str, ...]: + """Names of the tools this view offers the current caller, in list order.""" + return tuple(tool.name for tool in self.visible_tools()) + + @property + def enabled_names(self) -> Tuple[str, ...]: + """The names this session enabled, whether or not they are still offered.""" + with self._lock: + return tuple(self._enabled) + + def visible_tools(self) -> Tuple[MCPTool, ...]: + """The tools this view offers the current caller: pinned first, then enabled.""" + return self._visible(self._catalog()) + + def allows(self, name: str) -> bool: + """Whether ``name`` is offered to the current caller by this view.""" + catalog = self._catalog() + if name not in catalog.tools: + return False + with self._lock: + if name in self._enabled: + return True + return name in self._pinned_names(catalog) + + def enable(self, names: Sequence[str]) -> DisclosureResult: + """Add tools to this session's list; ``category:`` adds a whole category. + + An entry that names nothing the current caller may use is reported + as unavailable, without saying whether it exists. Raises + :class:`ToolDisclosureError` for malformed input or a static profile. + """ + entries = _checked_names(names) + self._require_mutable() + catalog = self._catalog() + pinned = set(self._pinned_names(catalog)) + enabled: List[str] = [] + already: List[str] = [] + unavailable: List[str] = [] + with self._lock: + for entry in entries: + targets = catalog.expand(entry) + if not targets: + unavailable.append(entry) + for name in targets: + if name in pinned or name in self._enabled: + already.append(name) + else: + self._enabled[name] = None + enabled.append(name) + if enabled: + self._revision += 1 + return DisclosureResult( + enabled=tuple(enabled), already_enabled=tuple(already), + unavailable=tuple(unavailable), visible_count=len(self._visible(catalog)), + list_changed_sent=self._announce(bool(enabled)), + ) + + def disable(self, names: Sequence[str]) -> DisclosureResult: + """Remove tools this session enabled; pinned tools cannot be removed.""" + entries = _checked_names(names) + self._require_mutable() + catalog = self._catalog() + disabled: List[str] = [] + unavailable: List[str] = [] + with self._lock: + for entry in entries: + targets = [name for name in (catalog.expand(entry) or [entry]) + if name in self._enabled] + if not targets: + unavailable.append(entry) + for name in targets: + del self._enabled[name] + disabled.append(name) + if disabled: + self._revision += 1 + return DisclosureResult( + disabled=tuple(disabled), unavailable=tuple(unavailable), + visible_count=len(self._visible(catalog)), + list_changed_sent=self._announce(bool(disabled)), + ) + + def forget_tool(self, name: str) -> None: + """Drop ``name`` from the enabled set: the registry no longer has it. + + Without this a tool registered later under the same name would be + enabled in a session that never asked for *that* tool. + """ + with self._lock: + if self._enabled.pop(name, 0) is None: + self._revision += 1 + + def list_page(self, cursor: Optional[str]) -> ToolPage: + """The first page of this view, or the page ``cursor`` continues. + + Raises :class:`ToolDisclosureError` for a cursor that is malformed, + belongs to another session or caller, or names a snapshot this view + no longer keeps. + """ + catalog = self._catalog() + size = max(1, int(self._page_size())) + if cursor is None: + tools = self._visible(catalog) + snapshot_id = self._snapshot_id(catalog) + if len(tools) > size: + self._remember(snapshot_id, _Snapshot(tools, catalog.caller)) + return self._page(snapshot_id, tools, 0, size) + snapshot_id, offset = self._decode(cursor) + with self._lock: + snapshot = self._snapshots.get(snapshot_id) + if (snapshot is None or snapshot.caller != catalog.caller + or not 0 < offset < len(snapshot.tools)): + raise ToolDisclosureError( + "stale or unknown cursor; request tools/list again without one") + return self._page(snapshot_id, snapshot.tools, offset, size) + + # --- internals ---------------------------------------------------------------- + + def _require_mutable(self) -> None: + if self._fixed: + raise ToolDisclosureError("this tool list is a static profile; nothing can be enabled") + + def _announce(self, changed: bool) -> bool: + if not changed or self._on_change is None: + return False + return bool(self._on_change()) + + def _pinned_names(self, catalog: Catalog) -> List[str]: + names: Dict[str, None] = {} + for entry in self._pinned: + names.update(dict.fromkeys(catalog.expand(entry))) + return list(names) + + def _visible(self, catalog: Catalog) -> Tuple[MCPTool, ...]: + pinned = self._pinned_names(catalog) + with self._lock: + enabled = set(self._enabled) - set(pinned) + tools = catalog.tools + return tuple([tools[name] for name in pinned] + + [tool for name, tool in tools.items() if name in enabled]) + + def _snapshot_id(self, catalog: Catalog) -> str: + with self._lock: + state = f"{catalog.revision}|{catalog.caller}|{self._revision}" + return self._mac(state) + + def _mac(self, text: str) -> str: + return hmac.new(self._secret, text.encode("utf-8"), hashlib.sha256).hexdigest()[:16] + + def _remember(self, snapshot_id: str, snapshot: _Snapshot) -> None: + with self._lock: + self._snapshots[snapshot_id] = snapshot + self._snapshots.move_to_end(snapshot_id) + while len(self._snapshots) > self.max_snapshots: + self._snapshots.popitem(last=False) + + def _page(self, snapshot_id: str, tools: Tuple[MCPTool, ...], + offset: int, size: int) -> ToolPage: + end = offset + size + cursor = self._encode(snapshot_id, end) if end < len(tools) else None + return ToolPage(tools=tools[offset:end], next_cursor=cursor, snapshot_id=snapshot_id) + + def _encode(self, snapshot_id: str, offset: int) -> str: + body = f"{snapshot_id}.{offset}" + token = f"{body}.{self._mac(body)}".encode("ascii") + return base64.urlsafe_b64encode(token).decode("ascii").rstrip("=") + + def _decode(self, cursor: str) -> Tuple[str, int]: + """``(snapshot id, offset)`` of a cursor this view issued.""" + malformed = ToolDisclosureError("malformed cursor") + if not isinstance(cursor, str) or not cursor or len(cursor) > 4 * _MAX_NAME_CHARS: + raise malformed + try: + padded = cursor + "=" * (-len(cursor) % 4) + text = base64.b64decode(padded, altchars=b"-_", validate=True).decode("ascii") + except (binascii.Error, ValueError) as error: + raise malformed from error + snapshot_id, _, rest = text.partition(".") + number, _, mac = rest.partition(".") + if not number.isdigit() or not hmac.compare_digest( + mac, self._mac(f"{snapshot_id}.{number}")): + raise malformed + return snapshot_id, int(number) + + +class ToolDisclosure: + """The tool mode of one :class:`MCPServer` and the views of its sessions. + + ``mode``, ``read_only`` and ``profile`` default to what the environment + says (``JE_AUTOCONTROL_MCP_TOOL_MODE``, ``JE_AUTOCONTROL_MCP_READONLY``, + ``JE_AUTOCONTROL_MCP_TOOL_PROFILE``). ``page_size`` may be reassigned. + """ + + def __init__(self, server: "MCPServer", + mode: Union[str, ToolMode, None] = None, *, + read_only: Optional[bool] = None, + profile: Optional[Sequence[str]] = None, + page_size: int = DEFAULT_PAGE_SIZE) -> None: + self._server = server + self.mode = resolve_mode(mode) + self.read_only = read_only_env_flag() if read_only is None else bool(read_only) + configured = profile_from_env() if profile is None else tuple(profile) + #: The configured profile; ``None`` when the default one applies. + self.profile: Optional[Tuple[str, ...]] = configured + self.page_size = page_size + self._revision = 0 + self._lock = threading.Lock() + self._views: "OrderedDict[Any, ToolView]" = OrderedDict() + self._catalogs: Dict[Optional[str], Catalog] = {} + self._indexes: Dict[Optional[str], ToolIndex] = {} + self._static_view = ToolView( + self.catalog, pinned=DEFAULT_PROFILE if configured is None else configured, + page_size=self._current_page_size, fixed=True) + + @property + def argument_policy(self) -> Any: + """The server's path roots and ``env://`` allowlist, which calls are held to.""" + return self._server.argument_policy + + @property + def session_count(self) -> int: + """How many sessions currently hold a view.""" + with self._lock: + return len(self._views) + + # --- the registry, as the current caller may use it ----------------------------- + + def catalog(self) -> Catalog: + """The tools the current caller may use, read from the live registry.""" + caller = current_authorization() + role = caller.role if caller is not None else None + with self._lock: + revision = self._revision + cached = self._catalogs.get(role) + if cached is not None and cached.revision == revision: + return cached + with self._server._tools_lock: + registered = list(self._server._tools.values()) + allowed = visible_tools(registered) + if self.read_only: + allowed = [tool for tool in allowed if tool.annotations.read_only] + catalog = Catalog({tool.name: tool for tool in allowed}, revision, role) + with self._lock: + if len(self._catalogs) >= _MAX_CACHED_CALLERS: + self._catalogs.clear() + self._catalogs[role] = catalog + return catalog + + def index(self) -> ToolIndex: + """The search index over :meth:`catalog`, rebuilt when the registry changes.""" + catalog = self.catalog() + with self._lock: + cached = self._indexes.get(catalog.caller) + if cached is not None and cached.version == catalog.revision: + return cached + index = ToolIndex((tool for name, tool in catalog.tools.items() + if name not in CORE_TOOL_NAMES), version=catalog.revision) + with self._lock: + if len(self._indexes) >= _MAX_CACHED_CALLERS: + self._indexes.clear() + self._indexes[catalog.caller] = index + return index + + def check_name(self, name: str) -> None: + """Refuse to register or remove a tool under a core tool's name.""" + if self.mode is ToolMode.PROGRESSIVE and name in CORE_TOOL_NAMES: + raise ToolDisclosureError(f"{name!r} is a core tool of the progressive mode") + + def registry_changed(self, removed: Optional[str] = None) -> None: + """Note that the registry changed; ``removed`` is forgotten by every session.""" + with self._lock: + self._revision += 1 + views = list(self._views.values()) + if removed is not None: + for view in views: + view.forget_tool(removed) + + # --- sessions ------------------------------------------------------------------ + + def view_for(self, connection_id: Any) -> ToolView: + """The view of the session ``connection_id``; ``None`` is the stdio session.""" + with self._lock: + view = self._views.get(connection_id) + if view is not None: + return view + view = ToolView(self.catalog, pinned=CORE_TOOL_NAMES + (self.profile or ()), + page_size=self._current_page_size, on_change=self._announce) + self._views[connection_id] = view + while len(self._views) > _MAX_VIEWS: + dropped, _view = self._views.popitem(last=False) + autocontrol_logger.warning( + "MCP tool views exceeded %d; dropping the oldest (%r)", _MAX_VIEWS, dropped) + return view + + def current_view(self) -> ToolView: + """The view the request on this thread is served from.""" + if self.mode is not ToolMode.PROGRESSIVE or self._server._stateless_request is not None: + return self._static_view + return self.view_for(self._server._connection_id) + + def forget(self, connection_id: Any) -> None: + """Release the view of a session that ended.""" + with self._lock: + self._views.pop(connection_id, None) + + def initial_tools(self) -> List[MCPTool]: + """What ``tools/list`` offers a session that has enabled nothing.""" + if self.mode is ToolMode.FULL: + return self._full_list() + return list(self.current_view().visible_tools()) + + # --- the dispatcher's two questions ---------------------------------------------- + + def handle_list(self, params: Dict[str, Any]) -> Dict[str, Any]: + """Answer ``tools/list``: the whole registry in full mode, else one page.""" + if self.mode is ToolMode.FULL: + return {"tools": [tool.to_descriptor() for tool in self._full_list()]} + try: + return self.current_view().list_page(params.get("cursor")).to_result() + except ToolDisclosureError as error: + raise _MCPError(-32602, f"Invalid params: {error}") from error + + def require_available(self, tool: MCPTool, arguments: Dict[str, Any]) -> None: + """Refuse a call to a tool the session's view does not offer. + + A no-op in full mode. Otherwise this runs after the role check and + before every other gate, and a refusal is recorded like a role one. + """ + if self.mode is ToolMode.FULL: + return + view = self.current_view() + if view.allows(tool.name): + return + message = f"Tool {tool.name!r} is not available in this session" + if not view.fixed and tool.name in self.catalog().tools: + message = (f"Tool {tool.name!r} is not enabled in this session; " + f"enable it with {ENABLE_TOOL} first") + self._server._audit.record(tool=tool.name, arguments=arguments, status="denied", + duration_seconds=0.0, error_text=message) + raise _MCPError(-32602, message) + + def _full_list(self) -> List[MCPTool]: + # Snapshot under the lock: PluginWatcher re-registers tools from its + # own thread, and mutating the dict mid-iteration surfaced to the + # client as "-32603 dictionary changed size during iteration". + with self._server._tools_lock: + tools = list(self._server._tools.values()) + return visible_tools(tools) + + def _current_page_size(self) -> int: + return self.page_size + + def _announce(self) -> bool: + """Tell the session being served that its list changed; ``False`` if it cannot hear.""" + notifier = self._server._unsolicited_notifier() + if notifier is None: + return False + try: + notifier(_LIST_CHANGED, {}) + except _SEND_ERRORS: + autocontrol_logger.exception("MCP failed to send tools/list_changed") + return False + return True + + +__all__ = [ + "CATEGORY_PREFIX", "CORE_TOOL_NAMES", "Catalog", "DEFAULT_PAGE_SIZE", "DEFAULT_PROFILE", + "DISABLE_TOOL", "DisclosureResult", "ENABLE_TOOL", "MAX_NAMES_PER_CALL", "MODE_ENV", + "PROFILE_ENV", "SCHEMA_TOOL", "SEARCH_TOOL", "SNAPSHOT_META", "STATE_TOOL", + "ToolDisclosure", "ToolDisclosureError", "ToolMode", "ToolPage", "ToolView", + "profile_from_env", "resolve_mode", +] diff --git a/je_auto_control/utils/mcp_server/discovery.py b/je_auto_control/utils/mcp_server/discovery.py new file mode 100644 index 000000000..4ff925ab5 --- /dev/null +++ b/je_auto_control/utils/mcp_server/discovery.py @@ -0,0 +1,264 @@ +"""Search index over the MCP tool registry: summaries first, one schema on request. + +The registry holds several hundred tools and their descriptors run to a +couple of hundred kilobytes, which is what ``tools/list`` sends when a server +runs in its default, full mode. :class:`ToolIndex` is the other way to learn +what exists: a query returns a bounded number of :class:`ToolSummary` rows -- +a name, one sentence, a category and the capability the tool needs -- and +:meth:`ToolIndex.get_schema` returns the full descriptor of one tool. + +An index is an immutable snapshot of the tools it was built from. It never +reads the registry itself, so whoever builds it decides what is in it: +:class:`~je_auto_control.utils.mcp_server.disclosure.ToolDisclosure` builds +one per registry revision from the tools the current caller may call, which +is why a tool the caller's role or the server's read-only mode rules out +cannot be found here either. +""" +import re +from dataclasses import dataclass +from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.mcp_server._argument_policy import ( + PATH_FORMAT, VALUE_REF_FORMAT, +) +from je_auto_control.utils.mcp_server.tools._base import MCPTool, MCPToolDescriptor +from je_auto_control.utils.rbac.policy import capability_for_tool +from je_auto_control.utils.rbac.users import Capability + +DEFAULT_SEARCH_LIMIT = 10 +#: The most rows one search may return, whatever the caller asks for. +MAX_SEARCH_LIMIT = 50 +MAX_QUERY_CHARS = 200 +#: Longest summary sentence; a longer first sentence is cut at a word. +SUMMARY_CHARS = 160 +#: Category of a tool a plugin registered, which no factory named. +PLUGIN_CATEGORY = "plugin" +UNCATEGORISED = "general" +_PLUGIN_PREFIX = "plugin_" + +_WORD = re.compile(r"[a-z0-9]+") +_SENTENCE_END = re.compile(r"(?<=[.!?])\s") +_PATH_FORMATS = frozenset({PATH_FORMAT, VALUE_REF_FORMAT}) +_MAX_SCHEMA_DEPTH = 32 + +_EXACT_NAME, _NAME_WORD, _NAME_PART, _CATEGORY_WORD, _TEXT_WORD, _TEXT_PART = 100, 10, 6, 5, 2, 1 + + +class ToolDiscoveryError(AutoControlException, ValueError): + """A search or schema request the index cannot answer as asked.""" + + +@dataclass(frozen=True) +class ToolSummary: + """What a search says about one tool: enough to choose it, not to call it.""" + + name: str + summary: str + category: str + #: The :class:`~je_auto_control.utils.rbac.users.Capability` a call needs. + capability: str + read_only: bool + #: Whether an argument names a file, which the server's path roots confine. + takes_paths: bool + + def to_dict(self) -> Dict[str, Any]: + """Return the JSON shape of one search row.""" + return { + "name": self.name, "summary": self.summary, "category": self.category, + "capability": self.capability, "read_only": self.read_only, + "takes_paths": self.takes_paths, + } + + +@dataclass(frozen=True) +class _Entry: + """One indexed tool with the word sets a query is scored against.""" + + tool: MCPTool + summary: ToolSummary + name: str + name_words: FrozenSet[str] + category_words: FrozenSet[str] + text: str + text_words: FrozenSet[str] + + +def category_of(tool: MCPTool) -> str: + """The category ``tool`` is filed under in the index.""" + if tool.category: + return tool.category + return PLUGIN_CATEGORY if tool.name.startswith(_PLUGIN_PREFIX) else UNCATEGORISED + + +def summarise(description: str) -> str: + """The first sentence of ``description``, cut to :data:`SUMMARY_CHARS`.""" + text = " ".join(description.split()) + sentence = _SENTENCE_END.split(text, maxsplit=1)[0] + if len(sentence) <= SUMMARY_CHARS: + return sentence + return sentence[:SUMMARY_CHARS].rsplit(" ", 1)[0] + "..." + + +def _names_a_path(node: Any, depth: int = 0) -> bool: + """Whether a schema node, or anything under it, is a path or value reference.""" + if depth > _MAX_SCHEMA_DEPTH: + return False + if isinstance(node, list): + return any(_names_a_path(item, depth + 1) for item in node) + if not isinstance(node, dict): + return False + if node.get("format") in _PATH_FORMATS: + return True + return any(_names_a_path(value, depth + 1) for value in node.values()) + + +def _entry(tool: MCPTool) -> _Entry: + category = category_of(tool) + name = tool.name.lower() + text = tool.description.lower() + summary = ToolSummary( + name=tool.name, summary=summarise(tool.description), category=category, + capability=capability_for_tool(tool.name, tool.annotations.read_only), + read_only=tool.annotations.read_only, + takes_paths=_names_a_path(tool.input_schema), + ) + return _Entry( + tool=tool, summary=summary, name=name, + name_words=frozenset(_WORD.findall(name)), + category_words=frozenset(_WORD.findall(category.lower())), + text=text, text_words=frozenset(_WORD.findall(text)), + ) + + +def _word_score(entry: _Entry, word: str) -> int: + """How strongly one query word points at ``entry``; 0 when it does not.""" + if word in entry.name_words: + return _NAME_WORD + if word in entry.name: + return _NAME_PART + if word in entry.category_words: + return _CATEGORY_WORD + if word in entry.text_words: + return _TEXT_WORD + return _TEXT_PART if word in entry.text else 0 + + +def _score(entry: _Entry, query: str, words: List[str]) -> int: + score = sum(_word_score(entry, word) for word in words) + return score + _EXACT_NAME if entry.name == query else score + + +def checked_limit(limit: Any) -> int: + """``limit`` when it is an integer from 1 to :data:`MAX_SEARCH_LIMIT`, else an error.""" + if isinstance(limit, bool) or not isinstance(limit, int): + raise ToolDiscoveryError(f"limit must be an integer, got {type(limit).__name__}") + if not 1 <= limit <= MAX_SEARCH_LIMIT: + raise ToolDiscoveryError(f"limit must be between 1 and {MAX_SEARCH_LIMIT}, got {limit}") + return limit + + +def _checked_query(query: Any) -> str: + if not isinstance(query, str): + raise ToolDiscoveryError(f"query must be a string, got {type(query).__name__}") + if len(query) > MAX_QUERY_CHARS: + raise ToolDiscoveryError(f"query is longer than {MAX_QUERY_CHARS} characters") + return query.strip().lower() + + +class ToolIndex: + """An immutable, searchable snapshot of a set of tools. + + ``version`` is whatever revision of the registry the tools were read at; + two indexes with different versions may disagree, one index never changes. + """ + + def __init__(self, tools: Iterable[MCPTool], *, version: int = 0) -> None: + self._entries: Tuple[_Entry, ...] = tuple(_entry(tool) for tool in tools) + self._by_name: Dict[str, _Entry] = {entry.tool.name: entry for entry in self._entries} + self._version = int(version) + categories: Dict[str, int] = {} + for entry in self._entries: + categories[entry.summary.category] = categories.get(entry.summary.category, 0) + 1 + self._categories = categories + + def __len__(self) -> int: + return len(self._entries) + + def __contains__(self, name: object) -> bool: + return name in self._by_name + + @property + def version(self) -> int: + """The registry revision this snapshot was taken at.""" + return self._version + + def categories(self) -> Dict[str, int]: + """Every category in the index with the number of tools in it.""" + return dict(self._categories) + + def matches(self, query: str, *, category: Optional[str] = None, + capability: Optional[str] = None) -> List[ToolSummary]: + """Every tool matching ``query`` and the filters, best match first. + + An empty query matches everything the filters leave, in registry + order. Raises :class:`ToolDiscoveryError` for a query that is not + text or is too long, a category the index does not hold, or a + capability that is not one of + :class:`~je_auto_control.utils.rbac.users.Capability`. + """ + text = _checked_query(query) + candidates = self._filtered(category, capability) + words = _WORD.findall(text) + if not words: + return [entry.summary for entry in candidates] if not text else [] + scored = [(_score(entry, text, words), position, entry) + for position, entry in enumerate(candidates)] + ranked = sorted((item for item in scored if item[0] > 0), + key=lambda item: (-item[0], item[1])) + return [entry.summary for _score_value, _position, entry in ranked] + + def search(self, query: str, *, limit: int = DEFAULT_SEARCH_LIMIT, + category: Optional[str] = None, + capability: Optional[str] = None) -> List[ToolSummary]: + """The ``limit`` best matches for ``query``: summaries, never schemas. + + ``limit`` must be an integer from 1 to :data:`MAX_SEARCH_LIMIT`; the + other arguments are checked as :meth:`matches` checks them. + """ + bounded = checked_limit(limit) + return self.matches(query, category=category, capability=capability)[:bounded] + + def get_schema(self, name: str) -> MCPToolDescriptor: + """The full descriptor of the one tool called ``name``. + + Raises :class:`ToolDiscoveryError` when the index has no such tool, + which is also the answer for one the caller may not use. + """ + return self.tool(name).descriptor() + + def tool(self, name: str) -> MCPTool: + """The indexed tool called ``name``; :class:`ToolDiscoveryError` if absent.""" + entry = self._by_name.get(name) if isinstance(name, str) else None + if entry is None: + raise ToolDiscoveryError(f"no tool named {name!r} is available to this caller") + return entry.tool + + def _filtered(self, category: Optional[str], + capability: Optional[str]) -> List[_Entry]: + if category is not None and category not in self._categories: + raise ToolDiscoveryError( + f"unknown category {category!r}; known: {sorted(self._categories)}") + if capability is not None and capability not in Capability.all(): + raise ToolDiscoveryError( + f"unknown capability {capability!r}; known: {Capability.all()}") + return [entry for entry in self._entries + if (category is None or entry.summary.category == category) + and (capability is None or entry.summary.capability == capability)] + + +__all__ = [ + "DEFAULT_SEARCH_LIMIT", "MAX_QUERY_CHARS", "MAX_SEARCH_LIMIT", "PLUGIN_CATEGORY", + "ToolDiscoveryError", "ToolIndex", "ToolSummary", "category_of", "checked_limit", + "summarise", +] diff --git a/je_auto_control/utils/mcp_server/server.py b/je_auto_control/utils/mcp_server/server.py index c36644f04..4b9e161b0 100644 --- a/je_auto_control/utils/mcp_server/server.py +++ b/je_auto_control/utils/mcp_server/server.py @@ -4,24 +4,27 @@ the handshake-based revisions (``initialize``, up to 2025-11-25) and the stateless 2026-07-28, chosen per request (see :mod:`._stateless`). Each stdio line is one JSON-RPC message — no Content-Length framing — matching -the MCP stdio spec. +the MCP stdio spec. Which tools a session is offered -- all of them by +default -- is :mod:`.disclosure`'s. """ -import contextlib import itertools import json import sys import threading import time -from typing import Any, Callable, Dict, List, Optional, TextIO +from typing import Any, Callable, Dict, List, Optional, TextIO, Union from je_auto_control.utils.cli_output import utf8_stream from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger -from je_auto_control.utils.mcp_server._authz import authorize_tool_call, visible_tools +from je_auto_control.utils.mcp_server._authz import authorize_tool_call +from je_auto_control.utils.mcp_server._connection_state import ConnectionStateMixin +from je_auto_control.utils.mcp_server._core_tools import build_core_tools from je_auto_control.utils.mcp_server.audit import AuditLogger from je_auto_control.utils.mcp_server.context import ( OperationCancelledError, ToolCallContext, ) +from je_auto_control.utils.mcp_server.disclosure import ToolDisclosure, ToolMode from je_auto_control.utils.mcp_server.log_bridge import ( MCPLogBridge, mcp_level_to_logging, ) @@ -61,8 +64,15 @@ WORKER_DRAIN_TIMEOUT = 10.0 -class MCPServer(StatelessDispatchMixin, SubscriptionMixin, ClientRequestMixin): - """JSON-RPC 2.0 MCP server with a configurable tool registry.""" +class MCPServer(ConnectionStateMixin, StatelessDispatchMixin, SubscriptionMixin, + ClientRequestMixin): + """JSON-RPC 2.0 MCP server with a configurable tool registry. + + ``tool_mode`` is ``full`` (every tool in ``tools/list``, the default), + ``progressive`` or ``static``; ``None`` reads ``JE_AUTOCONTROL_MCP_TOOL_MODE``. + ``read_only`` builds the default registry without mutating tools and, in + the two non-full modes, keeps any registered later out of every session. + """ def __init__(self, tools: Optional[List[MCPTool]] = None, resource_provider: Optional[ResourceProvider] = None, @@ -71,9 +81,15 @@ def __init__(self, tools: Optional[List[MCPTool]] = None, audit_logger: Optional[AuditLogger] = None, rate_limiter: Optional[RateLimiter] = None, log_bridge: Optional[MCPLogBridge] = None, + tool_mode: Union[str, ToolMode, None] = None, + read_only: Optional[bool] = None, ) -> None: - registry = tools if tools is not None else build_default_tool_registry() + registry = tools if tools is not None else build_default_tool_registry(read_only=read_only) self._tools: Dict[str, MCPTool] = {tool.name: tool for tool in registry} + #: The tool mode and each session's view of the registry. + self.disclosure = ToolDisclosure(self, tool_mode, read_only=read_only) + if self.disclosure.mode is ToolMode.PROGRESSIVE: + self._tools.update((tool.name, tool) for tool in build_core_tools(self.disclosure)) self._resources = (resource_provider if resource_provider is not None else default_resource_provider()) self._prompts = (prompt_provider if prompt_provider is not None @@ -97,12 +113,7 @@ def __init__(self, tools: Optional[List[MCPTool]] = None, # the writer out from under them. self._workers: List[threading.Thread] = [] self._workers_lock = threading.Lock() - # Connection-scoped state. The stdio transport has exactly one peer, so - # a server-wide default is right for it; an HTTP transport has many, and - # each request runs on its own thread. Keeping the notifier/writer in - # thread-local storage is what stops one client's progress - # notifications from being emitted down another client's socket. - # See :meth:`connection_scope`. + # Connection-scoped state; see :mod:`._connection_state`. self._default_notifier: Optional[ Callable[[str, Dict[str, Any]], None]] = None self._default_writer: Optional[Callable[[str], None]] = None @@ -128,123 +139,26 @@ def __init__(self, tools: Optional[List[MCPTool]] = None, self._listeners: Dict[Any, Any] = {} # open subscriptions/listen, by (connection, id) self._listeners_lock = threading.Lock() - # --- connection-scoped state ------------------------------------------ - # - # Each property prefers a value set for the current thread (one HTTP - # request = one thread) and falls back to the server-wide default that - # stdio and set_notifier() use. A plain POST therefore sees *no* notifier - # rather than inheriting whichever SSE connection happens to be open — - # correct, since a plain POST has no stream to deliver notifications on. - - @property - def _notifier(self) -> Optional[Callable[[str, Dict[str, Any]], None]]: - return getattr(self._local, "notifier", None) or self._default_notifier - - @_notifier.setter - def _notifier(self, value) -> None: - self._default_notifier = value - - @property - def _writer(self) -> Optional[Callable[[str], None]]: - return getattr(self._local, "writer", None) or self._default_writer - - @_writer.setter - def _writer(self, value) -> None: - self._default_writer = value - - @property - def _concurrent_tools(self) -> bool: - scoped = getattr(self._local, "concurrent_tools", None) - if scoped is None: - return self._default_concurrent_tools - return scoped - - @_concurrent_tools.setter - def _concurrent_tools(self, value) -> None: - self._default_concurrent_tools = bool(value) - - @property - def _connection_id(self) -> Any: - """Identity of the peer served on this thread (None for stdio).""" - return getattr(self._local, "connection_id", None) - - @property - def _client_capabilities(self) -> Dict[str, Any]: - """Capabilities advertised by the peer served on this thread.""" - if self._stateless_request is not None: - return self._stateless_request.capabilities - conn = self._connection_id - if conn is None: - return self._default_client_capabilities - with self._caps_lock: - return self._client_caps_by_conn.get(conn, {}) - - @_client_capabilities.setter - def _client_capabilities(self, value: Dict[str, Any]) -> None: - conn = self._connection_id - if conn is None: - self._default_client_capabilities = value - return - with self._caps_lock: - self._client_caps_by_conn[conn] = value - - @contextlib.contextmanager - def connection_scope(self, *, notifier=None, writer=None, - concurrent_tools=None, connection_id=None): - """Bind notifier/writer/concurrency/identity to the calling thread only. - - Transports that serve more than one peer must wrap each request in - this. Previously they swapped the attributes on the shared server, so - any concurrent request bound to the wrong peer's socket. ``connection_id`` - scopes active-call slots and client capabilities per peer. - """ - prior = (getattr(self._local, "notifier", None), - getattr(self._local, "writer", None), - getattr(self._local, "concurrent_tools", None), - getattr(self._local, "connection_id", None)) - self._local.notifier = notifier - self._local.writer = writer - self._local.concurrent_tools = concurrent_tools - self._local.connection_id = connection_id - try: - yield self - finally: - (self._local.notifier, self._local.writer, - self._local.concurrent_tools, self._local.connection_id) = prior - - def forget_connection(self, connection_id: Any) -> None: - """Drop per-connection state when a transport connection closes. - - HTTP connections are transient; without this their capabilities and - any stray active-call contexts would accumulate for the server's life. - """ - if connection_id is None: - return - with self._caps_lock: - self._client_caps_by_conn.pop(connection_id, None) - self._end_listeners(lambda conn: conn == connection_id, graceful=False) - with self._calls_lock: - stale = [key for key in self._active_calls - if isinstance(key, tuple) and key[0] == connection_id] - for key in stale: - self._active_calls.pop(key, None) - def register_tool(self, tool: MCPTool) -> None: """Add or replace a tool in the live registry. Emits ``notifications/tools/list_changed`` to the connected client so it knows to refresh its cached tool list. """ + self.disclosure.check_name(tool.name) with self._tools_lock: self._tools[tool.name] = tool + self.disclosure.registry_changed() self._notify_tools_list_changed() def unregister_tool(self, name: str) -> bool: """Remove a tool by name. Returns True if it existed.""" + self.disclosure.check_name(name) with self._tools_lock: if name not in self._tools: return False del self._tools[name] + self.disclosure.registry_changed(removed=name) self._notify_tools_list_changed() return True @@ -289,6 +203,7 @@ def serve_stdio(self, stdin: Optional[TextIO] = None, # writer is swapped back. self._join_workers() self.end_subscriptions(stdio=True) + self.disclosure.forget(None) # the stdio peer's one implicit session self._detach_log_bridge_if_configured() self._notifier = prior_notifier self._writer = prior_writer @@ -499,7 +414,6 @@ def _run_method(self, msg_id: Any, method: Optional[str], raise _MCPError(-32601, f"Method not found: {method}") nullary = { "ping": self._handle_ping, - "tools/list": self._handle_tools_list, "resources/list": self._handle_resources_list, "prompts/list": self._handle_prompts_list, }.get(method) @@ -507,6 +421,7 @@ def _run_method(self, msg_id: Any, method: Optional[str], return nullary() handler = { "initialize": self._handle_initialize, + "tools/list": self.disclosure.handle_list, "resources/read": self._handle_resources_read, "resources/subscribe": self._handle_resources_subscribe, "resources/unsubscribe": self._handle_resources_unsubscribe, @@ -521,15 +436,6 @@ def _handle_ping(self) -> Dict[str, Any]: """Liveness probe; returns an empty result per the MCP spec.""" return {} - def _handle_tools_list(self) -> Dict[str, Any]: - """List descriptors for every registered tool.""" - # Snapshot under the lock: PluginWatcher re-registers tools from its - # own thread, and mutating the dict mid-iteration surfaced to the - # client as "-32603 dictionary changed size during iteration". - with self._tools_lock: - tools = list(self._tools.values()) - return {"tools": [tool.to_descriptor() for tool in visible_tools(tools)]} - def _handle_resources_list(self) -> Dict[str, Any]: """List descriptors for every registered resource.""" return {"resources": [resource.to_descriptor() @@ -624,6 +530,7 @@ def _prepare_tool_call( if tool is None: raise _MCPError(-32602, f"Unknown tool: {name}") authorize_tool_call(tool, arguments, self._audit) # RBAC; a no-op without a user + self.disclosure.require_available(tool, arguments) # a no-op in full mode violation = (validate_arguments(tool.input_schema, arguments) or undeclared_arguments(tool.input_schema, arguments)) if violation is not None: @@ -721,14 +628,16 @@ def _parse_request(line: str) -> "tuple[Optional[Dict[str, Any]], Optional[str]] return (None, refusal) if refusal is not None else (message, None) -def start_mcp_stdio_server(read_only: Optional[bool] = None) -> MCPServer: +def start_mcp_stdio_server(read_only: Optional[bool] = None, + tool_mode: Union[str, ToolMode, None] = None) -> MCPServer: """Start a stdio MCP server in the foreground; blocks until EOF. ``read_only=True`` offers only tools marked read-only; ``None`` leaves the choice to ``JE_AUTOCONTROL_MCP_READONLY``. ``je_auto_control_mcp --read-only`` used to reach only its ``--list-*`` output, so the server - it started offered every tool, clicks and typing included. + it started offered every tool, clicks and typing included. ``tool_mode`` + is :class:`MCPServer`'s; ``None`` reads ``JE_AUTOCONTROL_MCP_TOOL_MODE``. """ - server = MCPServer(tools=build_default_tool_registry(read_only=read_only)) + server = MCPServer(read_only=read_only, tool_mode=tool_mode) server.serve_stdio() return server diff --git a/je_auto_control/utils/mcp_server/tools/__init__.py b/je_auto_control/utils/mcp_server/tools/__init__.py index 49cdf5c59..cbf6677bd 100644 --- a/je_auto_control/utils/mcp_server/tools/__init__.py +++ b/je_auto_control/utils/mcp_server/tools/__init__.py @@ -7,10 +7,10 @@ """ import os from dataclasses import replace -from typing import Dict, List, Optional +from typing import Callable, Dict, List, Optional from je_auto_control.utils.mcp_server.tools._base import ( - MCPContent, MCPTool, MCPToolAnnotations, read_only_env_flag, + MCPContent, MCPTool, MCPToolAnnotations, MCPToolDescriptor, read_only_env_flag, ) from je_auto_control.utils.mcp_server.tools._factories import ALL_FACTORIES from je_auto_control.utils.mcp_server.tools.plugin_tools import ( @@ -43,6 +43,17 @@ } +#: The canonical tools behind the aliases: the handful most sessions need. +COMMON_TOOL_NAMES = tuple(_DEFAULT_ALIASES.values()) +_FACTORY_SUFFIX = "_tools" + + +def _category_of(factory: Callable[[], List[MCPTool]]) -> str: + """The category of a factory's tools: its name without ``_tools``.""" + name = getattr(factory, "__name__", "") + return name[:-len(_FACTORY_SUFFIX)] if name.endswith(_FACTORY_SUFFIX) else name + + def _aliases_enabled(explicit: Optional[bool]) -> bool: if explicit is not None: return bool(explicit) @@ -83,7 +94,10 @@ def build_default_tool_registry(read_only: Optional[bool] = None, ) tools: List[MCPTool] = [] for factory in ALL_FACTORIES: - tools.extend(factory()) + # A factory may name a category itself; otherwise its own name is one. + category = _category_of(factory) + tools.extend(tool if tool.category else replace(tool, category=category) + for tool in factory()) if enforce_read_only: tools = [tool for tool in tools if tool.annotations.read_only] if _aliases_enabled(aliases): @@ -92,7 +106,7 @@ def build_default_tool_registry(read_only: Optional[bool] = None, __all__ = [ - "MCPContent", "MCPTool", "MCPToolAnnotations", - "build_default_tool_registry", "make_plugin_tool", + "COMMON_TOOL_NAMES", "MCPContent", "MCPTool", "MCPToolAnnotations", + "MCPToolDescriptor", "build_default_tool_registry", "make_plugin_tool", "register_plugin_tools", ] diff --git a/je_auto_control/utils/mcp_server/tools/_base.py b/je_auto_control/utils/mcp_server/tools/_base.py index 7bb26ed63..99d06095f 100644 --- a/je_auto_control/utils/mcp_server/tools/_base.py +++ b/je_auto_control/utils/mcp_server/tools/_base.py @@ -77,9 +77,37 @@ def to_dict(self) -> Dict[str, Any]: return annotations +@dataclass(frozen=True) +class MCPToolDescriptor: + """One tool as ``tools/list`` describes it: the schema without the callable.""" + + name: str + description: str + input_schema: Dict[str, Any] + annotations: Dict[str, Any] + output_schema: Optional[Dict[str, Any]] = None + + def to_dict(self) -> Dict[str, Any]: + """Return the JSON shape of one ``tools/list`` entry.""" + descriptor: Dict[str, Any] = { + "name": self.name, + "description": self.description, + "inputSchema": self.input_schema, + "annotations": self.annotations, + } + if self.output_schema is not None: + descriptor["outputSchema"] = self.output_schema + return descriptor + + @dataclass(frozen=True) class MCPTool: - """A single MCP tool — public name, schema, and Python callable.""" + """A single MCP tool — public name, schema, and Python callable. + + ``category`` groups tools for the search index (:mod:`..discovery`); it + is the registry's own bookkeeping and never part of the descriptor a + client is sent. + """ name: str description: str @@ -87,6 +115,15 @@ class MCPTool: handler: Callable[..., Any] annotations: MCPToolAnnotations = MCPToolAnnotations() output_schema: Optional[Dict[str, Any]] = None + category: str = "" + + def descriptor(self) -> MCPToolDescriptor: + """Return the typed form of :meth:`to_descriptor`.""" + return MCPToolDescriptor( + name=self.name, description=self.description, + input_schema=self.input_schema, annotations=self.annotations.to_dict(), + output_schema=self.output_schema, + ) def to_descriptor(self) -> Dict[str, Any]: """Return the dict shape MCP clients expect from ``tools/list``.""" diff --git a/test/unit_test/headless/test_mcp_disclosure_policy.py b/test/unit_test/headless/test_mcp_disclosure_policy.py new file mode 100644 index 000000000..7c2f01b27 --- /dev/null +++ b/test/unit_test/headless/test_mcp_disclosure_policy.py @@ -0,0 +1,329 @@ +"""Disclosure changes availability only: the existing gates still decide (plan G3). + +Fake tools whose handlers record that they ran. Nothing here drives the +desktop, and the benchmark is run against a small fake registry. +""" +import importlib.util +import json +from pathlib import Path + +import pytest + +from je_auto_control.utils.mcp_server import __main__ as mcp_cli +from je_auto_control.utils.mcp_server._argument_policy import ArgumentPolicy +from je_auto_control.utils.mcp_server.audit import AuditLogger +from je_auto_control.utils.mcp_server.disclosure import ( + CORE_TOOL_NAMES, DEFAULT_PROFILE, ToolDisclosureError, ToolMode, +) +from je_auto_control.utils.mcp_server.rate_limit import RateLimiter +from je_auto_control.utils.mcp_server.server import MCPServer, start_mcp_stdio_server +from je_auto_control.utils.mcp_server.tools import MCPTool, MCPToolAnnotations +from je_auto_control.utils.path_guard.policy import PathPolicy +from je_auto_control.utils.rbac.authorization import ( + AuthorizationContext, authorization_scope, +) + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_STATELESS_META = { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": {}, +} + + +class _Recorder: + """Fake tools that only note which of them ran.""" + + def __init__(self) -> None: + self.ran = [] + + def tool(self, name, *, read_only=False, properties=None, category="fake"): + def handler(**kwargs): + self.ran.append((name, kwargs)) + return {"ran": name} + return MCPTool( + name=name, description=f"Fake tool {name}.", + input_schema={"type": "object", "properties": properties or {}}, + handler=handler, annotations=MCPToolAnnotations(read_only=read_only), + category=category, + ) + + def tools(self): + return [ + self.tool("fx_write", properties={"file_path": {"type": "string", "format": "path"}}), + self.tool("fx_read", read_only=True), + self.tool("fx_click"), + ] + + +def _rpc(server, method, params=None, msg_id=1): + return json.loads(server.handle_line(json.dumps({ + "jsonrpc": "2.0", "id": msg_id, "method": method, "params": params or {}}))) + + +def _call(server, name, arguments=None, meta=None): + params = {"name": name, "arguments": arguments or {}} + if meta is not None: + params["_meta"] = meta + return _rpc(server, "tools/call", params) + + +def _names(reply): + return [tool["name"] for tool in reply["result"]["tools"]] + + +def _enable(server, *names): + result = _call(server, "ac_tools_enable", {"names": list(names)})["result"] + assert result["isError"] is False, result + return json.loads(result["content"][0]["text"]) + + +# --- every call still goes through the existing gates ------------------------------- + + +def test_discovered_call_uses_existing_policy(tmp_path, monkeypatch): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + policy_checks = [] + real_prepare = server._prepare_tool_call + + def spying_prepare(params): + policy_checks.append(params.get("name")) + return real_prepare(params) + monkeypatch.setattr(server, "_prepare_tool_call", spying_prepare) + server.argument_policy = ArgumentPolicy(PathPolicy(roots=[tmp_path])) + _enable(server, "fx_write") + inside = _call(server, "fx_write", {"file_path": str(tmp_path / "ok.txt")}) + assert inside["result"]["isError"] is False + outside = _call(server, "fx_write", {"file_path": str(tmp_path.parent / "escape.txt")}) + assert outside["result"]["isError"] is True + undeclared = _call(server, "fx_write", {"surprise": 1}) + assert undeclared["result"]["isError"] is True + expected_checks = ["ac_tools_enable", "fx_write", "fx_write", "fx_write"] + assert policy_checks == expected_checks + # Only the call inside the root ran, and with the canonical path. + assert [name for name, _kwargs in recorder.ran] == ["fx_write"] + assert Path(recorder.ran[0][1]["file_path"]) == (tmp_path / "ok.txt").resolve() + + +def test_a_tool_that_is_not_enabled_is_refused_before_it_runs(tmp_path): + recorder = _Recorder() + audit_path = tmp_path / "audit.jsonl" + server = MCPServer(tools=recorder.tools(), tool_mode="progressive", + audit_logger=AuditLogger(str(audit_path))) + reply = _call(server, "fx_click") + assert reply["error"]["code"] == -32602 + assert "ac_tools_enable" in reply["error"]["message"] + assert recorder.ran == [] + entries = [json.loads(line) for line in audit_path.read_text(encoding="utf-8").splitlines()] + assert [(entry["tool"], entry["status"]) for entry in entries] == [("fx_click", "denied")] + + +def test_rbac_is_checked_for_search_enable_and_call(): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + with authorization_scope(AuthorizationContext(user_id="v", role="viewer")): + outcome = _enable(server, "fx_click", "fx_read") + assert outcome["enabled"] == ["fx_read"] + assert outcome["unavailable"] == ["fx_click"] + assert "fx_click" not in _names(_rpc(server, "tools/list")) + # The role refusal is the existing one, with its own code. + assert _call(server, "fx_click")["error"]["code"] == -32003 + assert _call(server, "ac_tools_schema", {"name": "fx_click"})["result"]["isError"] is True + assert _call(server, "fx_read")["result"]["isError"] is False + assert recorder.ran == [("fx_read", {})] + + +def test_a_tool_enabled_by_an_operator_is_not_open_to_a_viewer_on_the_same_session(): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + with authorization_scope(AuthorizationContext(user_id="o", role="operator")): + assert _enable(server, "fx_click")["enabled"] == ["fx_click"] + with authorization_scope(AuthorizationContext(user_id="v", role="viewer")): + assert "fx_click" not in _names(_rpc(server, "tools/list")) + assert _call(server, "fx_click")["error"]["code"] == -32003 + assert recorder.ran == [] + + +def test_rate_limit_still_applies_to_an_enabled_tool(): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive", + rate_limiter=RateLimiter(rate_per_sec=0.001, capacity=2)) + _enable(server, "fx_click") # first token + assert "result" in _call(server, "fx_click") # second token + assert _call(server, "fx_click")["error"]["code"] == -32000 + + +# --- read-only --------------------------------------------------------------------- + + +def test_readonly_forbids_mutating_enable(): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive", read_only=True) + # A plugin registers a mutating tool on the live, read-only server. + server.register_tool(recorder.tool("plugin_fx_mutate")) + outcome = _enable(server, "fx_click", "plugin_fx_mutate", "fx_read") + assert outcome["enabled"] == ["fx_read"] + assert outcome["unavailable"] == ["fx_click", "plugin_fx_mutate"] + listed = _names(_rpc(server, "tools/list")) + mutating_tool_visible = "fx_click" in listed or "plugin_fx_mutate" in listed + assert mutating_tool_visible is False + assert _call(server, "plugin_fx_mutate")["error"]["code"] == -32602 + assert recorder.ran == [] + # The core stays usable: a read-only session can still search and enable. + assert listed == list(CORE_TOOL_NAMES) + ["fx_read"] + + +def test_read_only_argument_reaches_the_stdio_entry_point(monkeypatch): + served = [] + monkeypatch.setattr(MCPServer, "serve_stdio", lambda self: served.append(self)) + server = start_mcp_stdio_server(read_only=True, tool_mode="progressive") + assert served == [server] + assert server.disclosure.mode is ToolMode.PROGRESSIVE + assert server.disclosure.read_only is True + assert all(tool.annotations.read_only for tool in server._tools.values()) + + +# --- plugins ----------------------------------------------------------------------- + + +def test_removed_tool_cannot_run(): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + server.register_tool(recorder.tool("plugin_fx_gone")) + _enable(server, "plugin_fx_gone") + assert server.unregister_tool("plugin_fx_gone") is True + assert "plugin_fx_gone" not in _names(_rpc(server, "tools/list")) + assert _call(server, "plugin_fx_gone")["error"]["code"] == -32602 + # The same name coming back is a new tool: the session has to ask again. + server.register_tool(recorder.tool("plugin_fx_gone")) + assert _call(server, "plugin_fx_gone")["error"]["code"] == -32602 + removed_tool_executed = any(name == "plugin_fx_gone" for name, _kwargs in recorder.ran) + assert removed_tool_executed is False + _enable(server, "plugin_fx_gone") + assert _call(server, "plugin_fx_gone")["result"]["isError"] is False + + +def test_the_core_tools_cannot_be_shadowed_or_removed(): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + with pytest.raises(ToolDisclosureError): + server.register_tool(recorder.tool("ac_tools_enable")) + with pytest.raises(ToolDisclosureError): + server.unregister_tool("ac_tools_search") + assert _names(_rpc(server, "tools/list")) == list(CORE_TOOL_NAMES) + + +# --- clients that cannot follow a changing list ------------------------------------- + + +def test_plain_client_static_profile(monkeypatch): + recorder = _Recorder() + monkeypatch.setenv("JE_AUTOCONTROL_MCP_TOOL_PROFILE", "fx_read, category:nope, fx_click") + server = MCPServer(tools=recorder.tools(), tool_mode="static") + sent = [] + server.set_notifier(lambda method, params: sent.append(method)) + assert _names(_rpc(server, "tools/list")) == ["fx_read", "fx_click"] + assert not any(name in server._tools for name in CORE_TOOL_NAMES) + assert _call(server, "fx_read")["result"]["isError"] is False + assert _call(server, "fx_write")["error"]["code"] == -32602 + assert sent == [] + + +def test_static_profile_defaults_to_the_common_tools(monkeypatch): + monkeypatch.delenv("JE_AUTOCONTROL_MCP_TOOL_PROFILE", raising=False) + recorder = _Recorder() + tools = recorder.tools() + [recorder.tool(name) for name in DEFAULT_PROFILE[:3]] + server = MCPServer(tools=tools, tool_mode="static") + assert _names(_rpc(server, "tools/list")) == list(DEFAULT_PROFILE[:3]) + + +def test_a_stateless_request_gets_the_static_profile_in_progressive_mode(monkeypatch): + monkeypatch.setenv("JE_AUTOCONTROL_MCP_TOOL_PROFILE", "fx_read") + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + _enable(server, "fx_click") # the handshake-era stdio session + listed = _rpc(server, "tools/list", {"_meta": _STATELESS_META}) + # 2026-07-28 has no session to keep an enabled set in. + assert _names(listed) == ["fx_read"] + assert listed["result"]["resultType"] == "complete" + assert _call(server, "fx_read", meta=_STATELESS_META)["result"]["isError"] is False + assert _call(server, "fx_click", meta=_STATELESS_META)["error"]["code"] == -32602 + assert _call(server, "ac_tools_enable", {"names": ["fx_click"]}, + meta=_STATELESS_META)["error"]["code"] == -32602 + # The session's own view is untouched: core, its profile, what it enabled. + assert _names(_rpc(server, "tools/list")) == list(CORE_TOOL_NAMES) + ["fx_read", "fx_click"] + + +def test_protocol_negotiation_and_cancel_are_unchanged_by_the_mode(): + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + for version in ("2024-11-05", "2025-06-18", "2025-11-25"): + reply = _rpc(server, "initialize", {"protocolVersion": version, "capabilities": {}}) + assert reply["result"]["protocolVersion"] == version + assert reply["result"]["capabilities"]["tools"] == {"listChanged": True} + # A cancel for a call that is not running is ignored, as before. + assert server.handle_line(json.dumps({ + "jsonrpc": "2.0", "method": "notifications/cancelled", + "params": {"requestId": 99}})) is None + + +def test_stdio_loop_serves_one_implicit_session_and_reclaims_it(): + import io + recorder = _Recorder() + server = MCPServer(tools=recorder.tools(), tool_mode="progressive") + lines = [ + {"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-11-25", "capabilities": {}}}, + {"jsonrpc": "2.0", "method": "notifications/initialized"}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/call", + "params": {"name": "ac_tools_enable", "arguments": {"names": ["fx_read"]}}}, + ] + stdout = io.StringIO() + server.serve_stdio(stdin=io.StringIO("".join(json.dumps(line) + "\n" for line in lines)), + stdout=stdout) + written = [json.loads(line) for line in stdout.getvalue().splitlines()] + assert sum(1 for message in written + if message.get("method") == "notifications/tools/list_changed") == 1 + assert server.disclosure.session_count == 0 # the loop ended, so did its session + + +# --- the CLI and the benchmark ------------------------------------------------------ + + +def test_cli_lists_what_a_new_progressive_session_sees(capsys, monkeypatch): + monkeypatch.delenv("JE_AUTOCONTROL_MCP_TOOL_PROFILE", raising=False) + mcp_cli.main(["--list-tools", "--tool-mode", "progressive", "--read-only"]) + listed = json.loads(capsys.readouterr().out) + assert [tool["name"] for tool in listed] == list(CORE_TOOL_NAMES) + + +def test_cli_passes_the_mode_to_the_server(monkeypatch): + seen = {} + monkeypatch.setattr(mcp_cli, "start_mcp_stdio_server", + lambda **kwargs: seen.update(kwargs)) + mcp_cli.main(["--tool-mode", "progressive"]) + assert seen == {"read_only": None, "tool_mode": "progressive"} + seen.clear() + # Without the flag the call is the one it always was. + mcp_cli.main([]) + assert seen == {"read_only": None} + + +def _load_benchmark(): + spec = importlib.util.spec_from_file_location( + "mcp_discovery_benchmark", _REPO_ROOT / "benchmarks" / "mcp_discovery.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_benchmark_reports_the_four_figures_for_both_modes(): + recorder = _Recorder() + report = _load_benchmark().measure(tools=recorder.tools(), rounds=2) + assert set(report["modes"]) == {"full", "progressive"} + for figures in report["modes"].values(): + assert {"count", "json_bytes", "handshake_ms", "search_ms"} <= set(figures) + assert report["modes"]["full"]["count"] == 3 + assert report["modes"]["progressive"]["count"] == len(CORE_TOOL_NAMES) + assert report["modes"]["full"]["search_ms"] is None # full mode has no search tool + assert report["modes"]["progressive"]["search_ms"] >= 0 diff --git a/test/unit_test/headless/test_mcp_discovery_index.py b/test/unit_test/headless/test_mcp_discovery_index.py new file mode 100644 index 000000000..3c384283e --- /dev/null +++ b/test/unit_test/headless/test_mcp_discovery_index.py @@ -0,0 +1,219 @@ +"""The MCP tool search index: summaries first, one schema on request (plan G1). + +Every tool here is a fake whose handler only records that it ran, so nothing +touches the desktop; the registry-wide checks build descriptors and never +invoke a handler. +""" +import json + +import pytest + +from je_auto_control.utils.mcp_server.discovery import ( + DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT, ToolDiscoveryError, ToolIndex, + ToolSummary, +) +from je_auto_control.utils.mcp_server.server import MCPServer +from je_auto_control.utils.mcp_server.tools import ( + MCPTool, MCPToolAnnotations, build_default_tool_registry, +) +from je_auto_control.utils.mcp_server.tools._base import MCPToolDescriptor +from je_auto_control.utils.rbac.authorization import ( + AuthorizationContext, authorization_scope, +) +from je_auto_control.utils.rbac.users import Capability + +_READ = MCPToolAnnotations(read_only=True, idempotent=True) + + +def _tool(name, description, *, read_only=False, category="", properties=None): + return MCPTool( + name=name, description=description, + input_schema={"type": "object", "properties": properties or {"value": {"type": "string"}}}, + handler=lambda **kwargs: {"ran": name, **kwargs}, + annotations=_READ if read_only else MCPToolAnnotations(), + category=category, + ) + + +def _fake_tools(): + return [ + _tool("fx_click", "Click a mouse button at a point. Long tail that is not the summary.", + category="mouse"), + _tool("fx_where", "Return the mouse pointer position.", read_only=True, category="mouse"), + _tool("fx_shot", "Capture the screen to a PNG file.", read_only=True, category="screen", + properties={"file_path": {"type": "string", "format": "path"}}), + _tool("fx_type", "Type text with the keyboard.", category="keyboard"), + _tool("plugin_fx_extra", "A plugin command."), + ] + + +def _progressive(tools=None, **kwargs): + return MCPServer(tools=_fake_tools() if tools is None else tools, + tool_mode="progressive", **kwargs) + + +def _call(server, name, arguments=None, msg_id=1): + reply = json.loads(server.handle_line(json.dumps({ + "jsonrpc": "2.0", "id": msg_id, "method": "tools/call", + "params": {"name": name, "arguments": arguments or {}}}))) + return reply + + +def _payload(reply): + result = reply["result"] + assert result["isError"] is False, result + return json.loads(result["content"][0]["text"]) + + +# --- the index ---------------------------------------------------------------- + + +def test_search_returns_summary_not_full_schema(): + index = ToolIndex(_fake_tools()) + found = index.search("mouse") + assert all(isinstance(item, ToolSummary) for item in found) + search_results = [item.to_dict() for item in found] + assert [item["name"] for item in search_results] == ["fx_click", "fx_where"] + assert 'inputSchema' not in search_results[0] + assert set(search_results[0]) == { + "name", "summary", "category", "capability", "read_only", "takes_paths"} + # The summary is the first sentence, not the whole description. + assert search_results[0]["summary"] == "Click a mouse button at a point." + + +def test_summary_carries_typed_category_and_capability(): + index = ToolIndex(_fake_tools()) + by_name = {item.name: item for item in index.search("", limit=MAX_SEARCH_LIMIT)} + assert by_name["fx_click"].capability == Capability.DRIVE_INPUT + assert by_name["fx_where"].capability == Capability.READ_SCREEN + assert by_name["fx_shot"].takes_paths is True + assert by_name["fx_click"].takes_paths is False + assert by_name["plugin_fx_extra"].category == "plugin" + assert index.categories() == {"mouse": 2, "screen": 1, "keyboard": 1, "plugin": 1} + + +def test_search_filters_are_typed(): + index = ToolIndex(_fake_tools()) + assert [item.name for item in index.search("", category="screen")] == ["fx_shot"] + assert [item.name for item in index.search("", capability=Capability.READ_SCREEN)] == [ + "fx_where", "fx_shot"] + with pytest.raises(ToolDiscoveryError, match="unknown category"): + index.search("", category="nope") + with pytest.raises(ToolDiscoveryError, match="unknown capability"): + index.search("", capability="root") + + +@pytest.mark.parametrize("limit", [0, -1, MAX_SEARCH_LIMIT + 1, True, "5", 2.5]) +def test_limit_is_bounded(limit): + index = ToolIndex(_fake_tools()) + with pytest.raises(ToolDiscoveryError): + index.search("mouse", limit=limit) + + +def test_default_limit_caps_the_reply(): + tools = [_tool(f"fx_many_{number}", "Many mouse tools.") for number in range(40)] + index = ToolIndex(tools) + assert len(index.search("mouse")) == DEFAULT_SEARCH_LIMIT + assert len(index.search("mouse", limit=MAX_SEARCH_LIMIT)) == 40 + + +def test_query_is_bounded_and_must_be_text(): + index = ToolIndex(_fake_tools()) + with pytest.raises(ToolDiscoveryError): + index.search("x" * 10_000) + with pytest.raises(ToolDiscoveryError): + index.search(None) + + +def test_an_exact_name_ranks_first(): + index = ToolIndex(_fake_tools() + [_tool("fx_type_fast", "Type text quickly.")]) + assert index.search("fx_type")[0].name == "fx_type" + + +def test_schema_lookup_is_single_tool(): + index = ToolIndex(_fake_tools()) + selected_tool = "fx_shot" + schema_reply = index.get_schema(selected_tool) + assert isinstance(schema_reply, MCPToolDescriptor) + assert schema_reply.name == selected_tool + assert schema_reply.to_dict() == _fake_tools()[2].to_descriptor() + with pytest.raises(ToolDiscoveryError): + index.get_schema("fx_missing") + + +# --- the registry is the single source ----------------------------------------- + + +def test_only_authorized_tools_are_indexed(): + server = _progressive() + hidden_tool = "fx_click" + with authorization_scope(AuthorizationContext(user_id="v", role="viewer")): + search_names = [item.name for item in server.disclosure.index().search( + "", limit=MAX_SEARCH_LIMIT)] + assert hidden_tool not in search_names + assert "fx_where" in search_names + with pytest.raises(ToolDiscoveryError): + server.disclosure.index().get_schema(hidden_tool) + # No user store, no RBAC: the same server indexes everything. + assert hidden_tool in [item.name for item in server.disclosure.index().search( + "", limit=MAX_SEARCH_LIMIT)] + + +def test_read_only_mode_indexes_no_mutating_tool(): + server = _progressive(read_only=True) + names = [item.name for item in server.disclosure.index().search("", limit=MAX_SEARCH_LIMIT)] + assert names == ["fx_where", "fx_shot"] + + +def test_index_snapshot_follows_plugin_changes(): + server = _progressive() + before = server.disclosure.index() + assert server.disclosure.index() is before # cached while nothing changed + server.register_tool(_tool("plugin_fx_late", "Arrived later.")) + after = server.disclosure.index() + assert after.version > before.version + assert [item.name for item in after.search("later")] == ["plugin_fx_late"] + assert before.search("later") == [] # the old snapshot is not mutated + server.unregister_tool("plugin_fx_late") + assert server.disclosure.index().search("later") == [] + + +def test_discover_and_schema_tools_go_through_the_dispatcher(): + server = _progressive() + found = _payload(_call(server, "ac_tools_search", {"query": "mouse", "limit": 1})) + assert [item["name"] for item in found["tools"]] == ["fx_click"] + assert found["total"] == 2 and found["truncated"] is True + assert "inputSchema" not in found["tools"][0] + assert found["tools"][0]["enabled"] is False + described = _payload(_call(server, "ac_tools_schema", {"name": "fx_shot"})) + assert described == _fake_tools()[2].to_descriptor() + + +def test_discover_tool_rejects_bad_input_as_a_tool_error(): + server = _progressive() + for arguments in ({"query": "mouse", "limit": 9999}, {"query": "mouse", "limit": "3"}, + {"query": 5}, {"category": "nope"}, {"surprise": 1}): + result = _call(server, "ac_tools_search", arguments)["result"] + assert result["isError"] is True, arguments + assert _call(server, "ac_tools_schema", {"name": "fx_missing"})["result"]["isError"] is True + + +# --- the real registry ---------------------------------------------------------- + + +def test_every_default_tool_has_a_category_and_the_descriptor_is_unchanged(): + tools = build_default_tool_registry(read_only=False, aliases=True) + assert all(tool.category for tool in tools) + assert all("category" not in tool.to_descriptor() for tool in tools) + index = ToolIndex(tools) + assert len(index) == len(tools) + assert "mouse" in index.categories() + + +def test_search_over_the_real_registry_is_small(): + tools = build_default_tool_registry(read_only=False, aliases=False) + index = ToolIndex(tools) + reply = json.dumps([item.to_dict() for item in index.search("screenshot")]) + full = json.dumps([tool.to_descriptor() for tool in tools]) + assert index.search("screenshot") + assert len(reply) * 20 < len(full) diff --git a/test/unit_test/headless/test_mcp_progressive_sessions.py b/test/unit_test/headless/test_mcp_progressive_sessions.py new file mode 100644 index 000000000..b7bcdb222 --- /dev/null +++ b/test/unit_test/headless/test_mcp_progressive_sessions.py @@ -0,0 +1,314 @@ +"""Session-local tool disclosure and paged ``tools/list`` (plan G2). + +Fake tools only. The HTTP half runs a real ``HttpMCPServer`` on a loopback +port, because "two sessions do not see each other's tools" is a statement +about session ids and connections, not about a dict. +""" +import http.client +import json + +import pytest + +from je_auto_control.utils.mcp_server.disclosure import ( + CORE_TOOL_NAMES, SNAPSHOT_META, DisclosureResult, ToolDisclosureError, + ToolMode, ToolPage, ToolView, resolve_mode, +) +from je_auto_control.utils.mcp_server.http_sessions import SESSION_HEADER +from je_auto_control.utils.mcp_server.http_transport import DEFAULT_PATH, HttpMCPServer +from je_auto_control.utils.mcp_server.server import MCPServer +from je_auto_control.utils.mcp_server.tools import ( + MCPTool, MCPToolAnnotations, build_default_tool_registry, +) + +_LIST_CHANGED = "notifications/tools/list_changed" + + +def _tool(name, *, read_only=False, category="fake"): + return MCPTool( + name=name, description=f"Fake tool {name}.", + input_schema={"type": "object", "properties": {}}, + handler=lambda: name, + annotations=MCPToolAnnotations(read_only=read_only), category=category, + ) + + +def _tools(count=6): + return [_tool(f"fx_{number:03d}") for number in range(count)] + + +def _rpc(server, method, params=None, msg_id=1): + return json.loads(server.handle_line(json.dumps({ + "jsonrpc": "2.0", "id": msg_id, "method": method, "params": params or {}}))) + + +def _names(reply): + return [tool["name"] for tool in reply["result"]["tools"]] + + +def _call(server, name, arguments=None): + reply = _rpc(server, "tools/call", {"name": name, "arguments": arguments or {}}) + result = reply["result"] + assert result["isError"] is False, result + return json.loads(result["content"][0]["text"]) + + +def _all_pages(server): + names, cursor, pages = [], None, 0 + while True: + reply = _rpc(server, "tools/list", {"cursor": cursor} if cursor else {}) + names.extend(_names(reply)) + pages += 1 + cursor = reply["result"].get("nextCursor") + if cursor is None: + return names, pages + + +# --- mode selection --------------------------------------------------------------- + + +def test_full_mode_is_compatible(monkeypatch): + monkeypatch.delenv("JE_AUTOCONTROL_MCP_TOOL_MODE", raising=False) + registry = build_default_tool_registry(read_only=False) + server = MCPServer(tools=registry) + assert server.disclosure.mode is ToolMode.FULL + reply = _rpc(server, "tools/list") + full_names = _names(reply) + registry_names = [tool.name for tool in registry] + assert full_names == registry_names + # Byte for byte what the server answered before the mode existed: one + # page, no cursor, no _meta, and a cursor argument is ignored as it was. + expected = {"tools": [tool.to_descriptor() for tool in registry]} + assert reply["result"] == expected + assert json.dumps(reply["result"]) == json.dumps(expected) + assert _rpc(server, "tools/list", {"cursor": "anything"})["result"] == expected + assert not any(name in server._tools for name in CORE_TOOL_NAMES) + + +def test_mode_comes_from_the_environment_and_rejects_typos(monkeypatch): + monkeypatch.setenv("JE_AUTOCONTROL_MCP_TOOL_MODE", "progressive") + assert MCPServer(tools=_tools()).disclosure.mode is ToolMode.PROGRESSIVE + assert MCPServer(tools=_tools(), tool_mode="full").disclosure.mode is ToolMode.FULL + monkeypatch.setenv("JE_AUTOCONTROL_MCP_TOOL_MODE", "progresive") + with pytest.raises(ToolDisclosureError): + resolve_mode(None) + monkeypatch.setenv("JE_AUTOCONTROL_MCP_TOOL_MODE", "") + assert resolve_mode(None) is ToolMode.FULL + + +def test_progressive_session_starts_with_the_core_only(): + server = MCPServer(tools=_tools(), tool_mode="progressive") + reply = _rpc(server, "tools/list") + assert _names(reply) == list(CORE_TOOL_NAMES) + assert "nextCursor" not in reply["result"] + assert isinstance(reply["result"]["_meta"][SNAPSHOT_META], str) + + +# --- enabling --------------------------------------------------------------------- + + +def test_enable_adds_to_the_list_and_reports_what_happened(): + server = MCPServer(tools=_tools(), tool_mode="progressive") + outcome = _call(server, "ac_tools_enable", {"names": ["fx_001", "fx_404", "fx_001"]}) + assert outcome["enabled"] == ["fx_001"] + assert outcome["unavailable"] == ["fx_404"] + assert _names(_rpc(server, "tools/list")) == list(CORE_TOOL_NAMES) + ["fx_001"] + again = _call(server, "ac_tools_enable", {"names": ["fx_001"]}) + assert again["enabled"] == [] and again["already_enabled"] == ["fx_001"] + assert _call(server, "ac_tools_disable", {"names": ["fx_001"]})["disabled"] == ["fx_001"] + assert _names(_rpc(server, "tools/list")) == list(CORE_TOOL_NAMES) + + +def test_enable_accepts_a_category(): + tools = _tools(3) + [_tool("fx_other", category="other")] + server = MCPServer(tools=tools, tool_mode="progressive") + outcome = _call(server, "ac_tools_enable", {"names": ["category:fake"]}) + assert outcome["enabled"] == ["fx_000", "fx_001", "fx_002"] + assert _call(server, "ac_tools_enable", {"names": ["category:nope"]})["unavailable"] == [ + "category:nope"] + + +def test_enable_emits_list_changed(): + server = MCPServer(tools=_tools(), tool_mode="progressive") + sent = [] + server.set_notifier(lambda method, params: sent.append(method)) + outcome = _call(server, "ac_tools_enable", {"names": ["fx_001", "fx_002"]}) + list_changed_count = sent.count(_LIST_CHANGED) + assert list_changed_count == 1 + assert outcome["list_changed_sent"] is True + # Enabling what is already enabled changes nothing, so it says nothing. + _call(server, "ac_tools_enable", {"names": ["fx_001"]}) + assert sent.count(_LIST_CHANGED) == 1 + _call(server, "ac_tools_disable", {"names": ["fx_001"]}) + assert sent.count(_LIST_CHANGED) == 2 + + +def test_enable_without_a_channel_says_the_client_must_list_again(): + server = MCPServer(tools=_tools(), tool_mode="progressive") + outcome = _call(server, "ac_tools_enable", {"names": ["fx_001"]}) + assert outcome["list_changed_sent"] is False + + +def test_view_api_returns_typed_results(): + server = MCPServer(tools=_tools(), tool_mode="progressive") + view = server.disclosure.view_for("session-a") + assert isinstance(view, ToolView) + result = view.enable(["fx_002"]) + assert isinstance(result, DisclosureResult) + assert result.enabled == ("fx_002",) + page = view.list_page(None) + assert isinstance(page, ToolPage) + assert page.next_cursor is None and page.snapshot_id + assert [tool.name for tool in page.tools] == list(CORE_TOOL_NAMES) + ["fx_002"] + with pytest.raises(ToolDisclosureError): + view.enable("fx_003") # a bare string is not a list of names + with pytest.raises(ToolDisclosureError): + view.enable([f"fx_{number}" for number in range(10_000)]) + + +def test_enable_is_session_local(): + server = MCPServer(tools=_tools(), tool_mode="progressive") + tool = "fx_003" + with server.connection_scope(connection_id="session-a"): + _call(server, "ac_tools_enable", {"names": [tool]}) + assert tool in _names(_rpc(server, "tools/list")) + other_session = server.disclosure.view_for("session-b") + assert tool not in other_session.visible_names + with server.connection_scope(connection_id="session-b"): + assert tool not in _names(_rpc(server, "tools/list")) + refused = _rpc(server, "tools/call", {"name": tool, "arguments": {}}) + assert refused["error"]["code"] == -32602 + # stdio's one implicit session is a third, separate view. + assert tool not in _names(_rpc(server, "tools/list")) + + +def test_a_session_is_reclaimed_when_it_ends(): + server = MCPServer(tools=_tools(), tool_mode="progressive") + with server.connection_scope(connection_id="session-a"): + _call(server, "ac_tools_enable", {"names": ["fx_001"]}) + assert server.disclosure.session_count == 1 + server.forget_connection("session-a") + assert server.disclosure.session_count == 0 + with server.connection_scope(connection_id="session-a"): + assert "fx_001" not in _names(_rpc(server, "tools/list")) + + +# --- pagination ------------------------------------------------------------------- + + +def _paged_server(count=7, page_size=3): + server = MCPServer(tools=_tools(count), tool_mode="progressive") + server.disclosure.page_size = page_size + _call(server, "ac_tools_enable", {"names": ["category:fake"]}) + return server + + +def test_list_pages_cover_the_view_exactly_once(): + server = _paged_server() + names, pages = _all_pages(server) + assert names == list(CORE_TOOL_NAMES) + [f"fx_{number:03d}" for number in range(7)] + assert pages == 4 + + +def test_cursor_snapshot_survives_plugin_change(): + server = _paged_server() + first = _rpc(server, "tools/list") + snapshot = first["result"]["_meta"][SNAPSHOT_META] + cursor = first["result"]["nextCursor"] + # A plugin arrives and another leaves between two pages. + server.register_tool(_tool("plugin_fx_new")) + server.unregister_tool("fx_005") + _call(server, "ac_tools_enable", {"names": ["plugin_fx_new"]}) + names = _names(first) + while cursor is not None: + page = _rpc(server, "tools/list", {"cursor": cursor}) + assert page["result"]["_meta"][SNAPSHOT_META] == snapshot + names.extend(_names(page)) + cursor = page["result"].get("nextCursor") + # The original snapshot, whole: no page mixes the two registries. + assert names == list(CORE_TOOL_NAMES) + [f"fx_{number:03d}" for number in range(7)] + fresh, _pages = _all_pages(server) + assert "plugin_fx_new" in fresh and "fx_005" not in fresh + assert _rpc(server, "tools/list")["result"]["_meta"][SNAPSHOT_META] != snapshot + + +def test_stale_and_malformed_cursors_fail_explicitly(): + server = _paged_server() + cursor = _rpc(server, "tools/list")["result"]["nextCursor"] + for bad in ("", "not-a-cursor", cursor[:-2] + "zz", cursor + "AAAA", 7, ["x"]): + reply = _rpc(server, "tools/list", {"cursor": bad}) + assert reply["error"]["code"] == -32602, bad + # Enough newer snapshots push the first one out; its cursor then fails + # instead of silently paging a different list. + for number in range(server.disclosure.view_for(None).max_snapshots + 1): + server.register_tool(_tool(f"fx_churn_{number}")) + _rpc(server, "tools/list") + assert _rpc(server, "tools/list", {"cursor": cursor})["error"]["code"] == -32602 + + +def test_a_cursor_belongs_to_its_session(): + server = _paged_server() + cursor = _rpc(server, "tools/list")["result"]["nextCursor"] + with server.connection_scope(connection_id="session-b"): + assert _rpc(server, "tools/list", {"cursor": cursor})["error"]["code"] == -32602 + + +# --- over HTTP -------------------------------------------------------------------- + + +def _post(port, payload, session_id=None): + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=10) + headers = {"Content-Type": "application/json"} + if session_id is not None: + headers[SESSION_HEADER] = session_id + try: + connection.request("POST", DEFAULT_PATH, body=json.dumps(payload), headers=headers) + response = connection.getresponse() + body = response.read().decode("utf-8") + return response.status, response.getheader(SESSION_HEADER), body + finally: + connection.close() + + +def _open_session(port): + status, session_id, _body = _post(port, { + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-11-25", "capabilities": {}}}) + assert status == 200 and session_id + return session_id + + +def _http_names(port, session_id): + _status, _sid, body = _post(port, {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, + session_id) + return [tool["name"] for tool in json.loads(body)["result"]["tools"]] + + +def test_two_http_sessions_do_not_share_enabled_tools(): + mcp = MCPServer(tools=_tools(), tool_mode="progressive") + transport = HttpMCPServer(mcp=mcp, port=0) + transport.start() + try: + port = transport.address[1] + first, second = _open_session(port), _open_session(port) + _post(port, {"jsonrpc": "2.0", "id": 3, "method": "tools/call", + "params": {"name": "ac_tools_enable", "arguments": {"names": ["fx_002"]}}}, + first) + assert "fx_002" in _http_names(port, first) + assert "fx_002" not in _http_names(port, second) + status, _sid, body = _post(port, { + "jsonrpc": "2.0", "id": 4, "method": "tools/call", + "params": {"name": "fx_002", "arguments": {}}}, second) + assert status == 200 and json.loads(body)["error"]["code"] == -32602 + assert mcp.disclosure.session_count == 2 + # Ending the session releases its view. + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=10) + try: + connection.request("DELETE", DEFAULT_PATH, headers={SESSION_HEADER: first}) + assert connection.getresponse().status == 200 + finally: + connection.close() + assert mcp.disclosure.session_count == 1 + transport.stop() + assert mcp.disclosure.session_count == 0 + finally: + transport.stop() From bfa60c9c9daa967a0a7f44007677124041580186 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:58:00 +0800 Subject: [PATCH 17/44] Report a refused minimise, maximise or restore to the MCP client show_window answers False when the handle is no longer a window or Windows refused to bring an activating command's window forward, and the MCP handlers returned the handle regardless, so a client was told a refused request had worked. It is a tool error now, as it already is for ac_focus_window. --- .../Eng/doc/mcp_server/mcp_server_doc.rst | 6 +- .../Zh/doc/mcp_server/mcp_server_doc.rst | 4 +- .../utils/mcp_server/tools/_factories.py | 9 +- .../mcp_server/tools/_handlers_system.py | 22 ++++- .../headless/test_mcp_window_show_refusal.py | 87 +++++++++++++++++++ 5 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 test/unit_test/headless/test_mcp_window_show_refusal.py diff --git a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst index 316f691ca..4410e26f8 100644 --- a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst @@ -38,7 +38,11 @@ Screen / image / OCR Window management (Windows) ``ac_list_windows``, ``ac_focus_window``, ``ac_wait_for_window``, ``ac_close_window``, ``ac_window_move``, ``ac_window_minimize``, - ``ac_window_maximize``, ``ac_window_restore``. + ``ac_window_maximize``, ``ac_window_restore``. The last three answer with + a tool error (``isError``) when the window is gone or, for maximise and + restore, when Windows refuses to bring it to the foreground -- the window + may then have changed state without becoming the active one. They used to + return the window handle either way. Semantic locators ``ac_a11y_list``, ``ac_a11y_find``, ``ac_a11y_click``, diff --git a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst index 6118174b4..f8a9546a1 100644 --- a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst @@ -37,7 +37,9 @@ list-changed 通知與 elicitation。 視窗管理 (Windows) ``ac_list_windows``、``ac_focus_window``、``ac_wait_for_window``、 ``ac_close_window``、``ac_window_move``、``ac_window_minimize``、 - ``ac_window_maximize``、``ac_window_restore``。 + ``ac_window_maximize``、``ac_window_restore``。最後三個在視窗已不存在時， + 或（最大化與還原）Windows 拒絕把它帶到前景時，會回工具錯誤（``isError``） + ——此時視窗可能已改變狀態但沒有成為作用中視窗。以前不論結果都回傳視窗代碼。 語意定位 ``ac_a11y_list``、``ac_a11y_find``、``ac_a11y_click``、 diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fecbc25ae..ddcc7d736 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -446,7 +446,8 @@ def window_tools() -> List[MCPTool]: ), MCPTool( name="ac_window_minimize", - description="Minimise the first matching window.", + description=("Minimise the first matching window. Fails if the " + "window is gone by the time the request is made."), input_schema=schema({ "title_substring": {"type": "string"}, "case_sensitive": {"type": "boolean"}, @@ -456,7 +457,8 @@ def window_tools() -> List[MCPTool]: ), MCPTool( name="ac_window_maximize", - description="Maximise the first matching window.", + description=("Maximise the first matching window. Fails if the " + "window is gone or Windows refuses to activate it."), input_schema=schema({ "title_substring": {"type": "string"}, "case_sensitive": {"type": "boolean"}, @@ -467,7 +469,8 @@ def window_tools() -> List[MCPTool]: MCPTool( name="ac_window_restore", description=("Restore the first matching window to its previous " - "size and position."), + "size and position. Fails if the window is gone or " + "Windows refuses to activate it."), input_schema=schema({ "title_substring": {"type": "string"}, "case_sensitive": {"type": "boolean"}, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_system.py b/je_auto_control/utils/mcp_server/tools/_handlers_system.py index b995e26ff..d22923a64 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_system.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_system.py @@ -141,12 +141,30 @@ def window_move(title_substring: str, x: int, y: int, "width": int(width), "height": int(height)} +#: What each ShowWindow command the tools below send was asked to do. +_SHOW_VERBS = {3: "maximise", 6: "minimise", 9: "restore"} + + def _show_command(title_substring: str, case_sensitive: bool, cmd_show: int) -> int: - """Resolve the window then call ShowWindow with the given cmd.""" + """Resolve the window then call ShowWindow with the given cmd. + + ``show_window`` answers ``False`` when the handle stopped being a window + or when a command that activates the window (maximise, restore) was + refused the foreground. The tool used to return the handle either way, + so a client was told a refused request had worked; it is an error now, + as it is for ``ac_focus_window``. ``None`` (a backend that cannot tell) + is not a failure. + """ + from je_auto_control.utils.exception.exceptions import AutoControlActionException from je_auto_control.windows.window import windows_window_manage as wm hwnd = _resolve_window_hwnd(title_substring, bool(case_sensitive)) - wm.show_window(hwnd, int(cmd_show)) + if wm.show_window(hwnd, int(cmd_show)) is False: + verb = _SHOW_VERBS.get(int(cmd_show), f"show (command {int(cmd_show)})") + raise AutoControlActionException( + f"could not {verb} {title_substring!r} (hwnd {hwnd}): the window is " + "gone, or Windows refused to bring it to the foreground -- it may " + "have changed state without becoming the active window") return hwnd diff --git a/test/unit_test/headless/test_mcp_window_show_refusal.py b/test/unit_test/headless/test_mcp_window_show_refusal.py new file mode 100644 index 000000000..f3fca3c91 --- /dev/null +++ b/test/unit_test/headless/test_mcp_window_show_refusal.py @@ -0,0 +1,87 @@ +"""``ac_window_minimize`` / ``maximize`` / ``restore`` report a refused request. + +``show_window`` answers ``False`` when the handle is no longer a window or +when Windows refused to bring an activating command's window forward. The MCP +handlers ignored that and returned the handle, so a client was told the +request had worked. + +The window lookup and ``show_window`` are fakes: no real window is touched. +""" +import json +import sys + +import pytest + +pytestmark = pytest.mark.skipif( + sys.platform not in ("win32", "cygwin", "msys"), + reason="windows_window_manage uses ctypes.WINFUNCTYPE; Win32-only.") + +_TOOLS = [("ac_window_minimize", 6, "minimise"), ("ac_window_maximize", 3, "maximise"), + ("ac_window_restore", 9, "restore")] + + +@pytest.fixture +def show(monkeypatch): + """Fake window lookup and ``show_window``; ``show.answer`` is what it returns.""" + import je_auto_control.wrapper.auto_control_window as window_module + from je_auto_control.windows.window import windows_window_manage + + class _Show: + answer = True + calls = [] + + def fake_show(hwnd, cmd_show): + _Show.calls.append((int(hwnd), int(cmd_show))) + return _Show.answer + + monkeypatch.setattr(window_module, "find_window", + lambda title, case_sensitive=False: (456, title)) + monkeypatch.setattr(windows_window_manage, "show_window", fake_show) + return _Show + + +def _tool(name): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + return {tool.name: tool for tool in build_default_tool_registry()}[name] + + +@pytest.mark.parametrize("name, command, verb", _TOOLS) +def test_a_refused_show_command_is_an_error(show, name, command, verb): + from je_auto_control.utils.exception.exceptions import AutoControlActionException + show.answer = False + with pytest.raises(AutoControlActionException) as refused: + _tool(name).invoke({"title_substring": "Notepad"}) + assert show.calls == [(456, command)] + message = str(refused.value) + assert f"could not {verb} 'Notepad' (hwnd 456)" in message + assert "refused" in message and "gone" in message + + +@pytest.mark.parametrize("name, command, _verb", _TOOLS) +@pytest.mark.parametrize("answer", [True, None], ids=["succeeded", "cannot-tell"]) +def test_a_show_command_that_was_not_refused_returns_the_handle(show, name, command, _verb, + answer): + show.answer = answer + assert _tool(name).invoke({"title_substring": "Notepad"}) == 456 + assert show.calls == [(456, command)] + + +def test_the_refusal_reaches_the_client_as_a_tool_error(show): + from je_auto_control.utils.mcp_server.server import MCPServer + show.answer = False + server = MCPServer(tools=[_tool("ac_window_restore")]) + reply = json.loads(server.handle_line(json.dumps({ + "jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": "ac_window_restore", "arguments": {"title_substring": "Notepad"}}, + }))) + result = reply["result"] + assert result["isError"] is True + assert "could not restore 'Notepad'" in result["content"][0]["text"] + + +def test_a_window_that_does_not_match_is_still_its_own_error(show, monkeypatch): + import je_auto_control.wrapper.auto_control_window as window_module + monkeypatch.setattr(window_module, "find_window", lambda title, case_sensitive=False: None) + with pytest.raises(ValueError, match="no window matches"): + _tool("ac_window_minimize").invoke({"title_substring": "Nope"}) + assert show.calls == [] From f277e285528cfe58d50c2c7eabe1f92d94a2e1bf Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 01:59:25 +0800 Subject: [PATCH 18/44] Add mobile app lifecycle and report missing device features instead of imitating them A script could not start, wait for or stop the app under test, nor answer an alert, and nothing said which of install / files / clipboard / recording a backend really has. launch_app, wait_for_app, stop_app and the alert calls now work on any DeviceSession with a fixed timeout and the session's cancel signal. MobileExtension is the contract for the optional features: adb covers them on Android, WebDriverAgent covers only the pasteboard on iOS, and the rest report needs_dependency and raise with the reason rather than borrowing adb. A device-matrix spec with app_id stops the app it started even when the run fails. --- docs/source/Eng/doc/mobile/mobile_doc.rst | 108 +++++++ docs/source/Zh/doc/mobile/mobile_doc.rst | 102 +++++++ je_auto_control/__init__.py | 8 + je_auto_control/android/__init__.py | 4 +- je_auto_control/android/adb_client.py | 33 ++- je_auto_control/android/apps.py | 249 ++++++++++++++++ je_auto_control/android/session.py | 35 ++- je_auto_control/ios/__init__.py | 2 + je_auto_control/ios/apps.py | 145 ++++++++++ je_auto_control/ios/session.py | 37 ++- je_auto_control/utils/device_matrix/matrix.py | 57 +++- je_auto_control/wrapper/device_context.py | 41 ++- je_auto_control/wrapper/mobile_extensions.py | 182 ++++++++++++ test/unit_test/headless/_mobile_doubles.py | 47 ++- .../headless/test_mobile_app_lifecycle.py | 269 ++++++++++++++++++ 15 files changed, 1293 insertions(+), 26 deletions(-) create mode 100644 je_auto_control/android/apps.py create mode 100644 je_auto_control/ios/apps.py create mode 100644 je_auto_control/wrapper/mobile_extensions.py create mode 100644 test/unit_test/headless/test_mobile_app_lifecycle.py diff --git a/docs/source/Eng/doc/mobile/mobile_doc.rst b/docs/source/Eng/doc/mobile/mobile_doc.rst index 98747fc13..8015a4c16 100644 --- a/docs/source/Eng/doc/mobile/mobile_doc.rst +++ b/docs/source/Eng/doc/mobile/mobile_doc.rst @@ -235,3 +235,111 @@ every answer is in device points, ready for a gesture:: ``self_heal_locate(..., frame=frame)`` runs the same template-then-VLM fallback and writes the same heal log; ``screen_region`` does not apply to a frame. ``self_heal_click`` still clicks the desktop mouse and takes no frame. + +App lifecycle and alerts +======================== + +:: + + from je_auto_control import ( + AppState, accept_alert, launch_app, stop_app, wait_for_app, + ) + + launch_app(session, "com.example.shop") # Android package / iOS bundle id + wait_for_app(session, "com.example.shop", timeout_s=15) + accept_alert(session) + assert stop_app(session, "com.example.shop") == AppState.NOT_RUNNING + +``AppState`` is ``not_installed``, ``not_running``, ``background`` or +``foreground``, and compares equal to those strings. + +.. list-table:: + :header-rows: 1 + :widths: 22 39 39 + + * - Call + - Android + - iOS + * - ``launch_app`` + - ``monkey -p -c android.intent.category.LAUNCHER 1``, or + ``am start -n`` for a ``package/activity`` id + - WDA app launch + * - ``stop_app`` + - ``am force-stop`` + - WDA app terminate + * - ``app_state`` + - ``pidof``, ``pm path`` and the resumed activity from ``dumpsys`` + - WDA app state. WDA reports an app that is not installed as not + running, so ``not_installed`` is never returned. + * - ``wait_for_app`` + - Polls ``app_state`` until the timeout, raising ``DeviceTimeoutError``; + cancelling the session ends the wait. + - Same. + * - ``accept_alert`` / ``dismiss_alert`` + - Presses the standard dialog or permission button through + ``uiautomator2`` (``android:id/button1`` / ``button2``, the permission + controller's allow / deny buttons). Without ``uiautomator2`` it raises + ``DeviceUnsupportedError``; locate the button in a frame and tap it + instead. + - WDA alert accept / dismiss; returns the alert text. + +With no alert showing, both raise ``AlertNotPresentError``. An app id is +validated before it reaches the device shell. + +A device-matrix spec may name an ``app_id``: the app is launched and in front +before the first step and stopped afterwards whether the steps passed or not +(``"keep_app": true`` leaves it running). ``DeviceResult.app_state`` records +where it ended up. + +Install, files, clipboard, recording +==================================== + +These are the features a backend may not have. ``mobile_extension(session)`` +returns a :class:`MobileExtension`; ``capability(feature)`` says whether each +of ``install``, ``files``, ``clipboard`` and ``recording`` can be used, and a +method whose feature is missing raises ``DeviceUnsupportedError`` with the +reason:: + + from je_auto_control import mobile_extension + + extension = mobile_extension(session) + if extension.capability("recording").available: + extension.start_recording(time_limit_s=60) + ... + extension.stop_recording("run.mp4") + +.. list-table:: + :header-rows: 1 + :widths: 18 41 41 + + * - Feature + - Android + - iOS (WebDriverAgent) + * - ``install`` + - ``adb install -r`` + - ``needs_dependency``: WDA has no install endpoint. + * - ``files`` + - ``adb push`` / ``adb pull`` + - ``needs_dependency``: WDA has no file endpoint. + * - ``clipboard`` + - Through ``uiautomator2``; ``needs_dependency`` without it, because + ``adb`` cannot reach the clipboard on current Android. + - Set only. Reading is allowed by WDA only while WDA itself is in front, + and raises ``DeviceUnsupportedError`` on a ``facebook-wda`` build + without ``get_clipboard``. + * - ``recording`` + - ``adb shell screenrecord`` (180 s at most), stopped with ``SIGINT`` so + the file is finalised, then pulled and removed from the device. A + recording is device-side state: it is not stopped by closing the + session, only by ``stop_recording``. + - ``needs_dependency``: WDA has no recording endpoint. + +Nothing from ``adb`` is used for an iOS device. To add the missing iOS +features, register an adapter around a host-side tool; it is asked first and +WebDriverAgent covers what it does not provide:: + + from je_auto_control import register_mobile_extension + + register_mobile_extension("ios", lambda session: MyTideviceAdapter(session)) + +No such adapter ships with AutoControl. diff --git a/docs/source/Zh/doc/mobile/mobile_doc.rst b/docs/source/Zh/doc/mobile/mobile_doc.rst index 5f15f2a93..ff5d9ec21 100644 --- a/docs/source/Zh/doc/mobile/mobile_doc.rst +++ b/docs/source/Zh/doc/mobile/mobile_doc.rst @@ -215,3 +215,105 @@ iOS 只請求 WDA 的 ``/status``\ 。 ``self_heal_locate(..., frame=frame)`` 跑的是同一套「先樣板、後 VLM」的退路， 寫的也是同一份 heal log；\ ``screen_region`` 不適用於 frame。 ``self_heal_click`` 仍然是點桌面的滑鼠，不接受 frame。 + +App 生命週期與 alert +==================== + +:: + + from je_auto_control import ( + AppState, accept_alert, launch_app, stop_app, wait_for_app, + ) + + launch_app(session, "com.example.shop") # Android package / iOS bundle id + wait_for_app(session, "com.example.shop", timeout_s=15) + accept_alert(session) + assert stop_app(session, "com.example.shop") == AppState.NOT_RUNNING + +``AppState`` 是 ``not_installed``\ 、\ ``not_running``\ 、\ ``background`` 或 +``foreground``\ ，可以直接與這些字串比較。 + +.. list-table:: + :header-rows: 1 + :widths: 22 39 39 + + * - 呼叫 + - Android + - iOS + * - ``launch_app`` + - ``monkey -p -c android.intent.category.LAUNCHER 1``\ ； + id 寫成 ``package/activity`` 時用 ``am start -n`` + - WDA app launch + * - ``stop_app`` + - ``am force-stop`` + - WDA app terminate + * - ``app_state`` + - ``pidof``\ 、\ ``pm path``\ ，以及 ``dumpsys`` 裡的 resumed activity + - WDA app state。WDA 把沒安裝的 App 回報成沒在執行，所以不會回傳 + ``not_installed``\ 。 + * - ``wait_for_app`` + - 輪詢 ``app_state`` 直到逾時，逾時丟 ``DeviceTimeoutError``\ ； + 取消 session 會結束等待。 + - 相同。 + * - ``accept_alert`` / ``dismiss_alert`` + - 透過 ``uiautomator2`` 按下標準對話框或權限按鈕（``android:id/button1`` / + ``button2``\ 、permission controller 的允許／拒絕按鈕）。沒有 + ``uiautomator2`` 時丟 ``DeviceUnsupportedError``\ ；請改在 frame 裡定位 + 按鈕再點它。 + - WDA alert accept / dismiss；回傳 alert 的文字。 + +沒有 alert 時兩者都丟 ``AlertNotPresentError``\ 。App id 在送進裝置 shell 之前 +會先驗證。 + +Device matrix 的裝置規格可以寫 ``app_id``\ ：第一個步驟之前會啟動 App 並等它到 +前景，步驟結束後不論成功與否都會停止它（\ ``"keep_app": true`` 則保留執行）。 +``DeviceResult.app_state`` 記錄它最後的狀態。 + +安裝、檔案、剪貼簿、錄影 +======================== + +這些是後端可能沒有的功能。\ ``mobile_extension(session)`` 回傳 +:class:`MobileExtension`\ ；\ ``capability(feature)`` 說明 ``install``\ 、 +``files``\ 、\ ``clipboard``\ 、\ ``recording`` 各自能不能用，缺少的功能呼叫時 +會丟 ``DeviceUnsupportedError`` 並附上原因:: + + from je_auto_control import mobile_extension + + extension = mobile_extension(session) + if extension.capability("recording").available: + extension.start_recording(time_limit_s=60) + ... + extension.stop_recording("run.mp4") + +.. list-table:: + :header-rows: 1 + :widths: 18 41 41 + + * - 功能 + - Android + - iOS（WebDriverAgent） + * - ``install`` + - ``adb install -r`` + - ``needs_dependency``\ ：WDA 沒有安裝的端點。 + * - ``files`` + - ``adb push`` / ``adb pull`` + - ``needs_dependency``\ ：WDA 沒有檔案的端點。 + * - ``clipboard`` + - 透過 ``uiautomator2``\ ；沒裝時是 ``needs_dependency``\ ，因為新版 + Android 上 ``adb`` 碰不到剪貼簿。 + - 只能寫入。WDA 只在 WDA 自己位於前景時才允許讀取；\ ``facebook-wda`` + 版本沒有 ``get_clipboard`` 時丟 ``DeviceUnsupportedError``\ 。 + * - ``recording`` + - ``adb shell screenrecord``\ （最長 180 秒），以 ``SIGINT`` 停止好讓檔案 + 收尾，再拉回主機並從裝置刪除。錄影是裝置端的狀態：關閉 session 不會 + 停止它，只有 ``stop_recording`` 會。 + - ``needs_dependency``\ ：WDA 沒有錄影的端點。 + +iOS 裝置不會用到任何 ``adb`` 的東西。要補上 iOS 缺少的功能，請針對主機端工具 +註冊 adapter；它會先被詢問，它沒提供的部分仍由 WebDriverAgent 負責:: + + from je_auto_control import register_mobile_extension + + register_mobile_extension("ios", lambda session: MyTideviceAdapter(session)) + +AutoControl 本身沒有附帶這樣的 adapter。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 91421adda..b03e41df5 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1040,12 +1040,17 @@ ) # Mobile device contexts (one isolated session per Android / iOS device) from je_auto_control.wrapper.device_context import ( + AlertNotPresentError, AppState, DeviceCancelledError, DeviceCapability, DeviceClosedError, DeviceContext, DeviceError, DevicePermissionError, DeviceSession, DeviceTimeoutError, DeviceUnavailableError, DeviceUnsupportedError, Drag, LongPress, Pinch, Swipe, Tap, open_device, use_device, ) from je_auto_control.wrapper.device_frame import DeviceFrame +from je_auto_control.wrapper.mobile_extensions import ( + MobileExtension, accept_alert, app_state, dismiss_alert, launch_app, + mobile_extension, register_mobile_extension, stop_app, wait_for_app, +) # Media assertions (audio activity, video motion) from je_auto_control.utils.media_assert import ( MediaAssertionResult, assert_audio_activity, assert_video_changes, @@ -1947,6 +1952,9 @@ def start_autocontrol_gui(*args, **kwargs): "DeviceError", "DevicePermissionError", "DeviceSession", "DeviceTimeoutError", "DeviceUnavailableError", "DeviceUnsupportedError", "open_device", "use_device", "DeviceFrame", "Drag", "LongPress", "Pinch", "Swipe", "Tap", + "AlertNotPresentError", "AppState", "MobileExtension", "accept_alert", + "app_state", "dismiss_alert", "launch_app", "mobile_extension", + "register_mobile_extension", "stop_app", "wait_for_app", # Media assertions "MediaAssertionResult", "assert_audio_activity", "assert_video_changes", "measure_audio_rms", "video_segment_motion", diff --git a/je_auto_control/android/__init__.py b/je_auto_control/android/__init__.py index 4007fc6d2..c8b0beda5 100644 --- a/je_auto_control/android/__init__.py +++ b/je_auto_control/android/__init__.py @@ -42,6 +42,7 @@ from je_auto_control.android.find import ( ElementNotFoundError, click_element, dump_hierarchy, find_element, ) +from je_auto_control.android.apps import AndroidExtension from je_auto_control.android.input import ADB_KEYBOARD_IME, type_text from je_auto_control.android.screen import capture_frame from je_auto_control.android.session import AndroidSession @@ -49,7 +50,8 @@ __all__ = [ "ADB_KEYBOARD_IME", "AdbClient", "AdbDeviceMissingError", "AdbError", "AdbNotAvailable", "AdbTimeoutError", "AdbUnauthorizedError", - "AdbUnsupportedError", "AndroidDevice", "AndroidSession", "capture_frame", + "AdbUnsupportedError", "AndroidDevice", "AndroidExtension", "AndroidSession", + "capture_frame", "type_text", "ElementNotFoundError", "UIAutomatorDevice", "UIAutomatorUnavailableError", diff --git a/je_auto_control/android/adb_client.py b/je_auto_control/android/adb_client.py index 24b737c04..ecc2f4e1f 100644 --- a/je_auto_control/android/adb_client.py +++ b/je_auto_control/android/adb_client.py @@ -7,7 +7,7 @@ import subprocess # nosec B404 # reason: required to invoke the adb binary from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Sequence +from typing import List, Optional, Sequence, Tuple from je_auto_control.wrapper.device_context import ( DeviceError, DevicePermissionError, DeviceTimeoutError, DeviceUnavailableError, @@ -159,6 +159,37 @@ def shell(self, command: str, *, serial: Optional[str] = None, ) return result.stdout.decode("utf-8", errors="replace") + def shell_status(self, command: str, *, serial: Optional[str] = None, + timeout: Optional[float] = None) -> Tuple[int, str]: + """Run a shell command whose exit code is the answer: ``(code, stdout)``. + + ``pidof`` and ``pm path`` exit non-zero to say "no". A non-zero exit + because the *device* failed (unauthorised, gone) is still raised, so + an unreachable device is never read as "not running". + """ + result = self.run(["shell", command], serial=serial, timeout=timeout, check=False) + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace").strip() + error = _exit_error(["shell"], result.returncode, stderr) + if type(error) is not AdbError: + raise error + return result.returncode, result.stdout.decode("utf-8", errors="replace") + + def spawn(self, args: Sequence[str], *, + serial: Optional[str] = None) -> "subprocess.Popen[bytes]": + """Start adb without waiting for it (a screen recording); the caller owns the process.""" + cmd: List[str] = [self._adb] + target = serial if serial is not None else self._default_serial + if target: + cmd.extend(["-s", target]) + cmd.extend(args) + try: + return subprocess.Popen( # nosec B603 # nosemgrep # reason: argv list, no shell, same adb path as run() + cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + except OSError as error: + raise AdbError(f"{_describe(args)} failed: {error}") from error + # --- device discovery --------------------------------------------- def list_devices(self) -> List[AndroidDevice]: diff --git a/je_auto_control/android/apps.py b/je_auto_control/android/apps.py new file mode 100644 index 000000000..9033a42fd --- /dev/null +++ b/je_auto_control/android/apps.py @@ -0,0 +1,249 @@ +"""Android app lifecycle, system dialogs and device extensions over adb. + +Lifecycle and file transfer need nothing but ``adb``. Dialog buttons and the +clipboard need the uiautomator2 daemon, because ``adb shell`` can neither read +the widget tree nor reach the clipboard on current Android; without it those +calls raise and say so. +""" +from __future__ import annotations + +import os +import re +import shlex +import subprocess # nosec B404 # reason: holds the adb process of a running screen recording +import threading +from time import monotonic, sleep +from typing import Any, Dict, Optional, Sequence + +from je_auto_control.android.adb_client import AdbError, AdbUnsupportedError +from je_auto_control.android.client import UIAutomatorDevice, translate_device_errors +from je_auto_control.wrapper.device_context import ( + STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, AlertNotPresentError, AppState, + DeviceCapability, DeviceError, +) + +_PACKAGE = re.compile(r"[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+)+") +_COMPONENT = re.compile(r"[A-Za-z0-9_.]+/[A-Za-z0-9_.$]+") +_RESUMED = re.compile( + r"(?:topResumedActivity|mResumedActivity|ResumedActivity)[=:]\s*ActivityRecord\{\S+ \S+ ([\w.]+)/") +_NEEDS_UI = "uiautomator2 is not installed: pip install uiautomator2" + +#: Buttons that accept a system dialog, most specific first. +ACCEPT_BUTTONS = ( + "com.android.permissioncontroller:id/permission_allow_foreground_only_button", + "com.android.permissioncontroller:id/permission_allow_button", + "com.android.packageinstaller:id/permission_allow_button", + "android:id/button1", +) +#: Buttons that dismiss a system dialog. +DISMISS_BUTTONS = ( + "com.android.permissioncontroller:id/permission_deny_button", + "com.android.packageinstaller:id/permission_deny_button", + "android:id/button2", +) +DEFAULT_RECORDING_PATH = "/sdcard/autocontrol_recording.mp4" +#: ``screenrecord`` refuses anything longer. +MAX_RECORDING_S = 180 +_FINALIZE_TIMEOUT_S = 10.0 + +#: adb processes of running recordings, by device. A recording is device-side +#: state that outlives the command (and the session) that started it, so the +#: command that stops it has to be able to find it again. +_RECORDINGS: Dict[str, Any] = {} +_RECORDINGS_LOCK = threading.Lock() + + +def _package(app_id: str) -> str: + """``app_id`` checked as a package name: it goes into a device shell command.""" + if not isinstance(app_id, str) or not _PACKAGE.fullmatch(app_id): + raise AdbError(f"not an Android package name: {app_id!r}") + return app_id + + +def launch(adb: Any, app_id: str) -> None: + """Start an app by package (its launcher activity) or by ``package/activity``.""" + if isinstance(app_id, str) and _COMPONENT.fullmatch(app_id): + out = adb.shell(f"am start -n {app_id}") + if "Error" in out: + raise AdbError(f"could not start {app_id}: {out.strip().splitlines()[-1]}") + return + package = _package(app_id) + out = adb.shell(f"monkey -p {package} -c android.intent.category.LAUNCHER 1") + if "No activities found" in out or "monkey aborted" in out: + raise AdbError(f"{package} is not installed or has no launcher activity") + + +def stop(adb: Any, app_id: str) -> None: + """Force-stop an app.""" + adb.shell(f"am force-stop {_package(app_id.split('/')[0])}") + + +def state(adb: Any, app_id: str) -> AppState: + """Whether the app is installed, running, and in front.""" + package = _package(app_id.split("/")[0]) + code, _out = adb.shell_status(f"pidof {package}") + if code != 0: + installed, _path = adb.shell_status(f"pm path {package}") + return AppState.NOT_RUNNING if installed == 0 else AppState.NOT_INSTALLED + resumed = _RESUMED.search(adb.shell("dumpsys activity activities")) + if resumed is not None and resumed.group(1) == package: + return AppState.FOREGROUND + return AppState.BACKGROUND + + +@translate_device_errors +def _press_dialog_button(ui_device: UIAutomatorDevice, buttons: Sequence[str]) -> str: + handle = ui_device.handle + for resource_id in buttons: + button = handle(resourceId=resource_id) + if button.exists: + button.click() + return resource_id + raise AlertNotPresentError("no system dialog with a matching button is showing") + + +def answer_dialog(ui_device: Optional[UIAutomatorDevice], accept: bool) -> str: + """Press the accepting or dismissing button of a system dialog; returns its resource id.""" + if ui_device is None: + raise AdbUnsupportedError( + "pressing a dialog button needs the widget tree, which adb alone cannot read", + reason=_NEEDS_UI, + alternative="locate the button in a captured frame (find_text / find_image) and tap it") + return _press_dialog_button(ui_device, ACCEPT_BUTTONS if accept else DISMISS_BUTTONS) + + +def _existing_file(path: str) -> str: + resolved = os.path.realpath(os.fspath(path)) + if not os.path.isfile(resolved): + raise AdbError(f"no such file: {path}") + return resolved + + +def _remote_path(path: str) -> str: + if not isinstance(path, str) or not path.startswith("/") or "\x00" in path: + raise AdbError(f"a device path must be absolute, got {path!r}") + return path + + +class AndroidExtension: + """Install, files, clipboard and screen recording for one Android device.""" + + name = "adb" + + def __init__(self, adb_source: Any, ui_source: Any, device_id: str = "") -> None: + # Callables, so nothing is built (and no adb is needed) until a feature is used. + self._adb_source = adb_source + self._ui_source = ui_source + self._key = device_id + + @property + def _adb(self) -> Any: + return self._adb_source() + + def capability(self, feature: str) -> DeviceCapability: + """Whether ``feature`` can be used on this device.""" + if feature != "clipboard" or self._ui_source() is not None: + return DeviceCapability(feature, STATE_AVAILABLE) + return DeviceCapability( + feature, STATE_NEEDS_DEPENDENCY, _NEEDS_UI, + alternative="type the text with type_text instead of pasting it") + + def install_app(self, source: str) -> str: + """Install (or replace) an APK from the host; returns the path installed.""" + apk = _existing_file(source) + out = self._adb.run(["install", "-r", apk]).stdout.decode("utf-8", errors="replace") + if "Success" not in out: + # adb exits 0 for some refusals and prints "Failure [REASON]". + raise AdbError(f"install failed: {out.strip().splitlines()[-1] if out.strip() else ''}") + return apk + + def push_file(self, local_path: str, remote_path: str) -> str: + """Copy a host file to the device; returns the device path.""" + self._adb.run(["push", _existing_file(local_path), _remote_path(remote_path)]) + return remote_path + + def pull_file(self, remote_path: str, local_path: str) -> str: + """Copy a device file to the host; returns the host path.""" + target = os.path.realpath(os.fspath(local_path)) + os.makedirs(os.path.dirname(target) or ".", exist_ok=True) + self._adb.run(["pull", _remote_path(remote_path), target]) + return target + + def _clipboard_device(self) -> UIAutomatorDevice: + ui_device = self._ui_source() + if ui_device is None: + raise AdbUnsupportedError( + "the clipboard cannot be reached through adb on current Android", + reason=_NEEDS_UI, + alternative="type the text with type_text instead of pasting it") + return ui_device + + def get_clipboard(self) -> str: + """The device clipboard's text.""" + return _read_clipboard(self._clipboard_device()) + + def set_clipboard(self, text: str) -> None: + """Put ``text`` on the device clipboard.""" + _write_clipboard(self._clipboard_device(), text) + + def start_recording(self, remote_path: str = DEFAULT_RECORDING_PATH, + time_limit_s: int = MAX_RECORDING_S) -> str: + """Start ``screenrecord`` on the device; returns the device path it writes.""" + limit = int(time_limit_s) + if not 1 <= limit <= MAX_RECORDING_S: + raise AdbError(f"time_limit_s must be 1..{MAX_RECORDING_S}, got {time_limit_s!r}") + target = _remote_path(remote_path) + with _RECORDINGS_LOCK: + if self._key in _RECORDINGS and _RECORDINGS[self._key].poll() is None: + raise AdbError("a recording is already running on this device") + _RECORDINGS[self._key] = self._adb.spawn( + ["shell", f"screenrecord --time-limit {limit} {shlex.quote(target)}"]) + return target + + def stop_recording(self, local_path: str, + remote_path: str = DEFAULT_RECORDING_PATH) -> str: + """Stop the recording, copy it to the host and delete it from the device.""" + adb = self._adb + # SIGINT, not SIGKILL: screenrecord has to write the MP4 index on the way out. + adb.shell_status("pkill -INT screenrecord") + with _RECORDINGS_LOCK: + process = _RECORDINGS.pop(self._key, None) + _wait_finalized(adb, process) + saved = self.pull_file(remote_path, local_path) + adb.shell_status(f"rm -f {shlex.quote(_remote_path(remote_path))}") + return saved + + +def _wait_finalized(adb: Any, process: Any) -> None: + """Wait until ``screenrecord`` has exited, so the file it wrote is complete.""" + if process is not None: + try: + process.wait(timeout=_FINALIZE_TIMEOUT_S) + except subprocess.TimeoutExpired as error: + process.kill() + raise AdbError("screenrecord did not stop; the recording is incomplete") from error + return + # Started by another process: all that can be watched is the device side. + deadline = monotonic() + _FINALIZE_TIMEOUT_S + while adb.shell_status("pidof screenrecord")[0] == 0: + if monotonic() >= deadline: + raise DeviceError("screenrecord did not stop; the recording is incomplete") + sleep(0.1) + + +@translate_device_errors +def _read_clipboard(ui_device: UIAutomatorDevice) -> str: + return str(ui_device.handle.clipboard or "") + + +@translate_device_errors +def _write_clipboard(ui_device: UIAutomatorDevice, text: str) -> None: + if not isinstance(text, str): + raise TypeError("clipboard text must be a string") + ui_device.handle.set_clipboard(text) + + +__all__ = [ + "ACCEPT_BUTTONS", "AndroidExtension", "DEFAULT_RECORDING_PATH", "DISMISS_BUTTONS", + "MAX_RECORDING_S", "answer_dialog", "launch", "state", "stop", +] diff --git a/je_auto_control/android/session.py b/je_auto_control/android/session.py index 133857bf8..5a7a6bb8b 100644 --- a/je_auto_control/android/session.py +++ b/je_auto_control/android/session.py @@ -4,6 +4,7 @@ from importlib.util import find_spec from typing import Any, Callable, Dict, Optional, Tuple +from je_auto_control.android import apps as android_apps from je_auto_control.android import input as android_input from je_auto_control.android.adb_client import AdbClient, AdbError from je_auto_control.android.client import UIAutomatorDevice @@ -11,15 +12,16 @@ from je_auto_control.android.screen import capture_frame from je_auto_control.wrapper.device_context import ( CAPABILITY_NAMES, STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, STATE_NEEDS_PERMISSION, - DeviceCapability, DeviceContext, DeviceError, DeviceSession, Drag, LongPress, - Pinch, Swipe, Tap, bound_session, + AppState, DeviceCapability, DeviceContext, DeviceError, DeviceSession, Drag, + LongPress, Pinch, Swipe, Tap, bound_session, ) from je_auto_control.wrapper.device_frame import DeviceFrame +from je_auto_control.wrapper.mobile_extensions import EXTENSION_FEATURES, mobile_extension #: Capabilities plain ``adb`` provides once the device is authorised. -_ADB_CAPABILITIES = ("input", "screenshot", "app_lifecycle", "install", "files", "recording") +_ADB_CAPABILITIES = ("input", "screenshot", "app_lifecycle") #: Capabilities that need the uiautomator2 daemon on top of adb. -_UI_CAPABILITIES = ("ui_tree", "multi_touch", "alerts", "clipboard") +_UI_CAPABILITIES = ("ui_tree", "multi_touch", "alerts") _NEEDS_UI = "uiautomator2 is not installed: pip install uiautomator2" _UNAUTHORIZED = ("the device has not authorised this host: accept the USB debugging " "prompt on the device, then check `adb devices`") @@ -80,6 +82,9 @@ def _probe(self) -> Dict[str, DeviceCapability]: for name in _UI_CAPABILITIES: found[name] = self._ui_capability(name) found["unicode_text"] = self._unicode_capability() + extension = mobile_extension(self) + for name in EXTENSION_FEATURES: + found[name] = extension.capability(name) return {name: found[name] for name in CAPABILITY_NAMES} def _blocker(self) -> Optional[Tuple[str, str]]: @@ -124,6 +129,28 @@ def _ui_or_none(self) -> Optional[UIAutomatorDevice]: """The uiautomator2 wrapper when that path can be used, else ``None``.""" return self.ui_device if self.has_ui_automator else None + def launch_app(self, app_id: str) -> None: + """Launch a package's launcher activity, or a ``package/activity`` component.""" + self.invoke("launch_app", lambda: android_apps.launch(self.adb, app_id)) + + def stop_app(self, app_id: str) -> None: + """Force-stop a package.""" + self.invoke("stop_app", lambda: android_apps.stop(self.adb, app_id)) + + def app_state(self, app_id: str) -> AppState: + """Whether a package is installed, running, and in front.""" + return self.invoke("app_state", lambda: android_apps.state(self.adb, app_id)) + + def answer_alert(self, accept: bool) -> str: + """Press the accepting or dismissing button of a system dialog (needs uiautomator2).""" + return self.invoke("answer_alert", lambda: android_apps.answer_dialog( + self._ui_or_none(), accept)) + + def builtin_extension(self) -> android_apps.AndroidExtension: + """Install, files, clipboard and recording over adb / uiautomator2.""" + return android_apps.AndroidExtension( + lambda: self.adb, self._ui_or_none, self.device_id) + def capture(self) -> DeviceFrame: """The current screen, upright, in the coordinates ``input tap`` takes.""" return self.invoke("capture", lambda: capture_frame(self.adb, self.device_id)) diff --git a/je_auto_control/ios/__init__.py b/je_auto_control/ios/__init__.py index f996af60a..1b8d220a6 100644 --- a/je_auto_control/ios/__init__.py +++ b/je_auto_control/ios/__init__.py @@ -14,6 +14,7 @@ dependency that loads lazily so importing this module on a non-Mac host does not fail. """ +from je_auto_control.ios.apps import WdaExtension from je_auto_control.ios.client import ( IOSDevice, IOSUnavailableError, default_ios_device, reset_default_ios_device, @@ -30,6 +31,7 @@ __all__ = [ "ElementNotFoundError", "IOSDevice", "IOSSession", "IOSUnavailableError", + "WdaExtension", "capture_frame", "click_element", "default_ios_device", "drag", "dump_source", "find_element", "long_press", "pinch", "press_key", "reset_default_ios_device", "screen_size", "screenshot", "swipe", diff --git a/je_auto_control/ios/apps.py b/je_auto_control/ios/apps.py new file mode 100644 index 000000000..3dd5135ee --- /dev/null +++ b/je_auto_control/ios/apps.py @@ -0,0 +1,145 @@ +"""iOS app lifecycle, alerts and the part of the extension surface WebDriverAgent has. + +WebDriverAgent can launch, terminate and query apps, answer alerts and set the +pasteboard. It has no endpoint for installing an app, moving files or +recording the screen: those need a host-side tool, and are reported as a +missing dependency here rather than imitated. +""" +from __future__ import annotations + +from typing import Any, Optional + +from je_auto_control.ios.client import IOSDevice, default_ios_device, translate_device_errors +from je_auto_control.wrapper.device_context import ( + STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, STATE_UNSUPPORTED, AlertNotPresentError, + AppState, DeviceCapability, DeviceUnsupportedError, +) + +#: ``XCUIApplicationState``: 0 unknown, 1 not running, 2 suspended, 3 background, 4 foreground. +_APP_STATES = {2: AppState.BACKGROUND, 3: AppState.BACKGROUND, 4: AppState.FOREGROUND} +NEEDS_ADAPTER = ("WebDriverAgent has no API for this; it needs a host-side adapter " + "(for example tidevice, pymobiledevice3 or Appium) registered with " + "register_mobile_extension('ios', ...)") +_NO_PASTEBOARD_READ = ("this facebook-wda build cannot read the pasteboard " + "(WebDriverAgent only allows it while WDA itself is in front)") + + +def _bundle(app_id: str) -> str: + if not isinstance(app_id, str) or not app_id.strip(): + raise ValueError("bundle id must be a non-empty string") + return app_id + + +@translate_device_errors +def launch(app_id: str, *, device: Optional[IOSDevice] = None) -> None: + """Launch (or bring to the front) the app with bundle id ``app_id``.""" + (device or default_ios_device()).handle.app_launch(_bundle(app_id)) + + +@translate_device_errors +def stop(app_id: str, *, device: Optional[IOSDevice] = None) -> None: + """Terminate the app with bundle id ``app_id``.""" + (device or default_ios_device()).handle.app_terminate(_bundle(app_id)) + + +@translate_device_errors +def state(app_id: str, *, device: Optional[IOSDevice] = None) -> AppState: + """Whether the app is running and in front. + + WebDriverAgent reports an app that is not installed as not running, so + this never answers ``NOT_INSTALLED``. + """ + reply = (device or default_ios_device()).handle.app_state(_bundle(app_id)) + value: Any = reply.get("value") if isinstance(reply, dict) else getattr(reply, "value", reply) + try: + return _APP_STATES.get(int(value), AppState.NOT_RUNNING) + except (TypeError, ValueError): + return AppState.NOT_RUNNING + + +@translate_device_errors +def answer_alert(accept: bool, *, device: Optional[IOSDevice] = None) -> str: + """Accept or dismiss the alert that is showing; returns its text.""" + alert = (device or default_ios_device()).handle.alert + if not alert.exists: + raise AlertNotPresentError("no alert is showing") + text = str(alert.text or "") + if accept: + alert.accept() + else: + alert.dismiss() + return text + + +class WdaExtension: + """What WebDriverAgent offers of the extension surface: the pasteboard.""" + + name = "wda" + + def __init__(self, device_source: Any) -> None: + # A callable, so no client is built until a feature is used. + self._device_source = device_source + + def capability(self, feature: str) -> DeviceCapability: + """Whether ``feature`` can be used through WebDriverAgent.""" + if feature == "clipboard": + return DeviceCapability( + feature, STATE_AVAILABLE, + "set only: reading the pasteboard is allowed only while WDA is in front") + if feature in ("install", "files", "recording"): + return DeviceCapability(feature, STATE_NEEDS_DEPENDENCY, NEEDS_ADAPTER) + return DeviceCapability(feature, STATE_UNSUPPORTED, f"unknown feature {feature!r}") + + def _missing(self, feature: str) -> DeviceUnsupportedError: + return DeviceUnsupportedError( + f"{feature} is not available on iOS through WebDriverAgent", + reason=NEEDS_ADAPTER, + alternative="register an adapter with register_mobile_extension('ios', factory)") + + def install_app(self, source: str) -> str: + """Not available through WebDriverAgent.""" + raise self._missing("install") + + def push_file(self, local_path: str, remote_path: str) -> str: + """Not available through WebDriverAgent.""" + raise self._missing("files") + + def pull_file(self, remote_path: str, local_path: str) -> str: + """Not available through WebDriverAgent.""" + raise self._missing("files") + + def start_recording(self, remote_path: str = "", time_limit_s: int = 0) -> str: + """Not available through WebDriverAgent.""" + raise self._missing("recording") + + def stop_recording(self, local_path: str, remote_path: str = "") -> str: + """Not available through WebDriverAgent.""" + raise self._missing("recording") + + def set_clipboard(self, text: str) -> None: + """Put ``text`` on the device pasteboard.""" + _set_pasteboard(self._device_source(), text) + + def get_clipboard(self) -> str: + """The pasteboard's text, where this facebook-wda build can read it.""" + return _get_pasteboard(self._device_source()) + + +@translate_device_errors +def _set_pasteboard(device: IOSDevice, text: str) -> None: + if not isinstance(text, str): + raise TypeError("clipboard text must be a string") + device.handle.set_clipboard(text) + + +@translate_device_errors +def _get_pasteboard(device: IOSDevice) -> str: + reader = getattr(device.handle, "get_clipboard", None) + if not callable(reader): + raise DeviceUnsupportedError( + "the pasteboard cannot be read on this device", reason=_NO_PASTEBOARD_READ, + alternative="read the text from the screen (find_text) or the page source") + return str(reader() or "") + + +__all__ = ["NEEDS_ADAPTER", "WdaExtension", "answer_alert", "launch", "state", "stop"] diff --git a/je_auto_control/ios/session.py b/je_auto_control/ios/session.py index 2ab86a012..2f13825bc 100644 --- a/je_auto_control/ios/session.py +++ b/je_auto_control/ios/session.py @@ -3,23 +3,21 @@ from typing import Any, Callable, Dict, Optional, Tuple +from je_auto_control.ios import apps as ios_apps from je_auto_control.ios import input as ios_input from je_auto_control.ios.client import IOSDevice, translate_device_errors from je_auto_control.ios.screen import capture_frame from je_auto_control.wrapper.device_context import ( CAPABILITY_NAMES, STATE_AVAILABLE, STATE_NEEDS_DEPENDENCY, - DeviceCapability, DeviceContext, DeviceError, DeviceSession, Drag, LongPress, - Pinch, Swipe, Tap, bound_session, + AppState, DeviceCapability, DeviceContext, DeviceError, DeviceSession, Drag, + LongPress, Pinch, Swipe, Tap, bound_session, ) from je_auto_control.wrapper.device_frame import DeviceFrame +from je_auto_control.wrapper.mobile_extensions import EXTENSION_FEATURES, mobile_extension #: Capabilities WebDriverAgent itself provides. _WDA_CAPABILITIES = ("input", "unicode_text", "multi_touch", "screenshot", "ui_tree", "app_lifecycle", "alerts") -#: Capabilities WebDriverAgent has no endpoint for. -_ADAPTER_CAPABILITIES = ("install", "files", "clipboard", "recording") -_NEEDS_ADAPTER = ("WebDriverAgent has no API for this; it needs a host-side adapter " - "(for example tidevice, pymobiledevice3 or Appium)") @translate_device_errors @@ -66,8 +64,9 @@ def _probe(self) -> Dict[str, DeviceCapability]: state, reason = blocker return {name: DeviceCapability(name, state, reason) for name in CAPABILITY_NAMES} found = {name: DeviceCapability(name, STATE_AVAILABLE) for name in _WDA_CAPABILITIES} - for name in _ADAPTER_CAPABILITIES: - found[name] = self._adapter_capability(name) + extension = mobile_extension(self) + for name in EXTENSION_FEATURES: + found[name] = extension.capability(name) return {name: found[name] for name in CAPABILITY_NAMES} def _blocker(self) -> Optional[Tuple[str, str]]: @@ -101,8 +100,26 @@ def _gesture_handlers(self) -> Dict[type, Callable[[Any], None]]: Pinch: lambda g: ios_input.pinch(g.scale, g.duration_s, device=device), } - def _adapter_capability(self, name: str) -> DeviceCapability: - return DeviceCapability(name, STATE_NEEDS_DEPENDENCY, _NEEDS_ADAPTER) + def launch_app(self, app_id: str) -> None: + """Launch (or bring to the front) the app with this bundle id.""" + self.invoke("launch_app", lambda: ios_apps.launch(app_id, device=self.device)) + + def stop_app(self, app_id: str) -> None: + """Terminate the app with this bundle id.""" + self.invoke("stop_app", lambda: ios_apps.stop(app_id, device=self.device)) + + def app_state(self, app_id: str) -> AppState: + """Whether the app is running and in front.""" + return self.invoke("app_state", lambda: ios_apps.state(app_id, device=self.device)) + + def answer_alert(self, accept: bool) -> str: + """Accept or dismiss the alert that is showing; returns its text.""" + return self.invoke("answer_alert", lambda: ios_apps.answer_alert( + accept, device=self.device)) + + def builtin_extension(self) -> ios_apps.WdaExtension: + """What WebDriverAgent offers of install / files / clipboard / recording.""" + return ios_apps.WdaExtension(lambda: self.device) def _release(self) -> None: if not self._injected: diff --git a/je_auto_control/utils/device_matrix/matrix.py b/je_auto_control/utils/device_matrix/matrix.py index 1e7ebc5c8..c075be4bd 100644 --- a/je_auto_control/utils/device_matrix/matrix.py +++ b/je_auto_control/utils/device_matrix/matrix.py @@ -31,7 +31,9 @@ from typing import Any, Dict, List, Optional from je_auto_control.utils.exception.exceptions import AutoControlException -from je_auto_control.wrapper.device_context import DeviceContext, open_device, use_device +from je_auto_control.wrapper.device_context import ( + DeviceContext, DeviceError, DeviceSession, open_device, use_device, +) @dataclass @@ -43,6 +45,8 @@ class DeviceResult: success: bool duration_s: float = 0.0 error: Optional[str] = None + #: State of the spec's ``app_id`` after the run (``None`` when it names none). + app_state: Optional[str] = None def to_dict(self) -> Dict[str, Any]: return asdict(self) @@ -84,6 +88,41 @@ def _device_id(device: Dict[str, Any], index: int) -> str: return str(device.get("serial") or device.get("url") or f"device-{index}") +def _run_with_app(runner: Any, actions: List[Any], session: DeviceSession, + device: Dict[str, Any]) -> Optional[str]: + """Run ``actions`` around the spec's app; returns the app's state afterwards. + + A spec with ``app_id`` gets the app launched and in front before the first + step, and stopped afterwards whether the steps passed or not — a failed or + cancelled run does not leave the app wherever the failure found it. Set + ``"keep_app": true`` to leave it running. + """ + app_id = device.get("app_id") + if not app_id: + runner.execute_action(actions, raise_on_error=True) + return None + from je_auto_control.wrapper.mobile_extensions import launch_app, stop_app, wait_for_app + launch_app(session, str(app_id)) + try: + wait_for_app(session, str(app_id), timeout_s=float(device.get("app_timeout_s", 15.0))) + runner.execute_action(actions, raise_on_error=True) + finally: + if not device.get("keep_app") and session.connected: + stop_app(session, str(app_id)) + return str(session.app_state(str(app_id)).value) + + +def _final_app_state(session: Optional[DeviceSession], device: Dict[str, Any]) -> Optional[str]: + """The spec's app state after a failed run, when it can still be asked.""" + app_id = device.get("app_id") + if session is None or not app_id or not session.connected: + return None + try: + return str(session.app_state(str(app_id)).value) + except DeviceError: + return None + + def _run_one_device(actions: List[Any], device: Dict[str, Any], index: int, var_name: str) -> DeviceResult: """Run ``actions`` against a single device on a fresh executor.""" @@ -93,22 +132,28 @@ def _run_one_device(actions: List[Any], device: Dict[str, Any], device_id = _device_id(device, index) platform = str(device.get("platform", "")) started = time.monotonic() + session: Optional[DeviceSession] = None try: context = DeviceContext.from_spec(device) if context is None: # Not a mobile spec: nothing to bind, the list runs as written. runner.execute_action(actions, raise_on_error=True) - else: - with open_device(context) as session, use_device(session): - runner.execute_action(actions, raise_on_error=True) + return DeviceResult(device_id, platform, True, time.monotonic() - started) + session = open_device(context) + with use_device(session): + app_state = _run_with_app(runner, actions, session, device) return DeviceResult(device_id, platform, True, - time.monotonic() - started) + time.monotonic() - started, app_state=app_state) # The executor's own containment set: an ImageNotFoundException or a # KeyError on one device used to abort the whole matrix. except (AutoControlException, OSError, RuntimeError, ArithmeticError, AttributeError, TypeError, ValueError, LookupError) as error: return DeviceResult(device_id, platform, False, - time.monotonic() - started, repr(error)) + time.monotonic() - started, repr(error), + app_state=_final_app_state(session, device)) + finally: + if session is not None: + session.close() def run_on_devices(actions: List[Any], diff --git a/je_auto_control/wrapper/device_context.py b/je_auto_control/wrapper/device_context.py index 73e4c5869..52e059096 100644 --- a/je_auto_control/wrapper/device_context.py +++ b/je_auto_control/wrapper/device_context.py @@ -22,6 +22,7 @@ import threading from contextlib import contextmanager from dataclasses import asdict, dataclass +from enum import Enum from time import monotonic from typing import Any, Callable, Dict, Iterator, Mapping, Optional, TypeVar, Union @@ -72,6 +73,19 @@ class DeviceClosedError(DeviceError): """The session was closed (or broken by a timeout) and must be reopened.""" +class AlertNotPresentError(DeviceError, LookupError): + """No alert or system dialog is showing to accept or dismiss.""" + + +class AppState(str, Enum): + """Where an app is in its lifecycle. Compares equal to its string value.""" + + NOT_INSTALLED = "not_installed" + NOT_RUNNING = "not_running" + BACKGROUND = "background" + FOREGROUND = "foreground" + + class DeviceUnsupportedError(DeviceError): """The backend cannot do what was asked; says why and what to use instead.""" @@ -339,6 +353,30 @@ def _gesture_handlers(self) -> Dict[type, Callable[[Any], None]]: """Gesture type to the backend call that performs it; overridden per platform.""" return {} + def launch_app(self, app_id: str) -> None: + """Launch an app (Android package, iOS bundle id).""" + raise NotImplementedError + + def stop_app(self, app_id: str) -> None: + """Stop an app.""" + raise NotImplementedError + + def app_state(self, app_id: str) -> AppState: + """Where an app is in its lifecycle.""" + raise NotImplementedError + + def answer_alert(self, accept: bool) -> str: + """Accept or dismiss the alert that is showing; returns what was pressed or read.""" + raise NotImplementedError + + def builtin_extension(self) -> Any: + """The backend's own install / files / clipboard / recording implementation.""" + raise NotImplementedError + + def wait_cancelled(self, timeout_s: float) -> bool: + """Sleep up to ``timeout_s``; ``True`` as soon as the session is cancelled.""" + return self._cancel.wait(timeout_s) + def invoke(self, operation: str, function: Callable[[], _Result], *, timeout_s: Optional[float] = None) -> _Result: """Run one device call under the session's timeout and cancel signal. @@ -433,7 +471,8 @@ def bound_session(platform: str, device_id: Optional[str] = None) -> Optional[De __all__ = [ - "CAPABILITY_NAMES", "DEFAULT_TIMEOUT_S", "DeviceCancelledError", "DeviceCapability", + "AlertNotPresentError", "AppState", "CAPABILITY_NAMES", "DEFAULT_TIMEOUT_S", + "DeviceCancelledError", "DeviceCapability", "DeviceClosedError", "DeviceContext", "DeviceError", "DevicePermissionError", "DeviceSession", "DeviceTimeoutError", "DeviceUnavailableError", "DeviceUnsupportedError", "Drag", "Gesture", "LongPress", "MOBILE_PLATFORMS", diff --git a/je_auto_control/wrapper/mobile_extensions.py b/je_auto_control/wrapper/mobile_extensions.py new file mode 100644 index 000000000..8fa53a80f --- /dev/null +++ b/je_auto_control/wrapper/mobile_extensions.py @@ -0,0 +1,182 @@ +"""App lifecycle and the optional device extensions, for any mobile session. + +Two layers: + +* **Lifecycle and alerts** — :func:`launch_app`, :func:`stop_app`, + :func:`app_state`, :func:`wait_for_app`, :func:`accept_alert`, + :func:`dismiss_alert` — work on every :class:`DeviceSession`. +* **Extensions** — install, files, clipboard, recording — are the features a + backend may simply not have. :class:`MobileExtension` is their contract and + :func:`mobile_extension` returns the implementation for a session. A feature + the backend lacks reports ``needs_dependency`` with the reason, and calling + it raises :class:`DeviceUnsupportedError`; nothing is borrowed from another + platform's tooling to make it look supported. + +An adapter for a host-side tool is plugged in with +:func:`register_mobile_extension`. It is asked first, and the built-in +extension covers whatever it does not provide. +""" +from __future__ import annotations + +import threading +from time import monotonic +from typing import Any, Callable, Dict, Optional, Protocol, runtime_checkable + +from je_auto_control.wrapper.device_context import ( + MOBILE_PLATFORMS, AppState, DeviceCapability, DeviceError, DeviceSession, + DeviceTimeoutError, +) + +#: The features an extension may or may not provide. +EXTENSION_FEATURES = ("install", "files", "clipboard", "recording") +_WAIT_POLL_S = 0.25 + + +@runtime_checkable +class MobileExtension(Protocol): + """Install, files, clipboard and recording for one device. + + ``capability(feature)`` answers for each of :data:`EXTENSION_FEATURES` + without touching the device. A method whose feature is not ``available`` + raises :class:`DeviceUnsupportedError` carrying the same reason. + """ + + name: str + + def capability(self, feature: str) -> DeviceCapability: + """Whether ``feature`` can be used, and if not, what is missing.""" + + def install_app(self, source: str) -> str: + """Install the app package at host path ``source``.""" + + def push_file(self, local_path: str, remote_path: str) -> str: + """Copy a host file to the device.""" + + def pull_file(self, remote_path: str, local_path: str) -> str: + """Copy a device file to the host.""" + + def get_clipboard(self) -> str: + """The device clipboard's text.""" + + def set_clipboard(self, text: str) -> None: + """Put ``text`` on the device clipboard.""" + + def start_recording(self, remote_path: str = ..., time_limit_s: int = ...) -> str: + """Start recording the screen.""" + + def stop_recording(self, local_path: str, remote_path: str = ...) -> str: + """Stop recording and save the result to host path ``local_path``.""" + + +ExtensionFactory = Callable[[DeviceSession], MobileExtension] + +_FACTORIES: Dict[str, ExtensionFactory] = {} +_FACTORIES_LOCK = threading.Lock() + +#: Which feature each extension method belongs to. +_METHOD_FEATURES = { + "install_app": "install", "push_file": "files", "pull_file": "files", + "get_clipboard": "clipboard", "set_clipboard": "clipboard", + "start_recording": "recording", "stop_recording": "recording", +} + + +def register_mobile_extension(platform: str, factory: Optional[ExtensionFactory]) -> None: + """Plug in an adapter for ``platform``; ``None`` removes it. + + ``factory(session)`` returns a :class:`MobileExtension`. It is how iOS + gets install / files / recording, which WebDriverAgent does not provide. + """ + if platform not in MOBILE_PLATFORMS: + raise DeviceError(f"unknown device platform {platform!r}") + with _FACTORIES_LOCK: + if factory is None: + _FACTORIES.pop(platform, None) + else: + _FACTORIES[platform] = factory + + +class _Layered: + """An adapter first, the built-in extension for what the adapter lacks.""" + + def __init__(self, adapter: MobileExtension, builtin: MobileExtension) -> None: + self._adapter = adapter + self._builtin = builtin + self.name = f"{adapter.name}+{builtin.name}" + + def _pick(self, feature: str) -> MobileExtension: + return self._adapter if self._adapter.capability(feature).available else self._builtin + + def capability(self, feature: str) -> DeviceCapability: + """The adapter's answer when it provides ``feature``, else the built-in's.""" + return self._pick(feature).capability(feature) + + def __getattr__(self, method: str) -> Any: + feature = _METHOD_FEATURES.get(method) + if feature is None: + raise AttributeError(method) + return getattr(self._pick(feature), method) + + +def mobile_extension(session: DeviceSession) -> MobileExtension: + """The extension for ``session``: a registered adapter layered over the built-in one.""" + builtin: MobileExtension = session.builtin_extension() + with _FACTORIES_LOCK: + factory = _FACTORIES.get(session.platform) + if factory is None: + return builtin + layered: MobileExtension = _Layered(factory(session), builtin) + return layered + + +def launch_app(session: DeviceSession, app_id: str) -> AppState: + """Launch ``app_id`` (Android package, iOS bundle id); returns its state afterwards.""" + session.launch_app(app_id) + return session.app_state(app_id) + + +def stop_app(session: DeviceSession, app_id: str) -> AppState: + """Stop ``app_id``; returns its state afterwards.""" + session.stop_app(app_id) + return session.app_state(app_id) + + +def app_state(session: DeviceSession, app_id: str) -> AppState: + """Whether ``app_id`` is installed, running, and in front.""" + return session.app_state(app_id) + + +def wait_for_app(session: DeviceSession, app_id: str, *, timeout_s: float = 10.0, + state: AppState = AppState.FOREGROUND) -> AppState: + """Wait until ``app_id`` reaches ``state``; raises :class:`DeviceTimeoutError` if it does not. + + Cancelling the session ends the wait at the next poll. + """ + wanted = AppState(state) + deadline = monotonic() + float(timeout_s) + while True: + current = session.app_state(app_id) + if current == wanted: + return current + if monotonic() >= deadline: + raise DeviceTimeoutError( + f"{app_id} is {current.value}, not {wanted.value}, after {float(timeout_s):g}s") + if session.wait_cancelled(_WAIT_POLL_S): + session.check_usable() + + +def accept_alert(session: DeviceSession) -> str: + """Accept the alert or system dialog that is showing; returns what was pressed or read.""" + return session.answer_alert(True) + + +def dismiss_alert(session: DeviceSession) -> str: + """Dismiss the alert or system dialog that is showing.""" + return session.answer_alert(False) + + +__all__ = [ + "EXTENSION_FEATURES", "ExtensionFactory", "MobileExtension", "accept_alert", + "app_state", "dismiss_alert", "launch_app", "mobile_extension", + "register_mobile_extension", "stop_app", "wait_for_app", +] diff --git a/test/unit_test/headless/_mobile_doubles.py b/test/unit_test/headless/_mobile_doubles.py index 48b825754..3f87d0731 100644 --- a/test/unit_test/headless/_mobile_doubles.py +++ b/test/unit_test/headless/_mobile_doubles.py @@ -29,7 +29,7 @@ #: Shell command prefixes that put input into the device. _INPUT_PREFIXES = ("input ", "am broadcast", "am start", "am force-stop", "monkey ", - "pm ", "screenrecord", "pkill") + "pm install", "pm uninstall", "pm clear", "screenrecord", "pkill") def png_bytes(width: int, height: int, color: Tuple[int, int, int] = (0, 0, 0), @@ -112,6 +112,7 @@ def _handlers(self) -> List[Tuple[str, Callable[[str], Tuple[int, str]]]]: ("monkey -p ", self._launch), ("am force-stop ", self._stop), ("pidof ", self._pidof), + ("pm path ", self._pm_path), ("dumpsys activity activities", self._resumed), ("pkill -INT screenrecord", self._stop_recording), ("rm ", lambda _c: (0, "")), @@ -152,8 +153,16 @@ def _stop(self, command: str) -> Tuple[int, str]: def _pidof(self, command: str) -> Tuple[int, str]: package = shlex.split(command)[-1] + if package == "screenrecord": + return (0, "777\n") if self.recording else (1, "") return (0, "4242\n") if package in self.running else (1, "") + def _pm_path(self, command: str) -> Tuple[int, str]: + package = shlex.split(command)[-1] + if package in self.installed: + return 0, f"package:/data/app/{package}/base.apk\n" + return 1, "" + def _resumed(self, _command: str) -> Tuple[int, str]: package = getattr(self, "foreground", "") if not package: @@ -161,12 +170,33 @@ def _resumed(self, _command: str) -> Tuple[int, str]: return 0, f" topResumedActivity=ActivityRecord{{1 u0 {package}/.Main t7}}\n" def _stop_recording(self, _command: str) -> Tuple[int, str]: - self.recording = False + if self.recording: + self.recording = False + self.files[self.recording_path] = b"mp4-bytes" return 0, "" +class FakeRecordingProcess: + """The ``Popen`` of ``adb shell screenrecord``: runs until the device stops recording.""" + + def __init__(self, device: FakeAndroidDevice) -> None: + self._device = device + self.killed = False + + def poll(self) -> Optional[int]: + return None if self._device.recording else 0 + + def wait(self, timeout: Optional[float] = None) -> int: + if self._device.recording: + raise subprocess.TimeoutExpired("adb", timeout or 0) + return 0 + + def kill(self) -> None: + self.killed = True + + class FakeAdbHost: - """Replacement for ``subprocess.run`` in ``android.adb_client``.""" + """Replacement for ``subprocess.run`` (and ``Popen``) in ``android.adb_client``.""" def __init__(self) -> None: self.devices: Dict[str, FakeAndroidDevice] = {} @@ -184,6 +214,17 @@ def calls_for(self, serial: str) -> List[List[str]]: """The adb argv (after ``-s serial``) of every call addressed to ``serial``.""" return [args for target, args in self.calls if target == serial] + def popen(self, cmd: List[str], **_kwargs: Any) -> FakeRecordingProcess: + """Start a long-running adb invocation: only ``shell screenrecord`` is one.""" + serial, args = self._split(cmd) + with self._lock: + self.calls.append((serial, list(args))) + device = self.devices[serial] if serial else next(iter(self.devices.values())) + device.shell_commands.append(args[1]) + device.recording = True + device.recording_path = shlex.split(args[1])[-1] + return FakeRecordingProcess(device) + def run(self, cmd: List[str], **_kwargs: Any) -> subprocess.CompletedProcess: """Answer one adb invocation.""" serial, args = self._split(cmd) diff --git a/test/unit_test/headless/test_mobile_app_lifecycle.py b/test/unit_test/headless/test_mobile_app_lifecycle.py new file mode 100644 index 000000000..651dfc398 --- /dev/null +++ b/test/unit_test/headless/test_mobile_app_lifecycle.py @@ -0,0 +1,269 @@ +"""Mobile app lifecycle, alerts and the optional extensions, against the fake backends. + +No device is involved: these prove which commands are sent and that a missing +capability is reported instead of imitated. +""" +import pytest + +from headless._mobile_doubles import FakeAdbHost, FakeU2, FakeWda +from je_auto_control.android import UIAutomatorDevice +from je_auto_control.ios import IOSDevice +from je_auto_control.utils.device_matrix import run_on_devices +from je_auto_control.wrapper.device_context import ( + AlertNotPresentError, AppState, DeviceCapability, DeviceContext, DeviceError, + DeviceTimeoutError, DeviceUnsupportedError, open_device, +) +from je_auto_control.wrapper.mobile_extensions import ( + EXTENSION_FEATURES, MobileExtension, accept_alert, app_state, dismiss_alert, + launch_app, mobile_extension, register_mobile_extension, stop_app, wait_for_app, +) + +APP = "com.example.shop" + + +@pytest.fixture +def adb_host(monkeypatch): + """A fake ADB host with one attached device, ``phone``, with the app installed.""" + host = FakeAdbHost() + host.add("phone").installed.add(APP) + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", host.run) + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.Popen", host.popen) + monkeypatch.setattr("je_auto_control.android.apps._RECORDINGS", {}) + return host + + +@pytest.fixture +def no_uiautomator(monkeypatch): + """Pretend uiautomator2 is not installed, whatever the host has.""" + monkeypatch.setattr("je_auto_control.android.input.find_spec", lambda _name: None) + monkeypatch.setattr("je_auto_control.android.session.find_spec", lambda _name: None) + + +@pytest.fixture(autouse=True) +def _no_adapters(): + yield + register_mobile_extension("ios", None) + register_mobile_extension("android", None) + + +def _android(**transports): + return open_device(DeviceContext("android", "phone", adb_path="fake-adb"), **transports) + + +def _ios(handle): + return open_device(DeviceContext("ios", "http://wda.test:8100"), + device=IOSDevice(url="http://wda.test:8100", handle=handle)) + + +# --- lifecycle --------------------------------------------------------- + +def test_launch_wait_stop(adb_host): + session = _android() + assert app_state(session, APP) == AppState.NOT_RUNNING + assert launch_app(session, APP) == "foreground" + assert wait_for_app(session, APP, timeout_s=1.0) is AppState.FOREGROUND + app_state_after_stop = stop_app(session, APP) + assert app_state_after_stop == 'not_running' + assert adb_host.devices["phone"].input_calls == [ + f"monkey -p {APP} -c android.intent.category.LAUNCHER 1", + f"am force-stop {APP}", + ] + + +def test_ios_launch_wait_stop(): + handle = FakeWda() + session = _ios(handle) + assert launch_app(session, APP) is AppState.FOREGROUND + assert stop_app(session, APP) == "not_running" + assert [call["op"] for call in handle.input_calls] == ["app_launch", "app_terminate"] + + +def test_app_that_is_not_installed(adb_host): + session = _android() + assert app_state(session, "com.example.absent") is AppState.NOT_INSTALLED + with pytest.raises(DeviceError, match="not installed"): + launch_app(session, "com.example.absent") + + +def test_app_id_cannot_inject_a_shell_command(adb_host): + session = _android() + for bad in ("com.example; reboot", "$(reboot)", "", "nopackage"): + with pytest.raises(DeviceError): + launch_app(session, bad) + assert adb_host.devices["phone"].shell_commands == [] + + +def test_wait_for_app_times_out_and_stops_on_cancel(adb_host): + session = _android() + with pytest.raises(DeviceTimeoutError, match="not_running"): + wait_for_app(session, APP, timeout_s=0.05) + session.cancel() + with pytest.raises(DeviceError): + wait_for_app(session, APP, timeout_s=30.0) + + +def test_unreachable_device_is_not_read_as_not_running(adb_host): + adb_host.devices["phone"].state = "unauthorized" + with pytest.raises(DeviceError, match="unauthorized"): + app_state(_android(), APP) + + +def test_matrix_stops_the_app_it_started_even_when_the_run_fails(adb_host): + phone = adb_host.devices["phone"] + spec = {"platform": "android", "serial": "phone", "adb_path": "fake-adb", "app_id": APP} + passed = run_on_devices([["AC_android_tap", {"x": 1, "y": 2}]], [dict(spec)]) + assert passed.results[0].success and passed.results[0].app_state == "not_running" + failed = run_on_devices([["AC_android_key", {"key": "HOME; reboot"}]], [dict(spec)]) + assert failed.results[0].success is False + assert failed.results[0].app_state == "not_running" + assert APP not in phone.running + kept = run_on_devices([], [dict(spec, keep_app=True)]) + assert kept.results[0].app_state == "foreground" + + +# --- alerts ------------------------------------------------------------ + +def test_alert_accept_and_dismiss(adb_host): + wda = FakeWda() + ios = _ios(wda) + wda.alert_text = "Allow notifications?" + assert accept_alert(ios) == "Allow notifications?" + wda.alert_text = "Rate this app" + assert dismiss_alert(ios) == "Rate this app" + assert [call["op"] for call in wda.input_calls] == ["alert.accept", "alert.dismiss"] + with pytest.raises(AlertNotPresentError): + accept_alert(ios) + + handle = FakeU2() + android = _android(ui_device=UIAutomatorDevice(handle=handle)) + handle.buttons = {"android:id/button1", "android:id/button2"} + assert accept_alert(android) == "android:id/button1" + assert dismiss_alert(android) == "android:id/button2" + with pytest.raises(AlertNotPresentError): + dismiss_alert(android) + + +def test_android_alert_without_the_widget_tree_says_why(adb_host, no_uiautomator): + with pytest.raises(DeviceUnsupportedError) as caught: + accept_alert(_android()) + assert "uiautomator2" in caught.value.reason + assert caught.value.alternative + assert adb_host.devices["phone"].input_calls == [] + + +# --- extensions -------------------------------------------------------- + +def test_clipboard_file_recording_capabilities(adb_host, tmp_path, monkeypatch): + phone = adb_host.devices["phone"] + handle = FakeU2() + session = _android(ui_device=UIAutomatorDevice(handle=handle)) + extension = mobile_extension(session) + assert isinstance(extension, MobileExtension) + assert all(extension.capability(name).available for name in EXTENSION_FEATURES) + + apk = tmp_path / "shop.apk" + apk.write_bytes(b"apk") + assert extension.install_app(str(apk)) == str(apk.resolve()) + adb_host.host_files[str(apk.resolve())] = b"apk" + assert extension.push_file(str(apk), "/sdcard/shop.apk") == "/sdcard/shop.apk" + assert phone.files["/sdcard/shop.apk"] == b"apk" + pulled = extension.pull_file("/sdcard/shop.apk", str(tmp_path / "out" / "copy.apk")) + assert adb_host.host_files[pulled] == b"apk" + + extension.set_clipboard("copied 文字") + assert handle.clipboard == "copied 文字" + assert extension.get_clipboard() == "copied 文字" + + remote = extension.start_recording(time_limit_s=30) + assert phone.recording and remote == "/sdcard/autocontrol_recording.mp4" + with pytest.raises(DeviceError, match="already running"): + extension.start_recording() + saved = mobile_extension(session).stop_recording(str(tmp_path / "run.mp4")) + assert phone.recording is False + assert adb_host.host_files[saved] == b"mp4-bytes" + + # iOS through WebDriverAgent: the pasteboard, and nothing borrowed from adb. + def no_adb(*_args, **_kwargs): + raise AssertionError("adb was invoked for an iOS device") + + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", no_adb) + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.Popen", no_adb) + wda = FakeWda() + ios_extension = mobile_extension(_ios(wda)) + ios_extension.set_clipboard("hello") + assert wda.clipboard == "hello" + for call in (lambda: ios_extension.install_app(str(apk)), + lambda: ios_extension.push_file(str(apk), "/tmp/x"), + lambda: ios_extension.pull_file("/tmp/x", str(tmp_path / "x")), + lambda: ios_extension.start_recording(), + lambda: ios_extension.stop_recording(str(tmp_path / "x.mp4")), + ios_extension.get_clipboard): + with pytest.raises(DeviceUnsupportedError) as caught: + call() + unsupported_result = caught.value + assert unsupported_result.reason + assert unsupported_result.alternative + assert [call["op"] for call in wda.calls] == ["set_clipboard"] + + +def test_bad_install_and_paths_are_rejected(adb_host, tmp_path): + extension = mobile_extension(_android()) + with pytest.raises(DeviceError, match="no such file"): + extension.install_app(str(tmp_path / "absent.apk")) + with pytest.raises(DeviceError, match="absolute"): + extension.pull_file("relative/path", str(tmp_path / "x")) + with pytest.raises(DeviceError, match="time_limit_s"): + extension.start_recording(time_limit_s=999) + assert adb_host.calls == [] + + +def test_adapter_absent_reports_dependency(adb_host, no_uiautomator): + session = _ios(FakeWda()) + absent_adapter = mobile_extension(session).capability("install") + assert absent_adapter.state == 'needs_dependency' + assert "adapter" in absent_adapter.reason + capabilities = session.capabilities() + assert capabilities["files"].state == "needs_dependency" + assert capabilities["recording"].state == "needs_dependency" + assert capabilities["clipboard"].available + # Android without uiautomator2: adb covers files, nothing covers the clipboard. + android = _android().capabilities() + assert android["files"].available and android["recording"].available + assert android["clipboard"].state == "needs_dependency" + assert "uiautomator2" in android["clipboard"].reason + with pytest.raises(DeviceUnsupportedError): + mobile_extension(_android()).get_clipboard() + + +class _HostToolAdapter: + """A stand-in for a tidevice / pymobiledevice3 adapter: install only.""" + + name = "hosttool" + + def __init__(self) -> None: + self.installed = [] + + def capability(self, feature): + state = "available" if feature == "install" else "unsupported" + return DeviceCapability(feature, state) + + def install_app(self, source): + self.installed.append(source) + return source + + +def test_registered_adapter_fills_what_wda_lacks(): + adapter = _HostToolAdapter() + register_mobile_extension("ios", lambda _session: adapter) + wda = FakeWda() + session = _ios(wda) + extension = mobile_extension(session) + assert extension.install_app("shop.ipa") == "shop.ipa" + assert adapter.installed == ["shop.ipa"] + assert session.capabilities()["install"].available + # What the adapter lacks still comes from WebDriverAgent, or is still missing. + extension.set_clipboard("x") + assert wda.clipboard == "x" + assert extension.capability("files").state == "needs_dependency" + with pytest.raises(DeviceError): + register_mobile_extension("windows", None) From 2bfb2cb0739bd243ca6cc048888c4c12a46654c4 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:00:44 +0800 Subject: [PATCH 19/44] Split the computer-use backend by request shape before it reaches the file limit anthropic_computer_use.py was 27 lines from the 750-line limit, so the next change to it would have needed a split first. The action translation, the message blocks, the beta tool path and the toolset path each get a module; the backend keeps construction, the conversation and the API call. Behaviour is unchanged and every name imported from the module is re-exported from it. --- .../utils/agent/backends/_computer_actions.py | 324 +++++++++++ .../agent/backends/_computer_beta_path.py | 83 +++ .../agent/backends/_computer_messages.py | 101 ++++ .../agent/backends/_computer_toolset_path.py | 90 +++ .../agent/backends/anthropic_computer_use.py | 535 +----------------- .../test_computer_use_module_layout.py | 61 ++ 6 files changed, 682 insertions(+), 512 deletions(-) create mode 100644 je_auto_control/utils/agent/backends/_computer_actions.py create mode 100644 je_auto_control/utils/agent/backends/_computer_beta_path.py create mode 100644 je_auto_control/utils/agent/backends/_computer_messages.py create mode 100644 je_auto_control/utils/agent/backends/_computer_toolset_path.py create mode 100644 test/unit_test/headless/test_computer_use_module_layout.py diff --git a/je_auto_control/utils/agent/backends/_computer_actions.py b/je_auto_control/utils/agent/backends/_computer_actions.py new file mode 100644 index 000000000..025c9de33 --- /dev/null +++ b/je_auto_control/utils/agent/backends/_computer_actions.py @@ -0,0 +1,324 @@ +"""Anthropic computer-use actions as ``AgentLoop`` decisions. + +One action of the ``computer`` tool -- or one member call of the computer +toolset, which uses the same verbs -- becomes the equivalent ``AC_*`` +invocation here. Both request shapes of :mod:`.anthropic_computer_use` share +this translation; nothing in it talks to the API. +""" +from __future__ import annotations + +import math +from typing import Any, Callable, Dict, List, Tuple + +from je_auto_control.utils.agent.backends.base import AgentBackendError +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: Upper bounds on what one model action may ask for. The spec caps ``wait`` +#: at 100 s; a hold longer than that, or a scroll of more notches, is not a +#: step an agent needs, and an unbounded scroll overflowed the platform call. +_MAX_WAIT_S = 100.0 +_MAX_SCROLL_NOTCHES = 100 +_MAX_KEY_REPEAT = 100 + + +# Map xdotool-style key names (used by Anthropic's tool spec) to the +# names AutoControl's keyboard wrappers expect. Anything outside this +# table is forwarded as-is; the wrapper rejects unknown keys cleanly. +_XDOTOOL_KEY_ALIAS = { + "return": "enter", + "escape": "esc", + "page_up": "pageup", + "page_down": "pagedown", + "back_space": "backspace", + "bksp": "backspace", + "ctrl_l": "ctrl", + "ctrl_r": "ctrl", + "shift_l": "shift", + "shift_r": "shift", + "alt_l": "alt", + "alt_r": "alt", + "super_l": "win", + "super_r": "win", + "meta_l": "win", + "meta_r": "win", +} + + +def _normalise_key(name: str) -> str: + """This platform's name for an xdotool key. + + The fixed aliases ("return" -> "enter", "escape" -> "esc", "page_down" -> + "pagedown") are not names the Windows key table knows, so Enter, Esc, + paging, Alt and Super failed there; cua_action's resolver knows each + platform's spelling. + """ + from je_auto_control.utils.cua_action.cua_action import resolve_key_name + return resolve_key_name(_XDOTOOL_KEY_ALIAS.get(name.lower(), name.lower())) + + +def _parse_combo(combo: str) -> List[str]: + """Split an xdotool-style hotkey (``ctrl+shift+T``) into key names. + + ``ctrl++`` is Ctrl and the plus key: splitting on "+" dropped the plus. + """ + from je_auto_control.utils.cua_action.cua_action import split_key_combo + return [_normalise_key(part) for part in split_key_combo(combo)] + + +def _click_button(action: str) -> str: + return { + "left_click": "mouse_left", + "right_click": "mouse_right", + "middle_click": "mouse_middle", + "double_click": "mouse_left", + "triple_click": "mouse_left", + }.get(action, "mouse_left") + + +def _click_repeats(action: str) -> int: + return {"double_click": 2, "triple_click": 3}.get(action, 1) + + +def _action_screenshot(_payload): + return {"tool": "AC_screenshot", "input": {}} + + +def _action_cursor_position(_payload): + return {"tool": "AC_get_mouse_position", "input": {}} + + +def _action_mouse_move(payload): + x, y = _xy(payload.get("coordinate")) + return {"tool": "AC_set_mouse_position", "input": {"x": x, "y": y}} + + +def _action_type(payload): + return { + "tool": "AC_write", + "input": {"write_string": str(payload.get("text") or "")}, + } + + +def _action_wait(payload): + duration = payload.get("duration") + # An explicit 0 is a valid "don't wait" — only fall back when unset. + seconds = _number(duration, "duration") if duration is not None else 1.0 + return _sequence([["AC_sleep", {"seconds": _bounded_seconds(seconds)}]]) + + +def _bounded_seconds(seconds: float) -> float: + """A model-chosen duration within ``[0, _MAX_WAIT_S]``. + + The run's wall-clock budget is checked between steps, so one unbounded + ``wait`` or ``hold_key`` blocked far past it. + """ + return min(max(float(seconds), 0.0), _MAX_WAIT_S) + + +_CLICK_ACTIONS = frozenset({ + "left_click", "right_click", "middle_click", + "double_click", "triple_click", +}) + + +def _decision_from_computer_action(payload: Dict[str, Any]) -> Dict[str, Any]: + """Turn one ``computer`` tool payload into an ``AgentLoop`` decision.""" + action = str(payload.get("action") or "").lower() + if action in _CLICK_ACTIONS: + decision = _click_decision(action, payload.get("coordinate")) + else: + handler = _ACTION_HANDLERS.get(action) + if handler is None: + raise AgentBackendError( + f"computer-use action {action!r} is not recognised", + ) + decision = handler(payload) + if action in _MODIFIER_ACTIONS and payload.get("text"): + return _with_modifiers(decision, str(payload["text"])) + return decision + + +#: Actions whose ``text`` names modifier keys to hold (``shift`` for a +#: shift-click). The field was ignored, so a ctrl-click was a plain click. +_MODIFIER_ACTIONS = _CLICK_ACTIONS | {"left_click_drag", "scroll"} + + +def _with_modifiers(decision: Dict[str, Any], combo: str) -> Dict[str, Any]: + """Run ``decision`` with the keys of ``combo`` held; they are released even on failure.""" + keys = _parse_combo(combo) + if not keys: + return decision + inner = decision["input"] + actions = (inner["action_list"] if decision["tool"] == "AC_execute_action" + else [[decision["tool"], inner]]) + return {"tool": "AC_with_modifiers", + "input": {"modifiers": keys, "actions": actions}} + + +def _click_decision(action: str, coordinate) -> Dict[str, Any]: + button = _click_button(action) + repeats = _click_repeats(action) + inputs: Dict[str, Any] = {"mouse_keycode": button} + if coordinate is not None: + x, y = _xy(coordinate) + inputs["x"], inputs["y"] = x, y + if repeats == 1: + return {"tool": "AC_click_mouse", "input": inputs} + # AC_click_mouse takes no repeat count (a 'repeat' argument made every + # click a TypeError): move once, then click the given number of times. + clicks = [["AC_click_mouse", inputs]] + clicks += [["AC_click_mouse", {"mouse_keycode": button}]] * (repeats - 1) + return _sequence(clicks) + + +def _sequence(actions: List[List[Any]]) -> Dict[str, Any]: + """Run several executor actions as one step, failing the step on any error.""" + return {"tool": "AC_execute_action", + "input": {"action_list": actions, "raise_on_error": True}} + + +def _drag_decision(payload: Dict[str, Any]) -> Dict[str, Any]: + # The spec's end point is ``coordinate`` (``start_coordinate`` is the + # start); reading only ``end_coordinate`` failed every drag the model made. + start = payload.get("start_coordinate") + end = payload.get("end_coordinate") or payload.get("coordinate") + if start is None or end is None: + raise AgentBackendError( + "left_click_drag requires start_coordinate and coordinate", + ) + sx, sy = _xy(start) + ex, ey = _xy(end) + # No AC_drag command exists: press at the start, move, release at the end. + return _sequence([ + ["AC_press_mouse", {"mouse_keycode": "mouse_left", "x": sx, "y": sy}], + ["AC_set_mouse_position", {"x": ex, "y": ey}], + ["AC_release_mouse", {"mouse_keycode": "mouse_left", "x": ex, "y": ey}], + ]) + + +def _scroll_decision(payload: Dict[str, Any]) -> Dict[str, Any]: + direction = str(payload.get("scroll_direction") or "down").lower() + raw_amount = payload.get("scroll_amount") + # An explicit 0 means "no scroll" — only default when the key is absent. + amount = int(_number(raw_amount, "scroll_amount")) if raw_amount is not None else 3 + amount = min(max(amount, 0), _MAX_SCROLL_NOTCHES) + # The direction goes to AC_mouse_scroll as cua_action sends it: left out, + # X11 and Wayland scrolled their default way ("up" went down there), and + # left / right became a vertical scroll everywhere. + from je_auto_control.utils.cua_action.cua_action import scroll_params + try: + inputs: Dict[str, Any] = scroll_params({"direction": direction, "amount": amount}) + except AutoControlException as error: + raise AgentBackendError(str(error)) from error + # The scroll happens where the model pointed, not wherever the cursor was; + # _clamp_decision keeps the point on the display. + if payload.get("coordinate") is not None: + inputs["x"], inputs["y"] = _xy(payload["coordinate"]) + return {"tool": "AC_mouse_scroll", "input": inputs} + + +def _key_decision(payload: Dict[str, Any]) -> Dict[str, Any]: + combo = str(payload.get("text") or payload.get("key") or "") + keys = _parse_combo(combo) + if not keys: + raise AgentBackendError("key action missing 'text'") + press = ({"tool": "AC_type_keyboard", "input": {"keycode": keys[0]}} if len(keys) == 1 + else {"tool": "AC_hotkey", "input": {"key_code_list": keys}}) + raw_repeat = payload.get("repeat") + repeat = int(_number(raw_repeat, "repeat")) if raw_repeat is not None else 1 + repeat = min(max(repeat, 1), _MAX_KEY_REPEAT) + if repeat == 1: + return press + # ``repeat`` (1..100) was ignored, so "press Down 5 times" pressed once. + return _sequence([[press["tool"], press["input"]]] * repeat) + + +def _hold_key_decision(payload: Dict[str, Any]) -> Dict[str, Any]: + key = _normalise_key(str(payload.get("text") or payload.get("key") or "")) + if not key: + raise AgentBackendError("hold_key action missing 'text'") + duration = _bounded_seconds(_number(payload.get("duration") or 0.0, "duration")) + return { + "tool": "AC_hold_key", + "input": {"key": key, "duration_s": duration}, + } + + +def _mouse_button_decision(tool: str, payload: Dict[str, Any]) -> Dict[str, Any]: + """Map a bare press/release verb to an ``AC_press/release_mouse`` call.""" + inputs: Dict[str, Any] = {"mouse_keycode": "mouse_left"} + coordinate = payload.get("coordinate") + if coordinate is not None: + inputs["x"], inputs["y"] = _xy(coordinate) + return {"tool": tool, "input": inputs} + + +def _mouse_down_decision(payload: Dict[str, Any]) -> Dict[str, Any]: + return _mouse_button_decision("AC_press_mouse", payload) + + +def _mouse_up_decision(payload: Dict[str, Any]) -> Dict[str, Any]: + return _mouse_button_decision("AC_release_mouse", payload) + + +# Dispatch table — populated after every handler is defined so the +# table reads its targets at module load time. +_ACTION_HANDLERS: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = { + "screenshot": _action_screenshot, + "cursor_position": _action_cursor_position, + "mouse_move": _action_mouse_move, + "type": _action_type, + "wait": _action_wait, + "left_click_drag": _drag_decision, + "scroll": _scroll_decision, + "key": _key_decision, + "hold_key": _hold_key_decision, + # computer_20250124 press/release verbs — without these an otherwise + # valid action raised AgentBackendError outside the create() try and + # aborted the run. + "left_mouse_down": _mouse_down_decision, + "left_mouse_up": _mouse_up_decision, +} + + +def _xy(coordinate: Any) -> Tuple[int, int]: + if (not isinstance(coordinate, (list, tuple)) + or len(coordinate) != 2): + raise AgentBackendError( + f"coordinate must be [x, y]; got {coordinate!r}", + ) + return int(_number(coordinate[0], "x")), int(_number(coordinate[1], "y")) + + +def _number(value: Any, what: str) -> float: + """``value`` as a finite float, or :class:`AgentBackendError`.""" + try: + number = float(value) + except (TypeError, ValueError) as error: + raise AgentBackendError(f"{what} must be a number, got {value!r}") from error + if not math.isfinite(number): + raise AgentBackendError(f"{what} must be finite, got {value!r}") + return number + + +def _clamp_decision(decision: Dict[str, Any], width: int, height: int) -> Dict[str, Any]: + """Keep every ``x`` / ``y`` in the decision on the declared display. + + The model's coordinates went to the mouse unchecked: [-500, 99999] on a + 100x100 display was dispatched as given. + """ + inputs = decision.get("input") or {} + _clamp_inputs(inputs, width, height) + for key in ("action_list", "actions"): + for action in inputs.get(key) or []: + if isinstance(action, (list, tuple)) and len(action) == 2 and isinstance(action[1], dict): + _clamp_inputs(action[1], width, height) + return decision + + +def _clamp_inputs(inputs: Dict[str, Any], width: int, height: int) -> None: + if "x" in inputs: + inputs["x"] = min(max(int(inputs["x"]), 0), width - 1) + if "y" in inputs: + inputs["y"] = min(max(int(inputs["y"]), 0), height - 1) diff --git a/je_auto_control/utils/agent/backends/_computer_beta_path.py b/je_auto_control/utils/agent/backends/_computer_beta_path.py new file mode 100644 index 000000000..8979ef062 --- /dev/null +++ b/je_auto_control/utils/agent/backends/_computer_beta_path.py @@ -0,0 +1,83 @@ +"""The beta ``computer`` tool path of the Anthropic computer-use backend. + +``computer_20251124`` (and the older ``computer_20250124``): one tool named +``computer``, declared with a display size, sent under a beta, answering with +one ``tool_use`` per turn whose ``action`` field is the verb. The toolset +path is :mod:`._computer_toolset_path`. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.agent.agent_loop import AgentStep +from je_auto_control.utils.agent.backends._computer_actions import ( + _clamp_decision, _decision_from_computer_action, +) +from je_auto_control.utils.agent.backends._computer_messages import ( + _attr, _block_type, _final_answer, _raise_if_truncated, _tool_result_content, +) +from je_auto_control.utils.agent.backends._computer_toolset import unscale_decision +from je_auto_control.utils.agent.backends.base import AgentBackendError + +_DEFAULT_TOOL_TYPE = "computer_20251124" + +#: The beta each computer-use tool version is sent under. Every version is +#: beta-only: posted through the plain ``messages.create`` without one, the +#: API rejects the request, so the backend could never run. +_TOOL_BETAS = { + "computer_20250124": "computer-use-2025-01-24", + "computer_20251124": "computer-use-2025-11-24", +} + + +def beta_for(tool_type: str, beta: Optional[str]) -> str: + """The beta header for ``tool_type``: the caller's, else the known one.""" + chosen = beta or _TOOL_BETAS.get(tool_type) + if not chosen: + raise AgentBackendError( + f"no known beta for computer-use tool {tool_type!r}; pass beta=") + return chosen + + +class BetaToolPath: + """Reads the beta tool's responses and answers its one call per turn.""" + + _conversation: List[Dict[str, Any]] + _pending_tool_use_id: Optional[str] + _scale: Tuple[float, float] + _display: Tuple[int, int] + + def _handle_response(self, response: Any) -> Dict[str, Any]: + content = list(getattr(response, "content", []) or []) + self._conversation.append({"role": "assistant", "content": content}) + _raise_if_truncated(response) # before running any call it holds + for block in content: + if _block_type(block) != "tool_use": + continue + name = _attr(block, "name") + if name != "computer": + # Skipping it made the turn look like a final answer: the run + # reported success having done nothing the model asked for. + raise AgentBackendError( + f"model called tool {name!r}; only 'computer' was offered", + ) + payload = _attr(block, "input") or {} + self._pending_tool_use_id = _attr(block, "id") + decision = unscale_decision(_decision_from_computer_action(payload), self._scale) + return _clamp_decision(decision, *self._display) + return _final_answer(response, content) + + def _pending_result(self, history: Sequence[AgentStep], + screenshot: Optional[bytes]) -> List[Dict[str, Any]]: + """The ``tool_result`` answering the last turn's call, if one is pending.""" + if not history or self._pending_tool_use_id is None: + return [] + last = history[-1] + content = _tool_result_content(last, screenshot, self._scale) + tool_use_id, self._pending_tool_use_id = self._pending_tool_use_id, None + return [{ + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": content, + "is_error": bool(last.error), + }] diff --git a/je_auto_control/utils/agent/backends/_computer_messages.py b/je_auto_control/utils/agent/backends/_computer_messages.py new file mode 100644 index 000000000..e3de02835 --- /dev/null +++ b/je_auto_control/utils/agent/backends/_computer_messages.py @@ -0,0 +1,101 @@ +"""Message blocks of the Anthropic computer-use conversation. + +Reading a response (block type and attributes, truncation, the final answer) +and building what goes back (the first user turn, a ``tool_result``). Shared +by both request shapes of :mod:`.anthropic_computer_use`. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.agent.agent_loop import AgentStep +from je_auto_control.utils.agent.backends.base import ( + AgentBackendError, encode_screenshot_b64, +) + + +def _final_answer(response: Any, content: List[Any]) -> Dict[str, Any]: + """A turn without tool calls: the final answer (a truncated turn was refused earlier).""" + text_parts: List[str] = [ + _attr(b, "text") or "" + for b in content if _block_type(b) == "text" + ] + return {"stop": True, "message": "\n".join(text_parts).strip()} + + +_TRUNCATION_STOP_REASONS = frozenset({"max_tokens", "refusal"}) + + +def _raise_if_truncated(response: Any) -> None: + """Reject an incomplete turn instead of treating it as a final answer.""" + stop_reason = _attr(response, "stop_reason") + if stop_reason in _TRUNCATION_STOP_REASONS: + raise AgentBackendError( + "anthropic computer-use response was incomplete " + f"(stop_reason={stop_reason!r})", + ) + + +def _block_type(block: Any) -> Optional[str]: + if isinstance(block, dict): + return block.get("type") + return getattr(block, "type", None) + + +def _attr(block: Any, name: str) -> Any: + if isinstance(block, dict): + return block.get(name) + return getattr(block, name, None) + + +def _initial_user_content(goal: str, + screenshot: Optional[bytes]) -> List[Dict[str, Any]]: + blocks: List[Dict[str, Any]] = [] + encoded = encode_screenshot_b64(screenshot) + if encoded: + blocks.append({ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": encoded, + }, + }) + blocks.append({"type": "text", "text": goal}) + return blocks + + +def _tool_result_content(step: AgentStep, screenshot: Optional[bytes], + scale: Tuple[float, float] = (1.0, 1.0)) -> List[Dict[str, Any]]: + """Build a ``tool_result`` content payload for the last ``AC_*`` call. + + Anthropic's spec expects the screenshot tool to return the image + *itself* — text-only results just describe what happened. The cursor + position is given in the screenshot's pixels, where the model works: + in screen pixels it named a point off the image it was shown. + """ + if not step.error and step.tool == "AC_get_mouse_position": + return [{"type": "text", "text": _scaled_point(step.result, scale)}] + if step.error: + return [{"type": "text", "text": f"error: {step.error}"}] + if step.tool == "AC_screenshot": + encoded = encode_screenshot_b64(screenshot) + if encoded: + return [{ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": encoded, + }, + }] + text = repr(step.result) if step.result is not None else "ok" + return [{"type": "text", "text": text[:4000]}] + + +def _scaled_point(point: Any, scale: Tuple[float, float]) -> str: + """A screen point in screenshot pixels, as text; anything else as it came.""" + if (isinstance(point, (list, tuple)) and len(point) == 2 + and all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in point)): + return repr((int(round(point[0] * scale[0])), int(round(point[1] * scale[1])))) + return repr(point) diff --git a/je_auto_control/utils/agent/backends/_computer_toolset_path.py b/je_auto_control/utils/agent/backends/_computer_toolset_path.py new file mode 100644 index 000000000..6fa1d57d6 --- /dev/null +++ b/je_auto_control/utils/agent/backends/_computer_toolset_path.py @@ -0,0 +1,90 @@ +"""The GA computer-toolset path of the Anthropic computer-use backend. + +``computer_toolset_20260801``: no beta, no declared display size, one call +per member name and possibly several per turn, plus ``zoom``. The request +schema and the batch of queued calls live in :mod:`._computer_toolset`; this +is the part of the backend that reads the toolset's responses and builds its +results. The beta path is :mod:`._computer_beta_path`. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.agent.agent_loop import AgentStep +from je_auto_control.utils.agent.backends._computer_actions import ( + _ACTION_HANDLERS, _CLICK_ACTIONS, _clamp_decision, + _decision_from_computer_action, +) +from je_auto_control.utils.agent.backends._computer_messages import ( + _attr, _block_type, _final_answer, _raise_if_truncated, _tool_result_content, +) +from je_auto_control.utils.agent.backends._computer_toolset import ( + ToolsetBatch, fit_screenshot, screen_region, unscale_decision, zoom_image, +) +from je_auto_control.utils.agent.backends.base import AgentBackendError + + +class ToolsetPath: + """Reads the toolset's responses and builds the results of its calls.""" + + _conversation: List[Dict[str, Any]] + _scale: Tuple[float, float] + _tier: Any + _display: Tuple[int, int] + #: tool_use id -> the region a queued ``zoom`` asked for, in screenshot pixels. + _zooms: Dict[str, Tuple[int, int, int, int]] + + def _fit(self, screenshot: Optional[bytes]) -> Optional[bytes]: + """``screenshot`` within the toolset's image limits; remembers the scale.""" + if not screenshot: + return screenshot + fitted, self._scale = fit_screenshot(screenshot, self._tier) + return fitted + + def _toolset_result_content(self, step: AgentStep, screenshot: Optional[bytes], + tool_use_id: str) -> List[Dict[str, Any]]: + region = self._zooms.pop(tool_use_id, None) + if step.tool != "AC_screenshot" or not screenshot: + return _tool_result_content(step, screenshot, self._scale) + # A zoom is answered from the full-resolution frame; the scale of the + # full screenshot stays, since later coordinates are still in its space. + image = (zoom_image(screenshot, region, self._tier) if region is not None + else self._fit(screenshot)) + return _tool_result_content(step, image) + + def _handle_toolset_response(self, response: Any, + batch: ToolsetBatch) -> Dict[str, Any]: + content = list(getattr(response, "content", []) or []) + self._conversation.append({"role": "assistant", "content": content}) + self._zooms.clear() + _raise_if_truncated(response) # before running any call it holds + calls = [(_attr(block, "id"), self._toolset_decision(block)) + for block in content if _block_type(block) == "tool_use"] + if not calls: + return _final_answer(response, content) + batch.load(calls) + return batch.next_decision() + + def _toolset_decision(self, block: Any) -> Dict[str, Any]: + """A member call as a decision, in screen pixels and on the display.""" + name = str(_attr(block, "name") or "") + if name == "zoom": + return self._zoom_decision(block) + if name not in _CLICK_ACTIONS and name not in _ACTION_HANDLERS: + raise AgentBackendError( + f"model called tool {name!r}; only computer toolset members were offered") + payload = dict(_attr(block, "input") or {}) + payload["action"] = name # the member name is the action + decision = unscale_decision(_decision_from_computer_action(payload), self._scale) + return _clamp_decision(decision, *self._display) + + def _zoom_decision(self, block: Any) -> Dict[str, Any]: + """A ``zoom`` runs as a screenshot; its region crops the result.""" + payload = _attr(block, "input") or {} + try: + region = screen_region(payload.get("region") if isinstance(payload, dict) else None, + self._scale) + except (TypeError, ValueError) as error: + raise AgentBackendError(f"model sent an invalid zoom: {error}") from error + self._zooms[str(_attr(block, "id"))] = region + return {"tool": "AC_screenshot", "input": {}} diff --git a/je_auto_control/utils/agent/backends/anthropic_computer_use.py b/je_auto_control/utils/agent/backends/anthropic_computer_use.py index 1fc5b844d..700bcdca0 100644 --- a/je_auto_control/utils/agent/backends/anthropic_computer_use.py +++ b/je_auto_control/utils/agent/backends/anthropic_computer_use.py @@ -11,6 +11,13 @@ Either way each action becomes the equivalent ``AC_*`` invocation. +Layout. This module is the backend itself: construction, the conversation and +the API call. The two request shapes are read in :mod:`._computer_beta_path` +and :mod:`._computer_toolset_path`; turning an action into ``AC_*`` calls is +:mod:`._computer_actions`, and the message blocks are +:mod:`._computer_messages`. The private names tests and callers import from +here are re-exported below. + Why a second backend? :mod:`anthropic.py` exposes our full ``AC_*`` schema and lets the model pick any of ~100 tools. That works, but it foregoes Claude's specifically-trained computer-use behaviour. With @@ -24,101 +31,35 @@ """ from __future__ import annotations -import math from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple from je_auto_control.utils.agent.agent_loop import AgentBackend, AgentStep -from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.agent.backends._computer_actions import ( # noqa: F401 # reason: re-exported + _ACTION_HANDLERS, _CLICK_ACTIONS, _MAX_KEY_REPEAT, _MAX_SCROLL_NOTCHES, + _MAX_WAIT_S, _action_wait, _clamp_decision, _decision_from_computer_action, + _hold_key_decision, _normalise_key, _parse_combo, _scroll_decision, +) +from je_auto_control.utils.agent.backends._computer_beta_path import ( # noqa: F401 # reason: re-exported + _DEFAULT_TOOL_TYPE, _TOOL_BETAS, BetaToolPath, beta_for, +) +from je_auto_control.utils.agent.backends._computer_messages import ( # noqa: F401 # reason: re-exported + _initial_user_content, _tool_result_content, +) from je_auto_control.utils.agent.backends._computer_toolset import ( TOOLSET_ONLY_MODELS, TOOLSET_SCHEMA, TOOLSET_TYPE, ToolsetBatch, - fit_screenshot, fitted_size, image_tier, resize_png, screen_region, - unscale_decision, zoom_image, + fitted_size, image_tier, resize_png, ) +from je_auto_control.utils.agent.backends._computer_toolset_path import ToolsetPath from je_auto_control.utils.agent.backends.base import ( REQUEST_TIMEOUT_S, AgentBackendError, build_default_system_prompt, - compact_history, encode_screenshot_b64, image_block, needs_compaction, - summarise_steps, + compact_history, image_block, needs_compaction, summarise_steps, ) _DEFAULT_MODEL = "claude-opus-5" -_DEFAULT_TOOL_TYPE = "computer_20251124" - -#: The beta each computer-use tool version is sent under. Every version is -#: beta-only: posted through the plain ``messages.create`` without one, the -#: API rejects the request, so the backend could never run. -_TOOL_BETAS = { - "computer_20250124": "computer-use-2025-01-24", - "computer_20251124": "computer-use-2025-11-24", -} - -#: Upper bounds on what one model action may ask for. The spec caps ``wait`` -#: at 100 s; a hold longer than that, or a scroll of more notches, is not a -#: step an agent needs, and an unbounded scroll overflowed the platform call. -_MAX_WAIT_S = 100.0 -_MAX_SCROLL_NOTCHES = 100 -_MAX_KEY_REPEAT = 100 - -# Map xdotool-style key names (used by Anthropic's tool spec) to the -# names AutoControl's keyboard wrappers expect. Anything outside this -# table is forwarded as-is; the wrapper rejects unknown keys cleanly. -_XDOTOOL_KEY_ALIAS = { - "return": "enter", - "escape": "esc", - "page_up": "pageup", - "page_down": "pagedown", - "back_space": "backspace", - "bksp": "backspace", - "ctrl_l": "ctrl", - "ctrl_r": "ctrl", - "shift_l": "shift", - "shift_r": "shift", - "alt_l": "alt", - "alt_r": "alt", - "super_l": "win", - "super_r": "win", - "meta_l": "win", - "meta_r": "win", -} - -def _normalise_key(name: str) -> str: - """This platform's name for an xdotool key. - - The fixed aliases ("return" -> "enter", "escape" -> "esc", "page_down" -> - "pagedown") are not names the Windows key table knows, so Enter, Esc, - paging, Alt and Super failed there; cua_action's resolver knows each - platform's spelling. - """ - from je_auto_control.utils.cua_action.cua_action import resolve_key_name - return resolve_key_name(_XDOTOOL_KEY_ALIAS.get(name.lower(), name.lower())) - - -def _parse_combo(combo: str) -> List[str]: - """Split an xdotool-style hotkey (``ctrl+shift+T``) into key names. - - ``ctrl++`` is Ctrl and the plus key: splitting on "+" dropped the plus. - """ - from je_auto_control.utils.cua_action.cua_action import split_key_combo - return [_normalise_key(part) for part in split_key_combo(combo)] - - -def _click_button(action: str) -> str: - return { - "left_click": "mouse_left", - "right_click": "mouse_right", - "middle_click": "mouse_middle", - "double_click": "mouse_left", - "triple_click": "mouse_left", - }.get(action, "mouse_left") - - -def _click_repeats(action: str) -> int: - return {"double_click": 2, "triple_click": 3}.get(action, 1) - - -class ComputerUseAgentBackend(AgentBackend): +class ComputerUseAgentBackend(BetaToolPath, ToolsetPath, AgentBackend): """Drive ``AgentLoop`` through Anthropic's native computer-use tool. The backend exposes the beta ``computer`` tool or the GA computer @@ -178,10 +119,7 @@ def __init__(self, } if display_number is not None: self._tool_schema["display_number"] = int(display_number) - self._beta = beta or _TOOL_BETAS.get(tool_type) - if not self._beta: - raise AgentBackendError( - f"no known beta for computer-use tool {tool_type!r}; pass beta=") + self._beta = beta_for(tool_type, beta) self._client = client self._api_key = api_key self._model = model @@ -280,98 +218,6 @@ def _decide_with_toolset(self, batch: ToolsetBatch, goal: str, self._extend(results, goal, screenshot, history) return self._handle_toolset_response(self._create(goal, beta=False), batch) - def _fit(self, screenshot: Optional[bytes]) -> Optional[bytes]: - """``screenshot`` within the toolset's image limits; remembers the scale.""" - if not screenshot: - return screenshot - fitted, self._scale = fit_screenshot(screenshot, self._tier) - return fitted - - def _toolset_result_content(self, step: AgentStep, screenshot: Optional[bytes], - tool_use_id: str) -> List[Dict[str, Any]]: - region = self._zooms.pop(tool_use_id, None) - if step.tool != "AC_screenshot" or not screenshot: - return _tool_result_content(step, screenshot, self._scale) - # A zoom is answered from the full-resolution frame; the scale of the - # full screenshot stays, since later coordinates are still in its space. - image = (zoom_image(screenshot, region, self._tier) if region is not None - else self._fit(screenshot)) - return _tool_result_content(step, image) - - def _handle_toolset_response(self, response: Any, - batch: ToolsetBatch) -> Dict[str, Any]: - content = list(getattr(response, "content", []) or []) - self._conversation.append({"role": "assistant", "content": content}) - self._zooms.clear() - _raise_if_truncated(response) # before running any call it holds - calls = [(_attr(block, "id"), self._toolset_decision(block)) - for block in content if _block_type(block) == "tool_use"] - if not calls: - return _final_answer(response, content) - batch.load(calls) - return batch.next_decision() - - def _toolset_decision(self, block: Any) -> Dict[str, Any]: - """A member call as a decision, in screen pixels and on the display.""" - name = str(_attr(block, "name") or "") - if name == "zoom": - return self._zoom_decision(block) - if name not in _CLICK_ACTIONS and name not in _ACTION_HANDLERS: - raise AgentBackendError( - f"model called tool {name!r}; only computer toolset members were offered") - payload = dict(_attr(block, "input") or {}) - payload["action"] = name # the member name is the action - decision = unscale_decision(_decision_from_computer_action(payload), self._scale) - return _clamp_decision(decision, *self._display) - - def _zoom_decision(self, block: Any) -> Dict[str, Any]: - """A ``zoom`` runs as a screenshot; its region crops the result.""" - payload = _attr(block, "input") or {} - try: - region = screen_region(payload.get("region") if isinstance(payload, dict) else None, - self._scale) - except (TypeError, ValueError) as error: - raise AgentBackendError(f"model sent an invalid zoom: {error}") from error - self._zooms[str(_attr(block, "id"))] = region - return {"tool": "AC_screenshot", "input": {}} - - # --- response → AgentLoop decision ------------------------------- - - def _handle_response(self, response: Any) -> Dict[str, Any]: - content = list(getattr(response, "content", []) or []) - self._conversation.append({"role": "assistant", "content": content}) - _raise_if_truncated(response) # before running any call it holds - for block in content: - if _block_type(block) != "tool_use": - continue - name = _attr(block, "name") - if name != "computer": - # Skipping it made the turn look like a final answer: the run - # reported success having done nothing the model asked for. - raise AgentBackendError( - f"model called tool {name!r}; only 'computer' was offered", - ) - payload = _attr(block, "input") or {} - self._pending_tool_use_id = _attr(block, "id") - decision = unscale_decision(_decision_from_computer_action(payload), self._scale) - return _clamp_decision(decision, *self._display) - return _final_answer(response, content) - - def _pending_result(self, history: Sequence[AgentStep], - screenshot: Optional[bytes]) -> List[Dict[str, Any]]: - """The ``tool_result`` answering the last turn's call, if one is pending.""" - if not history or self._pending_tool_use_id is None: - return [] - last = history[-1] - content = _tool_result_content(last, screenshot, self._scale) - tool_use_id, self._pending_tool_use_id = self._pending_tool_use_id, None - return [{ - "type": "tool_result", - "tool_use_id": tool_use_id, - "content": content, - "is_error": bool(last.error), - }] - def _resolve_client(self) -> Any: if self._client is not None: return self._client @@ -385,339 +231,4 @@ def _resolve_client(self) -> Any: return self._client -# --- action translation --------------------------------------------- - -def _final_answer(response: Any, content: List[Any]) -> Dict[str, Any]: - """A turn without tool calls: the final answer (a truncated turn was refused earlier).""" - text_parts: List[str] = [ - _attr(b, "text") or "" - for b in content if _block_type(b) == "text" - ] - return {"stop": True, "message": "\n".join(text_parts).strip()} - - -def _action_screenshot(_payload): - return {"tool": "AC_screenshot", "input": {}} - - -def _action_cursor_position(_payload): - return {"tool": "AC_get_mouse_position", "input": {}} - - -def _action_mouse_move(payload): - x, y = _xy(payload.get("coordinate")) - return {"tool": "AC_set_mouse_position", "input": {"x": x, "y": y}} - - -def _action_type(payload): - return { - "tool": "AC_write", - "input": {"write_string": str(payload.get("text") or "")}, - } - - -def _action_wait(payload): - duration = payload.get("duration") - # An explicit 0 is a valid "don't wait" — only fall back when unset. - seconds = _number(duration, "duration") if duration is not None else 1.0 - return _sequence([["AC_sleep", {"seconds": _bounded_seconds(seconds)}]]) - - -def _bounded_seconds(seconds: float) -> float: - """A model-chosen duration within ``[0, _MAX_WAIT_S]``. - - The run's wall-clock budget is checked between steps, so one unbounded - ``wait`` or ``hold_key`` blocked far past it. - """ - return min(max(float(seconds), 0.0), _MAX_WAIT_S) - - -_CLICK_ACTIONS = frozenset({ - "left_click", "right_click", "middle_click", - "double_click", "triple_click", -}) - - -def _decision_from_computer_action(payload: Dict[str, Any]) -> Dict[str, Any]: - """Turn one ``computer`` tool payload into an ``AgentLoop`` decision.""" - action = str(payload.get("action") or "").lower() - if action in _CLICK_ACTIONS: - decision = _click_decision(action, payload.get("coordinate")) - else: - handler = _ACTION_HANDLERS.get(action) - if handler is None: - raise AgentBackendError( - f"computer-use action {action!r} is not recognised", - ) - decision = handler(payload) - if action in _MODIFIER_ACTIONS and payload.get("text"): - return _with_modifiers(decision, str(payload["text"])) - return decision - - -#: Actions whose ``text`` names modifier keys to hold (``shift`` for a -#: shift-click). The field was ignored, so a ctrl-click was a plain click. -_MODIFIER_ACTIONS = _CLICK_ACTIONS | {"left_click_drag", "scroll"} - - -def _with_modifiers(decision: Dict[str, Any], combo: str) -> Dict[str, Any]: - """Run ``decision`` with the keys of ``combo`` held; they are released even on failure.""" - keys = _parse_combo(combo) - if not keys: - return decision - inner = decision["input"] - actions = (inner["action_list"] if decision["tool"] == "AC_execute_action" - else [[decision["tool"], inner]]) - return {"tool": "AC_with_modifiers", - "input": {"modifiers": keys, "actions": actions}} - - -def _click_decision(action: str, coordinate) -> Dict[str, Any]: - button = _click_button(action) - repeats = _click_repeats(action) - inputs: Dict[str, Any] = {"mouse_keycode": button} - if coordinate is not None: - x, y = _xy(coordinate) - inputs["x"], inputs["y"] = x, y - if repeats == 1: - return {"tool": "AC_click_mouse", "input": inputs} - # AC_click_mouse takes no repeat count (a 'repeat' argument made every - # click a TypeError): move once, then click the given number of times. - clicks = [["AC_click_mouse", inputs]] - clicks += [["AC_click_mouse", {"mouse_keycode": button}]] * (repeats - 1) - return _sequence(clicks) - - -def _sequence(actions: List[List[Any]]) -> Dict[str, Any]: - """Run several executor actions as one step, failing the step on any error.""" - return {"tool": "AC_execute_action", - "input": {"action_list": actions, "raise_on_error": True}} - - -def _drag_decision(payload: Dict[str, Any]) -> Dict[str, Any]: - # The spec's end point is ``coordinate`` (``start_coordinate`` is the - # start); reading only ``end_coordinate`` failed every drag the model made. - start = payload.get("start_coordinate") - end = payload.get("end_coordinate") or payload.get("coordinate") - if start is None or end is None: - raise AgentBackendError( - "left_click_drag requires start_coordinate and coordinate", - ) - sx, sy = _xy(start) - ex, ey = _xy(end) - # No AC_drag command exists: press at the start, move, release at the end. - return _sequence([ - ["AC_press_mouse", {"mouse_keycode": "mouse_left", "x": sx, "y": sy}], - ["AC_set_mouse_position", {"x": ex, "y": ey}], - ["AC_release_mouse", {"mouse_keycode": "mouse_left", "x": ex, "y": ey}], - ]) - - -def _scroll_decision(payload: Dict[str, Any]) -> Dict[str, Any]: - direction = str(payload.get("scroll_direction") or "down").lower() - raw_amount = payload.get("scroll_amount") - # An explicit 0 means "no scroll" — only default when the key is absent. - amount = int(_number(raw_amount, "scroll_amount")) if raw_amount is not None else 3 - amount = min(max(amount, 0), _MAX_SCROLL_NOTCHES) - # The direction goes to AC_mouse_scroll as cua_action sends it: left out, - # X11 and Wayland scrolled their default way ("up" went down there), and - # left / right became a vertical scroll everywhere. - from je_auto_control.utils.cua_action.cua_action import scroll_params - try: - inputs: Dict[str, Any] = scroll_params({"direction": direction, "amount": amount}) - except AutoControlException as error: - raise AgentBackendError(str(error)) from error - # The scroll happens where the model pointed, not wherever the cursor was; - # _clamp_decision keeps the point on the display. - if payload.get("coordinate") is not None: - inputs["x"], inputs["y"] = _xy(payload["coordinate"]) - return {"tool": "AC_mouse_scroll", "input": inputs} - - -def _key_decision(payload: Dict[str, Any]) -> Dict[str, Any]: - combo = str(payload.get("text") or payload.get("key") or "") - keys = _parse_combo(combo) - if not keys: - raise AgentBackendError("key action missing 'text'") - press = ({"tool": "AC_type_keyboard", "input": {"keycode": keys[0]}} if len(keys) == 1 - else {"tool": "AC_hotkey", "input": {"key_code_list": keys}}) - raw_repeat = payload.get("repeat") - repeat = int(_number(raw_repeat, "repeat")) if raw_repeat is not None else 1 - repeat = min(max(repeat, 1), _MAX_KEY_REPEAT) - if repeat == 1: - return press - # ``repeat`` (1..100) was ignored, so "press Down 5 times" pressed once. - return _sequence([[press["tool"], press["input"]]] * repeat) - - -def _hold_key_decision(payload: Dict[str, Any]) -> Dict[str, Any]: - key = _normalise_key(str(payload.get("text") or payload.get("key") or "")) - if not key: - raise AgentBackendError("hold_key action missing 'text'") - duration = _bounded_seconds(_number(payload.get("duration") or 0.0, "duration")) - return { - "tool": "AC_hold_key", - "input": {"key": key, "duration_s": duration}, - } - - -def _mouse_button_decision(tool: str, payload: Dict[str, Any]) -> Dict[str, Any]: - """Map a bare press/release verb to an ``AC_press/release_mouse`` call.""" - inputs: Dict[str, Any] = {"mouse_keycode": "mouse_left"} - coordinate = payload.get("coordinate") - if coordinate is not None: - inputs["x"], inputs["y"] = _xy(coordinate) - return {"tool": tool, "input": inputs} - - -def _mouse_down_decision(payload: Dict[str, Any]) -> Dict[str, Any]: - return _mouse_button_decision("AC_press_mouse", payload) - - -def _mouse_up_decision(payload: Dict[str, Any]) -> Dict[str, Any]: - return _mouse_button_decision("AC_release_mouse", payload) - - -# Dispatch table — populated after every handler is defined so the -# table reads its targets at module load time. -_ACTION_HANDLERS: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = { - "screenshot": _action_screenshot, - "cursor_position": _action_cursor_position, - "mouse_move": _action_mouse_move, - "type": _action_type, - "wait": _action_wait, - "left_click_drag": _drag_decision, - "scroll": _scroll_decision, - "key": _key_decision, - "hold_key": _hold_key_decision, - # computer_20250124 press/release verbs — without these an otherwise - # valid action raised AgentBackendError outside the create() try and - # aborted the run. - "left_mouse_down": _mouse_down_decision, - "left_mouse_up": _mouse_up_decision, -} - - -_TRUNCATION_STOP_REASONS = frozenset({"max_tokens", "refusal"}) - - -def _raise_if_truncated(response: Any) -> None: - """Reject an incomplete turn instead of treating it as a final answer.""" - stop_reason = _attr(response, "stop_reason") - if stop_reason in _TRUNCATION_STOP_REASONS: - raise AgentBackendError( - "anthropic computer-use response was incomplete " - f"(stop_reason={stop_reason!r})", - ) - - -# --- helpers -------------------------------------------------------- - -def _xy(coordinate: Any) -> Tuple[int, int]: - if (not isinstance(coordinate, (list, tuple)) - or len(coordinate) != 2): - raise AgentBackendError( - f"coordinate must be [x, y]; got {coordinate!r}", - ) - return int(_number(coordinate[0], "x")), int(_number(coordinate[1], "y")) - - -def _number(value: Any, what: str) -> float: - """``value`` as a finite float, or :class:`AgentBackendError`.""" - try: - number = float(value) - except (TypeError, ValueError) as error: - raise AgentBackendError(f"{what} must be a number, got {value!r}") from error - if not math.isfinite(number): - raise AgentBackendError(f"{what} must be finite, got {value!r}") - return number - - -def _clamp_decision(decision: Dict[str, Any], width: int, height: int) -> Dict[str, Any]: - """Keep every ``x`` / ``y`` in the decision on the declared display. - - The model's coordinates went to the mouse unchecked: [-500, 99999] on a - 100x100 display was dispatched as given. - """ - inputs = decision.get("input") or {} - _clamp_inputs(inputs, width, height) - for key in ("action_list", "actions"): - for action in inputs.get(key) or []: - if isinstance(action, (list, tuple)) and len(action) == 2 and isinstance(action[1], dict): - _clamp_inputs(action[1], width, height) - return decision - - -def _clamp_inputs(inputs: Dict[str, Any], width: int, height: int) -> None: - if "x" in inputs: - inputs["x"] = min(max(int(inputs["x"]), 0), width - 1) - if "y" in inputs: - inputs["y"] = min(max(int(inputs["y"]), 0), height - 1) - - -def _block_type(block: Any) -> Optional[str]: - if isinstance(block, dict): - return block.get("type") - return getattr(block, "type", None) - - -def _attr(block: Any, name: str) -> Any: - if isinstance(block, dict): - return block.get(name) - return getattr(block, name, None) - - -def _initial_user_content(goal: str, - screenshot: Optional[bytes]) -> List[Dict[str, Any]]: - blocks: List[Dict[str, Any]] = [] - encoded = encode_screenshot_b64(screenshot) - if encoded: - blocks.append({ - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": encoded, - }, - }) - blocks.append({"type": "text", "text": goal}) - return blocks - - -def _tool_result_content(step: AgentStep, screenshot: Optional[bytes], - scale: Tuple[float, float] = (1.0, 1.0)) -> List[Dict[str, Any]]: - """Build a ``tool_result`` content payload for the last ``AC_*`` call. - - Anthropic's spec expects the screenshot tool to return the image - *itself* — text-only results just describe what happened. The cursor - position is given in the screenshot's pixels, where the model works: - in screen pixels it named a point off the image it was shown. - """ - if not step.error and step.tool == "AC_get_mouse_position": - return [{"type": "text", "text": _scaled_point(step.result, scale)}] - if step.error: - return [{"type": "text", "text": f"error: {step.error}"}] - if step.tool == "AC_screenshot": - encoded = encode_screenshot_b64(screenshot) - if encoded: - return [{ - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": encoded, - }, - }] - text = repr(step.result) if step.result is not None else "ok" - return [{"type": "text", "text": text[:4000]}] - - -def _scaled_point(point: Any, scale: Tuple[float, float]) -> str: - """A screen point in screenshot pixels, as text; anything else as it came.""" - if (isinstance(point, (list, tuple)) and len(point) == 2 - and all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in point)): - return repr((int(round(point[0] * scale[0])), int(round(point[1] * scale[1])))) - return repr(point) - - __all__ = ["ComputerUseAgentBackend"] diff --git a/test/unit_test/headless/test_computer_use_module_layout.py b/test/unit_test/headless/test_computer_use_module_layout.py new file mode 100644 index 000000000..eea7de8c1 --- /dev/null +++ b/test/unit_test/headless/test_computer_use_module_layout.py @@ -0,0 +1,61 @@ +"""The computer-use backend was split by request shape; its names did not move. + +``anthropic_computer_use.py`` was 27 lines from the file-length limit. The +action translation, the message blocks, the beta tool path and the toolset +path now have a module each, and every name that was imported from +``anthropic_computer_use`` -- by callers and by the tests of its behaviour -- +is still importable from there and is the same object. +""" +from je_auto_control.utils.agent import backends +from je_auto_control.utils.agent.backends import ( + _computer_actions, _computer_beta_path, _computer_messages, _computer_toolset_path, + anthropic_computer_use as cu, +) + +#: name -> the module that now defines it. +_MOVED = { + "_decision_from_computer_action": _computer_actions, + "_scroll_decision": _computer_actions, + "_hold_key_decision": _computer_actions, + "_action_wait": _computer_actions, + "_clamp_decision": _computer_actions, + "_parse_combo": _computer_actions, + "_normalise_key": _computer_actions, + "_ACTION_HANDLERS": _computer_actions, + "_CLICK_ACTIONS": _computer_actions, + "_MAX_WAIT_S": _computer_actions, + "_MAX_SCROLL_NOTCHES": _computer_actions, + "_MAX_KEY_REPEAT": _computer_actions, + "_tool_result_content": _computer_messages, + "_initial_user_content": _computer_messages, + "_TOOL_BETAS": _computer_beta_path, + "_DEFAULT_TOOL_TYPE": _computer_beta_path, +} + + +def test_every_moved_name_is_still_importable_from_the_backend_module(): + for name, home in _MOVED.items(): + assert getattr(cu, name) is getattr(home, name), name + + +def test_the_public_name_is_unchanged(): + assert cu.__all__ == ["ComputerUseAgentBackend"] + assert backends.ComputerUseAgentBackend is cu.ComputerUseAgentBackend + assert cu.ComputerUseAgentBackend.__module__ == cu.__name__ + + +def test_the_backend_is_both_paths(): + backend = cu.ComputerUseAgentBackend + assert issubclass(backend, _computer_beta_path.BetaToolPath) + assert issubclass(backend, _computer_toolset_path.ToolsetPath) + for method in ("decide_next_action", "_handle_response", "_pending_result", + "_handle_toolset_response", "_toolset_decision", "_zoom_decision", + "_fit", "_toolset_result_content", "_decide_with_toolset"): + assert callable(getattr(backend, method)), method + + +def test_default_tool_and_betas_are_what_they_were(): + assert cu._DEFAULT_MODEL == "claude-opus-5" + assert cu._DEFAULT_TOOL_TYPE == "computer_20251124" + assert cu._TOOL_BETAS == {"computer_20250124": "computer-use-2025-01-24", + "computer_20251124": "computer-use-2025-11-24"} From 8db0de5757b5cd906d62fa685578952ccaef8a93 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:01:23 +0800 Subject: [PATCH 20/44] Run deferred work as its owner, give user management a surface and keep data tools from viewers A scheduler job, trigger, hotkey, webhook, e-mail trigger or watchdog rule registered by an authenticated user ran later with no identity, so an operator could schedule a privileged command. Each entry now stores its owner and runs inside the owner's scope, with the role looked up when it fires. manage_users had nothing behind it: add AC_user_*, the ac_user_* MCP tools, 'je_auto_control users' for the first admin and a users group in the REST API tab. A token is returned once and masked wherever the result is formatted. Read-only MCP tools that return files, databases, the clipboard, references or the process list need the new read_data capability, which a viewer does not hold. The REST status no longer reports a shared token that RBAC refuses, a gate with only check() is accepted again, the Ed25519 private key can carry a passphrase, and a file refused by enforced signatures is a typed error answered with 403. --- docs/source/Eng/doc/cli/cli_doc.rst | 21 + .../Eng/doc/mcp_server/mcp_server_doc.rst | 28 +- .../Eng/doc/new_features/v4_features_doc.rst | 16 +- .../operations_layer/operations_layer_doc.rst | 77 ++- docs/source/Zh/doc/cli/cli_doc.rst | 18 + .../Zh/doc/mcp_server/mcp_server_doc.rst | 24 +- .../Zh/doc/new_features/v4_features_doc.rst | 12 +- .../operations_layer/operations_layer_doc.rst | 62 ++- je_auto_control/__init__.py | 14 +- je_auto_control/cli.py | 6 + .../gui/language_wrapper/english.py | 29 ++ .../gui/language_wrapper/japanese.py | 29 ++ .../language_wrapper/simplified_chinese.py | 29 ++ .../language_wrapper/traditional_chinese.py | 29 ++ je_auto_control/gui/rbac_users_panel.py | 228 +++++++++ je_auto_control/gui/rest_api_tab.py | 45 +- .../gui/script_builder/command_schema.py | 4 + .../utils/action_signing/asymmetric.py | 68 ++- .../utils/action_signing/config.py | 15 +- .../utils/action_signing/signer.py | 22 +- je_auto_control/utils/exception/exceptions.py | 11 + .../utils/executor/action_executor.py | 51 +- je_auto_control/utils/hotkey/hotkey_daemon.py | 8 +- .../utils/mcp_server/tools/_factories.py | 56 ++- .../mcp_server/tools/_handlers_operations.py | 25 + je_auto_control/utils/rbac/__init__.py | 25 +- je_auto_control/utils/rbac/__main__.py | 5 + je_auto_control/utils/rbac/admin.py | 167 +++++++ je_auto_control/utils/rbac/authorization.py | 11 +- je_auto_control/utils/rbac/cli.py | 116 +++++ je_auto_control/utils/rbac/deferred.py | 105 +++++ je_auto_control/utils/rbac/policy.py | 47 +- je_auto_control/utils/rbac/users.py | 8 +- je_auto_control/utils/rest_api/rest_auth.py | 22 +- .../utils/rest_api/rest_handlers.py | 6 + .../utils/rest_api/rest_registry.py | 15 +- je_auto_control/utils/rest_api/rest_server.py | 8 +- .../utils/run_history/run_outcome.py | 21 +- je_auto_control/utils/scheduler/scheduler.py | 8 +- .../utils/triggers/email_trigger.py | 6 +- .../utils/triggers/trigger_engine.py | 7 +- .../utils/triggers/webhook_server.py | 8 +- .../utils/watchdog/popup_watchdog.py | 16 +- .../headless/test_rbac_deferred_owner.py | 287 +++++++++++ .../headless/test_rbac_mcp_transports.py | 410 ++++++++++++++++ .../headless/test_rbac_server_wiring.py | 5 +- .../headless/test_rbac_user_admin.py | 444 ++++++++++++++++++ .../unit_test/headless/test_rbac_users_gui.py | 228 +++++++++ .../headless/test_signing_execution_roles.py | 2 +- .../headless/test_signing_passphrase_e2e.py | 285 +++++++++++ 50 files changed, 3077 insertions(+), 112 deletions(-) create mode 100644 je_auto_control/gui/rbac_users_panel.py create mode 100644 je_auto_control/utils/rbac/__main__.py create mode 100644 je_auto_control/utils/rbac/admin.py create mode 100644 je_auto_control/utils/rbac/cli.py create mode 100644 je_auto_control/utils/rbac/deferred.py create mode 100644 test/unit_test/headless/test_rbac_deferred_owner.py create mode 100644 test/unit_test/headless/test_rbac_mcp_transports.py create mode 100644 test/unit_test/headless/test_rbac_user_admin.py create mode 100644 test/unit_test/headless/test_rbac_users_gui.py create mode 100644 test/unit_test/headless/test_signing_passphrase_e2e.py diff --git a/docs/source/Eng/doc/cli/cli_doc.rst b/docs/source/Eng/doc/cli/cli_doc.rst index 808f10132..bd0dc9836 100644 --- a/docs/source/Eng/doc/cli/cli_doc.rst +++ b/docs/source/Eng/doc/cli/cli_doc.rst @@ -58,6 +58,27 @@ Start the REST API server Endpoints: ``GET /health``, ``GET /jobs``, ``POST /execute`` with ``{"actions": [...]}``. +Manage RBAC users +----------------- + +.. code-block:: bash + + python -m je_auto_control.cli users --users users.json add alice --role admin + python -m je_auto_control.cli users --users users.json list + python -m je_auto_control.cli users --users users.json set-role bob operator + python -m je_auto_control.cli users --users users.json rotate-token bob + python -m je_auto_control.cli users --users users.json remove bob + +The users of the REST API and MCP server when RBAC is on (see *Roles* in the +operations-layer chapter). ``--users`` names the store file and defaults to +``JE_AUTOCONTROL_RBAC_USERS``; ``--json`` (before the sub-command) prints the +result as JSON. ``add`` (``--role viewer|operator|admin``, ``--name``, +``--tag``) and ``rotate-token`` print the token once -- it is not stored and +cannot be shown again. The last admin cannot be removed or demoted; +``remove`` exits 1 when there is no such user. ``python -m +je_auto_control.utils.rbac`` takes the same arguments without the leading +``users``. + Legacy flag-style CLI (``python -m je_auto_control``) ===================================================== diff --git a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst index 316f691ca..d7b8dcc69 100644 --- a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst @@ -302,11 +302,33 @@ share one file. Without a store nothing changes. The stdio transport has no bearer token and is never subject to RBAC. - A tool needs ``read_screen`` when it is marked ``readOnlyHint`` and - ``drive_input`` otherwise, so a ``viewer`` gets exactly the read-only - tools and an ``operator`` the rest. ``ac_remote_host_start`` / ``_stop``, + ``drive_input`` otherwise. ``ac_remote_host_start`` / ``_stop``, ``ac_usb_acl_add`` / ``_remove`` / ``_set_default``, ``ac_usb_passthrough_enable``, ``ac_egress_allow`` / ``_reset`` and - ``ac_load_plugins`` need ``manage_hosts`` (``admin``). + ``ac_load_plugins`` need ``manage_hosts`` (``admin``); ``ac_user_add`` / + ``_remove`` / ``_set_role`` / ``_rotate_token`` / ``_list`` need + ``manage_users`` (``admin``). +- Read-only is not the same as harmless to show: the read-only tools that + return the host's data rather than the state of its screen need + ``read_data``, which ``operator`` and ``admin`` hold and ``viewer`` does + not. They are the clipboard tools (``ac_get_clipboard`` and its + ``_csv`` / ``_files`` / ``_html`` / ``_image`` / ``_rtf`` variants, + ``ac_clipboard_formats``, ``ac_assert_clipboard``, ``ac_clip_history_list`` + / ``_search``); the file readers (``ac_load_dotenv``, ``ac_load_data``, + ``ac_read_action_file``, ``ac_read_document``, ``ac_read_presentation``, + ``ac_read_workbook``, ``ac_extract_pdf_text``, ``ac_assert_pdf_text``, + ``ac_assert_file``, ``ac_build_provenance``, ``ac_verify_provenance``); the + database and store readers (``ac_sql_query``, ``ac_assert_db``, + ``ac_get_asset``, ``ac_list_assets``, ``ac_cas_get``, ``ac_outbox_pending``, + ``ac_checkpoint_status``, ``ac_memory_recall``, ``ac_memory_recent``, + ``ac_s3_list``); references and tokens (``ac_resolve_ref``, + ``ac_resolve_refs``, ``ac_generate_otp``, ``ac_jwt_encode``, + ``ac_jwt_decode``); and the process list, network and microphone probes + (``ac_list_processes``, ``ac_assert_process``, ``ac_wait_for_process``, + ``ac_assert_http``, ``ac_wait_for_port``, ``ac_assert_audio``). The list is + ``DATA_TOOLS`` in ``je_auto_control.utils.rbac.policy``. A ``viewer`` keeps + every other read-only tool: screen size, windows, pixels, image and text + location, accessibility reads, waits. - ``tools/list`` returns only the tools the caller may call, and ``tools/call`` on any other answers JSON-RPC error ``-32003`` (``Forbidden: ...``, ``data.required_capability``) without running it. diff --git a/docs/source/Eng/doc/new_features/v4_features_doc.rst b/docs/source/Eng/doc/new_features/v4_features_doc.rst index fa6297798..92671414f 100644 --- a/docs/source/Eng/doc/new_features/v4_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v4_features_doc.rst @@ -151,11 +151,20 @@ File security & safety verifies and cannot sign. ``sign_action_file(path, private_key_path=...)`` writes a JSON envelope (``version`` 2, ``algorithm`` ``ed25519``) into the same ``.sig`` sidecar; ``verify_action_file(path, public_key_path=...)`` - checks it. Three environment variables configure a process, and + checks it. ``create_signing_keypair(..., passphrase="...")`` (the + ``passphrase`` field of ``AC_create_signing_keypair``) encrypts the private + key file, so a copy of the file alone signs nothing; the signer then passes + ``sign_action_file(..., passphrase="...")`` or sets + ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE``. A key created without a + passphrase keeps loading as it is, and verifying never needs one. These + environment variables configure a process, and ``action_signing_config()`` reports what it sees: * ``JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY`` — path of the private key; signing machine only. ``sign_action_file(path)`` then signs with it. + * ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE`` — passphrase of an encrypted + private key; signing machine only, and not part of what + ``action_signing_config()`` reports. * ``JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY`` — path of the public key, on every execution endpoint. With this and no private key the endpoint is verify-only: ``AC_sign_action_file`` and ``AC_create_signing_keypair`` @@ -171,7 +180,10 @@ File security & safety on the endpoint; sign every file again with the private key; then unset the migration variable. Key-pair signing needs ``cryptography`` -- where it is absent (Windows arm64) these calls raise ``CryptographyUnavailableError`` - with the install hint, and HMAC signing still works. + with the install hint, and HMAC signing still works. A file refused by + enforced signatures raises ``AutoControlSignatureException`` (an + ``AutoControlException``); REST ``POST /execute_file`` answers it with 403 + and the MCP ``ac_execute_action_file`` tool with an error result. * **Action-file encryption** — ``encrypt_action_file`` / ``decrypt_action_file`` keep a script's contents secret at rest with Fernet (AES-128-CBC + HMAC), keyed by a per-user 0600 key or by a diff --git a/docs/source/Eng/doc/operations_layer/operations_layer_doc.rst b/docs/source/Eng/doc/operations_layer/operations_layer_doc.rst index 7782d1108..82982030d 100644 --- a/docs/source/Eng/doc/operations_layer/operations_layer_doc.rst +++ b/docs/source/Eng/doc/operations_layer/operations_layer_doc.rst @@ -134,6 +134,10 @@ CLI:: python -m je_auto_control.utils.rest_api --host 127.0.0.1 --port 9939 +``POST /execute_file`` answers ``403`` with the reason when +``JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS`` is set and the file's signature is +missing or does not verify; the file is not run. + Roles (opt-in RBAC) ------------------- @@ -150,16 +154,35 @@ their own and a role instead: one user of the store; a store that is empty or unreadable admits nobody. - **Without it, nothing changes**: the single shared token, no roles, no 403. -Create the users from Python; the plain token is shown once and only its -hash is stored:: - - import je_auto_control as ac - - store = ac.UserStore("/etc/autocontrol/users.json") - token = store.add_user(user_id="alice", display_name="Alice", role="operator") - store.set_role("alice", "viewer") # roles: viewer / operator / admin - store.rotate_token("alice") - store.remove_user("alice") +Create the first admin on the host, from a terminal; the plain token is +printed once and only its hash is stored:: + + je_auto_control users --users /etc/autocontrol/users.json add alice --role admin + je_auto_control users list # store from JE_AUTOCONTROL_RBAC_USERS + je_auto_control users set-role bob operator # roles: viewer / operator / admin + je_auto_control users rotate-token bob + je_auto_control users remove bob + +(``python -m je_auto_control.utils.rbac ...`` runs the same commands; ``--json`` +prints the result as JSON.) After that an admin can manage users through any +surface, each of which needs the ``manage_users`` capability: + +- ``AC_user_add`` (``user_id``, ``role``, ``display_name``, ``tags``), + ``AC_user_remove``, ``AC_user_set_role``, ``AC_user_rotate_token`` and + ``AC_user_list`` in an action list -- so also through ``POST /execute``; +- the MCP tools ``ac_user_add`` / ``ac_user_remove`` / ``ac_user_set_role`` / + ``ac_user_rotate_token`` / ``ac_user_list``; +- the **Users (RBAC)** group of the REST API tab (commands in the Actions + menu), which also lets the tab start the server against that user store; +- Python: ``ac.rbac_add_user`` / ``rbac_remove_user`` / ``rbac_set_user_role`` / + ``rbac_rotate_user_token`` / ``rbac_list_users``, or ``UserStore`` directly. + +An authenticated caller always works on the store that authenticated it; the +optional ``users_path`` argument is for scripts and the CLI. The last admin +cannot be removed or demoted. ``add`` and ``rotate`` return the token once: it +is in the reply, masked wherever the reply is logged, and never in an audit +entry (which records ``rbac_user_added`` and the like, with the user and who +did it). A running server re-reads the file when it changes, so a removed user, a rotated token or a changed role applies to the next request. @@ -173,6 +196,10 @@ rotated token or a changed role applies to the next request. * - ``read_screen`` - viewer, operator, admin - every ``GET`` except the two below, ``/metrics`` included + * - ``read_data`` + - operator, admin + - no REST route; the MCP tools that return the host's data rather than + its screen (see the MCP chapter) * - ``drive_input`` - operator, admin - ``POST /execute``, ``/execute_file``, ``/usb/loopback/open``, @@ -188,6 +215,9 @@ rotated token or a changed role applies to the next request. * - ``sign_actions`` - admin - the ``AC_sign_action_file`` command (see below) + * - ``manage_users`` + - admin + - the ``AC_user_*`` commands (see above) A role that lacks the capability gets ``403`` ``{"error": "forbidden", "required_capability": "...", "role": "..."}``. @@ -202,16 +232,35 @@ webhook servers, ``AC_usb_acl_*``, ``AC_usb_passthrough_enable``, ``AC_config_import`` / ``AC_config_export``, ``AC_egress_allow`` / ``AC_egress_reset``, ``AC_load_plugins``, ``AC_add_package_to_executor``, ``AC_secret_init`` / ``set`` / ``remove`` / ``lock`` / ``unlock``, -``AC_audit_log_clear`` (all ``manage_hosts``) -- need their own capability +``AC_audit_log_clear`` (all ``manage_hosts``) -- and ``AC_user_*`` +(``manage_users``) need their own capability wherever they appear in the action list, however deeply nested. The request is answered 403 with the ``command`` named before its first action runs, and the executor refuses the same commands when they come from an action file. The roles are not a sandbox: an operator drives the real keyboard and can launch programs, so whatever the desktop user could do by hand is within -reach. They protect the host's own privileged state. Work an operator defers -to another thread -- a scheduler job, a trigger, a hotkey binding -- runs -later without the caller's role attached. +reach. They protect the host's own privileged state. + +**Deferred work keeps its owner.** A scheduler job, a trigger, a hotkey +binding, a webhook, an e-mail trigger or a watchdog rule registered by an +authenticated caller is stored with that user (``job.owner`` and the like, a +``DeferredOwner``) and later runs as them. The role is looked up in the user +store when the work fires, not remembered: a user demoted since loses the +privilege for work already registered, a user who no longer has +``drive_input`` or was removed gets none of it run (the run is recorded as +failed), and a promoted user gains the new role. Work registered without RBAC +-- from the GUI, a script, a server with no user store -- has no owner and +runs as before. + +Status calls (``AC_rest_api_status``, the REST API tab) report +``"rbac": true``, ``"users_path"`` and ``"token": null`` while a user store is +in use: the shared token is not shown where it is refused. + +A custom gate object that only implements ``check(client_ip=, header_value=)`` +is accepted: its verdict is used and the request is served with no user +identity, as under the shared token. Implement ``authenticate(...)`` +returning an ``AuthResult`` to supply one. Every audit row of an RBAC request names the user: the ``viewer_id`` column holds the user id and the detail reads diff --git a/docs/source/Zh/doc/cli/cli_doc.rst b/docs/source/Zh/doc/cli/cli_doc.rst index e76b14f70..88754f2c0 100644 --- a/docs/source/Zh/doc/cli/cli_doc.rst +++ b/docs/source/Zh/doc/cli/cli_doc.rst @@ -68,3 +68,21 @@ AutoControl 可以直接從命令列執行自動化腳本。 啟動 GUI 需要安裝 ``[gui]`` 額外套件： ``pip install je_auto_control[gui]`` + +管理 RBAC 使用者 +================ + +.. code-block:: bash + + python -m je_auto_control.cli users --users users.json add alice --role admin + python -m je_auto_control.cli users --users users.json list + python -m je_auto_control.cli users --users users.json set-role bob operator + python -m je_auto_control.cli users --users users.json rotate-token bob + python -m je_auto_control.cli users --users users.json remove bob + +RBAC 啟用時 REST API 與 MCP server 的使用者（見維運層章節的「角色」）。``--users`` 指定 +存放檔，預設取自 ``JE_AUTOCONTROL_RBAC_USERS``；``--json``\ （放在子指令之前）以 JSON +輸出結果。``add``\ （``--role viewer|operator|admin``、``--name``、``--tag``）與 +``rotate-token`` 只印出 token 一次——它不會被存下來，之後無法再顯示。最後一位 admin +不能被移除或降級；``remove`` 找不到使用者時結束碼為 1。 +``python -m je_auto_control.utils.rbac`` 接受相同的參數，只是不需要開頭的 ``users``。 diff --git a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst index 6118174b4..f84e673d2 100644 --- a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst @@ -277,11 +277,29 @@ Bearer token 也可從 ``JE_AUTOCONTROL_MCP_TOKEN`` 環境變數讀取。 同一個檔案。沒有設定存放檔時行為完全不變。stdio 傳輸沒有 bearer token，不受 RBAC 約束。 -- 工具標示 ``readOnlyHint`` 時需要 ``read_screen``，否則需要 ``drive_input``，所以 - ``viewer`` 拿到的正好是唯讀工具，``operator`` 再加上其餘工具。 +- 工具標示 ``readOnlyHint`` 時需要 ``read_screen``，否則需要 ``drive_input``。 ``ac_remote_host_start``／``_stop``、``ac_usb_acl_add``／``_remove``／ ``_set_default``、``ac_usb_passthrough_enable``、``ac_egress_allow``／``_reset`` 與 - ``ac_load_plugins`` 需要 ``manage_hosts``\ （``admin``）。 + ``ac_load_plugins`` 需要 ``manage_hosts``\ （``admin``）；``ac_user_add``／ + ``_remove``／``_set_role``／``_rotate_token``／``_list`` 需要 ``manage_users``\ + （``admin``）。 +- 唯讀不等於可以給人看：回傳主機資料而不是畫面狀態的唯讀工具需要 ``read_data``， + ``operator`` 與 ``admin`` 有這個能力，``viewer`` 沒有。包括剪貼簿工具 + （``ac_get_clipboard`` 及其 ``_csv``／``_files``／``_html``／``_image``／``_rtf`` + 變體、``ac_clipboard_formats``、``ac_assert_clipboard``、``ac_clip_history_list``／ + ``_search``）；讀檔工具（``ac_load_dotenv``、``ac_load_data``、 + ``ac_read_action_file``、``ac_read_document``、``ac_read_presentation``、 + ``ac_read_workbook``、``ac_extract_pdf_text``、``ac_assert_pdf_text``、 + ``ac_assert_file``、``ac_build_provenance``、``ac_verify_provenance``）；資料庫與 + 具名儲存區（``ac_sql_query``、``ac_assert_db``、``ac_get_asset``、``ac_list_assets``、 + ``ac_cas_get``、``ac_outbox_pending``、``ac_checkpoint_status``、``ac_memory_recall``、 + ``ac_memory_recent``、``ac_s3_list``）；參照與 token（``ac_resolve_ref``、 + ``ac_resolve_refs``、``ac_generate_otp``、``ac_jwt_encode``、``ac_jwt_decode``）； + 以及行程清單、網路與麥克風探測（``ac_list_processes``、``ac_assert_process``、 + ``ac_wait_for_process``、``ac_assert_http``、``ac_wait_for_port``、 + ``ac_assert_audio``）。完整清單是 ``je_auto_control.utils.rbac.policy`` 的 + ``DATA_TOOLS``。其餘唯讀工具 ``viewer`` 都保留：螢幕尺寸、視窗、像素、影像與文字 + 定位、無障礙讀取、等待。 - ``tools/list`` 只回呼叫者可以呼叫的工具；對其他工具 ``tools/call`` 會回 JSON-RPC 錯誤 ``-32003``\ （``Forbidden: ...``、``data.required_capability``），且不會執行。 - 接受動作清單的工具（``ac_execute_actions`` 等）在清單含有呼叫者角色沒有的指令時 diff --git a/docs/source/Zh/doc/new_features/v4_features_doc.rst b/docs/source/Zh/doc/new_features/v4_features_doc.rst index 201592a29..610a4645b 100644 --- a/docs/source/Zh/doc/new_features/v4_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v4_features_doc.rst @@ -124,11 +124,18 @@ Builder 項目。視覺與視窗功能的 geometry / IO 操作皆可注入,因 (``AC_create_signing_keypair``)產生一組 Ed25519 金鑰:私鑰留在簽章機,執行端只拿 公鑰,公鑰能驗章、不能簽章。``sign_action_file(path, private_key_path=...)`` 把 JSON envelope(``version`` 2、``algorithm`` ``ed25519``)寫進同一個 ``.sig`` sidecar; - ``verify_action_file(path, public_key_path=...)`` 負責驗證。行程由三個環境變數設定, + ``verify_action_file(path, public_key_path=...)`` 負責驗證。 + ``create_signing_keypair(..., passphrase="...")``(``AC_create_signing_keypair`` 的 + ``passphrase`` 欄位)會加密私鑰檔,只拿到檔案副本的人簽不了任何東西;簽章端之後傳 + ``sign_action_file(..., passphrase="...")`` 或設定 + ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE``。沒有通行碼建立的金鑰照舊可以載入, + 驗章也從不需要通行碼。行程由下列環境變數設定, ``action_signing_config()`` 會回報目前讀到的內容: * ``JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY`` — 私鑰路徑;只設在簽章機。設了之後 ``sign_action_file(path)`` 就用它簽。 + * ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE`` — 加密私鑰的通行碼;只設在簽章機, + ``action_signing_config()`` 不會回報它。 * ``JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY`` — 公鑰路徑,設在每個執行端。只有它而 沒有私鑰時,該端點只能驗章:``AC_sign_action_file`` 與 ``AC_create_signing_keypair`` 會拋出例外,不會退回每位使用者的 HMAC 金鑰,HMAC sidecar 也不再通過驗證。 @@ -139,6 +146,9 @@ Builder 項目。視覺與視窗功能的 geometry / IO 操作皆可注入,因 簽章機產生金鑰組;在端點設定公鑰變數與遷移變數;用私鑰把每個檔案重簽一次;最後取消 遷移變數。金鑰組簽章需要 ``cryptography`` —— 沒有該套件的平台(Windows arm64)上, 這些呼叫會拋出帶安裝提示的 ``CryptographyUnavailableError``,HMAC 簽章仍可使用。 + 被強制驗章拒絕的檔案會拋出 ``AutoControlSignatureException``(屬於 + ``AutoControlException``);REST 的 ``POST /execute_file`` 以 403 回應,MCP 的 + ``ac_execute_action_file`` 工具則回傳錯誤結果。 * **動作檔加密** — ``encrypt_action_file`` / ``decrypt_action_file`` 以 Fernet(AES-128-CBC + HMAC)讓腳本內容在靜態時保密,金鑰來自每位使用者的 0600 金鑰,或經 scrypt 與每個檔案各自的隨機鹽值衍生自通行碼。``AC_encrypt_action_file`` / diff --git a/docs/source/Zh/doc/operations_layer/operations_layer_doc.rst b/docs/source/Zh/doc/operations_layer/operations_layer_doc.rst index 7fcd83e1c..fd8ac8361 100644 --- a/docs/source/Zh/doc/operations_layer/operations_layer_doc.rst +++ b/docs/source/Zh/doc/operations_layer/operations_layer_doc.rst @@ -125,6 +125,9 @@ CLI:: python -m je_auto_control.utils.rest_api --host 127.0.0.1 --port 9939 +設定了 ``JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS`` 而檔案沒有簽章或驗章失敗時， +``POST /execute_file`` 回 ``403`` 並附上原因；該檔案不會被執行。 + 角色（選用的 RBAC） -------------------- @@ -139,15 +142,31 @@ CLI:: 存放檔是空的或讀不出來時，任何人都進不來。 - **沒有啟用時完全不變**：單一共用 token、沒有角色、不會有 403。 -使用者以 Python 建立；明文 token 只顯示一次，檔案裡只存雜湊:: +第一位 admin 在主機上從終端機建立；明文 token 只印出一次，檔案裡只存雜湊:: + + je_auto_control users --users /etc/autocontrol/users.json add alice --role admin + je_auto_control users list # 存放檔取自 JE_AUTOCONTROL_RBAC_USERS + je_auto_control users set-role bob operator # 角色：viewer / operator / admin + je_auto_control users rotate-token bob + je_auto_control users remove bob + +（``python -m je_auto_control.utils.rbac ...`` 是同一組指令；``--json`` 以 JSON 輸出 +結果。）之後 admin 可以從任何介面管理使用者，每一個都需要 ``manage_users`` 能力： - import je_auto_control as ac +- 動作清單裡的 ``AC_user_add``\ （``user_id``、``role``、``display_name``、``tags``）、 + ``AC_user_remove``、``AC_user_set_role``、``AC_user_rotate_token`` 與 + ``AC_user_list``——所以 ``POST /execute`` 也可以； +- MCP 工具 ``ac_user_add``／``ac_user_remove``／``ac_user_set_role``／ + ``ac_user_rotate_token``／``ac_user_list``； +- REST API 分頁的 **使用者（RBAC）** 群組（指令在 Actions 選單），該分頁也可以用這個 + 使用者存放檔啟動伺服器； +- Python：``ac.rbac_add_user``／``rbac_remove_user``／``rbac_set_user_role``／ + ``rbac_rotate_user_token``／``rbac_list_users``，或直接用 ``UserStore``。 - store = ac.UserStore("/etc/autocontrol/users.json") - token = store.add_user(user_id="alice", display_name="Alice", role="operator") - store.set_role("alice", "viewer") # 角色：viewer / operator / admin - store.rotate_token("alice") - store.remove_user("alice") +已驗證的呼叫者一律操作驗證它的那個存放檔；選用的 ``users_path`` 參數是給腳本與 CLI +用的。最後一位 admin 不能被移除或降級。``add`` 與 ``rotate`` 只回傳 token 一次：它在 +回覆裡，回覆被寫進記錄時會被遮蔽，也不會出現在稽核紀錄中（稽核只記 +``rbac_user_added`` 這類事件、對象與操作者）。 執行中的伺服器會在檔案變動時重新讀取，所以移除使用者、輪替 token 或改角色都在 下一個請求生效。 @@ -161,6 +180,9 @@ CLI:: * - ``read_screen`` - viewer、operator、admin - 除下列兩個之外的所有 ``GET``，包含 ``/metrics`` + * - ``read_data`` + - operator、admin + - 沒有 REST 路由；回傳主機資料而非畫面的 MCP 工具（見 MCP 章節） * - ``drive_input`` - operator、admin - ``POST /execute``、``/execute_file``、``/usb/loopback/open``、 @@ -176,6 +198,9 @@ CLI:: * - ``sign_actions`` - admin - ``AC_sign_action_file`` 指令（見下） + * - ``manage_users`` + - admin + - ``AC_user_*`` 指令（見上） 角色沒有該能力時回 ``403`` ``{"error": "forbidden", "required_capability": "...", "role": "..."}``。 @@ -187,13 +212,28 @@ CLI:: ``AC_usb_passthrough_enable``、``AC_config_import``／``AC_config_export``、 ``AC_egress_allow``／``AC_egress_reset``、``AC_load_plugins``、 ``AC_add_package_to_executor``、``AC_secret_init``／``set``／``remove``／``lock``／ -``unlock``、``AC_audit_log_clear``\ （皆為 ``manage_hosts``）——不論在動作清單裡 -巢狀多深，都需要各自的能力。請求會在第一個動作執行前就回 403 並指出 ``command``； +``unlock``、``AC_audit_log_clear``\ （皆為 ``manage_hosts``）與 ``AC_user_*``\ +（``manage_users``）——不論在動作清單裡巢狀多深，都需要各自的能力。請求會在第一個動作執行前就回 403 並指出 ``command``； 同樣的指令若來自動作檔，則由 executor 拒絕。 角色不是沙箱：operator 操作的是真實鍵盤，也能啟動程式，桌面使用者手動做得到的事 -它都做得到。角色保護的是主機自身的特權狀態。operator 交給其他執行緒延後執行的 -工作（排程工作、觸發器、熱鍵綁定）之後執行時不會帶著呼叫者的角色。 +它都做得到。角色保護的是主機自身的特權狀態。 + +**延後執行的工作會記住擁有者。** 已驗證的呼叫者註冊的排程工作、觸發器、熱鍵綁定、 +webhook、e-mail 觸發器或 watchdog 規則，會連同該使用者一起存下來（``job.owner`` 等， +型別為 ``DeferredOwner``），之後以該使用者的身分執行。角色是在工作觸發時到使用者 +存放檔查的，不是記下來的：之後被降級的使用者，已註冊的工作也失去特權；不再有 +``drive_input`` 或已被移除的使用者，其工作完全不會執行（該次執行記為失敗）；被升級 +的使用者則取得新角色。沒有 RBAC 時註冊的工作（GUI、腳本、沒有使用者存放檔的伺服器） +沒有擁有者，行為與以前相同。 + +使用者存放檔啟用時，狀態查詢（``AC_rest_api_status``、REST API 分頁）回報 +``"rbac": true``、``"users_path"`` 與 ``"token": null``：共用 token 會被拒絕，所以不再 +顯示它。 + +只實作 ``check(client_ip=, header_value=)`` 的自製 gate 物件仍可使用：採用它的判定， +請求以沒有使用者身分的方式處理，與共用 token 相同。要提供身分，請實作回傳 +``AuthResult`` 的 ``authenticate(...)``。 RBAC 請求的每一筆稽核紀錄都會記下使用者：``viewer_id`` 欄位存使用者 id，detail 為 ``POST /execute -> ok:200 user=alice role=operator``\ （被拒絕時為 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 475c40ca2..c1969a209 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1102,6 +1102,14 @@ AuthorizationContext, AuthorizationError, UserAuthError, UserRecord, UserStore, authorization_scope, ) +from je_auto_control.utils.rbac import ( + DeferredOwner, capture_owner, owner_scope, +) +from je_auto_control.utils.rbac.admin import ( + add_user as rbac_add_user, list_users as rbac_list_users, + remove_user as rbac_remove_user, rotate_user_token as rbac_rotate_user_token, + set_user_role as rbac_set_user_role, +) # Admin console (headless multi-host client) from je_auto_control.utils.admin import ( AdminConsoleClient, AdminHost, default_admin_console, @@ -1143,6 +1151,7 @@ require_signed_actions, sign_action_file, verify_action_file, ) from je_auto_control.utils.exception.exceptions import CryptographyUnavailableError +from je_auto_control.utils.exception.exceptions import AutoControlSignatureException # Observability (Prometheus metrics + OpenTelemetry traces, headless) from je_auto_control.utils.observability import ( Counter as MetricCounter, @@ -1767,6 +1776,9 @@ def start_autocontrol_gui(*args, **kwargs): # RBAC "AuthorizationContext", "AuthorizationError", "UserAuthError", "UserRecord", "UserStore", "authorization_scope", + "DeferredOwner", "capture_owner", "owner_scope", + "rbac_add_user", "rbac_list_users", "rbac_remove_user", + "rbac_rotate_user_token", "rbac_set_user_role", # Admin console "AdminConsoleClient", "AdminHost", "default_admin_console", # WebRTC inspector @@ -1798,7 +1810,7 @@ def start_autocontrol_gui(*args, **kwargs): # Action-file security (sign + encrypt) "VerifyResult", "sign_action_file", "verify_action_file", "SigningConfig", "action_signing_config", "create_signing_keypair", - "CryptographyUnavailableError", + "CryptographyUnavailableError", "AutoControlSignatureException", "require_signed_actions", "encrypt_action_file", "decrypt_action_file", # Observability (Prometheus + OpenTelemetry) "MetricCounter", "MetricGauge", "MetricHistogram", diff --git a/je_auto_control/cli.py b/je_auto_control/cli.py index ed6e99c65..d8d7f1a3b 100644 --- a/je_auto_control/cli.py +++ b/je_auto_control/cli.py @@ -17,6 +17,8 @@ je_auto_control list-jobs je_auto_control start-server --port 9938 je_auto_control start-rest --port 9939 + je_auto_control users [--users users.json] add alice --role admin + je_auto_control users list | set-role ID ROLE | rotate-token ID | remove ID The CLI is a thin wrapper around the headless APIs so every feature works without ever importing PySide6. @@ -338,6 +340,10 @@ def build_parser() -> argparse.ArgumentParser: p_rest.add_argument("--host", default="127.0.0.1") p_rest.add_argument("--port", type=_port, default=9939) p_rest.set_defaults(func=cmd_start_rest) + + from je_auto_control.utils.rbac.cli import add_users_arguments + add_users_arguments(sub.add_parser( + "users", help="Manage RBAC users of the REST API and MCP server")) return parser diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 6bc8d1ebe..9fd74f173 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -350,6 +350,35 @@ "Replace user config from this bundle? " "Existing files are renamed to .bak. first."), "rest_config_import_done": "Wrote {written} files; skipped {skipped}.", + "rest_token_rbac": "per-user tokens (user store: {path})", + "rest_running_rbac": "REST API is running with RBAC: the shared token is refused.", + "rest_users_group": "Users (RBAC)", + "rest_users_path": "User store:", + "rest_users_path_ph": "path of users.json (JE_AUTOCONTROL_RBAC_USERS)", + "rest_users_enable": "Authenticate REST requests against this user store", + "rest_users_col_id": "User ID", + "rest_users_col_name": "Name", + "rest_users_col_role": "Role", + "rest_users_id": "User ID:", + "rest_users_name": "Name:", + "rest_users_role": "Role:", + "rest_users_new_token": "New token (shown once):", + "rest_users_refresh": "Refresh users", + "rest_users_add": "Add user", + "rest_users_set_role": "Set role", + "rest_users_rotate": "Rotate token", + "rest_users_remove": "Remove user", + "rest_users_copy_token": "Copy new token", + "rest_users_remove_confirm": ( + "Remove user {user}? Their token stops working and work they " + "scheduled no longer runs." + ), + "rest_users_no_store": "Enter the path of a user store first.", + "rest_users_no_user": "Enter or select a user ID first.", + "rest_users_count": "{count} user(s) in {path}", + "rest_users_token_issued": ( + "Token for {user} issued. Copy it now: it cannot be shown again." + ), # Remote Desktop — WebRTC sub-tabs "rd_webrtc_host_tab": "WebRTC Host", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 0740bfc51..a1fdc86e7 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -239,6 +239,35 @@ "このバンドルでユーザー設定を置き換えますか？" "既存ファイルは .bak.<時刻> にリネームされます。"), "rest_config_import_done": "{written} ファイル書き込み、{skipped} スキップ。", + "rest_token_rbac": "ユーザーごとのトークン（ユーザーストア: {path}）", + "rest_running_rbac": "REST API は RBAC で実行中です。共有トークンは拒否されます。", + "rest_users_group": "ユーザー（RBAC）", + "rest_users_path": "ユーザーストア:", + "rest_users_path_ph": "users.json のパス（JE_AUTOCONTROL_RBAC_USERS）", + "rest_users_enable": "このユーザーストアで REST リクエストを認証する", + "rest_users_col_id": "ユーザー ID", + "rest_users_col_name": "名前", + "rest_users_col_role": "ロール", + "rest_users_id": "ユーザー ID:", + "rest_users_name": "名前:", + "rest_users_role": "ロール:", + "rest_users_new_token": "新しいトークン（一度だけ表示）:", + "rest_users_refresh": "ユーザーを再読み込み", + "rest_users_add": "ユーザーを追加", + "rest_users_set_role": "ロールを設定", + "rest_users_rotate": "トークンを再発行", + "rest_users_remove": "ユーザーを削除", + "rest_users_copy_token": "新しいトークンをコピー", + "rest_users_remove_confirm": ( + "ユーザー {user} を削除しますか？トークンは無効になり、" + "そのユーザーが登録した予約作業は実行されなくなります。" + ), + "rest_users_no_store": "先にユーザーストアのパスを入力してください。", + "rest_users_no_user": "先にユーザー ID を入力または選択してください。", + "rest_users_count": "{path} に {count} 人のユーザー", + "rest_users_token_issued": ( + "{user} のトークンを発行しました。今すぐコピーしてください。再表示はできません。" + ), # Remote Desktop — WebRTC サブタブ "rd_webrtc_host_tab": "WebRTC ホスト", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index d002ab239..ce54127bb 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -224,6 +224,35 @@ "rest_config_export_done": "已将 {count} 个文件写入 {path}。", "rest_config_import_confirm": "用此配置包覆盖用户配置？既有文件会先被改名为 .bak.<时间戳>。", "rest_config_import_done": "已写入 {written} 个文件；跳过 {skipped} 个。", + "rest_token_rbac": "每位用户各自的 token（用户文件：{path}）", + "rest_running_rbac": "REST API 以 RBAC 运行中：共用 token 会被拒绝。", + "rest_users_group": "用户（RBAC）", + "rest_users_path": "用户文件：", + "rest_users_path_ph": "users.json 的路径（JE_AUTOCONTROL_RBAC_USERS）", + "rest_users_enable": "REST 请求以这个用户文件验证", + "rest_users_col_id": "用户 ID", + "rest_users_col_name": "名称", + "rest_users_col_role": "角色", + "rest_users_id": "用户 ID：", + "rest_users_name": "名称：", + "rest_users_role": "角色：", + "rest_users_new_token": "新 token（只显示一次）：", + "rest_users_refresh": "重新加载用户", + "rest_users_add": "添加用户", + "rest_users_set_role": "设置角色", + "rest_users_rotate": "更换 token", + "rest_users_remove": "移除用户", + "rest_users_copy_token": "复制新 token", + "rest_users_remove_confirm": ( + "要移除用户 {user} 吗？其 token 会失效，" + "他安排的任务也不会再执行。" + ), + "rest_users_no_store": "请先输入用户文件的路径。", + "rest_users_no_user": "请先输入或选择用户 ID。", + "rest_users_count": "{path} 内有 {count} 位用户", + "rest_users_token_issued": ( + "已发出 {user} 的 token。请立即复制，之后无法再显示。" + ), # Remote Desktop — WebRTC 子分页 "rd_webrtc_host_tab": "WebRTC 被远程", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 046af99cd..3210a3c72 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -225,6 +225,35 @@ "rest_config_export_done": "已將 {count} 個檔案寫入 {path}。", "rest_config_import_confirm": "用此設定包覆寫使用者設定？既有檔案會先被改名為 .bak.<時間戳>。", "rest_config_import_done": "已寫入 {written} 個檔案；略過 {skipped} 個。", + "rest_token_rbac": "每位使用者各自的 token（使用者檔：{path}）", + "rest_running_rbac": "REST API 以 RBAC 執行中：共用 token 會被拒絕。", + "rest_users_group": "使用者（RBAC）", + "rest_users_path": "使用者檔：", + "rest_users_path_ph": "users.json 的路徑（JE_AUTOCONTROL_RBAC_USERS）", + "rest_users_enable": "REST 請求以這個使用者檔驗證", + "rest_users_col_id": "使用者 ID", + "rest_users_col_name": "名稱", + "rest_users_col_role": "角色", + "rest_users_id": "使用者 ID：", + "rest_users_name": "名稱：", + "rest_users_role": "角色：", + "rest_users_new_token": "新 token（只顯示一次）：", + "rest_users_refresh": "重新載入使用者", + "rest_users_add": "新增使用者", + "rest_users_set_role": "設定角色", + "rest_users_rotate": "更換 token", + "rest_users_remove": "移除使用者", + "rest_users_copy_token": "複製新 token", + "rest_users_remove_confirm": ( + "要移除使用者 {user} 嗎？其 token 會失效，" + "他排定的工作也不會再執行。" + ), + "rest_users_no_store": "請先輸入使用者檔的路徑。", + "rest_users_no_user": "請先輸入或選取使用者 ID。", + "rest_users_count": "{path} 內有 {count} 位使用者", + "rest_users_token_issued": ( + "已發出 {user} 的 token。請立即複製，之後無法再顯示。" + ), # Remote Desktop — WebRTC 子分頁 "rd_webrtc_host_tab": "WebRTC 被遠端", diff --git a/je_auto_control/gui/rbac_users_panel.py b/je_auto_control/gui/rbac_users_panel.py new file mode 100644 index 000000000..4976f6268 --- /dev/null +++ b/je_auto_control/gui/rbac_users_panel.py @@ -0,0 +1,228 @@ +"""RBAC users group of the REST API tab: list, add, re-role, rotate, remove. + +A thin Qt surface over :mod:`je_auto_control.utils.rbac.admin` -- the same +functions ``AC_user_*`` and ``je_auto_control users`` call. The desktop user +is not an authenticated caller, so nothing is authorised here: whoever can +open this window can already edit the store file. + +A token is shown once, in the "new token" field, right after ``add`` or +``rotate``; the next operation clears it. It is never written to a log. +""" +import os +from typing import List, Optional, Tuple + +from PySide6.QtCore import Qt +from PySide6.QtGui import QGuiApplication +from PySide6.QtWidgets import ( + QAbstractItemView, QCheckBox, QComboBox, QGroupBox, QHBoxLayout, QLabel, + QLineEdit, QMessageBox, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, +) + +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.rbac import admin +from je_auto_control.utils.rbac.authorization import USERS_ENV +from je_auto_control.utils.rbac.users import Role, UserStore + +_COLUMNS = ("rest_users_col_id", "rest_users_col_name", "rest_users_col_role") +_ERRORS = (AutoControlException, OSError, ValueError) + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class RbacUsersPanel(TranslatableMixin, QGroupBox): + """The user store behind the REST API: its path, its users, one-time tokens.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tr_init() + self._tr(self, "rest_users_group", "setTitle") + self._path_input = QLineEdit(os.environ.get(USERS_ENV, "").strip()) + self._path_input.setPlaceholderText(_t("rest_users_path_ph")) + self._enable_check = QCheckBox() + self._enable_check.setChecked(bool(self._path_input.text())) + self._table = QTableWidget(0, len(_COLUMNS)) + self._table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) + self._table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) + self._table.itemSelectionChanged.connect(self._on_row_selected) + self._id_input = QLineEdit() + self._name_input = QLineEdit() + self._role_input = QComboBox() + self._role_input.addItems(Role.all()) + self._token_value = QLineEdit() + self._token_value.setReadOnly(True) + self._status_label = QLabel() + self._status_label.setWordWrap(True) + self._status_label.setTextInteractionFlags(Qt.TextSelectableByMouse) + self._build_layout() + self._retitle_columns() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + path_row = QHBoxLayout() + path_row.addWidget(self._tr(QLabel(), "rest_users_path")) + path_row.addWidget(self._path_input, stretch=1) + root.addLayout(path_row) + self._tr(self._enable_check, "rest_users_enable") + root.addWidget(self._enable_check) + root.addWidget(self._table) + edit_row = QHBoxLayout() + edit_row.addWidget(self._tr(QLabel(), "rest_users_id")) + edit_row.addWidget(self._id_input, stretch=1) + edit_row.addWidget(self._tr(QLabel(), "rest_users_name")) + edit_row.addWidget(self._name_input, stretch=1) + edit_row.addWidget(self._tr(QLabel(), "rest_users_role")) + edit_row.addWidget(self._role_input) + root.addLayout(edit_row) + token_row = QHBoxLayout() + token_row.addWidget(self._tr(QLabel(), "rest_users_new_token")) + token_row.addWidget(self._token_value, stretch=1) + root.addLayout(token_row) + root.addWidget(self._status_label) + + def retranslate(self) -> None: + """Re-apply the registered keys and the table's column titles.""" + super().retranslate() + self._retitle_columns() + + def _retitle_columns(self) -> None: + self._table.setHorizontalHeaderLabels([_t(key) for key in _COLUMNS]) + + def menu_actions(self) -> List[Tuple[str, object]]: + """The panel's commands, for the tab's Actions menu.""" + return [ + ("rest_users_refresh", self.refresh), + ("rest_users_add", self._on_add), + ("rest_users_set_role", self._on_set_role), + ("rest_users_rotate", self._on_rotate), + ("rest_users_remove", self._on_remove), + ("rest_users_copy_token", self._on_copy_token), + ] + + def users_path(self) -> Optional[str]: + """The store path typed into the panel, or ``None`` when it is empty.""" + return self._path_input.text().strip() or None + + def user_store(self) -> Optional[UserStore]: + """The store the REST server should authenticate against, when ticked.""" + path = self.users_path() + if path is None or not self._enable_check.isChecked(): + return None + return admin.management_store(path) + + def refresh(self) -> None: + """Re-read the store and show its users; forgets a token still on show.""" + self._token_value.clear() + self._reload() + + def _reload(self) -> bool: + path = self.users_path() + if path is None: + self._table.setRowCount(0) + self._status_label.setText(_t("rest_users_no_store")) + return False + try: + users = admin.list_users(users_path=path) + except _ERRORS as error: + self._table.setRowCount(0) + self._status_label.setText(str(error)) + return False + self._table.setRowCount(len(users)) + for row, user in enumerate(users): + for column, key in enumerate(("user_id", "display_name", "role")): + self._table.setItem(row, column, QTableWidgetItem(str(user[key]))) + self._status_label.setText( + _t("rest_users_count").format(count=len(users), path=path)) + return True + + def _on_row_selected(self) -> None: + row = self._table.currentRow() + if row < 0 or self._table.item(row, 0) is None: + return + self._id_input.setText(self._table.item(row, 0).text()) + self._name_input.setText(self._table.item(row, 1).text()) + self._role_input.setCurrentText(self._table.item(row, 2).text()) + + def _target(self) -> Optional[Tuple[str, str]]: + """``(store path, user id)`` of the operation, or ``None`` with the reason shown.""" + self._token_value.clear() + path = self.users_path() + if path is None: + self._status_label.setText(_t("rest_users_no_store")) + return None + user_id = self._id_input.text().strip() + if not user_id: + self._status_label.setText(_t("rest_users_no_user")) + return None + return path, user_id + + def _show_token(self, issued: dict) -> None: + self._reload() + self._token_value.setText(str(issued["token"])) + self._status_label.setText( + _t("rest_users_token_issued").format(user=issued["user_id"])) + + def _on_add(self) -> None: + target = self._target() + if target is None: + return + try: + issued = admin.add_user( + target[1], role=self._role_input.currentText(), + display_name=self._name_input.text().strip(), users_path=target[0]) + except _ERRORS as error: + self._status_label.setText(str(error)) + return + self._show_token(issued) + + def _on_rotate(self) -> None: + target = self._target() + if target is None: + return + try: + issued = admin.rotate_user_token(target[1], users_path=target[0]) + except _ERRORS as error: + self._status_label.setText(str(error)) + return + self._show_token(issued) + + def _on_set_role(self) -> None: + target = self._target() + if target is None: + return + try: + admin.set_user_role(target[1], self._role_input.currentText(), + users_path=target[0]) + except _ERRORS as error: + self._status_label.setText(str(error)) + return + self._reload() + + def _on_remove(self) -> None: + target = self._target() + if target is None: + return + confirm = QMessageBox.question( + self, _t("rest_users_remove"), + _t("rest_users_remove_confirm").format(user=target[1])) + if confirm != QMessageBox.StandardButton.Yes: + return + try: + admin.remove_user(target[1], users_path=target[0]) + except _ERRORS as error: + self._status_label.setText(str(error)) + return + self._reload() + + def _on_copy_token(self) -> None: + text = self._token_value.text() + if text: + QGuiApplication.clipboard().setText(text) + + +__all__ = ["RbacUsersPanel"] diff --git a/je_auto_control/gui/rest_api_tab.py b/je_auto_control/gui/rest_api_tab.py index cd1274418..2927b6b77 100644 --- a/je_auto_control/gui/rest_api_tab.py +++ b/je_auto_control/gui/rest_api_tab.py @@ -1,4 +1,10 @@ -"""REST API tab: start/stop the HTTP front-end and surface URL + token.""" +"""REST API tab: start/stop the HTTP front-end, show how to authenticate, manage users. + +Without a user store the server has one shared bearer token and the tab +shows it. With one (RBAC) the shared token opens nothing, so the tab says +whose tokens are accepted instead of displaying a credential the server +refuses; the users themselves are managed in the group below. +""" from typing import Optional import json @@ -15,6 +21,7 @@ from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( language_wrapper, ) +from je_auto_control.gui.rbac_users_panel import RbacUsersPanel from je_auto_control.utils.config_bundle import ( ConfigBundleError, export_config_bundle, import_config_bundle, ) @@ -44,8 +51,12 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._token_value = QLabel("-") self._token_value.setTextInteractionFlags(Qt.TextSelectableByMouse) self._status_label = QLabel() + # The token label holds a copyable token only under the shared token. + self._shared_token: Optional[str] = None + self._users_panel = RbacUsersPanel() self._build_layout() self._refresh_status() + self._users_panel.refresh() self._timer = QTimer(self) self._timer.setInterval(2000) self._timer.timeout.connect(self._refresh_status) @@ -57,7 +68,7 @@ def _build_layout(self) -> None: root = QVBoxLayout(self) root.addWidget(self._build_config_group()) root.addWidget(self._build_status_group()) - root.addStretch(1) + root.addWidget(self._users_panel, stretch=1) def menu_actions(self) -> list: """Expose tab commands to the window-level Actions menu.""" @@ -68,8 +79,15 @@ def menu_actions(self) -> list: ("rest_copy_token", self._on_copy_token), ("rest_config_export", self._on_config_export), ("rest_config_import", self._on_config_import), + *self._users_panel.menu_actions(), ] + def retranslate(self) -> None: + """Re-translate the tab, its users group and the status texts.""" + super().retranslate() + self._users_panel.retranslate() + self._refresh_status() + def _build_config_group(self) -> QGroupBox: group = self._tr(QGroupBox(), "rest_config_group") form = QVBoxLayout(group) @@ -162,6 +180,7 @@ def _on_start(self) -> None: rest_api_registry.start( host=host, port=port, token=token, enable_audit=self._audit_check.isChecked(), + user_store=self._users_panel.user_store(), ) except OSError as error: QMessageBox.warning(self, _t("rest_start"), str(error)) @@ -178,20 +197,26 @@ def _on_copy_url(self) -> None: QGuiApplication.clipboard().setText(text) def _on_copy_token(self) -> None: - text = self._token_value.text() - if text and text != "-": - QGuiApplication.clipboard().setText(text) + if self._shared_token: + QGuiApplication.clipboard().setText(self._shared_token) def _refresh_status(self) -> None: status = rest_api_registry.status() - if status["running"]: - self._url_value.setText(status["url"]) - self._token_value.setText(status["token"]) - self._status_label.setText(_t("rest_running")) - else: + self._shared_token = None + if not status["running"]: self._url_value.setText("-") self._token_value.setText("-") self._status_label.setText(_t("rest_stopped")) + return + self._url_value.setText(status["url"]) + if status.get("rbac"): + self._token_value.setText( + _t("rest_token_rbac").format(path=status.get("users_path"))) + self._status_label.setText(_t("rest_running_rbac")) + return + self._shared_token = status["token"] + self._token_value.setText(status["token"]) + self._status_label.setText(_t("rest_running")) __all__ = ["RestApiTab"] diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index a9638555d..92944adb4 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -5062,6 +5062,8 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: fields=( FieldSpec("private_path", FieldType.STRING), FieldSpec("public_path", FieldType.STRING), + FieldSpec("passphrase", FieldType.STRING, optional=True, + placeholder="encrypts the private key file"), ), description="Create an Ed25519 key pair: keep the private key on the " "signing machine, give execution endpoints the public key.", @@ -5072,6 +5074,8 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: FieldSpec("path", FieldType.FILE_PATH), FieldSpec("key", FieldType.STRING, optional=True), FieldSpec("private_key_path", FieldType.FILE_PATH, optional=True), + FieldSpec("passphrase", FieldType.STRING, optional=True, + placeholder="of an encrypted private key"), ), description="Write a signature sidecar for an action file " "(Ed25519 with a private key, else HMAC-SHA256).", diff --git a/je_auto_control/utils/action_signing/asymmetric.py b/je_auto_control/utils/action_signing/asymmetric.py index a6b327eb8..e6f298887 100644 --- a/je_auto_control/utils/action_signing/asymmetric.py +++ b/je_auto_control/utils/action_signing/asymmetric.py @@ -11,8 +11,11 @@ ``signature`` is base64 of the Ed25519 signature over a fixed context line followed by the file's exact bytes; ``key_id`` names the public key (the first 16 hex digits of its SHA-256) so a file signed by another pair says so instead -of reading as tampered. Keys are unencrypted PEM: PKCS#8 for the private half -(created 0600), SubjectPublicKeyInfo for the public half. +of reading as tampered. Keys are PEM: PKCS#8 for the private half (created +0600), SubjectPublicKeyInfo for the public half. The private half is +encrypted when a passphrase is given at creation; the signer then needs the +same passphrase (an argument, or ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE``). +A key created without one stays loadable as it is. ``cryptography`` is imported lazily -- it has no Windows arm64 wheel, and HMAC signing must keep working without it. This module is GUI-free and imports no Qt. @@ -26,7 +29,9 @@ from typing import Any, Dict, Optional, Union from je_auto_control.utils.action_signing._key_file import write_new_file -from je_auto_control.utils.action_signing.config import action_signing_config +from je_auto_control.utils.action_signing.config import ( + PASSPHRASE_ENV, action_signing_config, signing_passphrase, +) from je_auto_control.utils.exception.exceptions import ( AutoControlException, CryptographyUnavailableError, ) @@ -38,6 +43,15 @@ _KEY_ID_LENGTH = 16 PathLike = Union[str, Path] +Passphrase = Optional[Union[str, bytes]] + + +def _passphrase_bytes(passphrase: Passphrase) -> Optional[bytes]: + """``passphrase`` as bytes; ``None`` and the empty string mean "none".""" + if passphrase is None: + return None + raw = passphrase if isinstance(passphrase, bytes) else str(passphrase).encode("utf-8") + return raw or None def _ed25519() -> SimpleNamespace: @@ -58,13 +72,17 @@ def _ed25519() -> SimpleNamespace: ) -def create_signing_keypair(private_path: PathLike, public_path: PathLike) -> None: +def create_signing_keypair(private_path: PathLike, public_path: PathLike, + *, passphrase: Passphrase = None) -> None: """Create an Ed25519 key pair as two PEM files; neither may exist yet. Keep ``private_path`` on the machine that signs and copy ``public_path`` - to every execution endpoint. Raises :class:`AutoControlException` when a - file is already there, or when this process is configured to verify only - -- a key pair minted on an execution endpoint separates nothing. + to every execution endpoint. With ``passphrase`` the private key file is + encrypted, so the file alone -- a backup, a copied disk -- signs + nothing; without one it is plain PEM as before. Raises + :class:`AutoControlException` when a file is already there, or when this + process is configured to verify only -- a key pair minted on an execution + endpoint separates nothing. """ if action_signing_config().verify_only: raise AutoControlException( @@ -78,9 +96,11 @@ def create_signing_keypair(private_path: PathLike, public_path: PathLike) -> Non crypto = _ed25519() key = crypto.private_key.generate() encoding = crypto.serialization.Encoding.PEM + secret = _passphrase_bytes(passphrase) + protection = (crypto.serialization.NoEncryption() if secret is None + else crypto.serialization.BestAvailableEncryption(secret)) write_new_file(private, key.private_bytes( - encoding, crypto.serialization.PrivateFormat.PKCS8, - crypto.serialization.NoEncryption()), 0o600) + encoding, crypto.serialization.PrivateFormat.PKCS8, protection), 0o600) try: write_new_file(public, key.public_key().public_bytes( encoding, crypto.serialization.PublicFormat.SubjectPublicKeyInfo), 0o644) @@ -98,15 +118,30 @@ def _read_key_file(path: Path, kind: str) -> bytes: f"cannot read the {kind} signing key {str(path)!r}: {error}") from error -def _load_private_key(path: PathLike) -> Any: +def _load_private_key(path: PathLike, passphrase: Passphrase = None) -> Any: + """Load the private key; an encrypted one needs ``passphrase`` or the variable. + + The passphrase is only handed to the loader for a file that says it is + encrypted: ``cryptography`` refuses a password for a plain key, and a + signer whose environment carries a passphrase must still load a key that + was created without one. + """ crypto = _ed25519() target = Path(path) + data = _read_key_file(target, "private") + secret: Optional[bytes] = None + if b"ENCRYPTED" in data: + secret = _passphrase_bytes(passphrase) or signing_passphrase() + if secret is None: + raise AutoControlException( + f"the private signing key {str(target)!r} is passphrase-protected; " + f"pass the passphrase or set {PASSPHRASE_ENV}") try: - key = crypto.serialization.load_pem_private_key( - _read_key_file(target, "private"), password=None) + key = crypto.serialization.load_pem_private_key(data, password=secret) except (ValueError, TypeError) as error: - raise AutoControlException( - f"{str(target)!r} is not an unencrypted PEM private key") from error + problem = ("could not be decrypted (wrong passphrase?)" if secret is not None + else "is not a PEM private key") + raise AutoControlException(f"{str(target)!r} {problem}") from error if not isinstance(key, crypto.private_key): raise AutoControlException(f"{str(target)!r} is not an Ed25519 private key") return key @@ -131,9 +166,10 @@ def _key_id(public_key: Any) -> str: return hashlib.sha256(raw).hexdigest()[:_KEY_ID_LENGTH] -def sign_envelope(data: bytes, private_key_path: PathLike) -> str: +def sign_envelope(data: bytes, private_key_path: PathLike, + passphrase: Passphrase = None) -> str: """Return the version-2 sidecar text for ``data``, signed by the private key.""" - key = _load_private_key(private_key_path) + key = _load_private_key(private_key_path, passphrase) signature = key.sign(_CONTEXT + data) return json.dumps({ "version": SIGNATURE_VERSION, diff --git a/je_auto_control/utils/action_signing/config.py b/je_auto_control/utils/action_signing/config.py index 97b15307d..84cf80099 100644 --- a/je_auto_control/utils/action_signing/config.py +++ b/je_auto_control/utils/action_signing/config.py @@ -1,6 +1,6 @@ """Which signing keys this process was configured with, read at call time. -Three environment variables separate the machine that signs from the machines +These environment variables separate the machine that signs from the machines that execute: ``JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY`` @@ -8,6 +8,9 @@ ``JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY`` Path of the matching public key. Set it on every execution endpoint; an endpoint that has this and no private key verifies and cannot sign. +``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE`` + Passphrase of the private key file, when it was created with one. Only + the signing machine needs it; verifying uses the public key. ``JE_AUTOCONTROL_ACCEPT_LEGACY_ACTION_SIGNATURES`` Migration mode: also accept HMAC sidecars written before version 2. Anyone who can execute actions can produce one, so switch it off again @@ -23,6 +26,7 @@ PRIVATE_KEY_ENV = "JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY" PUBLIC_KEY_ENV = "JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY" +PASSPHRASE_ENV = "JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE" # nosec B105 # reason: a variable's name ACCEPT_LEGACY_ENV = "JE_AUTOCONTROL_ACCEPT_LEGACY_ACTION_SIGNATURES" _SWITCHED_ON = frozenset({"1", "true", "yes", "on"}) @@ -52,6 +56,15 @@ def _path_from(name: str) -> Optional[Path]: return Path(value) if value else None +def signing_passphrase() -> Optional[bytes]: + """The private key's passphrase from the environment, or ``None`` when unset. + + Kept out of :class:`SigningConfig`, whose ``repr`` ends up in logs. + """ + value = os.environ.get(PASSPHRASE_ENV, "") + return value.encode("utf-8") if value else None + + def action_signing_config() -> SigningConfig: """Return the signing configuration the environment describes right now.""" legacy = os.environ.get(ACCEPT_LEGACY_ENV, "").strip().lower() in _SWITCHED_ON diff --git a/je_auto_control/utils/action_signing/signer.py b/je_auto_control/utils/action_signing/signer.py index eba364d6f..bef18d5d1 100644 --- a/je_auto_control/utils/action_signing/signer.py +++ b/je_auto_control/utils/action_signing/signer.py @@ -34,7 +34,9 @@ from je_auto_control.utils.action_signing.config import ( ACCEPT_LEGACY_ENV, PUBLIC_KEY_ENV, action_signing_config, ) -from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlSignatureException, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -89,14 +91,15 @@ def _digest(data: bytes, key: bytes) -> str: return hmac.new(key, data, hashlib.sha256).hexdigest() -def _signature_text(data: bytes, key: KeyType, private_key_path: KeyPath) -> str: +def _signature_text(data: bytes, key: KeyType, private_key_path: KeyPath, + passphrase: Optional[Union[bytes, str]] = None) -> str: """Sign ``data`` with the key pair when one applies, else with HMAC.""" config = action_signing_config() private = private_key_path if private is None and key is None: private = config.private_key_path if private is not None: - return asymmetric.sign_envelope(data, private) + return asymmetric.sign_envelope(data, private, passphrase) if config.verify_only: raise AutoControlException( "this endpoint verifies only (a public key and no private key are " @@ -106,7 +109,8 @@ def _signature_text(data: bytes, key: KeyType, private_key_path: KeyPath) -> str def sign_action_file(path: Union[str, Path], key: KeyType = None, - *, private_key_path: KeyPath = None) -> str: + *, private_key_path: KeyPath = None, + passphrase: Optional[Union[bytes, str]] = None) -> str: """Write a signature sidecar for the file at ``path``. With ``private_key_path`` -- or, when no ``key`` is given, the private key @@ -114,10 +118,12 @@ def sign_action_file(path: Union[str, Path], key: KeyType = None, version-2 Ed25519 envelope. Otherwise it is the HMAC-SHA256 of the file under ``key`` or the per-user key. An endpoint configured with a public key and no private key refuses instead of falling back to HMAC, raising - :class:`AutoControlException`. Returns the sidecar path (``.sig``). + :class:`AutoControlException`. ``passphrase`` unlocks a private key file + created with one; without it ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE`` + is used. Returns the sidecar path (``.sig``). """ target = Path(path) - signature = _signature_text(target.read_bytes(), key, private_key_path) + signature = _signature_text(target.read_bytes(), key, private_key_path, passphrase) sig_path = _sig_path(target) sig_path.write_text(signature, encoding="utf-8") autocontrol_logger.info("signed action file %s", target) @@ -134,7 +140,7 @@ def verify_action_file(path: Union[str, Path], key: KeyType = None, ``key`` or the per-user key, and refused when a key pair is configured unless migration mode is on. Returns a :class:`VerifyResult`. With ``raise_on_fail`` set, an unverified file raises - :class:`AutoControlException` instead. + :class:`AutoControlSignatureException` (an ``AutoControlException``) instead. """ try: data = Path(path).read_bytes() @@ -185,7 +191,7 @@ def _verify_bytes(path: Union[str, Path], data: bytes, key: KeyType, def _fail(path: Union[str, Path], reason: str, raise_on_fail: bool) -> VerifyResult: if raise_on_fail: - raise AutoControlException( + raise AutoControlSignatureException( f"action file {path!r} failed verification: {reason}", ) autocontrol_logger.info("action file %r unverified: %s", path, reason) diff --git a/je_auto_control/utils/exception/exceptions.py b/je_auto_control/utils/exception/exceptions.py index 86e6ef541..25bff660f 100644 --- a/je_auto_control/utils/exception/exceptions.py +++ b/je_auto_control/utils/exception/exceptions.py @@ -131,6 +131,17 @@ class AutoControlUnsupportedOperationException( """ +# Action-file signatures +class AutoControlSignatureException(AutoControlException): + """An action file that had to be signed failed verification and was not run. + + Raised where a file is refused -- a missing sidecar, a mismatch, a key + this endpoint does not trust -- so a server can answer "refused" rather + than "crashed". Still an :class:`AutoControlException`, which is what + this was before the type existed. + """ + + # Optional dependency class CryptographyUnavailableError(AutoControlException, RuntimeError): """A feature needs ``cryptography`` and the package is not installed. diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cc475cf85..812a39c43 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -1422,6 +1422,40 @@ def _admin_broadcast_execute(actions: List[Any], ) +def _user_add(user_id: str, role: str = "viewer", display_name: str = "", + tags: Optional[List[str]] = None, + users_path: Optional[str] = None) -> Dict[str, Any]: + """Executor adapter: add an RBAC user; the reply carries its token, once.""" + from je_auto_control.utils.rbac.admin import add_user + return add_user(user_id, role=role, display_name=display_name, tags=tags, + users_path=users_path) + + +def _user_remove(user_id: str, users_path: Optional[str] = None) -> Dict[str, Any]: + """Executor adapter: remove an RBAC user.""" + from je_auto_control.utils.rbac.admin import remove_user + return remove_user(user_id, users_path=users_path) + + +def _user_set_role(user_id: str, role: str, + users_path: Optional[str] = None) -> Dict[str, Any]: + """Executor adapter: change an RBAC user's role.""" + from je_auto_control.utils.rbac.admin import set_user_role + return set_user_role(user_id, role, users_path=users_path) + + +def _user_rotate_token(user_id: str, users_path: Optional[str] = None) -> Dict[str, Any]: + """Executor adapter: replace an RBAC user's token; the reply carries it, once.""" + from je_auto_control.utils.rbac.admin import rotate_user_token + return rotate_user_token(user_id, users_path=users_path) + + +def _user_list(users_path: Optional[str] = None) -> List[Dict[str, Any]]: + """Executor adapter: list RBAC users (no tokens, no hashes).""" + from je_auto_control.utils.rbac.admin import list_users + return list_users(users_path=users_path) + + def _audit_log_list(event_type: Optional[str] = None, host_id: Optional[str] = None, limit: int = 200) -> List[Dict[str, Any]]: @@ -2214,17 +2248,19 @@ def _human_type(text: str, base_delay: float = 0.05, jitter: float = 0.04, def _sign_action_file(path: str, key: Optional[str] = None, - private_key_path: Optional[str] = None) -> Dict[str, Any]: + private_key_path: Optional[str] = None, + passphrase: Optional[str] = None) -> Dict[str, Any]: """Executor adapter: write an Ed25519 or HMAC signature sidecar for a file.""" from je_auto_control.utils.action_signing import sign_action_file return {"signature_path": sign_action_file( - path, key, private_key_path=private_key_path)} + path, key, private_key_path=private_key_path, passphrase=passphrase)} -def _create_signing_keypair(private_path: str, public_path: str) -> Dict[str, Any]: +def _create_signing_keypair(private_path: str, public_path: str, + passphrase: Optional[str] = None) -> Dict[str, Any]: """Executor adapter: create an Ed25519 action-signing key pair.""" from je_auto_control.utils.action_signing import create_signing_keypair - create_signing_keypair(private_path, public_path) + create_signing_keypair(private_path, public_path, passphrase=passphrase) return {"private_path": str(private_path), "public_path": str(public_path)} @@ -8027,6 +8063,13 @@ def __init__(self): "AC_admin_poll": _admin_poll, "AC_admin_broadcast_execute": _admin_broadcast_execute, + # RBAC user management (needs manage_users under RBAC) + "AC_user_add": _user_add, + "AC_user_remove": _user_remove, + "AC_user_set_role": _user_set_role, + "AC_user_rotate_token": _user_rotate_token, + "AC_user_list": _user_list, + # Audit log (tamper-evident security log) "AC_audit_log_list": _audit_log_list, "AC_audit_log_verify": _audit_log_verify, diff --git a/je_auto_control/utils/hotkey/hotkey_daemon.py b/je_auto_control/utils/hotkey/hotkey_daemon.py index e2211e03e..26aaa675a 100644 --- a/je_auto_control/utils/hotkey/hotkey_daemon.py +++ b/je_auto_control/utils/hotkey/hotkey_daemon.py @@ -14,7 +14,7 @@ import sys import threading import uuid -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Callable, Dict, FrozenSet, List, Optional, Tuple from je_auto_control.utils.exception.exceptions import AutoControlException @@ -23,6 +23,7 @@ from je_auto_control.utils.run_history.artifact_manager import ( capture_error_snapshot, ) +from je_auto_control.utils.rbac.deferred import DeferredOwner, capture_owner from je_auto_control.utils.run_history.run_outcome import run_counting_failures from je_auto_control.utils.run_history.history_store import ( SOURCE_HOTKEY, STATUS_ERROR, STATUS_OK, default_history_store, @@ -52,6 +53,8 @@ class HotkeyBinding: script_path: str enabled: bool = True fired: int = 0 + #: Who registered it under RBAC; the run is authorised as that user. + owner: Optional[DeferredOwner] = field(default=None, kw_only=True) def split_combo(combo: str) -> Tuple[FrozenSet[str], str]: @@ -157,6 +160,7 @@ def bind(self, combo: str, script_path: str, bid = binding_id or uuid.uuid4().hex[:8] binding = HotkeyBinding( binding_id=bid, combo=combo, script_path=script_path, + owner=capture_owner(), ) with self._lock: self._bindings[bid] = binding @@ -219,7 +223,7 @@ def _fire_binding(self, binding_id: str) -> None: error_text: Optional[str] = None try: actions = read_executable_action_json(match.script_path) - run_counting_failures(lambda: self._execute(actions)) + run_counting_failures(lambda: self._execute(actions), owner=match.owner) except Exception as error: # noqa: BLE001 # reason: this runs on the backend's listener thread; any escape ends every hotkey # AutoControlException covers the common cases — a missing/renamed # script (AutoControlJsonActionException) or an action that raises diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fecbc25ae..bbb2d4b92 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -5722,6 +5722,60 @@ def credential_lease_tools() -> List[MCPTool]: ] +def user_admin_tools() -> List[MCPTool]: + _ID = {"user_id": {"type": "string"}} + _ROLE = {"type": "string", "enum": ["viewer", "operator", "admin"]} + return [ + MCPTool( + name="ac_user_add", + description=("RBAC: add a user with a 'role' (viewer / operator / " + "admin). Returns {user_id, display_name, role, tags, " + "token}; the token is shown this once and only its " + "hash is stored. Needs the manage_users capability."), + input_schema=schema({**_ID, "role": _ROLE, + "display_name": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}}}, + ["user_id"]), + handler=h_ops.user_add, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_user_remove", + description=("RBAC: remove a user; their token stops working and " + "work they scheduled no longer runs. The last admin " + "cannot be removed. Returns {user_id, removed}."), + input_schema=schema(_ID, ["user_id"]), + handler=h_ops.user_remove, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_user_set_role", + description=("RBAC: change a user's role; it applies to their next " + "request and to work they already scheduled. The last " + "admin cannot be demoted. Returns {user_id, role}."), + input_schema=schema({**_ID, "role": _ROLE}, ["user_id", "role"]), + handler=h_ops.user_set_role, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_user_rotate_token", + description=("RBAC: replace a user's token; the old one stops " + "working. Returns {user_id, token}, shown this once."), + input_schema=schema(_ID, ["user_id"]), + handler=h_ops.user_rotate_token, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_user_list", + description=("RBAC: list users as {users:[{user_id, display_name, " + "role, tags}]} - no tokens. Needs manage_users."), + input_schema=schema({}), + handler=h_ops.user_list, + annotations=READ_ONLY, + ), + ] + + def egress_tools() -> List[MCPTool]: _LISTS = {"allow": {"type": "array", "items": {"type": "string"}}, "deny": {"type": "array", "items": {"type": "string"}}} @@ -9028,7 +9082,7 @@ def media_assert_tools() -> List[MCPTool]: observation_tools, action_grounding_tools, agent_replay_tools, element_diff_tools, element_scoring_tools, barcode_tools, plugin_sdk_tools, governance_tools, - credential_lease_tools, egress_tools, approval_testing_tools, + credential_lease_tools, egress_tools, user_admin_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, url_canon_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_operations.py b/je_auto_control/utils/mcp_server/tools/_handlers_operations.py index 60f1cfa3e..e299c4afb 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_operations.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_operations.py @@ -238,6 +238,31 @@ def lease_active(): return {"leases": default_broker.active()} +def user_add(user_id: str, role: str = "viewer", display_name: str = "", tags=None): + from je_auto_control.utils.rbac.admin import add_user + return dict(add_user(user_id, role=role, display_name=display_name, tags=tags)) + + +def user_remove(user_id: str): + from je_auto_control.utils.rbac.admin import remove_user + return remove_user(user_id) + + +def user_set_role(user_id: str, role: str): + from je_auto_control.utils.rbac.admin import set_user_role + return set_user_role(user_id, role) + + +def user_rotate_token(user_id: str): + from je_auto_control.utils.rbac.admin import rotate_user_token + return dict(rotate_user_token(user_id)) + + +def user_list(): + from je_auto_control.utils.rbac.admin import list_users + return {"users": list_users()} + + def egress_allow(allow=None, deny=None): from je_auto_control.utils.egress import set_egress_policy policy = set_egress_policy(allow, deny) diff --git a/je_auto_control/utils/rbac/__init__.py b/je_auto_control/utils/rbac/__init__.py index beb31a9d7..ba701f7bf 100644 --- a/je_auto_control/utils/rbac/__init__.py +++ b/je_auto_control/utils/rbac/__init__.py @@ -20,15 +20,28 @@ command is checked against the capability it needs (:mod:`.policy`), and the audit entries carry the ``user_id``. +Work registered by an authenticated user and run later -- a scheduler job, a +trigger, a hotkey, a webhook, a watchdog rule -- runs as that user, with the +role they hold when it fires (:mod:`.deferred`). Users are managed through +:mod:`.admin`: ``AC_user_*``, the ``ac_user_*`` MCP tools, +``je_auto_control users`` and the REST API tab, all gated by ``manage_users``. + The store is intentionally tiny — no LDAP, no OAuth, no row-level permissions. Operators who need more should stand up a proper IdP in front of the REST endpoint; this is the "good-enough-for-small-team" baseline. """ +from je_auto_control.utils.rbac.admin import ( + IssuedToken, add_user, list_users, management_store, remove_user, + rotate_user_token, set_user_role, +) from je_auto_control.utils.rbac.authorization import ( USERS_ENV, AuthorizationContext, AuthorizationError, authorization_scope, current_authorization, resolve_token, user_store_from_env, ) +from je_auto_control.utils.rbac.deferred import ( + DeferredOwner, capture_owner, owner_scope, resolve_owner, +) from je_auto_control.utils.rbac.policy import ( authorize_command, capability_for_command, capability_for_route, capability_for_tool, denied_command_in, @@ -39,10 +52,12 @@ ) __all__ = [ - "AuthorizationContext", "AuthorizationError", "Capability", "Role", - "USERS_ENV", "UserAuthError", "UserRecord", "UserStore", - "authorization_scope", "authorize_command", "can", + "AuthorizationContext", "AuthorizationError", "Capability", "DeferredOwner", + "IssuedToken", "Role", "USERS_ENV", "UserAuthError", "UserRecord", "UserStore", + "add_user", "authorization_scope", "authorize_command", "can", "capability_for_command", "capability_for_route", "capability_for_tool", - "current_authorization", "default_user_store", "denied_command_in", - "resolve_token", "role_capabilities", "user_store_from_env", + "capture_owner", "current_authorization", "default_user_store", + "denied_command_in", "list_users", "management_store", "owner_scope", + "remove_user", "resolve_owner", "resolve_token", "role_capabilities", + "rotate_user_token", "set_user_role", "user_store_from_env", ] diff --git a/je_auto_control/utils/rbac/__main__.py b/je_auto_control/utils/rbac/__main__.py new file mode 100644 index 000000000..daeff768e --- /dev/null +++ b/je_auto_control/utils/rbac/__main__.py @@ -0,0 +1,5 @@ +"""``python -m je_auto_control.utils.rbac``: the ``je_auto_control users`` commands.""" +from je_auto_control.utils.rbac.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/je_auto_control/utils/rbac/admin.py b/je_auto_control/utils/rbac/admin.py new file mode 100644 index 000000000..ae7e80de5 --- /dev/null +++ b/je_auto_control/utils/rbac/admin.py @@ -0,0 +1,167 @@ +"""Managing the users of a store: add, remove, change role, rotate token, list. + +One implementation behind every surface -- the ``AC_user_*`` commands, the +``ac_user_*`` MCP tools, ``je_auto_control users ...`` and the GUI panel -- +so the rules below hold whichever one is used: + +* **Who may.** Inside an RBAC scope the caller needs ``manage_users`` (the + admin role) and works on the store that authenticated them; naming another + file is refused. Outside any scope -- the CLI bootstrapping the first admin, + a script, the GUI of the desktop user -- nothing is checked, as with every + other privileged call made directly on the host. +* **A store keeps an admin.** Removing or demoting the last admin is refused: + afterwards nobody could manage users through any authenticated surface. +* **A token is shown once.** ``add`` and ``rotate`` return the plain token in + an :class:`IssuedToken`, whose text form masks it, so the executor's result + log and anything else that formats the result never holds it. Only the + hash is stored, and the audit entry names the user, never the token. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.rbac.authorization import ( + AuthorizationError, current_authorization, user_store_from_env, +) +from je_auto_control.utils.rbac.users import ( + Capability, Role, UserAuthError, UserRecord, UserStore, +) + +_MASK = "" + + +class IssuedToken(Dict[str, Any]): + """The reply of ``add`` / ``rotate``: a plain dict whose text form hides ``token``. + + It serialises as JSON with the token in it -- that is the one time the + caller sees it -- while ``str()`` / ``repr()``, which is what a log line + or a formatted run record uses, show a placeholder. + """ + + def __repr__(self) -> str: + return repr({key: _MASK if key == "token" else value for key, value in self.items()}) + + __str__ = __repr__ + + +def management_store(users_path: Optional[str] = None) -> UserStore: + """The store a management call works on, after checking the caller may. + + An authenticated caller gets the store that identified them. Otherwise it + is ``users_path``, else the file ``JE_AUTOCONTROL_RBAC_USERS`` names; + with neither, :class:`UserAuthError` -- the default location is never + assumed, for the reason :func:`user_store_from_env` gives. + """ + caller = current_authorization() + if caller is not None and not caller.allows(Capability.MANAGE_USERS): + raise AuthorizationError( + f"managing users needs the {Capability.MANAGE_USERS!r} capability; " + f"user {caller.user_id!r} has role {caller.role!r}", + capability=Capability.MANAGE_USERS) + requested = _real(users_path) if users_path else None + if caller is not None and caller.store is not None: + if requested is not None and requested != _real(str(caller.store.path)): + raise AuthorizationError( + "an authenticated caller manages the user store that authenticated it; " + "users_path may not name another file", capability=Capability.MANAGE_USERS) + caller.store.refresh() # another process may have changed the file + return caller.store + if requested is not None: + return UserStore(requested) + store = user_store_from_env() + if store is None: + raise UserAuthError( + "no user store: pass users_path or set JE_AUTOCONTROL_RBAC_USERS") + return store + + +def _real(path: str) -> Path: + return Path(os.path.realpath(os.path.expanduser(path))) + + +def _public(record: UserRecord) -> Dict[str, Any]: + """A user as it may be shown: everything but the token hash.""" + return {"user_id": record.user_id, "display_name": record.display_name, + "role": record.role, "tags": list(record.tags)} + + +def _audit(event: str, user_id: str, detail: str) -> None: + """Record a change to the user list; never the token, and never fatal.""" + caller = current_authorization() + actor = caller.user_id if caller is not None else "local" + autocontrol_logger.info("rbac %s user=%s by=%s %s", event, user_id, actor, detail) + try: + from je_auto_control.utils.remote_desktop.audit_log import default_audit_log + default_audit_log().log( + f"rbac_{event}", viewer_id=actor, detail=f"user={user_id} {detail}".strip()) + except Exception as error: # noqa: BLE001 # reason: the change is already saved; a failed audit write must not undo or hide it + autocontrol_logger.warning("rbac %s not written to the audit log: %r", event, error) + + +def _admins(store: UserStore) -> List[str]: + return [record.user_id for record in store.list_users() if record.role == Role.ADMIN] + + +def _keep_an_admin(store: UserStore, user_id: str, action: str) -> None: + """Refuse ``action`` on ``user_id`` when it is the store's only admin.""" + if _admins(store) == [user_id]: + raise UserAuthError( + f"cannot {action} {user_id!r}: it is the only admin, and nobody could " + "manage users afterwards; add another admin first") + + +def add_user(user_id: str, role: str = Role.VIEWER, display_name: str = "", + tags: Optional[Sequence[str]] = None, + users_path: Optional[str] = None) -> IssuedToken: + """Add a user and return it with its token -- the only time the token is shown.""" + store = management_store(users_path) + token = store.add_user(user_id=str(user_id), display_name=str(display_name or ""), + role=str(role), tags=list(tags) if tags else None) + _audit("user_added", str(user_id), f"role={role}") + record = store.get(str(user_id)) + shown = _public(record) if record is not None else {"user_id": str(user_id), "role": role} + return IssuedToken({**shown, "token": token}) + + +def remove_user(user_id: str, users_path: Optional[str] = None) -> Dict[str, Any]: + """Remove a user; ``{"removed": False}`` when there was no such user.""" + store = management_store(users_path) + _keep_an_admin(store, str(user_id), "remove") + removed = store.remove_user(str(user_id)) + if removed: + _audit("user_removed", str(user_id), "") + return {"user_id": str(user_id), "removed": removed} + + +def set_user_role(user_id: str, role: str, + users_path: Optional[str] = None) -> Dict[str, Any]: + """Change a user's role; it applies to their next request and their deferred work.""" + store = management_store(users_path) + if role != Role.ADMIN: + _keep_an_admin(store, str(user_id), "demote") + store.set_role(str(user_id), str(role)) + _audit("user_role_set", str(user_id), f"role={role}") + return {"user_id": str(user_id), "role": str(role)} + + +def rotate_user_token(user_id: str, users_path: Optional[str] = None) -> IssuedToken: + """Replace a user's token and return the new one -- shown this once.""" + store = management_store(users_path) + token = store.rotate_token(str(user_id)) + _audit("user_token_rotated", str(user_id), "") + return IssuedToken({"user_id": str(user_id), "token": token}) + + +def list_users(users_path: Optional[str] = None) -> List[Dict[str, Any]]: + """Every user as ``{user_id, display_name, role, tags}``; no token, no hash.""" + store = management_store(users_path) + return [_public(record) for record in store.list_users()] + + +__all__ = [ + "IssuedToken", "add_user", "list_users", "management_store", + "remove_user", "rotate_user_token", "set_user_role", +] diff --git a/je_auto_control/utils/rbac/authorization.py b/je_auto_control/utils/rbac/authorization.py index 1370c53c4..0fc1994e3 100644 --- a/je_auto_control/utils/rbac/authorization.py +++ b/je_auto_control/utils/rbac/authorization.py @@ -16,7 +16,7 @@ import contextlib import os from contextvars import ContextVar -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Iterator, Optional @@ -43,6 +43,9 @@ class AuthorizationContext: user_id: str role: str + #: The store that identified the caller. Deferred work registered by the + #: caller looks its role up there again when it fires. + store: Optional[UserStore] = field(default=None, compare=False, repr=False) def allows(self, capability: str) -> bool: """``True`` when this caller's role grants ``capability``.""" @@ -73,7 +76,7 @@ def resolve_token(store: UserStore, token: str) -> Optional[AuthorizationContext record = store.authenticate(token) except UserAuthError: return None - return AuthorizationContext(user_id=record.user_id, role=record.role) + return AuthorizationContext(user_id=record.user_id, role=record.role, store=store) _CURRENT: ContextVar[Optional[AuthorizationContext]] = ContextVar( @@ -90,7 +93,9 @@ def authorization_scope(context: Optional[AuthorizationContext]) -> Iterator[Non """Serve the enclosed work as ``context``; ``None`` clears any outer scope. The scope belongs to the calling thread. Work handed to another thread -- - a scheduler job, a trigger, a hotkey -- runs outside it. + a scheduler job, a trigger, a hotkey -- runs outside it, which is why + those registries store the caller and re-enter a scope when the work + fires (:mod:`je_auto_control.utils.rbac.deferred`). """ token = _CURRENT.set(context) try: diff --git a/je_auto_control/utils/rbac/cli.py b/je_auto_control/utils/rbac/cli.py new file mode 100644 index 000000000..770ede1bd --- /dev/null +++ b/je_auto_control/utils/rbac/cli.py @@ -0,0 +1,116 @@ +"""``je_auto_control users ...``: manage the RBAC user store from a terminal. + +This is how the first admin comes to exist. The REST API and the MCP server +refuse every token until the store has a user, and the ``AC_user_*`` +commands need an admin's token -- so the first one is created here, on the +host, by whoever can write the store file:: + + je_auto_control users --users /etc/autocontrol/users.json add alice --role admin + je_auto_control users list # store from JE_AUTOCONTROL_RBAC_USERS + je_auto_control users set-role bob operator + je_auto_control users rotate-token bob + je_auto_control users remove bob + +``add`` and ``rotate-token`` print the token once, on standard output; it is +not stored (only its hash is) and cannot be shown again. The same commands +run as ``python -m je_auto_control.utils.rbac ...``. +""" +from __future__ import annotations + +import argparse +import json +import sys +from typing import Any, List, Optional + +from je_auto_control.utils.rbac import admin +from je_auto_control.utils.rbac.authorization import USERS_ENV +from je_auto_control.utils.rbac.users import Role + + +def add_users_arguments(parser: argparse.ArgumentParser) -> None: + """Give ``parser`` the ``users`` sub-commands; :func:`cmd_users` runs them.""" + parser.add_argument( + "--users", metavar="PATH", + help=f"user store file (default: the one {USERS_ENV} names)") + parser.add_argument("--json", action="store_true", help="print the result as JSON") + actions = parser.add_subparsers(dest="users_command", required=True) + + p_add = actions.add_parser("add", help="Add a user and print its token once") + p_add.add_argument("user_id") + p_add.add_argument("--role", choices=Role.all(), default=Role.VIEWER) + p_add.add_argument("--name", default="", help="display name (default: the user id)") + p_add.add_argument("--tag", action="append", help="free-form tag; may be repeated") + + p_remove = actions.add_parser("remove", help="Remove a user") + p_remove.add_argument("user_id") + + p_role = actions.add_parser("set-role", help="Change a user's role") + p_role.add_argument("user_id") + p_role.add_argument("role", choices=Role.all()) + + p_rotate = actions.add_parser( + "rotate-token", help="Replace a user's token and print the new one once") + p_rotate.add_argument("user_id") + + actions.add_parser("list", help="List users (no tokens)") + parser.set_defaults(func=cmd_users) + + +def _run(args: argparse.Namespace) -> Any: + command = args.users_command + if command == "add": + return admin.add_user(args.user_id, role=args.role, display_name=args.name, + tags=args.tag, users_path=args.users) + if command == "remove": + return admin.remove_user(args.user_id, users_path=args.users) + if command == "set-role": + return admin.set_user_role(args.user_id, args.role, users_path=args.users) + if command == "rotate-token": + return admin.rotate_user_token(args.user_id, users_path=args.users) + return admin.list_users(users_path=args.users) + + +def _text(command: str, result: Any) -> str: + """The human form of ``result``; a token gets a line of its own.""" + if command == "list": + rows = [f"{user['user_id']}\t{user['role']}\t{user['display_name']}" for user in result] + return "\n".join(rows) + ("\n" if rows else "") + if command == "remove": + state = "removed" if result["removed"] else "no such user" + return f"{result['user_id']}: {state}\n" + if command == "set-role": + return f"{result['user_id']}: role is now {result['role']}\n" + return (f"{result['user_id']}: token (shown once, store it now)\n" + f"{result['token']}\n") + + +def cmd_users(args: argparse.Namespace) -> int: + """Run one ``users`` sub-command; 1 when ``remove`` found nobody to remove.""" + result = _run(args) + if args.json: + # dict(): an IssuedToken would serialise the same, this just says so. + payload = dict(result) if isinstance(result, dict) else result + sys.stdout.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n") + else: + sys.stdout.write(_text(args.users_command, result)) + if args.users_command == "remove" and not result["removed"]: + return 1 + return 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Entry point of ``python -m je_auto_control.utils.rbac``.""" + from je_auto_control.utils.exception.exceptions import AutoControlException + parser = argparse.ArgumentParser( + prog="python -m je_auto_control.utils.rbac", + description="Manage the RBAC user store of the REST API and MCP server.") + add_users_arguments(parser) + args = parser.parse_args(argv) + try: + return cmd_users(args) + except (AutoControlException, OSError, ValueError) as error: + sys.stderr.write(f"error: {error}\n") + return 1 + + +__all__ = ["add_users_arguments", "cmd_users", "main"] diff --git a/je_auto_control/utils/rbac/deferred.py b/je_auto_control/utils/rbac/deferred.py new file mode 100644 index 000000000..8dc905086 --- /dev/null +++ b/je_auto_control/utils/rbac/deferred.py @@ -0,0 +1,105 @@ +"""Work registered now and run later keeps the identity of whoever registered it. + +A scheduler job, a trigger, a hotkey binding, a webhook, an e-mail trigger or +a watchdog rule fires on a daemon thread, long after the request that +registered it has been answered. The authorisation scope belongs to the +request's thread, so that later run had no caller at all: an operator could +register a job whose action file held ``AC_sign_action_file`` and the executor +would run it unchecked. + +Each of those registries now stores a :class:`DeferredOwner` with the entry +(:func:`capture_owner`) and runs it inside :func:`owner_scope`. The role is +looked up again at that moment, not remembered: a user who was demoted loses +the privilege for work already registered, and work whose user was removed +does not run. + +Registered outside any RBAC scope -- no user store configured, the GUI, a +script -- there is no owner and nothing changes. +""" +from __future__ import annotations + +import contextlib +from dataclasses import dataclass, field +from typing import Any, Dict, Iterator, Optional + +from je_auto_control.utils.rbac.authorization import ( + AuthorizationContext, AuthorizationError, authorization_scope, + current_authorization, user_store_from_env, +) +from je_auto_control.utils.rbac.users import Capability, UserStore + + +@dataclass(frozen=True) +class DeferredOwner: + """The user a piece of deferred work runs as, and the role it had when registered.""" + + user_id: str + role: str + #: Where the role is looked up when the work fires. Absent on an owner + #: read back from a file; ``JE_AUTOCONTROL_RBAC_USERS`` is used then. + store: Optional[UserStore] = field(default=None, compare=False, repr=False) + + def to_dict(self) -> Dict[str, str]: + """The owner as plain JSON: the optional ``owner`` field of a saved entry.""" + return {"user_id": self.user_id, "role": self.role} + + @classmethod + def from_dict(cls, value: Any) -> Optional["DeferredOwner"]: + """The owner a saved entry names; ``None`` for an entry saved without one.""" + if not isinstance(value, dict): + return None + user_id, role = value.get("user_id"), value.get("role") + if not isinstance(user_id, str) or not user_id or not isinstance(role, str): + return None + return cls(user_id=user_id, role=role) + + +def capture_owner() -> Optional[DeferredOwner]: + """The current caller as an owner to store; ``None`` outside an RBAC scope.""" + caller = current_authorization() + if caller is None: + return None + return DeferredOwner(user_id=caller.user_id, role=caller.role, store=caller.store) + + +def resolve_owner(owner: DeferredOwner) -> AuthorizationContext: + """Who ``owner`` is right now, by the user store. + + Without any store to ask -- the owner came from a file and RBAC has since + been switched off -- the role recorded at registration stands: it is the + most the user was ever granted here. + """ + store = owner.store if owner.store is not None else user_store_from_env() + if store is None: + return AuthorizationContext(user_id=owner.user_id, role=owner.role) + store.refresh() + record = store.get(owner.user_id) + if record is None: + raise AuthorizationError( + f"user {owner.user_id!r}, who registered this work, is no longer in the user store") + return AuthorizationContext(user_id=record.user_id, role=record.role, store=store) + + +@contextlib.contextmanager +def owner_scope(owner: Optional[DeferredOwner], + capability: str = Capability.DRIVE_INPUT) -> Iterator[None]: + """Run the enclosed deferred work as ``owner``, with the role it holds now. + + ``capability`` is what registering the work needed in the first place; + an owner who no longer has it does not get the work run at all + (:class:`AuthorizationError`). ``None`` -- work nobody owns -- leaves the + thread as it is. + """ + if owner is None: + yield + return + caller = resolve_owner(owner) + if not caller.allows(capability): + raise AuthorizationError( + f"deferred work of user {caller.user_id!r} needs the {capability!r} capability; " + f"role {caller.role!r} no longer grants it", capability=capability) + with authorization_scope(caller): + yield + + +__all__ = ["DeferredOwner", "capture_owner", "owner_scope", "resolve_owner"] diff --git a/je_auto_control/utils/rbac/policy.py b/je_auto_control/utils/rbac/policy.py index b38466e56..e836bb55f 100644 --- a/je_auto_control/utils/rbac/policy.py +++ b/je_auto_control/utils/rbac/policy.py @@ -6,8 +6,10 @@ ``manage_hosts``, so a route added without a decision is admin-only rather than open. * **MCP tools** follow their own annotation -- a tool marked read-only needs - ``read_screen``, any other ``drive_input`` -- except the few named here - that change how the host itself is exposed. + ``read_screen``, any other ``drive_input`` -- except the ones named here: + those that change how the host itself is exposed, those that manage users, + and the read-only tools that hand back the host's data rather than what is + on its screen (``read_data``). * **``AC_*`` commands** reach the executor through anything that runs an action list (``POST /execute``, ``ac_execute_actions``, an action file). Being allowed to run actions is ``drive_input``; the commands named here @@ -33,6 +35,8 @@ _DRIVE = Capability.DRIVE_INPUT _HOSTS = Capability.MANAGE_HOSTS _AUDIT = Capability.READ_AUDIT +_USERS = Capability.MANAGE_USERS +_DATA = Capability.READ_DATA REST_ROUTE_CAPABILITIES: Dict[Tuple[str, str], str] = { ("GET", "/metrics"): _READ, @@ -113,8 +117,39 @@ "AC_secret_remove": _HOSTS, "AC_secret_unlock": _HOSTS, "AC_secret_lock": _HOSTS, + "AC_user_add": _USERS, + "AC_user_remove": _USERS, + "AC_user_set_role": _USERS, + "AC_user_rotate_token": _USERS, + "AC_user_list": _USERS, } +#: Read-only MCP tools that expose data rather than screen state. "Read-only" +#: says a tool changes nothing; it does not say that what it returns is +#: something a viewer of the desktop was meant to see. +DATA_TOOLS = frozenset({ + # the clipboard and its history + "ac_get_clipboard", "ac_get_clipboard_csv", "ac_get_clipboard_files", + "ac_get_clipboard_html", "ac_get_clipboard_image", "ac_get_clipboard_rtf", + "ac_clipboard_formats", "ac_assert_clipboard", + "ac_clip_history_list", "ac_clip_history_search", + # files: their content, or an oracle on it (substring, digest) + "ac_load_dotenv", "ac_load_data", "ac_read_action_file", "ac_read_document", + "ac_read_presentation", "ac_read_workbook", "ac_extract_pdf_text", + "ac_assert_pdf_text", "ac_assert_file", "ac_build_provenance", + "ac_verify_provenance", + # databases and the named stores kept on the host + "ac_sql_query", "ac_assert_db", "ac_get_asset", "ac_list_assets", + "ac_cas_get", "ac_outbox_pending", "ac_checkpoint_status", + "ac_memory_recall", "ac_memory_recent", "ac_s3_list", + # references into the environment, files and the secrets vault; tokens + "ac_resolve_ref", "ac_resolve_refs", "ac_generate_otp", + "ac_jwt_encode", "ac_jwt_decode", + # the process list, the network as seen from the host, the microphone + "ac_list_processes", "ac_assert_process", "ac_wait_for_process", + "ac_assert_http", "ac_wait_for_port", "ac_assert_audio", +}) + #: MCP tools whose capability is not the one their read-only hint implies. TOOL_CAPABILITIES: Dict[str, str] = { "ac_remote_host_start": _HOSTS, @@ -126,6 +161,12 @@ "ac_egress_allow": _HOSTS, "ac_egress_reset": _HOSTS, "ac_load_plugins": _HOSTS, + "ac_user_add": _USERS, + "ac_user_remove": _USERS, + "ac_user_set_role": _USERS, + "ac_user_rotate_token": _USERS, + "ac_user_list": _USERS, + **{name: _DATA for name in DATA_TOOLS}, } @@ -202,7 +243,7 @@ def _leading_names(node: Any, depth: int) -> Iterator[str]: __all__ = [ - "COMMAND_CAPABILITIES", "REST_ROUTE_CAPABILITIES", "TOOL_CAPABILITIES", + "COMMAND_CAPABILITIES", "DATA_TOOLS", "REST_ROUTE_CAPABILITIES", "TOOL_CAPABILITIES", "authorize_command", "capability_for_command", "capability_for_route", "capability_for_tool", "denied_command_in", ] diff --git a/je_auto_control/utils/rbac/users.py b/je_auto_control/utils/rbac/users.py index 898021165..16a3586e8 100644 --- a/je_auto_control/utils/rbac/users.py +++ b/je_auto_control/utils/rbac/users.py @@ -30,6 +30,10 @@ def all(cls) -> List[str]: class Capability: """Coarse capability tags checked by REST / MCP route guards.""" READ_SCREEN = "read_screen" + #: Reading what is not on the screen: files, databases, the clipboard, + #: stored secrets and references, the process list, the network. A + #: viewer watches the desktop; it does not get the host's data. + READ_DATA = "read_data" DRIVE_INPUT = "drive_input" MANAGE_HOSTS = "manage_hosts" MANAGE_USERS = "manage_users" @@ -41,13 +45,13 @@ class Capability: @classmethod def all(cls) -> List[str]: - return [cls.READ_SCREEN, cls.DRIVE_INPUT, cls.MANAGE_HOSTS, + return [cls.READ_SCREEN, cls.READ_DATA, cls.DRIVE_INPUT, cls.MANAGE_HOSTS, cls.MANAGE_USERS, cls.READ_AUDIT, cls.SIGN_ACTIONS] _ROLE_CAPABILITIES: Dict[str, Set[str]] = { Role.VIEWER: {Capability.READ_SCREEN}, - Role.OPERATOR: {Capability.READ_SCREEN, Capability.DRIVE_INPUT}, + Role.OPERATOR: {Capability.READ_SCREEN, Capability.READ_DATA, Capability.DRIVE_INPUT}, Role.ADMIN: set(Capability.all()), } diff --git a/je_auto_control/utils/rest_api/rest_auth.py b/je_auto_control/utils/rest_api/rest_auth.py index 69110f32e..25e1d2f06 100644 --- a/je_auto_control/utils/rest_api/rest_auth.py +++ b/je_auto_control/utils/rest_api/rest_auth.py @@ -26,7 +26,7 @@ import threading import time from dataclasses import dataclass -from typing import Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple from je_auto_control.utils.rbac.authorization import AuthorizationContext, resolve_token from je_auto_control.utils.rbac.users import UserStore @@ -182,6 +182,23 @@ def _reset_failures(self, client_ip: str) -> None: bucket.failed = 0 +def authenticate_with(gate: Any, *, client_ip: str, + header_value: Optional[str]) -> AuthResult: + """Ask ``gate`` who the caller is, whichever of the two methods it has. + + A gate written against the original interface implements only + ``check(...) -> str``. It cannot name a user, so its verdict is taken + with no identity -- the request is served as under the shared token. + """ + authenticate = getattr(gate, "authenticate", None) + if callable(authenticate): + result = authenticate(client_ip=client_ip, header_value=header_value) + if isinstance(result, AuthResult): + return result + return AuthResult(str(result)) + return AuthResult(str(gate.check(client_ip=client_ip, header_value=header_value))) + + def _bearer_token(header_value: Optional[str]) -> Optional[str]: """The token of an ``Authorization: Bearer `` header, or ``None``.""" if not header_value: @@ -198,5 +215,6 @@ def _matches_bearer(header_value: Optional[str], expected: str) -> bool: __all__ = [ - "AuthResult", "RestAuthGate", "generate_token", "constant_time_equal", + "AuthResult", "RestAuthGate", "authenticate_with", "generate_token", + "constant_time_equal", ] diff --git a/je_auto_control/utils/rest_api/rest_handlers.py b/je_auto_control/utils/rest_api/rest_handlers.py index 2623f1cee..01cdd3060 100644 --- a/je_auto_control/utils/rest_api/rest_handlers.py +++ b/je_auto_control/utils/rest_api/rest_handlers.py @@ -16,6 +16,7 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlActionException, AutoControlActionNullException, AutoControlException, AutoControlJsonActionException, + AutoControlSignatureException, ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.script_vars.execution import execution_scope @@ -225,6 +226,11 @@ def handle_execute_file(ctx: RouteContext) -> HandlerResult: # same reasoning as _reject_bad_action_list, reported the same way. autocontrol_logger.info("rest execute_file rejected: %r", error) return 400, {"error": str(error)} + except AutoControlSignatureException as error: + # Enforced signatures refused the file: nothing ran, and nothing + # crashed -- this used to be reported as a 500. + autocontrol_logger.warning("rest execute_file refused: %r", error) + return 403, {"error": str(error)} except Exception as error: # noqa: BLE001 # pylint: disable=broad-except # reason: REST boundary must always return JSON, never drop the HTTP response autocontrol_logger.error("rest execute_file failed: %r", error) return 500, {"error": "execute_files failed"} diff --git a/je_auto_control/utils/rest_api/rest_registry.py b/je_auto_control/utils/rest_api/rest_registry.py index e466ef5e3..1320b6320 100644 --- a/je_auto_control/utils/rest_api/rest_registry.py +++ b/je_auto_control/utils/rest_api/rest_registry.py @@ -57,17 +57,30 @@ def stop(self, timeout: float = 2.0) -> Dict[str, Any]: return self.status() def status(self) -> Dict[str, Any]: + """What is running and how a client authenticates to it. + + ``token`` is the shared bearer token, or ``None`` when ``rbac`` is + true -- then each user has their own, and ``users_path`` names the + store they are checked against. + """ with self._lock: server = self._server if server is None: return { "running": False, "host": None, "port": 0, "token": None, "url": None, # nosec B105 # reason: dict key, value None means server stopped + "rbac": False, "users_path": None, } host, port = server.address + users = server.user_store return { "running": server.is_running, "host": host, "port": int(port), - "token": server.token, "url": server.base_url, + # With a user store the shared token opens nothing: reporting it + # handed out a credential the server refuses. + "token": server.token if users is None else None, + "url": server.base_url, + "rbac": users is not None, + "users_path": None if users is None else str(users.path), } diff --git a/je_auto_control/utils/rest_api/rest_server.py b/je_auto_control/utils/rest_api/rest_server.py index af3f2268b..9fa0793a3 100644 --- a/je_auto_control/utils/rest_api/rest_server.py +++ b/je_auto_control/utils/rest_api/rest_server.py @@ -27,7 +27,9 @@ ) from je_auto_control.utils.rbac.policy import capability_for_route, denied_command_in from je_auto_control.utils.rbac.users import UserStore -from je_auto_control.utils.rest_api.rest_auth import RestAuthGate, generate_token +from je_auto_control.utils.rest_api.rest_auth import ( + RestAuthGate, authenticate_with, generate_token, +) from je_auto_control.utils.rest_api.rest_handlers import ( HandlerResult, RouteContext, handle_audit_list, handle_audit_verify, @@ -227,8 +229,8 @@ def _admit(self, method: str, path: str, client_ip: str, body: Any, The caller is ``None`` under the shared token: there is no user to hold a role, so there is nothing to authorise. """ - result = self._gate().authenticate( - client_ip=client_ip, + result = authenticate_with( + self._gate(), client_ip=client_ip, header_value=self.headers.get("Authorization"), ) if result.verdict != "ok": diff --git a/je_auto_control/utils/run_history/run_outcome.py b/je_auto_control/utils/run_history/run_outcome.py index e0872acd5..7088162eb 100644 --- a/je_auto_control/utils/run_history/run_outcome.py +++ b/je_auto_control/utils/run_history/run_outcome.py @@ -9,20 +9,33 @@ Each such run also gets its own variable scope: a job's ``AC_set_var`` used to stay in the module executor for whichever job, trigger or hotkey fired next. + +A run whose entry was registered by an authenticated user is executed as that +user (``owner``), with the role the user store gives them at that moment. """ -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable, Optional from je_auto_control.utils.exception.exceptions import AutoControlActionException +if TYPE_CHECKING: + from je_auto_control.utils.rbac.deferred import DeferredOwner + + +def run_counting_failures(run: Callable[[], Any], + owner: "Optional[DeferredOwner]" = None) -> Any: + """Call ``run()`` in a fresh variable scope; raise if an action it ran failed. -def run_counting_failures(run: Callable[[], Any]) -> Any: - """Call ``run()`` in a fresh variable scope; raise if an action it ran failed.""" + ``owner`` is who registered the work; ``None`` runs it with no identity, + as before. An owner who was removed, or whose role no longer allows + running actions, raises ``AuthorizationError`` before anything runs. + """ from je_auto_control.utils.executor.action_executor import ( recorded_failures, reset_recorded_failures, ) + from je_auto_control.utils.rbac.deferred import owner_scope from je_auto_control.utils.script_vars.execution import execution_scope reset_recorded_failures() - with execution_scope(): + with owner_scope(owner), execution_scope(): result = run() failures = recorded_failures() if failures: diff --git a/je_auto_control/utils/scheduler/scheduler.py b/je_auto_control/utils/scheduler/scheduler.py index c305ab386..0f94b41e5 100644 --- a/je_auto_control/utils/scheduler/scheduler.py +++ b/je_auto_control/utils/scheduler/scheduler.py @@ -15,6 +15,7 @@ from je_auto_control.utils.run_history.artifact_manager import ( capture_error_snapshot, ) +from je_auto_control.utils.rbac.deferred import DeferredOwner, capture_owner from je_auto_control.utils.run_history.run_outcome import run_counting_failures from je_auto_control.utils.run_history.history_store import ( SOURCE_SCHEDULER, STATUS_ERROR, STATUS_OK, default_history_store, @@ -49,6 +50,8 @@ class ScheduledJob: runs: int = 0 enabled: bool = True next_run_ts: float = field(default=0.0) + #: Who registered it under RBAC; the run is authorised as that user. + owner: Optional[DeferredOwner] = field(default=None, kw_only=True) @property def is_cron(self) -> bool: @@ -94,7 +97,7 @@ def add_job(self, script_path: str, interval_seconds: float, job_id=jid, script_path=script_path, interval_seconds=interval, repeat=repeat, max_runs=max_runs, - next_run_ts=now + interval, + next_run_ts=now + interval, owner=capture_owner(), ) with self._lock: self._jobs[jid] = job @@ -114,6 +117,7 @@ def add_cron_job(self, script_path: str, cron_expression: str, cron_expression=expression, repeat=True, max_runs=max_runs, next_run_ts=_next_cron_ts(expression, time.time()), + owner=capture_owner(), ) with self._lock: self._jobs[jid] = job @@ -231,7 +235,7 @@ def _fire(self, job: ScheduledJob, now_mono: float, now_wall: float) -> None: error_text: Optional[str] = None try: actions = read_executable_action_json(job.script_path) - run_counting_failures(lambda: self._execute(actions)) + run_counting_failures(lambda: self._execute(actions), owner=job.owner) # 一個排程工作失敗必須記錄為 STATUS_ERROR 並繼續輪詢,絕不能拖垮 # 排程執行緒。原本的 tuple 漏掉 AutoControlException——它是幾乎所有 # action 失敗(找不到視窗/圖片、輸入錯誤)的基底,直接繼承 diff --git a/je_auto_control/utils/triggers/email_trigger.py b/je_auto_control/utils/triggers/email_trigger.py index b74957259..c962c25e5 100644 --- a/je_auto_control/utils/triggers/email_trigger.py +++ b/je_auto_control/utils/triggers/email_trigger.py @@ -30,6 +30,7 @@ from je_auto_control.utils.run_history.artifact_manager import ( capture_error_snapshot, ) +from je_auto_control.utils.rbac.deferred import DeferredOwner, capture_owner from je_auto_control.utils.run_history.run_outcome import run_counting_failures from je_auto_control.utils.run_history.history_store import ( SOURCE_TRIGGER, STATUS_ERROR, STATUS_OK, default_history_store, @@ -62,6 +63,8 @@ class EmailTrigger: _seen_uids: set = field(default_factory=set, repr=False) _inflight: set = field(default_factory=set, repr=False) _uidvalidity: Optional[str] = field(default=None, repr=False) + #: Who registered it under RBAC; the run is authorised as that user. + owner: Optional[DeferredOwner] = field(default=None, kw_only=True) def _decode_header_value(value: Optional[str]) -> str: @@ -283,6 +286,7 @@ def add(self, search_criteria=str(search_criteria or "UNSEEN"), mark_seen=bool(mark_seen), poll_seconds=max(_MIN_POLL_SECONDS, float(poll_seconds)), + owner=capture_owner(), ) with self._lock: self._triggers[trigger.trigger_id] = trigger @@ -487,7 +491,7 @@ def _execute_with_history(self, trigger: EmailTrigger, error_text: Optional[str] = None try: actions = read_executable_action_json(trigger.script_path) - run_counting_failures(lambda: self._executor(actions, payload)) + run_counting_failures(lambda: self._executor(actions, payload), owner=trigger.owner) # Any failure is recorded as STATUS_ERROR -- not a bogus # STATUS_OK from the finally below -- before re-raising. except Exception as error: # noqa: BLE001 # reason: re-raised diff --git a/je_auto_control/utils/triggers/trigger_engine.py b/je_auto_control/utils/triggers/trigger_engine.py index 24568d6d9..4d422954d 100644 --- a/je_auto_control/utils/triggers/trigger_engine.py +++ b/je_auto_control/utils/triggers/trigger_engine.py @@ -21,6 +21,7 @@ from je_auto_control.utils.run_history.artifact_manager import ( capture_error_snapshot, ) +from je_auto_control.utils.rbac.deferred import DeferredOwner, capture_owner from je_auto_control.utils.run_history.run_outcome import run_counting_failures from je_auto_control.utils.run_history.history_store import ( SOURCE_TRIGGER, STATUS_ERROR, STATUS_OK, default_history_store, @@ -40,6 +41,8 @@ class _TriggerBase: fired: int = 0 cooldown_seconds: float = 0.5 _last_fire: float = field(default=0.0) + #: Who registered it under RBAC; the run is authorised as that user. + owner: Optional[DeferredOwner] = field(default=None, kw_only=True) #: True when checking the trigger uses up the event it reports (an edge: #: a cron minute, a file change, a sequence step), so a composite has to #: check it last. Not a dataclass field. @@ -253,6 +256,8 @@ def add(self, trigger: _TriggerBase) -> _TriggerBase: """Register ``trigger``; assigns an id when missing.""" if not trigger.trigger_id: trigger.trigger_id = uuid.uuid4().hex[:8] + if trigger.owner is None: + trigger.owner = capture_owner() with self._lock: self._triggers[trigger.trigger_id] = trigger return trigger @@ -365,7 +370,7 @@ def _fire(self, trigger: _TriggerBase, now: float) -> None: error_text: Optional[str] = None try: actions = read_executable_action_json(trigger.script_path) - run_counting_failures(lambda: self._execute(actions)) + run_counting_failures(lambda: self._execute(actions), owner=trigger.owner) # 這裡刻意攔截所有例外：一個 trigger 失敗必須記錄成 STATUS_ERROR # 並繼續，而不是拖垮輪詢執行緒。原本的 tuple 漏掉 # AutoControlJsonActionException，所以光是改名 script 檔就會讓 diff --git a/je_auto_control/utils/triggers/webhook_server.py b/je_auto_control/utils/triggers/webhook_server.py index 927353252..3ec35f5da 100644 --- a/je_auto_control/utils/triggers/webhook_server.py +++ b/je_auto_control/utils/triggers/webhook_server.py @@ -28,7 +28,7 @@ import json import threading import uuid -from dataclasses import dataclass +from dataclasses import dataclass, field from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, Callable, Dict, List, Optional, Tuple @@ -44,6 +44,7 @@ from je_auto_control.utils.run_history.artifact_manager import ( capture_error_snapshot, ) +from je_auto_control.utils.rbac.deferred import DeferredOwner, capture_owner from je_auto_control.utils.run_history.run_outcome import run_counting_failures from je_auto_control.utils.run_history.history_store import ( SOURCE_TRIGGER, STATUS_ERROR, STATUS_OK, default_history_store, @@ -82,6 +83,8 @@ class WebhookTrigger: enabled: bool = True fired: int = 0 last_status: int = 0 + #: Who registered it under RBAC; the run is authorised as that user. + owner: Optional[DeferredOwner] = field(default=None, kw_only=True) #: Verbs the request handler answers; anything else gets a 501 before routing. @@ -303,6 +306,7 @@ def add(self, script_path=str(script_path), methods=_normalize_methods(methods), token=token if token is not None else None, + owner=capture_owner(), ) with self._lock: for existing in self._triggers.values(): @@ -381,7 +385,7 @@ def fire(self, trigger: WebhookTrigger, # error and _dispatch still answers the request. try: actions = read_executable_action_json(trigger.script_path) - run_counting_failures(lambda: self._executor(actions, payload)) + run_counting_failures(lambda: self._executor(actions, payload), owner=trigger.owner) except Exception as error: # noqa: BLE001 # reason: any script failure must be recorded and answered status = STATUS_ERROR error_text = repr(error) diff --git a/je_auto_control/utils/watchdog/popup_watchdog.py b/je_auto_control/utils/watchdog/popup_watchdog.py index ec81a0d57..38b663018 100644 --- a/je_auto_control/utils/watchdog/popup_watchdog.py +++ b/je_auto_control/utils/watchdog/popup_watchdog.py @@ -16,11 +16,12 @@ import threading import time from collections import deque -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Callable, Deque, Dict, List, Optional from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.rbac.deferred import DeferredOwner, capture_owner, owner_scope from je_auto_control.utils.timeouts import clamp_poll_interval # Errors a rule's matcher/action may raise that must not kill the guard loop @@ -42,6 +43,8 @@ class WatchdogRule: name: str matcher: Callable[[], bool] action: Callable[[], None] + #: Who registered it under RBAC; it is checked and dismissed as that user. + owner: Optional[DeferredOwner] = field(default=None, kw_only=True) class PopupWatchdog: @@ -64,6 +67,8 @@ def __init__(self, poll_interval_s: float = 1.0) -> None: def add_rule(self, rule: WatchdogRule) -> None: """Register a generic detector/dismisser rule.""" + if rule.owner is None: + rule.owner = capture_owner() with self._lock: self._rules.append(rule) @@ -133,9 +138,12 @@ def check_once(self) -> int: def _apply(self, rule: WatchdogRule) -> bool: try: - if not rule.matcher(): - return False - rule.action() + # As the user who registered the rule, with the role they hold + # now: a removed or demoted user's rule is refused, not run. + with owner_scope(rule.owner): + if not rule.matcher(): + return False + rule.action() # Any rule error, as ScreenObserver does: a matcher raising something # off the list (subprocess.TimeoutExpired, sqlite3.Error) killed the # guard thread and every other rule with it. diff --git a/test/unit_test/headless/test_rbac_deferred_owner.py b/test/unit_test/headless/test_rbac_deferred_owner.py new file mode 100644 index 000000000..e5a8842c4 --- /dev/null +++ b/test/unit_test/headless/test_rbac_deferred_owner.py @@ -0,0 +1,287 @@ +"""Deferred work runs as whoever registered it, with the role they hold when it fires. + +A scheduler job, a trigger, a hotkey binding, a webhook, an e-mail trigger +and a watchdog rule all fire on a daemon thread, outside the request that +registered them. They used to run with no identity, so an operator could +register a job whose action file held a privileged command and the executor +ran it unchecked. + +Nothing here touches the real mouse, keyboard or screen: each engine is +fired by hand, the run history is a fake, and the privileged command is +replaced by a recorder. +""" +import json +import time + +import pytest + +from je_auto_control.utils.executor.action_executor import execute_action, executor +from je_auto_control.utils.hotkey import hotkey_daemon +from je_auto_control.utils.hotkey.hotkey_daemon import HotkeyDaemon +from je_auto_control.utils.rbac import ( + USERS_ENV, AuthorizationContext, AuthorizationError, DeferredOwner, Role, + UserStore, authorization_scope, capture_owner, current_authorization, + owner_scope, resolve_owner, resolve_token, +) +from je_auto_control.utils.run_history.run_outcome import run_counting_failures +from je_auto_control.utils.scheduler import scheduler as scheduler_module +from je_auto_control.utils.scheduler.scheduler import ScheduledJob, Scheduler +from je_auto_control.utils.triggers import email_trigger, trigger_engine, webhook_server +from je_auto_control.utils.triggers.email_trigger import EmailTriggerWatcher +from je_auto_control.utils.triggers.trigger_engine import TriggerEngine, WindowAppearsTrigger +from je_auto_control.utils.triggers.webhook_server import WebhookTriggerServer +from je_auto_control.utils.watchdog.popup_watchdog import PopupWatchdog, WatchdogRule + +_SIGN = "AC_sign_action_file" +_PROBE = "AC_deferred_probe" +_ENGINES = ("scheduler", "trigger", "hotkey", "webhook", "email") + + +class _FakeHistory: + """Stands in for the run-history database; keeps how each run finished.""" + + def __init__(self): + self.finished = [] + + def start_run(self, *_args): + return 1 + + def finish_run(self, _run_id, status, error_text=None, artifact_path=None): + self.finished.append((status, error_text)) + + +@pytest.fixture(autouse=True) +def _no_ambient_rbac(monkeypatch): + monkeypatch.delenv(USERS_ENV, raising=False) + + +@pytest.fixture() +def history(monkeypatch): + fake = _FakeHistory() + for module in (scheduler_module, trigger_engine, hotkey_daemon, webhook_server, + email_trigger): + monkeypatch.setattr(module, "default_history_store", fake) + monkeypatch.setattr(module, "capture_error_snapshot", lambda _run_id: None) + return fake + + +@pytest.fixture() +def ran(monkeypatch): + """Names of the commands that actually ran: the privileged one and a plain probe.""" + names = [] + monkeypatch.setitem(executor.event_dict, _SIGN, + lambda **_kwargs: names.append(_SIGN)) + monkeypatch.setitem(executor.event_dict, _PROBE, lambda: names.append(_PROBE)) + return names + + +@pytest.fixture() +def users(tmp_path): + store = UserStore(tmp_path / "users.json") + store.tokens = { + role: store.add_user(user_id=f"{role}-user", display_name=role, role=role) + for role in Role.all() + } + return store + + +@pytest.fixture() +def script(tmp_path): + path = tmp_path / "job.json" + path.write_text(json.dumps([[_PROBE], [_SIGN, {"path": "x"}]]), encoding="utf-8") + return str(path) + + +def _as(users, role): + """The scope a server puts a request of ``role``'s user in.""" + return authorization_scope(resolve_token(users, users.tokens[role])) + + +def _register(kind, script): + """Register ``script`` with a fresh engine; return a callable that fires it once.""" + def run(actions, _payload=None): + return execute_action(actions) + + if kind == "scheduler": + engine = Scheduler(executor=run) + job = engine.add_job(script, 60) + return job, lambda: engine._fire(job, time.monotonic(), time.time()) + if kind == "trigger": + engine = TriggerEngine(executor=run) + trigger = engine.add(WindowAppearsTrigger("", script, title_substring="x")) + return trigger, lambda: engine._fire(trigger, time.monotonic()) + if kind == "hotkey": + daemon = HotkeyDaemon(executor=run) + binding = daemon.bind("ctrl+alt+1", script) + return binding, lambda: daemon._fire_binding(binding.binding_id) + if kind == "webhook": + server = WebhookTriggerServer(executor=run) + hook = server.add("/hook", script) + return hook, lambda: server.fire(hook, {}) + watcher = EmailTriggerWatcher(executor=run) + mail = watcher.add("imap.invalid", "user", "pw", script) + + def fire(): + try: + watcher._execute_with_history(mail, {}) + except Exception: # noqa: BLE001 # reason: this watcher re-raises; the fake history holds the outcome + pass + return mail, fire + + +# --- the defect ------------------------------------------------------------- + +@pytest.mark.parametrize("kind", _ENGINES) +def test_an_operators_deferred_work_cannot_run_a_privileged_command( + kind, users, script, history, ran): + with _as(users, Role.OPERATOR): + entry, fire = _register(kind, script) + assert entry.owner == DeferredOwner("operator-user", Role.OPERATOR) + assert current_authorization() is None + fire() + assert ran == [_PROBE], "the plain command runs, the privileged one is refused" + assert history.finished[-1][0] == "error" + + +@pytest.mark.parametrize("kind", _ENGINES) +def test_an_admins_deferred_work_keeps_its_privilege(kind, users, script, history, ran): + with _as(users, Role.ADMIN): + _entry, fire = _register(kind, script) + fire() + assert ran == [_PROBE, _SIGN] + assert history.finished[-1][0] == "ok" + + +@pytest.mark.parametrize("kind", _ENGINES) +def test_work_registered_without_rbac_runs_as_before(kind, script, history, ran): + entry, fire = _register(kind, script) + assert entry.owner is None + fire() + assert ran == [_PROBE, _SIGN] + assert history.finished[-1][0] == "ok" + + +# --- the role is looked up when the work fires ------------------------------ + +@pytest.mark.parametrize("kind", _ENGINES) +def test_a_demoted_admins_work_loses_the_privilege(kind, users, script, history, ran): + with _as(users, Role.ADMIN): + _entry, fire = _register(kind, script) + users.set_role("admin-user", Role.OPERATOR) + fire() + assert ran == [_PROBE] + assert history.finished[-1][0] == "error" + + +@pytest.mark.parametrize("kind", _ENGINES) +def test_work_of_a_user_demoted_to_viewer_or_removed_does_not_run( + kind, users, script, history, ran): + with _as(users, Role.OPERATOR): + _entry, fire = _register(kind, script) + users.set_role("operator-user", Role.VIEWER) + fire() + assert ran == [] + status, error = history.finished[-1] + assert status == "error" and "drive_input" in error + users.remove_user("operator-user") + fire() + assert ran == [] + assert "no longer in the user store" in history.finished[-1][1] + + +def test_a_change_made_by_another_process_is_seen(users, script, history, ran): + """The store is re-read when its file changed, as it is for a request.""" + with _as(users, Role.ADMIN): + _entry, fire = _register("scheduler", script) + other = UserStore(users.path) + other.set_role("admin-user", Role.OPERATOR) + fire() + assert ran == [_PROBE] + + +def test_a_promoted_user_gains_the_privilege(users, script, history, ran): + with _as(users, Role.OPERATOR): + _entry, fire = _register("scheduler", script) + users.set_role("operator-user", Role.ADMIN) + fire() + assert ran == [_PROBE, _SIGN] + + +# --- the watchdog ----------------------------------------------------------- + +def test_a_watchdog_rule_runs_as_its_owner(users): + dismissed = [] + watchdog = PopupWatchdog() + with _as(users, Role.OPERATOR): + watchdog.add_rule(WatchdogRule( + "popup", matcher=lambda: True, + action=lambda: dismissed.append(current_authorization().user_id))) + watchdog.add_rule(WatchdogRule( + "unowned", matcher=lambda: True, + action=lambda: dismissed.append(current_authorization()))) + assert watchdog.check_once() == 2 + assert dismissed == ["operator-user", None] + users.remove_user("operator-user") + assert watchdog.check_once() == 1 + assert dismissed == ["operator-user", None, None] + + +def test_add_window_rule_records_the_caller(users): + watchdog = PopupWatchdog() + with _as(users, Role.OPERATOR): + watchdog.add_window_rule("Update available") + assert watchdog._rules[0].owner.user_id == "operator-user" + + +# --- the owner record ------------------------------------------------------- + +def test_capture_owner_is_none_outside_a_scope_and_carries_the_store_inside(users): + assert capture_owner() is None + with _as(users, Role.OPERATOR): + owner = capture_owner() + assert (owner.user_id, owner.role, owner.store) == ("operator-user", Role.OPERATOR, users) + + +def test_an_owner_survives_being_saved_and_is_optional_in_an_old_entry(users, monkeypatch): + saved = json.loads(json.dumps(DeferredOwner("admin-user", Role.ADMIN, users).to_dict())) + assert saved == {"user_id": "admin-user", "role": "admin"} + loaded = DeferredOwner.from_dict(saved) + assert loaded == DeferredOwner("admin-user", Role.ADMIN) and loaded.store is None + for old_entry in (None, {}, {"user_id": ""}, {"user_id": 3, "role": "admin"}, "admin"): + assert DeferredOwner.from_dict(old_entry) is None + # Read back from a file there is no store attached: the configured one is asked. + monkeypatch.setenv(USERS_ENV, str(users.path)) + users.set_role("admin-user", Role.VIEWER) + assert resolve_owner(loaded).role == Role.VIEWER + # With RBAC switched off since, the role recorded at registration stands. + monkeypatch.delenv(USERS_ENV) + assert resolve_owner(loaded).role == Role.ADMIN + + +def test_job_records_built_the_old_way_still_construct(): + """``owner`` is keyword-only, so positional construction is unchanged.""" + job = ScheduledJob("id", "script.json", 5.0, None, True, None, 0, True, 1.0) + assert job.owner is None and job.next_run_ts == pytest.approx(1.0) + trigger = WindowAppearsTrigger("id", "script.json", True, True, 0, 0.5, 0.0, "title") + assert trigger.owner is None and trigger.title_substring == "title" + + +def test_owner_scope_restores_the_thread_and_refuses_before_running(users): + owner = DeferredOwner("viewer-user", Role.VIEWER, users) + outer = AuthorizationContext("outer", Role.ADMIN) + with authorization_scope(outer): + with owner_scope(None): + assert current_authorization() is outer + with pytest.raises(AuthorizationError) as refused: + with owner_scope(owner): + pytest.fail("a viewer's deferred work must not start") + assert refused.value.capability == "drive_input" + assert current_authorization() is outer + + +def test_run_counting_failures_takes_the_owner(users): + seen = [] + owner = DeferredOwner("operator-user", Role.OPERATOR, users) + run_counting_failures(lambda: seen.append(current_authorization().role), owner=owner) + run_counting_failures(lambda: seen.append(current_authorization())) + assert seen == [Role.OPERATOR, None] diff --git a/test/unit_test/headless/test_rbac_mcp_transports.py b/test/unit_test/headless/test_rbac_mcp_transports.py new file mode 100644 index 000000000..a08b234d7 --- /dev/null +++ b/test/unit_test/headless/test_rbac_mcp_transports.py @@ -0,0 +1,410 @@ +"""RBAC on every way the MCP HTTP transport serves a request. + +The role checks were only exercised on the plain JSON POST. The same server +also answers a POST as an SSE stream, holds a session's standing GET stream, +serves the stateless 2026-07-28 revision, keeps ``subscriptions/listen`` open +and can listen over TLS -- each a separate code path that has to authenticate +the caller and run the tool inside the caller's scope. + +Loopback only, fake tools only: nothing touches the mouse, keyboard or screen. +""" +import datetime +import http.client +import ipaddress +import json +import ssl +import urllib.error +import urllib.request + +import pytest + +from je_auto_control.utils.mcp_server._stateless import STATELESS_PROTOCOL_VERSION +from je_auto_control.utils.mcp_server.audit import AuditLogger +from je_auto_control.utils.mcp_server.http_transport import DEFAULT_PATH, HttpMCPServer +from je_auto_control.utils.mcp_server.server import MCPServer +from je_auto_control.utils.mcp_server.tools import MCPTool +from je_auto_control.utils.mcp_server.tools._base import DESTRUCTIVE, READ_ONLY, schema +from je_auto_control.utils.rbac import ( + USERS_ENV, Capability, Role, UserStore, current_authorization, +) + +_SSE = "application/json, text/event-stream" +_ALL = ["peek", "poke", "ac_load_plugins"] +_VISIBLE = {Role.VIEWER: _ALL[:1], Role.OPERATOR: _ALL[:2], Role.ADMIN: _ALL} + + +@pytest.fixture(autouse=True) +def _no_ambient_rbac(monkeypatch): + monkeypatch.delenv(USERS_ENV, raising=False) + monkeypatch.delenv("JE_AUTOCONTROL_MCP_TOKEN", raising=False) + + +@pytest.fixture() +def users(tmp_path): + store = UserStore(tmp_path / "users.json") + store.tokens = { + role: store.add_user(user_id=f"{role}-user", display_name=role, role=role) + for role in Role.all() + } + return store + + +@pytest.fixture() +def serve(tmp_path, users): + """Start RBAC servers over three fake tools; ``calls`` records who ran what.""" + calls = [] + + def tool(name, annotations): + def handler(**_arguments): + caller = current_authorization() + calls.append((name, None if caller is None else caller.user_id)) + return {"ran": name} + return MCPTool(name=name, description=name, annotations=annotations, handler=handler, + input_schema=schema({"actions": {"type": "array"}})) + + audit_path = tmp_path / "mcp_audit.jsonl" + started = [] + + def start(**kwargs): + tools = [tool("peek", READ_ONLY), tool("poke", DESTRUCTIVE), + tool("ac_load_plugins", DESTRUCTIVE)] + server = HttpMCPServer( + mcp=MCPServer(tools=tools, audit_logger=AuditLogger(path=str(audit_path))), + host="127.0.0.1", port=0, user_store=users, **kwargs) + server.start() + server.calls = calls + server.audit_path = audit_path + started.append(server) + return server + + yield start + for server in started: + server.stop(timeout=2.0) + + +def _connection(server, context=None): + host, port = server.address + if context is not None: + return http.client.HTTPSConnection(host, port, timeout=5, context=context) + return http.client.HTTPConnection(host, port, timeout=5) + + +def _exchange(server, body, token=None, headers=None, method="POST", context=None): + """``(status, headers, raw body)`` of one request.""" + connection = _connection(server, context) + sent = {"Content-Type": "application/json", "Accept": "application/json", **(headers or {})} + if token is not None: + sent["Authorization"] = f"Bearer {token}" + payload = None if body is None else json.dumps(body) + connection.request(method, DEFAULT_PATH, body=payload, headers=sent) + response = connection.getresponse() + raw = response.read().decode("utf-8") + connection.close() + return response.status, dict(response.getheaders()), raw + + +def _events(raw): + return [json.loads(chunk[len("data: "):]) for chunk in raw.split("\n\n") + if chunk.startswith("data: ")] + + +def _request(method, params=None, msg_id=1): + return {"jsonrpc": "2.0", "id": msg_id, "method": method, "params": params or {}} + + +def _call(name): + return _request("tools/call", {"name": name, "arguments": {}}) + + +def _audit(server): + return [json.loads(line) for line in + server.audit_path.read_text(encoding="utf-8").splitlines()] + + +# --- a POST answered as an SSE stream --------------------------------------- + +def _sse(server, body, token): + status, headers, raw = _exchange(server, body, token, {"Accept": _SSE}) + return status, headers, _events(raw) + + +def test_sse_post_needs_a_users_token(serve): + server = serve(auth_token="shared-secret") + for token in (None, "shared-secret", "wrong"): + status, headers, raw = _exchange(server, _request("tools/list"), token, {"Accept": _SSE}) + assert status == 401 and "text/event-stream" not in headers.get("Content-Type", "") + assert "WWW-Authenticate" in headers and "tools" not in raw + + +@pytest.mark.parametrize("role", Role.all()) +def test_sse_post_lists_and_runs_only_what_the_role_grants(serve, users, role): + server = serve() + token = users.tokens[role] + status, headers, events = _sse(server, _request("tools/list"), token) + assert status == 200 and "text/event-stream" in headers["Content-Type"] + assert [tool["name"] for tool in events[-1]["result"]["tools"]] == _VISIBLE[role] + for name in _ALL: + _status, _headers, events = _sse(server, _call(name), token) + assert ("result" in events[-1]) is (name in _VISIBLE[role]), (role, name) + assert server.calls == [(name, f"{role}-user") for name in _VISIBLE[role]] + + +def test_sse_post_refusal_names_the_capability_and_is_audited(serve, users): + server = serve() + _status, _headers, events = _sse(server, _call("poke"), users.tokens[Role.VIEWER]) + assert events[-1]["error"]["code"] == -32003 + assert events[-1]["error"]["data"] == {"required_capability": Capability.DRIVE_INPUT} + _sse(server, _call("poke"), users.tokens[Role.OPERATOR]) + refused, ran = _audit(server) + assert (refused["user_id"], refused["status"]) == ("viewer-user", "denied") + assert (ran["user_id"], ran["role"], ran["status"]) == ("operator-user", "operator", "ok") + + +def test_sse_post_refuses_a_privileged_command_inside_the_arguments(serve, users): + server = serve() + body = _request("tools/call", {"name": "poke", "arguments": { + "actions": [["AC_sign_action_file", {"path": "x"}]]}}) + _status, _headers, events = _sse(server, body, users.tokens[Role.OPERATOR]) + assert events[-1]["error"]["data"] == {"required_capability": Capability.SIGN_ACTIONS} + assert server.calls == [] + + +# --- the session's standing GET stream -------------------------------------- + +def _initialize(server, token): + status, headers, raw = _exchange(server, _request("initialize", { + "protocolVersion": "2025-03-26", "capabilities": {}, + "clientInfo": {"name": "t", "version": "1"}}), token) + assert status == 200, raw + return headers["Mcp-Session-Id"] + + +def test_session_requests_are_authorised_one_by_one(serve, users): + """A session does not carry a role: every request in it is the caller's own.""" + server = serve() + session = {"Mcp-Session-Id": _initialize(server, users.tokens[Role.ADMIN])} + _status, _headers, raw = _exchange(server, _call("ac_load_plugins"), + users.tokens[Role.ADMIN], session) + assert "result" in json.loads(raw) + status, _headers, raw = _exchange(server, _call("ac_load_plugins"), + users.tokens[Role.OPERATOR], session) + assert status == 200 and json.loads(raw)["error"]["code"] == -32003 + assert _exchange(server, _call("peek"), None, session)[0] == 401 + assert server.calls == [("ac_load_plugins", "admin-user")] + + +def test_the_get_stream_and_delete_need_a_users_token(serve, users): + server = serve(auth_token="shared-secret") + session = {"Mcp-Session-Id": _initialize(server, users.tokens[Role.VIEWER])} + stream = {**session, "Accept": "text/event-stream"} + for token in (None, "shared-secret"): + assert _exchange(server, None, token, stream, method="GET")[0] == 401 + assert _exchange(server, None, token, session, method="DELETE")[0] == 401 + connection = _connection(server) + connection.request("GET", DEFAULT_PATH, headers={ + **stream, "Authorization": f"Bearer {users.tokens[Role.VIEWER]}"}) + response = connection.getresponse() + assert response.status == 200 + assert "text/event-stream" in response.getheader("Content-Type") + response.close() + connection.close() + assert _exchange(server, None, users.tokens[Role.VIEWER], session, method="DELETE")[0] == 200 + + +# --- the stateless revision ------------------------------------------------- + +def _stateless_headers(method, name=None): + headers = {"MCP-Protocol-Version": STATELESS_PROTOCOL_VERSION, "Mcp-Method": method} + if name is not None: + headers["Mcp-Name"] = name + return headers + + +def _stateless(method, params=None, msg_id=1): + meta = {"io.modelcontextprotocol/protocolVersion": STATELESS_PROTOCOL_VERSION, + "io.modelcontextprotocol/clientInfo": {"name": "t", "version": "1"}, + "io.modelcontextprotocol/clientCapabilities": {}} + return {"jsonrpc": "2.0", "id": msg_id, "method": method, + "params": {**(params or {}), "_meta": meta}} + + +def test_stateless_requests_need_a_users_token(serve): + server = serve(auth_token="shared-secret") + for token in (None, "shared-secret"): + status, _headers, raw = _exchange(server, _stateless("tools/list"), token, + _stateless_headers("tools/list")) + assert status == 401 and "tools" not in raw + + +@pytest.mark.parametrize("role", Role.all()) +def test_stateless_lists_and_runs_only_what_the_role_grants(serve, users, role): + server = serve() + token = users.tokens[role] + status, headers, raw = _exchange(server, _stateless("tools/list"), token, + _stateless_headers("tools/list")) + assert status == 200 and "Mcp-Session-Id" not in headers + assert [tool["name"] for tool in json.loads(raw)["result"]["tools"]] == _VISIBLE[role] + for name in ("peek", "ac_load_plugins"): + body = _stateless("tools/call", {"name": name, "arguments": {}}) + _status, _headers, raw = _exchange(server, body, token, + _stateless_headers("tools/call", name)) + reply = json.loads(raw) + if name in _VISIBLE[role]: + assert reply["result"]["isError"] is False + else: + assert reply["error"]["code"] == -32003 + expected = [name for name in ("peek", "ac_load_plugins") if name in _VISIBLE[role]] + assert server.calls == [(name, f"{role}-user") for name in expected] + + +def test_stateless_sse_reply_is_authorised_too(serve, users): + server = serve() + body = _stateless("tools/call", {"name": "peek", "arguments": {}}) + headers = {**_stateless_headers("tools/call", "peek"), "Accept": _SSE} + status, _headers, raw = _exchange(server, body, users.tokens[Role.VIEWER], headers) + assert status == 200 and _events(raw)[-1]["result"]["isError"] is False + assert _exchange(server, body, "wrong", headers)[0] == 401 + assert server.calls == [("peek", "viewer-user")] + + +# --- subscriptions/listen --------------------------------------------------- + +def _open_listen(server, token, context=None): + connection = _connection(server, context) + body = json.dumps(_stateless("subscriptions/listen", + {"notifications": {"toolsListChanged": True}}, msg_id=11)) + headers = {"Content-Type": "application/json", "Accept": _SSE, + **_stateless_headers("subscriptions/listen")} + if token is not None: + headers["Authorization"] = f"Bearer {token}" + connection.request("POST", DEFAULT_PATH, body=body, headers=headers) + return connection, connection.getresponse() + + +def _next_event(response): + while True: + line = response.fp.readline() + if not line: + return None + if line.startswith(b"data: "): + return json.loads(line[len(b"data: "):]) + + +def test_listen_is_refused_without_a_users_token_and_opens_no_stream(serve): + server = serve(auth_token="shared-secret") + for token in (None, "shared-secret"): + connection, response = _open_listen(server, token) + assert response.status == 401 + assert "text/event-stream" not in (response.getheader("Content-Type") or "") + response.read() + connection.close() + assert not server.mcp._listeners # noqa: SLF001 # reason: the registry is the observable + + +def test_a_viewer_may_listen_and_gets_only_the_change_signal(serve, users): + server = serve() + connection, response = _open_listen(server, users.tokens[Role.VIEWER]) + try: + assert response.status == 200 + assert _next_event(response)["params"]["notifications"] == {"toolsListChanged": True} + server.mcp.register_tool(MCPTool( + name="ac_egress_reset", description="d", handler=lambda: None, + annotations=DESTRUCTIVE, input_schema=schema({}))) + changed = _next_event(response) + assert changed["method"] == "notifications/tools/list_changed" + assert "ac_egress_reset" not in json.dumps(changed), "a signal, not the tool list" + finally: + response.close() + connection.close() + # What the signal leads to is still filtered by the viewer's role. + _status, _headers, raw = _exchange(server, _stateless("tools/list"), + users.tokens[Role.VIEWER], + _stateless_headers("tools/list")) + assert [tool["name"] for tool in json.loads(raw)["result"]["tools"]] == ["peek"] + + +# --- TLS -------------------------------------------------------------------- + +@pytest.fixture() +def tls(tmp_path): + """``(server context, client context)`` for a throwaway loopback certificate.""" + x509 = pytest.importorskip("cryptography.x509", exc_type=ImportError) + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(x509.NameOID.COMMON_NAME, "mcp-rbac-test")]) + now = datetime.datetime.now(datetime.timezone.utc) + certificate = ( + x509.CertificateBuilder().subject_name(name).issuer_name(name) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension(x509.SubjectAlternativeName( + [x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False) + .sign(private_key=key, algorithm=hashes.SHA256())) + cert_path, key_path = tmp_path / "cert.pem", tmp_path / "key.pem" + cert_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes(key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + server_context.load_cert_chain(certfile=str(cert_path), keyfile=str(key_path)) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.minimum_version = ssl.TLSVersion.TLSv1_2 + client_context.load_verify_locations(cafile=str(cert_path)) + return server_context, client_context + + +def test_rbac_over_tls_json_and_sse(serve, users, tls): + server_context, client_context = tls + server = serve(ssl_context=server_context, auth_token="shared-secret") + for token in (None, "shared-secret"): + assert _exchange(server, _request("tools/list"), token, + context=client_context)[0] == 401 + for role in Role.all(): + _status, _headers, raw = _exchange(server, _request("tools/list"), + users.tokens[role], context=client_context) + assert [tool["name"] for tool in json.loads(raw)["result"]["tools"]] == _VISIBLE[role] + status, _headers, raw = _exchange(server, _call("poke"), users.tokens[Role.VIEWER], + {"Accept": _SSE}, context=client_context) + assert status == 200 and _events(raw)[-1]["error"]["code"] == -32003 + _status, _headers, raw = _exchange(server, _call("poke"), users.tokens[Role.OPERATOR], + {"Accept": _SSE}, context=client_context) + assert "result" in _events(raw)[-1] + assert server.calls == [("poke", "operator-user")] + + +def test_rbac_over_tls_stateless_and_listen(serve, users, tls): + server_context, client_context = tls + server = serve(ssl_context=server_context) + body = _stateless("tools/call", {"name": "ac_load_plugins", "arguments": {}}) + headers = _stateless_headers("tools/call", "ac_load_plugins") + _status, _headers, raw = _exchange(server, body, users.tokens[Role.OPERATOR], headers, + context=client_context) + assert json.loads(raw)["error"]["data"] == {"required_capability": Capability.MANAGE_HOSTS} + _status, _headers, raw = _exchange(server, body, users.tokens[Role.ADMIN], headers, + context=client_context) + assert json.loads(raw)["result"]["isError"] is False + connection, response = _open_listen(server, None, context=client_context) + assert response.status == 401 + connection.close() + connection, response = _open_listen(server, users.tokens[Role.VIEWER], + context=client_context) + assert response.status == 200 + assert _next_event(response)["method"] == "notifications/subscriptions/acknowledged" + response.close() + connection.close() + + +def test_a_plain_client_cannot_reach_a_tls_rbac_server(serve, users, tls): + server = serve(ssl_context=tls[0]) + host, port = server.address + request = urllib.request.Request( + f"http://{host}:{port}{DEFAULT_PATH}", # NOSONAR the point: plaintext is refused + data=json.dumps(_request("tools/list")).encode("utf-8"), method="POST", + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {users.tokens[Role.ADMIN]}"}) + with pytest.raises((urllib.error.URLError, ConnectionError, http.client.HTTPException, + TimeoutError)): + urllib.request.urlopen(request, timeout=3) # nosec B310 # reason: loopback test server diff --git a/test/unit_test/headless/test_rbac_server_wiring.py b/test/unit_test/headless/test_rbac_server_wiring.py index da9d3c548..c0910aa77 100644 --- a/test/unit_test/headless/test_rbac_server_wiring.py +++ b/test/unit_test/headless/test_rbac_server_wiring.py @@ -418,10 +418,13 @@ def test_mcp_unconfigured_rbac_keeps_shared_token(mcp_http): def test_default_registry_offers_a_viewer_only_read_only_tools(users): + """Read-only, and not one of those listed as needing more than ``read_screen``.""" registry = build_default_tool_registry(read_only=False) with authorization_scope(AuthorizationContext("viewer-user", Role.VIEWER)): listed = MCPServer(tools=registry).handle_line( json.dumps({"jsonrpc": "2.0", "id": 1, "method": "tools/list"})) names = {tool["name"] for tool in json.loads(listed)["result"]["tools"]} - assert names == {tool.name for tool in registry if tool.annotations.read_only} + read_only = {tool.name for tool in registry if tool.annotations.read_only} + assert names == read_only - set(TOOL_CAPABILITIES) + assert names < read_only, "data tools and user listing are held back" assert "ac_execute_actions" not in names and "ac_click_mouse" not in names diff --git a/test/unit_test/headless/test_rbac_user_admin.py b/test/unit_test/headless/test_rbac_user_admin.py new file mode 100644 index 000000000..ffd7b86e3 --- /dev/null +++ b/test/unit_test/headless/test_rbac_user_admin.py @@ -0,0 +1,444 @@ +"""Managing RBAC users through AC_*, MCP, REST and the CLI; the viewer's tool set. + +``Capability.MANAGE_USERS`` existed with no route, tool or command behind it: +users could only be managed from Python. Tokens are shown once, when issued, +and reach neither the log nor an audit entry. + +Also here: read-only MCP tools that hand back data rather than screen state +need ``read_data``, a REST gate that only implements ``check()`` is accepted +again, and the REST status no longer reports a shared token RBAC refuses. + +Nothing here touches the real mouse, keyboard or screen. +""" +import json +import logging +import urllib.error +import urllib.request + +import pytest + +from je_auto_control import cli +from je_auto_control.utils.executor.action_executor import execute_action, executor +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.mcp_server.audit import AuditLogger +from je_auto_control.utils.mcp_server.http_transport import DEFAULT_PATH, HttpMCPServer +from je_auto_control.utils.mcp_server.server import MCPServer +from je_auto_control.utils.mcp_server.tools import build_default_tool_registry +from je_auto_control.utils.rbac import ( + USERS_ENV, AuthorizationContext, AuthorizationError, Capability, IssuedToken, + Role, UserAuthError, UserStore, admin, authorization_scope, can, + capability_for_tool, resolve_token, +) +from je_auto_control.utils.rbac import cli as rbac_cli +from je_auto_control.utils.rbac.policy import ( + COMMAND_CAPABILITIES, DATA_TOOLS, TOOL_CAPABILITIES, +) +from je_auto_control.utils.rest_api import rest_server +from je_auto_control.utils.rest_api.rest_auth import AuthResult, authenticate_with +from je_auto_control.utils.rest_api.rest_registry import rest_api_registry +from je_auto_control.utils.rest_api.rest_server import RestApiServer + +_SCHEME = "http" # NOSONAR localhost-only ephemeral test server; TLS out of scope +_USER_COMMANDS = ("AC_user_add", "AC_user_remove", "AC_user_set_role", + "AC_user_rotate_token", "AC_user_list") +_USER_TOOLS = tuple(name.replace("AC_", "ac_") for name in _USER_COMMANDS) +_REAL_AUDIT = admin._audit # the autouse fixture below replaces it + + +@pytest.fixture(autouse=True) +def _isolated(monkeypatch): + """No ambient RBAC, and no row written to the developer's real audit database.""" + monkeypatch.delenv(USERS_ENV, raising=False) + monkeypatch.delenv("JE_AUTOCONTROL_MCP_TOKEN", raising=False) + events = [] + monkeypatch.setattr(admin, "_audit", + lambda event, user_id, detail: events.append((event, user_id, detail))) + return events + + +@pytest.fixture() +def users(tmp_path): + store = UserStore(tmp_path / "users.json") + store.tokens = { + role: store.add_user(user_id=f"{role}-user", display_name=role, role=role) + for role in Role.all() + } + return store + + +def _as(users, role): + return authorization_scope(resolve_token(users, users.tokens[role])) + + +# --- the headless core ------------------------------------------------------ + +def test_bootstrap_then_manage_without_any_scope(tmp_path, _isolated): + path = str(tmp_path / "new" / "users.json") + issued = admin.add_user("alice", role=Role.ADMIN, display_name="Alice", + tags=["ops"], users_path=path) + assert issued["role"] == Role.ADMIN and issued["tags"] == ["ops"] + assert UserStore(path).authenticate(issued["token"]).user_id == "alice" + admin.add_user("bob", users_path=path) + assert admin.set_user_role("bob", Role.OPERATOR, users_path=path) == { + "user_id": "bob", "role": "operator"} + rotated = admin.rotate_user_token("bob", users_path=path) + assert UserStore(path).authenticate(rotated["token"]).role == Role.OPERATOR + assert admin.list_users(users_path=path) == [ + {"user_id": "alice", "display_name": "Alice", "role": "admin", "tags": ["ops"]}, + {"user_id": "bob", "display_name": "bob", "role": "operator", "tags": []}, + ] + assert admin.remove_user("bob", users_path=path) == {"user_id": "bob", "removed": True} + assert admin.remove_user("bob", users_path=path)["removed"] is False + assert [event for event, _user, _detail in _isolated] == [ + "user_added", "user_added", "user_role_set", "user_token_rotated", "user_removed"] + + +def test_no_store_named_is_an_error_not_the_default_location(): + with pytest.raises(UserAuthError, match="JE_AUTOCONTROL_RBAC_USERS"): + admin.list_users() + + +def test_the_environment_names_the_store(users, monkeypatch): + monkeypatch.setenv(USERS_ENV, str(users.path)) + assert len(admin.list_users()) == 3 + + +def test_the_last_admin_cannot_be_removed_or_demoted(users): + path = str(users.path) + with pytest.raises(UserAuthError, match="only admin"): + admin.remove_user("admin-user", users_path=path) + with pytest.raises(UserAuthError, match="only admin"): + admin.set_user_role("admin-user", Role.OPERATOR, users_path=path) + admin.set_user_role("admin-user", Role.ADMIN, users_path=path) + admin.set_user_role("operator-user", Role.ADMIN, users_path=path) + assert admin.remove_user("admin-user", users_path=path)["removed"] is True + + +def test_only_an_admin_manages_users_and_only_its_own_store(users, tmp_path): + for role in (Role.VIEWER, Role.OPERATOR): + with _as(users, role), pytest.raises(AuthorizationError) as refused: + admin.list_users() + assert refused.value.capability == Capability.MANAGE_USERS + with _as(users, Role.ADMIN): + assert len(admin.list_users()) == 3 + assert len(admin.list_users(users_path=str(users.path))) == 3 + with pytest.raises(AuthorizationError, match="another file"): + admin.add_user("x", users_path=str(tmp_path / "elsewhere.json")) + assert not (tmp_path / "elsewhere.json").exists() + + +def test_only_admin_holds_manage_users(): + assert [role for role in Role.all() if can(role, Capability.MANAGE_USERS)] == [Role.ADMIN] + + +# --- a token is shown once and never logged --------------------------------- + +def test_an_issued_token_serialises_but_does_not_print(tmp_path): + issued = admin.add_user("alice", role=Role.ADMIN, users_path=str(tmp_path / "u.json")) + assert isinstance(issued, IssuedToken) + token = issued["token"] + assert json.loads(json.dumps(issued))["token"] == token + for text in (str(issued), repr(issued), f"{issued}", "%s" % (issued,), str({"r": issued})): + assert token not in text and "" in text + + +def test_the_executor_returns_the_token_and_logs_none_of_it(tmp_path, caplog): + path = str(tmp_path / "u.json") + autocontrol_logger.addHandler(caplog.handler) + try: + with caplog.at_level(logging.DEBUG, logger=autocontrol_logger.name): + record = execute_action([ + ["AC_user_add", {"user_id": "alice", "role": "admin", "users_path": path}], + ["AC_user_rotate_token", {"user_id": "alice", "users_path": path}], + ["AC_user_list", {"users_path": path}], + ]) + finally: + autocontrol_logger.removeHandler(caplog.handler) + added, rotated, listed = record.values() + assert listed == [{"user_id": "alice", "display_name": "alice", "role": "admin", "tags": []}] + assert UserStore(path).authenticate(rotated["token"]).user_id == "alice" + logged = "\n".join(entry.getMessage() for entry in caplog.records) + assert "AC_user_add" in logged, "the run was logged" + assert added["token"] not in logged and rotated["token"] not in logged + assert added["token"] not in (tmp_path / "u.json").read_text(encoding="utf-8") + + +def test_the_real_audit_writer_is_given_no_token(tmp_path, monkeypatch): + rows = [] + + class _Audit: + def log(self, event_type, **fields): + rows.append((event_type, fields)) + + from je_auto_control.utils.remote_desktop import audit_log + monkeypatch.setattr(audit_log, "default_audit_log", _Audit) + monkeypatch.setattr(admin, "_audit", _REAL_AUDIT) + issued = admin.add_user("alice", role=Role.ADMIN, users_path=str(tmp_path / "u.json")) + rotated = admin.rotate_user_token("alice", users_path=str(tmp_path / "u.json")) + assert [event for event, _fields in rows] == ["rbac_user_added", "rbac_user_token_rotated"] + assert rows[0][1] == {"viewer_id": "local", "detail": "user=alice role=admin"} + assert issued["token"] not in json.dumps(rows) and rotated["token"] not in json.dumps(rows) + + +# --- AC_* commands ---------------------------------------------------------- + +def test_the_commands_exist_and_need_manage_users(): + for name in _USER_COMMANDS: + assert name in executor.known_commands() + assert COMMAND_CAPABILITIES[name] == Capability.MANAGE_USERS + + +def test_an_operator_cannot_run_a_user_command_and_an_admin_can(users): + action = [["AC_user_set_role", {"user_id": "viewer-user", "role": "operator"}]] + with _as(users, Role.OPERATOR): + refused = execute_action(action) + assert "manage_users" in str(list(refused.values())[0]) + assert users.get("viewer-user").role == Role.VIEWER + with _as(users, Role.ADMIN): + execute_action(action) + assert users.get("viewer-user").role == Role.OPERATOR + + +# --- REST ------------------------------------------------------------------- + +def _call(server, method, path, token=None, body=None): + host, port = server.address + data = json.dumps(body).encode("utf-8") if body is not None else None + headers = {"Content-Type": "application/json"} if data else {} + if token is not None: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request( + f"{_SCHEME}://{host}:{port}{path}", data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=5) as response: # nosec B310 # reason: loopback test server + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + +class _FakeAudit: + def __init__(self): + self.rows = [] + + def log(self, event_type, **fields): + self.rows.append({"event_type": event_type, **fields}) + + +@pytest.fixture() +def rest(): + started = [] + + def start(**kwargs): + server = RestApiServer(host="127.0.0.1", port=0, enable_audit=False, **kwargs) + server._audit_log = _FakeAudit() + server.start() + started.append(server) + return server + + yield start + for server in started: + server.stop(timeout=1.0) + + +def test_users_are_managed_over_rest_by_an_admin_only(rest, users): + server = rest(user_store=users) + body = {"actions": [["AC_user_add", {"user_id": "carol", "role": "operator"}]]} + status, reply = _call(server, "POST", "/execute", users.tokens[Role.OPERATOR], body) + assert status == 403 and reply["required_capability"] == Capability.MANAGE_USERS + assert users.get("carol") is None + status, reply = _call(server, "POST", "/execute", users.tokens[Role.ADMIN], body) + assert status == 200 + token = list(reply["result"].values())[0]["token"] + assert _call(server, "GET", "/commands", token)[0] == 200, "the new user signs in" + assert token not in json.dumps(server._audit_log.rows) + # Demoted through the same surface, it applies to the very next request. + demote = {"actions": [["AC_user_set_role", {"user_id": "carol", "role": "viewer"}]]} + assert _call(server, "POST", "/execute", users.tokens[Role.ADMIN], demote)[0] == 200 + assert _call(server, "POST", "/execute", token, {"actions": [["AC_user_list"]]})[0] == 403 + + +# --- MCP -------------------------------------------------------------------- + +@pytest.fixture() +def mcp_http(tmp_path): + registry = [tool for tool in build_default_tool_registry(read_only=False, aliases=False) + if tool.name in _USER_TOOLS] + audit_path = tmp_path / "mcp_audit.jsonl" + started = [] + + def start(**kwargs): + server = HttpMCPServer( + mcp=MCPServer(tools=registry, audit_logger=AuditLogger(path=str(audit_path))), + host="127.0.0.1", port=0, **kwargs) + server.start() + server.audit_path = audit_path + started.append(server) + return server + + yield start + for server in started: + server.stop(timeout=1.0) + + +def _rpc(server, method, params=None, token=None): + return _call(server, "POST", DEFAULT_PATH, token, + {"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}) + + +def _tool_result(body): + return json.loads(body["result"]["content"][0]["text"]) + + +def test_the_mcp_tools_are_offered_to_an_admin_only(mcp_http, users): + server = mcp_http(user_store=users) + for role in (Role.VIEWER, Role.OPERATOR): + _status, body = _rpc(server, "tools/list", token=users.tokens[role]) + assert body["result"]["tools"] == [] + _status, body = _rpc(server, "tools/call", + {"name": "ac_user_list", "arguments": {}}, users.tokens[role]) + assert body["error"]["data"] == {"required_capability": Capability.MANAGE_USERS} + _status, body = _rpc(server, "tools/list", token=users.tokens[Role.ADMIN]) + assert sorted(tool["name"] for tool in body["result"]["tools"]) == sorted(_USER_TOOLS) + + +def test_an_admin_manages_users_over_mcp_and_the_audit_holds_no_token(mcp_http, users): + server = mcp_http(user_store=users) + admin_token = users.tokens[Role.ADMIN] + _status, body = _rpc(server, "tools/call", { + "name": "ac_user_add", "arguments": {"user_id": "dave", "role": "viewer"}}, admin_token) + added = _tool_result(body) + assert added["user_id"] == "dave" and users.authenticate(added["token"]).role == Role.VIEWER + _status, body = _rpc(server, "tools/call", { + "name": "ac_user_rotate_token", "arguments": {"user_id": "dave"}}, admin_token) + rotated = _tool_result(body) + assert rotated["token"] != added["token"] + assert _rpc(server, "tools/list", token=added["token"])[0] == 401, "the old token is dead" + _status, body = _rpc(server, "tools/call", { + "name": "ac_user_set_role", "arguments": {"user_id": "dave", "role": "admin"}}, + admin_token) + assert _tool_result(body) == {"user_id": "dave", "role": "admin"} + _status, body = _rpc(server, "tools/call", {"name": "ac_user_list", "arguments": {}}, + rotated["token"]) + assert "dave" in [user["user_id"] for user in _tool_result(body)["users"]] + _status, body = _rpc(server, "tools/call", { + "name": "ac_user_remove", "arguments": {"user_id": "dave"}}, admin_token) + assert _tool_result(body) == {"user_id": "dave", "removed": True} + audit = server.audit_path.read_text(encoding="utf-8") + assert added["token"] not in audit and rotated["token"] not in audit + assert all(json.loads(line)["user_id"] for line in audit.splitlines()) + + +def test_without_rbac_the_tools_need_a_named_store(mcp_http): + """No user store on the server: nothing is authorised, and nothing is guessed.""" + server = mcp_http() + _status, body = _rpc(server, "tools/call", {"name": "ac_user_list", "arguments": {}}) + assert body["result"]["isError"] is True + assert "JE_AUTOCONTROL_RBAC_USERS" in body["result"]["content"][0]["text"] + + +# --- the CLI ---------------------------------------------------------------- + +def test_the_cli_bootstraps_the_first_admin(tmp_path, capsys): + path = str(tmp_path / "users.json") + assert cli.main(["users", "--users", path, "add", "alice", "--role", "admin", + "--name", "Alice"]) == 0 + out = capsys.readouterr().out.splitlines() + assert out[0].startswith("alice: token (shown once") + assert UserStore(path).authenticate(out[1]).role == Role.ADMIN + assert cli.main(["users", "--users", path, "--json", "add", "bob"]) == 0 + bob = json.loads(capsys.readouterr().out) + assert bob["role"] == Role.VIEWER and UserStore(path).authenticate(bob["token"]) + assert cli.main(["users", "--users", path, "set-role", "bob", "operator"]) == 0 + assert cli.main(["users", "--users", path, "rotate-token", "bob"]) == 0 + rotated = capsys.readouterr().out.splitlines()[-1] + assert UserStore(path).authenticate(rotated).role == Role.OPERATOR + assert cli.main(["users", "--users", path, "list"]) == 0 + assert capsys.readouterr().out.splitlines() == [ + "alice\tadmin\tAlice", "bob\toperator\tbob"] + assert cli.main(["users", "--users", path, "remove", "bob"]) == 0 + assert cli.main(["users", "--users", path, "remove", "bob"]) == 1 + + +def test_the_cli_reports_errors_without_a_traceback(tmp_path, capsys, monkeypatch): + path = str(tmp_path / "users.json") + assert cli.main(["users", "list"]) == 1 + assert "JE_AUTOCONTROL_RBAC_USERS" in capsys.readouterr().err + cli.main(["users", "--users", path, "add", "alice", "--role", "admin"]) + assert cli.main(["users", "--users", path, "remove", "alice"]) == 1 + assert "only admin" in capsys.readouterr().err + monkeypatch.setenv(USERS_ENV, path) + assert rbac_cli.main(["list"]) == 0, "python -m je_auto_control.utils.rbac" + assert capsys.readouterr().out == "alice\tadmin\talice\n" + assert rbac_cli.main(["set-role", "nobody", "viewer"]) == 1 + + +# --- the viewer's tool set -------------------------------------------------- + +def test_data_tools_need_more_than_read_screen(users): + registry = build_default_tool_registry(read_only=False) + by_name = {tool.name: tool for tool in registry} + assert DATA_TOOLS <= set(by_name), sorted(DATA_TOOLS - set(by_name)) + for name in ("ac_sql_query", "ac_load_dotenv", "ac_get_clipboard", "ac_jwt_encode"): + assert name in DATA_TOOLS + for name in DATA_TOOLS: + assert by_name[name].annotations.read_only, f"{name} is covered by drive_input already" + assert capability_for_tool(name, True) == Capability.READ_DATA + assert can(Role.VIEWER, Capability.READ_DATA) is False + assert can(Role.OPERATOR, Capability.READ_DATA) and can(Role.ADMIN, Capability.READ_DATA) + with authorization_scope(AuthorizationContext("viewer-user", Role.VIEWER)): + listed = MCPServer(tools=registry).handle_line( + json.dumps({"jsonrpc": "2.0", "id": 1, "method": "tools/list"})) + names = {tool["name"] for tool in json.loads(listed)["result"]["tools"]} + assert not names & DATA_TOOLS + assert {"ac_screen_size", "ac_list_windows", "ac_get_pixel", "screenshot"} - names == { + "screenshot"}, "screen-state tools stay; the screenshot tool is not read-only" + + +def test_an_alias_needs_what_its_tool_needs(): + from je_auto_control.utils.mcp_server.tools import _DEFAULT_ALIASES + registry = {tool.name: tool for tool in build_default_tool_registry(read_only=False)} + for alias, canonical in _DEFAULT_ALIASES.items(): + read_only = registry[canonical].annotations.read_only + assert capability_for_tool(alias, read_only) == capability_for_tool( + canonical, read_only), alias + assert set(TOOL_CAPABILITIES) <= set(registry), "every listed tool exists" + + +# --- a gate that only implements check() ------------------------------------ + +class _CheckOnlyGate: + """A gate written before ``authenticate`` existed.""" + + def check(self, *, client_ip, header_value): + return "ok" if header_value == "Bearer legacy" else "unauthorized" + + +def test_authenticate_with_falls_back_to_check(): + gate = _CheckOnlyGate() + assert authenticate_with(gate, client_ip="1", header_value="Bearer legacy") == AuthResult("ok") + assert authenticate_with(gate, client_ip="1", header_value=None).verdict == "unauthorized" + + +def test_a_check_only_gate_serves_requests(rest, monkeypatch): + monkeypatch.setitem(rest_server._POST_ROUTES, "/execute", + lambda ctx: (200, {"result": "ran"})) + server = rest(token="unused") + server._server.auth_gate = _CheckOnlyGate() + assert _call(server, "POST", "/execute", "legacy", {"actions": []}) == ( + 200, {"result": "ran"}) + assert _call(server, "POST", "/execute", "wrong", {"actions": []})[0] == 401 + + +# --- the REST status says how to authenticate ------------------------------- + +def test_status_reports_rbac_instead_of_a_token_that_is_refused(users): + try: + shared = rest_api_registry.start(port=0, token="shared", enable_audit=False) + assert (shared["token"], shared["rbac"], shared["users_path"]) == ("shared", False, None) + status = rest_api_registry.start(port=0, token="shared", enable_audit=False, + user_store=users) + assert status["running"] is True and status["rbac"] is True + assert status["token"] is None and status["users_path"] == str(users.path) + finally: + stopped = rest_api_registry.stop() + assert stopped["rbac"] is False and stopped["token"] is None diff --git a/test/unit_test/headless/test_rbac_users_gui.py b/test/unit_test/headless/test_rbac_users_gui.py new file mode 100644 index 000000000..d25eba96d --- /dev/null +++ b/test/unit_test/headless/test_rbac_users_gui.py @@ -0,0 +1,228 @@ +"""The REST API tab under RBAC: the real authentication state, and user management. + +The tab showed the shared bearer token even when a user store was configured, +where the server refuses that token. It now says whose tokens are accepted, +and its users group adds, re-roles, rotates and removes users through the +same headless functions as ``AC_user_*``. + +Offscreen Qt only; no window is shown, no dialog is opened and the clipboard +is not touched. +""" +import os + +import pytest + +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import QApplication, QMessageBox # noqa: E402 + +from je_auto_control.gui import rbac_users_panel # noqa: E402 +from je_auto_control.gui.language_wrapper import ( # noqa: E402 + english, japanese, simplified_chinese, traditional_chinese, +) +from je_auto_control.gui.rbac_users_panel import RbacUsersPanel # noqa: E402 +from je_auto_control.gui.rest_api_tab import RestApiTab # noqa: E402 +from je_auto_control.utils.rbac import USERS_ENV, Role, UserStore, admin # noqa: E402 +from je_auto_control.utils.rest_api.rest_registry import rest_api_registry # noqa: E402 + + +@pytest.fixture(scope="module") +def app(): + return QApplication.instance() or QApplication([]) + + +@pytest.fixture(autouse=True) +def _isolated(monkeypatch): + """No ambient RBAC, no audit row in the developer's database, no leftover server.""" + monkeypatch.delenv(USERS_ENV, raising=False) + monkeypatch.setattr(admin, "_audit", lambda *_args: None) + yield + rest_api_registry.stop() + + +@pytest.fixture() +def tab(app): + widget = RestApiTab() + yield widget + widget._timer.stop() + widget.deleteLater() + + +def _t(key): + """The string the GUI shows for ``key`` in whatever language is active.""" + return rbac_users_panel._t(key) + + +def _rows(panel): + table = panel._table + return [tuple(table.item(row, column).text() for column in range(3)) + for row in range(table.rowCount())] + + +def _fill(panel, user_id, role=Role.VIEWER, name=""): + panel._id_input.setText(user_id) + panel._name_input.setText(name) + panel._role_input.setCurrentText(role) + + +# --- the tab shows how to authenticate -------------------------------------- + +def test_shared_token_mode_shows_the_token(tab): + tab._port_input.setValue(0) + tab._token_input.setText("shared-secret") + tab._audit_check.setChecked(False) + tab._on_start() + assert tab._token_value.text() == "shared-secret" + assert tab._shared_token == "shared-secret" + assert tab._status_label.text() == _t("rest_running") + tab._on_stop() + assert tab._token_value.text() == "-" and tab._shared_token is None + + +def test_rbac_mode_names_the_user_store_instead_of_a_refused_token(tab, tmp_path): + path = tmp_path / "users.json" + token = UserStore(path).add_user(user_id="alice", display_name="Alice", role=Role.ADMIN) + tab._users_panel._path_input.setText(str(path)) + tab._users_panel._enable_check.setChecked(True) + tab._port_input.setValue(0) + tab._token_input.setText("shared-secret") + tab._audit_check.setChecked(False) + tab._on_start() + server = rest_api_registry.server + assert server.user_store is not None and server.user_store.path == path.resolve() + shown = tab._token_value.text() + assert "shared-secret" not in shown and server.token not in shown and token not in shown + assert str(path.resolve()) in shown + assert tab._shared_token is None, "nothing for Copy token to hand out" + assert tab._status_label.text() == _t("rest_running_rbac") + + +def test_the_user_store_is_used_only_when_ticked(tab, tmp_path): + tab._users_panel._path_input.setText(str(tmp_path / "users.json")) + tab._users_panel._enable_check.setChecked(False) + tab._port_input.setValue(0) + tab._audit_check.setChecked(False) + tab._on_start() + assert rest_api_registry.server.user_store is None + assert tab._shared_token == rest_api_registry.server.token + + +def test_the_tab_offers_the_user_commands_in_its_menu(tab): + keys = [key for key, _handler in tab.menu_actions()] + assert keys[:6] == ["rest_start", "rest_stop", "rest_copy_url", "rest_copy_token", + "rest_config_export", "rest_config_import"] + assert keys[6:] == ["rest_users_refresh", "rest_users_add", "rest_users_set_role", + "rest_users_rotate", "rest_users_remove", "rest_users_copy_token"] + assert all(callable(handler) for _key, handler in tab.menu_actions()) + + +# --- the users group -------------------------------------------------------- + +@pytest.fixture() +def panel(app, tmp_path): + widget = RbacUsersPanel() + widget._path_input.setText(str(tmp_path / "users.json")) + yield widget + widget.deleteLater() + + +def test_the_path_defaults_to_the_configured_store(app, tmp_path, monkeypatch): + monkeypatch.setenv(USERS_ENV, str(tmp_path / "configured.json")) + widget = RbacUsersPanel() + assert widget.users_path() == str(tmp_path / "configured.json") + assert widget._enable_check.isChecked() and widget.user_store() is not None + widget.deleteLater() + empty = RbacUsersPanel() + empty._path_input.clear() + assert empty.users_path() is None and empty.user_store() is None + empty.refresh() + assert empty._status_label.text() == _t("rest_users_no_store") + empty.deleteLater() + + +def test_add_shows_the_token_once(panel, tmp_path): + _fill(panel, "alice", Role.ADMIN, "Alice") + panel._on_add() + token = panel._token_value.text() + assert UserStore(tmp_path / "users.json").authenticate(token).role == Role.ADMIN + assert _rows(panel) == [("alice", "Alice", "admin")] + assert token not in panel._status_label.text() + panel.refresh() + assert panel._token_value.text() == "", "gone after the next operation" + assert _rows(panel) == [("alice", "Alice", "admin")] + + +def test_select_a_row_then_change_role_rotate_and_remove(panel, tmp_path, monkeypatch): + store_path = tmp_path / "users.json" + _fill(panel, "alice", Role.ADMIN) + panel._on_add() + _fill(panel, "bob", Role.VIEWER, "Bob") + panel._on_add() + first_token = panel._token_value.text() + panel._id_input.clear() + panel._table.selectRow(1) + assert panel._id_input.text() == "bob" and panel._role_input.currentText() == "viewer" + panel._role_input.setCurrentText(Role.OPERATOR) + panel._on_set_role() + assert _rows(panel)[1] == ("bob", "Bob", "operator") + assert panel._token_value.text() == "" + panel._on_rotate() + rotated = panel._token_value.text() + assert rotated and rotated != first_token + assert UserStore(store_path).authenticate(rotated).user_id == "bob" + answers = iter([QMessageBox.StandardButton.No, QMessageBox.StandardButton.Yes]) + monkeypatch.setattr(rbac_users_panel.QMessageBox, "question", + lambda *_args, **_kwargs: next(answers)) + panel._on_remove() + assert len(_rows(panel)) == 2, "declined" + panel._on_remove() + assert _rows(panel) == [("alice", "alice", "admin")] + + +def test_errors_are_shown_in_the_panel_not_in_a_dialog(panel, monkeypatch): + monkeypatch.setattr(rbac_users_panel.QMessageBox, "question", + lambda *_args, **_kwargs: QMessageBox.StandardButton.Yes) + panel._on_add() + assert panel._status_label.text() == _t("rest_users_no_user") + _fill(panel, "alice", Role.ADMIN) + panel._on_add() + panel._on_add() + assert "already exists" in panel._status_label.text() and panel._token_value.text() == "" + panel._on_remove() + assert "only admin" in panel._status_label.text() + _fill(panel, "nobody") + panel._on_rotate() + assert "unknown user_id" in panel._status_label.text() + + +def test_an_unreadable_store_is_reported(panel, tmp_path): + (tmp_path / "users.json").write_text("not json", encoding="utf-8") + panel.refresh() + assert _rows(panel) == [] + _fill(panel, "alice", Role.ADMIN) + panel._on_add() + assert "unreadable" in panel._status_label.text() + assert (tmp_path / "users.json").read_text(encoding="utf-8") == "not json" + + +# --- the strings exist in every language ------------------------------------ + +def test_every_new_key_is_translated_in_all_four_catalogues(): + catalogues = { + "english": english.english_word_dict, + "japanese": japanese.japanese_word_dict, + "traditional_chinese": traditional_chinese.traditional_chinese_word_dict, + "simplified_chinese": simplified_chinese.simplified_chinese_word_dict, + } + keys = [key for key in catalogues["english"] + if key.startswith("rest_users_") or key in ("rest_token_rbac", "rest_running_rbac")] + assert len(keys) == 24 + for language, words in catalogues.items(): + for key in keys: + assert words.get(key), f"{language} lacks {key}" + for words in catalogues.values(): + assert "{path}" in words["rest_token_rbac"] and "{user}" in words["rest_users_remove_confirm"] + assert "{count}" in words["rest_users_count"] and "{path}" in words["rest_users_count"] + assert "{user}" in words["rest_users_token_issued"] diff --git a/test/unit_test/headless/test_signing_execution_roles.py b/test/unit_test/headless/test_signing_execution_roles.py index db49bf380..b9c0ec387 100644 --- a/test/unit_test/headless/test_signing_execution_roles.py +++ b/test/unit_test/headless/test_signing_execution_roles.py @@ -241,7 +241,7 @@ def test_the_facade_and_the_script_builder_expose_the_key_pair_command(): assert name in je_auto_control.__all__ and hasattr(je_auto_control, name) specs = {spec.command: spec for spec in _build_specs()} assert [field.name for field in specs["AC_create_signing_keypair"].fields] == [ - "private_path", "public_path"] + "private_path", "public_path", "passphrase"] assert "private_key_path" in [f.name for f in specs["AC_sign_action_file"].fields] assert "public_key_path" in [f.name for f in specs["AC_verify_action_file"].fields] diff --git a/test/unit_test/headless/test_signing_passphrase_e2e.py b/test/unit_test/headless/test_signing_passphrase_e2e.py new file mode 100644 index 000000000..10c292233 --- /dev/null +++ b/test/unit_test/headless/test_signing_passphrase_e2e.py @@ -0,0 +1,285 @@ +"""A passphrase for the Ed25519 private key, and enforced signatures end to end. + +The private key was plain PEM: whoever copied the file could sign. It can +now be created with a passphrase, which the signer supplies as an argument or +through ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE``; a key created without +one loads as before. + +Enforced signatures were only exercised on the shared reader. Here they are +driven through the two network surfaces that run a file -- REST +``POST /execute_file`` and the MCP ``ac_execute_action_file`` tool -- over +loopback, with an execution endpoint that holds the public key only. + +Nothing here touches the real mouse, keyboard or screen: the action files +hold one probe command registered for the test. +""" +import json +import logging +import urllib.error +import urllib.request + +import pytest + +from je_auto_control.utils.action_signing import ( + create_signing_keypair, sign_action_file, signer, verify_action_file, +) +from je_auto_control.utils.action_signing.config import PASSPHRASE_ENV +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlSignatureException, +) +from je_auto_control.utils.executor.action_executor import execute_action, executor +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.mcp_server.http_transport import DEFAULT_PATH, HttpMCPServer +from je_auto_control.utils.mcp_server.server import MCPServer +from je_auto_control.utils.mcp_server.tools import build_default_tool_registry +from je_auto_control.utils.rbac import USERS_ENV +from je_auto_control.utils.rest_api.rest_server import RestApiServer + +pytest.importorskip("cryptography", exc_type=ImportError) + +PRIVATE_ENV = "JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY" +PUBLIC_ENV = "JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY" +LEGACY_ENV = "JE_AUTOCONTROL_ACCEPT_LEGACY_ACTION_SIGNATURES" +REQUIRE_ENV = "JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS" +_SCHEME = "http" # NOSONAR localhost-only ephemeral test server; TLS out of scope +_PROBE = "AC_signed_probe" +_SECRET = "correct horse battery staple" # NOSONAR a test passphrase for a throwaway key + + +@pytest.fixture(autouse=True) +def _clean_environment(monkeypatch, tmp_path): + """No signing or RBAC variable leaks in, and the per-user HMAC key lives in tmp_path.""" + for name in (PRIVATE_ENV, PUBLIC_ENV, LEGACY_ENV, REQUIRE_ENV, PASSPHRASE_ENV, USERS_ENV, + "JE_AUTOCONTROL_MCP_TOKEN", "JE_AUTOCONTROL_MCP_PATH_ROOTS"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(signer, "_default_key_path", + lambda: tmp_path / "home" / "action_signing_key") + + +@pytest.fixture() +def action_file(tmp_path): + path = tmp_path / "flow.json" + path.write_text(json.dumps([[_PROBE, {"value": "signed"}]]), encoding="utf-8") + return path + + +# --- the passphrase --------------------------------------------------------- + +@pytest.fixture() +def locked_keys(tmp_path): + private, public = tmp_path / "signer" / "private.pem", tmp_path / "public.pem" + create_signing_keypair(private, public, passphrase=_SECRET) + return private, public + + +def test_a_passphrase_encrypts_the_private_key_file(locked_keys, tmp_path): + private, _public = locked_keys + assert b"ENCRYPTED PRIVATE KEY" in private.read_bytes() + plain = tmp_path / "plain.pem" + create_signing_keypair(plain, tmp_path / "plain.pub") + assert b"ENCRYPTED" not in plain.read_bytes() + empty = tmp_path / "empty.pem" + create_signing_keypair(empty, tmp_path / "empty.pub", passphrase="") + assert b"ENCRYPTED" not in empty.read_bytes(), "an empty passphrase is no passphrase" + + +def test_signing_with_a_locked_key_needs_the_passphrase(locked_keys, action_file): + private, public = locked_keys + with pytest.raises(AutoControlException, match=PASSPHRASE_ENV): + sign_action_file(action_file, private_key_path=private) + with pytest.raises(AutoControlException, match="wrong passphrase"): + sign_action_file(action_file, private_key_path=private, passphrase="nope") + assert not action_file.with_name("flow.json.sig").exists() + sign_action_file(action_file, private_key_path=private, passphrase=_SECRET) + assert verify_action_file(action_file, public_key_path=public).verified + + +def test_the_signer_reads_the_passphrase_from_the_environment( + locked_keys, action_file, monkeypatch): + private, public = locked_keys + monkeypatch.setenv(PRIVATE_ENV, str(private)) + monkeypatch.setenv(PASSPHRASE_ENV, _SECRET) + sign_action_file(action_file) + assert verify_action_file(action_file, public_key_path=public).verified + # A signing machine verifies with the key it derives from its private half. + assert verify_action_file(action_file).verified + + +def test_bytes_and_text_passphrases_are_the_same(locked_keys, action_file): + private, public = locked_keys + sign_action_file(action_file, private_key_path=private, passphrase=_SECRET.encode("utf-8")) + assert verify_action_file(action_file, public_key_path=public).verified + + +def test_an_unencrypted_key_still_loads_even_with_a_passphrase_around( + tmp_path, action_file, monkeypatch): + private, public = tmp_path / "private.pem", tmp_path / "public.pem" + create_signing_keypair(private, public) + sign_action_file(action_file, private_key_path=private) + assert verify_action_file(action_file, public_key_path=public).verified + monkeypatch.setenv(PASSPHRASE_ENV, _SECRET) + sign_action_file(action_file, private_key_path=private) + sign_action_file(action_file, private_key_path=private, passphrase="anything") + assert verify_action_file(action_file, public_key_path=public).verified + + +def test_the_commands_take_a_passphrase_and_it_is_not_logged(tmp_path, action_file, caplog): + private, public = tmp_path / "k" / "private.pem", tmp_path / "k" / "public.pem" + private.parent.mkdir() + autocontrol_logger.addHandler(caplog.handler) + try: + with caplog.at_level(logging.DEBUG, logger=autocontrol_logger.name): + record = execute_action([ + ["AC_create_signing_keypair", {"private_path": str(private), + "public_path": str(public), + "passphrase": _SECRET}], + ["AC_sign_action_file", {"path": str(action_file), + "private_key_path": str(private), + "passphrase": _SECRET}], + ["AC_sign_action_file", {"path": str(action_file), + "private_key_path": str(private)}], + ]) + finally: + autocontrol_logger.removeHandler(caplog.handler) + created, signed, refused = record.values() + assert created == {"private_path": str(private), "public_path": str(public)} + assert signed == {"signature_path": str(action_file) + ".sig"} + assert PASSPHRASE_ENV in str(refused) + assert b"ENCRYPTED" in private.read_bytes() + assert verify_action_file(action_file, public_key_path=public).verified + assert _SECRET not in "\n".join(entry.getMessage() for entry in caplog.records) + + +# --- enforced signatures, end to end ---------------------------------------- + +@pytest.fixture() +def probe(monkeypatch): + values = [] + monkeypatch.setitem(executor.event_dict, _PROBE, + lambda value=None: values.append(value) or value) + return values + + +@pytest.fixture() +def endpoint(monkeypatch, locked_keys): + """An execution endpoint: signatures enforced, the public key and nothing else.""" + monkeypatch.setenv(PUBLIC_ENV, str(locked_keys[1])) + monkeypatch.setenv(REQUIRE_ENV, "1") + return locked_keys + + +def _sign(endpoint, path): + """What the signing machine does; it is the only place the private key is.""" + return sign_action_file(path, private_key_path=endpoint[0], passphrase=_SECRET) + + +def _post(server, path, body, token=None, headers=None): + host, port = server.address + request = urllib.request.Request( + f"{_SCHEME}://{host}:{port}{path}", data=json.dumps(body).encode("utf-8"), + method="POST", headers={"Content-Type": "application/json", **(headers or {})}) + if token is not None: + request.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(request, timeout=5) as response: # nosec B310 # reason: loopback test server + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read().decode("utf-8")) + + +@pytest.fixture() +def rest(): + server = RestApiServer(host="127.0.0.1", port=0, token="rest-token", enable_audit=False) + server.start() + yield server + server.stop(timeout=1.0) + + +def _rest_run(server, path): + return _post(server, "/execute_file", {"path": str(path)}, token="rest-token") + + +def test_rest_execute_file_runs_a_signed_file_and_refuses_the_rest( + rest, endpoint, action_file, probe, tmp_path): + status, body = _rest_run(rest, action_file) + assert status == 403 and "missing signature sidecar" in body["error"] + _sign(endpoint, action_file) + status, body = _rest_run(rest, action_file) + assert status == 200 and probe == ["signed"] + action_file.write_text(json.dumps([[_PROBE, {"value": "tampered"}]]), encoding="utf-8") + status, body = _rest_run(rest, action_file) + assert status == 403 and "signature mismatch" in body["error"] + assert probe == ["signed"], "the tampered file did not run" + + +def test_rest_execute_file_refuses_another_key_and_an_hmac_sidecar( + rest, endpoint, action_file, probe, tmp_path, monkeypatch): + # Both sidecars are written "elsewhere": this endpoint refuses to make either. + monkeypatch.delenv(PUBLIC_ENV) + other_private = tmp_path / "other.pem" + create_signing_keypair(other_private, tmp_path / "other.pub") + sign_action_file(action_file, private_key_path=other_private) + monkeypatch.setenv(PUBLIC_ENV, str(endpoint[1])) + status, body = _rest_run(rest, action_file) + assert status == 403 and "different key" in body["error"] + # An HMAC sidecar, which anyone able to execute could have written. + monkeypatch.delenv(PUBLIC_ENV) + sign_action_file(action_file, key="shared-secret") + monkeypatch.setenv(PUBLIC_ENV, str(endpoint[1])) + status, body = _rest_run(rest, action_file) + assert status == 403 and "legacy HMAC signature refused" in body["error"] + assert probe == [] + + +def test_rest_cannot_sign_on_an_execution_endpoint(rest, endpoint, action_file, probe): + status, body = _post(rest, "/execute", {"actions": [ + ["AC_sign_action_file", {"path": str(action_file)}]]}, token="rest-token") + assert status == 200 + assert "verifies only" in str(list(body["result"].values())[0]) + assert not action_file.with_name("flow.json.sig").exists() + assert _rest_run(rest, action_file)[0] == 403 and probe == [] + + +def test_rest_execute_file_without_enforcement_is_unchanged(rest, action_file, probe): + assert _rest_run(rest, action_file)[0] == 200 + assert probe == ["signed"] + + +@pytest.fixture() +def mcp(): + tools = [tool for tool in build_default_tool_registry(read_only=False, aliases=False) + if tool.name == "ac_execute_action_file"] + server = HttpMCPServer(mcp=MCPServer(tools=tools), host="127.0.0.1", port=0) + server.start() + yield server + server.stop(timeout=1.0) + + +def _mcp_run(server, path): + _status, body = _post(server, DEFAULT_PATH, { + "jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": "ac_execute_action_file", "arguments": {"file_path": str(path)}}}) + result = body["result"] + return result["isError"], result["content"][0]["text"] + + +def test_mcp_execute_action_file_runs_a_signed_file_and_refuses_the_rest( + mcp, endpoint, action_file, probe): + is_error, text = _mcp_run(mcp, action_file) + assert is_error and "missing signature sidecar" in text + _sign(endpoint, action_file) + is_error, _text = _mcp_run(mcp, action_file) + assert not is_error and probe == ["signed"] + action_file.write_text(json.dumps([[_PROBE, {"value": "tampered"}]]), encoding="utf-8") + is_error, text = _mcp_run(mcp, action_file) + assert is_error and "signature mismatch" in text + assert probe == ["signed"] + + +def test_the_refusal_is_a_typed_member_of_the_family(endpoint, action_file): + from je_auto_control.utils.json.json_file import read_executable_action_json + with pytest.raises(AutoControlSignatureException) as refused: + read_executable_action_json(str(action_file)) + assert isinstance(refused.value, AutoControlException) + with pytest.raises(AutoControlSignatureException): + verify_action_file(action_file, raise_on_fail=True) From dadd447094e8a5ad00943bc7566d035101f95452 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:07:34 +0800 Subject: [PATCH 21/44] Narrow the package name to str before the allowlist check would_allow takes any object, as the gate's own check does, and handed it to a helper typed for str: the typing contract failed on all three platforms. --- je_auto_control/utils/package_manager/package_manager_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/je_auto_control/utils/package_manager/package_manager_class.py b/je_auto_control/utils/package_manager/package_manager_class.py index 8f3cb9b5b..3f2be6e58 100644 --- a/je_auto_control/utils/package_manager/package_manager_class.py +++ b/je_auto_control/utils/package_manager/package_manager_class.py @@ -85,7 +85,7 @@ def would_allow(self, package: object) -> bool: Whether the gate would let ``package`` be loaded. Nothing is imported: this is what lets an action list be validated before the load command in it has run. """ - if not is_package_name(package): + if not isinstance(package, str) or not is_package_name(package): return False return self.allow_arbitrary_packages or self._is_allowlisted(package) From 35e6a0b2141b1ab9ea777a8edea78e26d73b4ea1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:08:51 +0800 Subject: [PATCH 22/44] Stop working around a refused or revoked Wayland session, and say what each capability can do A consent the portal denied became LibeiUnavailable, which keyboard and mouse read as 'use ydotool', so the desktop was driven through /dev/uinput behind the refusal. A session the compositor disconnected kept its devices marked as emulating, so the next emission wrote into a dropped context. Neither was visible anywhere an operator could look. Authorisation is now an explicit ledger per channel (input, capture). A denied consent and a revoked session raise WaylandAuthorisationError instead of falling back; every other failure falls back as before. probe_capabilities() reports input, capture, recording and the stop shortcut independently and without side effects, including the narrower reach of X11 serving a Wayland session; the Diagnostics tab, AC_probe_capabilities and the ac_probe_capabilities MCP tool read the same snapshot. Recording is split into what the program executed (ActionJournal, no hook) and what the user typed (PhysicalRecorder: opt-in per device, virtual devices excluded, a typed error on permission denial). StopShortcutSession registers one stop key through the GlobalShortcuts portal and closes its session. The libei session can run in a helper process behind JE_AUTOCONTROL_WAYLAND_EI_WORKER, off by default. The in-process teardown and its deliberate leak are unchanged; whether a helper is warranted can only be measured on Linux, so the verify scripts now classify the half-open teardown and measure the helper, as output that cannot change either job's exit status. --- docker/eis_verify.py | 242 ++++++++ docker/libei_verify.py | 173 ++++++ docker/portal_verify.py | 49 ++ docs/CAPABILITY_MATRIX.md | 29 + docs/source/Eng/doc/record/record_doc.rst | 8 + .../doc/wayland/wayland_capabilities_doc.rst | 213 +++++++ docs/source/Eng/eng_index.rst | 1 + docs/source/Zh/doc/record/record_doc.rst | 8 + .../doc/wayland/wayland_capabilities_doc.rst | 192 +++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 27 + je_auto_control/actions.pyi | 6 + je_auto_control/gui/diagnostics_tab.py | 109 +++- .../gui/language_wrapper/english.py | 51 ++ .../gui/language_wrapper/japanese.py | 53 ++ .../language_wrapper/simplified_chinese.py | 49 ++ .../language_wrapper/traditional_chinese.py | 49 ++ .../linux_wayland/_select_input.py | 165 +++++- .../linux_wayland/authorisation.py | 207 +++++++ je_auto_control/linux_wayland/ei_client.py | 518 ++++++++++++++++++ je_auto_control/linux_wayland/ei_transport.py | 247 +++++++++ je_auto_control/linux_wayland/ei_worker.py | 323 +++++++++++ .../linux_wayland/global_shortcuts.py | 275 ++++++++++ je_auto_control/linux_wayland/input_events.py | 395 +++++++++++++ je_auto_control/linux_wayland/libei.py | 69 ++- je_auto_control/linux_wayland/listener.py | 26 +- je_auto_control/linux_wayland/oeffis.py | 43 +- je_auto_control/linux_wayland/portal.py | 17 + je_auto_control/linux_wayland/record.py | 25 +- .../utils/executor/action_executor.py | 17 + .../utils/mcp_server/tools/_factories.py | 13 + .../mcp_server/tools/_handlers_system.py | 6 + je_auto_control/wrapper/capabilities.py | 441 +++++++++++++++ .../wayland_authorisation_checklist.md | 168 ++++++ .../test_diagnostics_capabilities_gui.py | 123 +++++ .../test_wayland_capability_states.py | 511 +++++++++++++++++ .../headless/test_wayland_recording_paths.py | 398 ++++++++++++++ .../headless/test_wayland_worker_cleanup.py | 517 +++++++++++++++++ 38 files changed, 5720 insertions(+), 44 deletions(-) create mode 100644 docs/source/Eng/doc/wayland/wayland_capabilities_doc.rst create mode 100644 docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst create mode 100644 je_auto_control/linux_wayland/authorisation.py create mode 100644 je_auto_control/linux_wayland/ei_client.py create mode 100644 je_auto_control/linux_wayland/ei_transport.py create mode 100644 je_auto_control/linux_wayland/ei_worker.py create mode 100644 je_auto_control/linux_wayland/global_shortcuts.py create mode 100644 je_auto_control/linux_wayland/input_events.py create mode 100644 je_auto_control/wrapper/capabilities.py create mode 100644 test/manual_test/wayland_authorisation_checklist.md create mode 100644 test/unit_test/headless/test_diagnostics_capabilities_gui.py create mode 100644 test/unit_test/headless/test_wayland_capability_states.py create mode 100644 test/unit_test/headless/test_wayland_recording_paths.py create mode 100644 test/unit_test/headless/test_wayland_worker_cleanup.py diff --git a/docker/eis_verify.py b/docker/eis_verify.py index acd3edb64..7ea91d21a 100644 --- a/docker/eis_verify.py +++ b/docker/eis_verify.py @@ -429,6 +429,243 @@ def _live_teardown_sentinel(path: str) -> str: return f"inconclusive (rc={finished.returncode}): {detail[-1] if detail else ''}" +def measure(name: str, fn: Callable[[], Any]) -> Any: + """Run a measurement and print what it found; never count it. + + The checks decide this script's exit status. What follows are numbers and + classifications about the *opt-in* helper process — printed for whoever + reads the log, and unable to turn the job red. + """ + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a measurement that cannot run is itself a finding + print(f"note {name} — could not be measured:") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + print(f"info {name} — {detail}") + return detail + + +def _describe_exit(returncode: Any) -> str: + """An exit status as a reader wants it: clean, a code, or a signal.""" + import signal + if returncode is None: + return "still running" + if returncode == 0: + return "clean (rc=0)" + if returncode < 0: + try: + name = signal.Signals(-returncode).name + except ValueError: + name = "signal" + return f"{name} (rc={returncode})" + return f"exit status {returncode}" + + +def _percentile(samples: List[float], fraction: float) -> float: + ordered = sorted(samples) + return ordered[min(len(ordered) - 1, int(len(ordered) * fraction))] + + +def _timed_keystrokes(backend, count: int) -> List[float]: + """Milliseconds per press+release pair, ``count`` times.""" + samples = [] + for _ in range(count): + started = time.perf_counter() + backend.press_key(KEY_A) + backend.release_key(KEY_A) + samples.append((time.perf_counter() - started) * 1000.0) + return samples + + +def _helper_session(path, server_class, ei_client): + """A helper process with a live session against a server of its own.""" + if os.path.exists(path): + os.unlink(path) + server = server_class(path) + server.start() + client = ei_client.EiWorkerClient(socket_path=path, + handshake_timeout_s=5.0, + start_timeout_s=120.0) + client.start() + return client, server + + +def _measure_helper_emits(path, server_class) -> str: + """Does input sent through the helper reach the EIS server at all?""" + from je_auto_control.linux_wayland import ei_client + client, server = _helper_session(path, server_class, ei_client) + try: + backend = ei_client.WorkerBackend(client) + backend.press_key(KEY_A) + backend.release_key(KEY_A) + backend.set_position(*TARGET_POSITION) + backend.click_button(BTN_LEFT) + backend.scroll(0, 1) + _wait_for(lambda: len(server.recording.keys) >= 2 + and server.recording.absolute_motions + and len(server.recording.buttons) >= 2 + and server.recording.scrolls, 3.0, + "the helper's events at the server") + record = server.recording + return (f"keys {record.keys[:2]}, motion " + f"{record.absolute_motions[-1]}, buttons {record.buttons[:2]}, " + f"scroll {record.scrolls[-1]} (in-process: (0, {SCROLL_UNIT}))") + finally: + client.close() + server.stop() + + +def _measure_helper_latency(path, libei, server_class) -> str: + """What a pipe round trip per emission costs, next to none.""" + from je_auto_control.linux_wayland import ei_client + count = 200 + if os.path.exists(path + "-direct"): + os.unlink(path + "-direct") + direct_server = server_class(path + "-direct") + direct_server.start() + direct = libei.LibeiBackend() + direct.connect(timeout=5.0, socket_path=(path + "-direct").encode()) + try: + in_process = _timed_keystrokes(direct, count) + finally: + direct.disconnect() + direct_server.stop() + + client, server = _helper_session(path, server_class, ei_client) + try: + helper = _timed_keystrokes(ei_client.WorkerBackend(client), count) + finally: + client.close() + server.stop() + + def summary(samples: List[float]) -> str: + return (f"median {_percentile(samples, 0.5):.3f} ms, " + f"p95 {_percentile(samples, 0.95):.3f} ms, " + f"max {max(samples):.3f} ms") + + return (f"{count} press+release pairs — in-process: {summary(in_process)}" + f"; through the helper: {summary(helper)}") + + +def _measure_helper_clean_close(path, server_class) -> str: + """A key held when the helper is closed: is it released, and how?""" + from je_auto_control.linux_wayland import ei_client + client, server = _helper_session(path, server_class, ei_client) + try: + ei_client.WorkerBackend(client).press_key(KEY_A) + _wait_for(lambda: server.recording.keys, 2.0, "the held key") + before = server.recording.disconnects + client.close() + try: + _wait_for(lambda: server.recording.disconnects > before, 3.0, + "the helper to disconnect") + except AssertionError: + pass + released = (KEY_A, False) in server.recording.keys + return (f"helper exit {_describe_exit(client.returncode)}; key-up " + f"{'reached' if released else 'did NOT reach'} the server " + f"before it left; server saw " + f"{server.recording.disconnects - before} disconnect(s)") + finally: + client.close() + server.stop() + + +def _measure_helper_killed(path, server_class) -> str: + """A key held when the helper is killed outright: who releases it? + + Nothing in the helper runs after SIGKILL, so whatever the server sees is + libeis reacting to a client that vanished. This is the measurement behind + "the compositor is expected to release them" in ``ei_worker``. + """ + import signal + from je_auto_control.linux_wayland import ei_transport, ei_client + client, server = _helper_session(path, server_class, ei_client) + try: + backend = ei_client.WorkerBackend(client) + backend.press_key(KEY_A) + _wait_for(lambda: server.recording.keys, 2.0, "the held key") + before = server.recording.disconnects + os.kill(client._process.pid, signal.SIGKILL) + try: + _wait_for(lambda: server.recording.disconnects > before, 3.0, + "the server to notice the helper is gone") + except AssertionError: + pass + try: + backend.release_key(KEY_A) + reported = "the parent did NOT notice the helper died" + except ei_transport.EiWorkerDied as died: + reported = (f"the parent raised EiWorkerDied with pressed_keys=" + f"{died.pressed_keys}, exit " + f"{_describe_exit(died.returncode)}") + released = (KEY_A, False) in server.recording.keys + return (f"{reported}; the server saw " + f"{server.recording.disconnects - before} disconnect(s) and " + f"{'a' if released else 'NO'} key-up for the held key — so " + "release on a dead client is " + f"{'done by libeis' if released else 'left to the compositor'}") + finally: + client.close() + server.stop() + + +_HELPER_FLAG = "--helper-measurements" + + +def _helper_measurements_in_a_child() -> None: + """Run the helper measurements where a crash cannot reach the checks. + + They kill a client mid-session on purpose, which is a thing libeis might + not survive; in a child of its own, the worst that can happen is one more + line in the log saying so. + """ + print("-" * 72) + print(" The libei helper process (JE_AUTOCONTROL_WAYLAND_EI_WORKER=1,") + print(" off by default). Measured here, never counted as a check:", + flush=True) + try: + # This interpreter running this file with a fixed flag; no shell. + finished = subprocess.run( # nosec B603 # nosemgrep + [sys.executable, os.path.abspath(__file__), _HELPER_FLAG], + timeout=900) + except subprocess.TimeoutExpired: + print("note the helper measurements did not finish within 900s") + return + if finished.returncode != 0: + print("note the helper measurements ended with " + f"{_describe_exit(finished.returncode)} — see above; this is " + "about the opt-in helper only and is not counted") + + +def _helper_measurements_main() -> int: + """Entry point of the measurement child.""" + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from eis_server import RecordingEisServer, load_symbols + from je_auto_control.linux_wayland import libei + if load_symbols() is None: + print("note libeis is not installed; nothing to measure") + return 0 + _run_helper_measurements(os.path.join(_scratch_dir(), "eis-helper"), + libei, RecordingEisServer) + return 0 + + +def _run_helper_measurements(path, libei, server_class) -> None: + """Everything about the opt-in helper, as measurements only.""" + measure("input sent through the helper reaches the EIS server", + lambda: _measure_helper_emits(path + "-emit", server_class)) + measure("emission latency, in-process against through the helper", + lambda: _measure_helper_latency(path + "-latency", libei, + server_class)) + measure("a key held when the helper is closed", + lambda: _measure_helper_clean_close(path + "-close", server_class)) + measure("a key held when the helper is killed", + lambda: _measure_helper_killed(path + "-kill", server_class)) + + #: An offset region layout: the shape a compositor must advertise for a #: desktop whose left-most monitor sits at a negative layout coordinate. #: Region offsets are ``uint32``, so it cannot advertise the negative @@ -650,6 +887,9 @@ def main() -> int: if server.error is not None: print(f" NOTE: the server thread ended with {server.error!r}") + # Last, and outside the checks: nothing below can change the exit status. + _helper_measurements_in_a_child() + failed = [name for name, ok in _results if not ok] print("=" * 72) print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") @@ -660,4 +900,6 @@ def main() -> int: if __name__ == "__main__": + if _HELPER_FLAG in sys.argv[1:]: + sys.exit(_helper_measurements_main()) sys.exit(main()) diff --git a/docker/libei_verify.py b/docker/libei_verify.py index 8a8eae768..21b55cc96 100644 --- a/docker/libei_verify.py +++ b/docker/libei_verify.py @@ -197,6 +197,169 @@ def _check_unref_sentinel(socket_path: str) -> str: return f"no longer crashes (rc={finished.returncode}) — see above" +def measure(name: str, fn: Callable[[], Any]) -> Any: + """Run a measurement and print what it found; never count it. + + The checks above decide this script's exit status. A measurement is + something the reader of the log needs and the job must not turn red over + — a classification, a number — so nothing here touches ``_results``. + """ + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a measurement that cannot run is itself a finding + print(f"note {name} — could not be measured:") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + print(f"info {name} — {detail}") + return detail + + +def _describe_exit(returncode: Any) -> str: + """An exit status as a reader wants it: clean, a code, or a signal.""" + import signal + if returncode is None: + return "still running" + if returncode == 0: + return "clean (rc=0)" + if returncode < 0: + try: + name = signal.Signals(-returncode).name + except ValueError: + name = "signal" + return f"{name} (rc={returncode})" + return f"exit status {returncode}" + + +#: Everything up to a context whose backend is open and whose handshake has +#: not progressed — the state ``LibeiBackend._teardown`` abandons. +_HALF_OPEN_PRELUDE = ( + "import ctypes, ctypes.util, os, socket;" + "lib = ctypes.CDLL(ctypes.util.find_library('ei'));" + "lib.ei_new_sender.restype = ctypes.c_void_p;" + "lib.ei_new_sender.argtypes = (ctypes.c_void_p,);" + "lib.ei_setup_backend_socket.restype = ctypes.c_int;" + "lib.ei_setup_backend_socket.argtypes = " + "(ctypes.c_void_p, ctypes.c_char_p);" + "lib.ei_unref.restype = ctypes.c_void_p;" + "lib.ei_unref.argtypes = (ctypes.c_void_p,);" + "lib.ei_dispatch.restype = None;" + "lib.ei_dispatch.argtypes = (ctypes.c_void_p,);" + "p = {path!r};" + "h = lib.ei_new_sender(None);" + "rc = lib.ei_setup_backend_socket(h, p.encode());" + "assert rc == 0, rc;" +) + +#: How a half-open context can be ended, and the code that ends it that way. +_HALF_OPEN_ENDINGS = ( + ("ei_unref", "lib.ei_unref(h)"), + ("ei_dispatch, then ei_unref", "lib.ei_dispatch(h); lib.ei_unref(h)"), + ("no release, interpreter exits", "pass"), + ("no release, os._exit(0)", "os._exit(0)"), +) + + +def _classify_half_open_teardown(socket_path: str) -> str: + """Which ways of ending a half-open context crash, and which do not. + + Each ending runs in its own interpreter because the answer is a signal. + This is what decides whether a helper process is a real fix: it only is + if *leaving without releasing* is clean while ``ei_unref`` is not. + """ + import subprocess # nosec B404 # reason: argv list, no shell + prelude = _HALF_OPEN_PRELUDE.format(path=socket_path) + outcomes = {} + print(" half-open context, ended by:") + for label, ending in _HALF_OPEN_ENDINGS: + # This interpreter, running a program built from literals; no shell. + finished = subprocess.run([sys.executable, "-c", prelude + ending], # nosec B603 # nosemgrep + capture_output=True, timeout=60) + outcomes[label] = finished.returncode + print(f" {label:<32} {_describe_exit(finished.returncode)}", + flush=True) + unref_crashes = outcomes["ei_unref"] < 0 + leaving_is_clean = (outcomes["no release, interpreter exits"] == 0 + and outcomes["no release, os._exit(0)"] == 0) + if unref_crashes and leaving_is_clean: + return ("CLASSIFIED: the crash is ei_unref on a half-open context; " + "a process that exits without releasing is clean, so the " + "state is reclaimable by a helper process") + if not unref_crashes: + return ("CLASSIFIED: ei_unref no longer crashes on a half-open " + "context, so neither the leak nor a helper is needed — see " + "the REVISIT banner above") + return ("UNCLASSIFIED: ei_unref crashes and exiting without release is " + f"not clean either ({outcomes}); a helper would not contain it") + + +def _open_descriptors() -> int: + return len(os.listdir("/proc/self/fd")) + + +def _leak_of(attempts: int, attempt: Callable[[], Any]) -> int: + """How many descriptors ``attempts`` runs of ``attempt`` leave open.""" + before = _open_descriptors() + for _ in range(attempts): + attempt() + return _open_descriptors() - before + + +def _half_open_in_process(socket_path: str) -> None: + """One handshake that never completes, on the default in-process path.""" + from je_auto_control.linux_wayland import libei + try: + libei.LibeiBackend().connect(timeout=0.3, + socket_path=socket_path.encode("utf-8")) + except libei.LibeiUnavailable: + pass + + +def _half_open_in_helper(socket_path: str) -> Any: + """The same handshake inside a helper; returns the helper's exit status.""" + from je_auto_control.linux_wayland import ei_client, libei + client = ei_client.EiWorkerClient( + socket_path=socket_path, handshake_timeout_s=0.3, + start_timeout_s=120.0) + try: + client.start() + except libei.LibeiUnavailable: + pass + finally: + client.close() + return client.returncode + + +def _measure_helper_reclaims(socket_path: str) -> str: + """Does the opt-in helper reclaim what the in-process path leaks? + + Three half-open handshakes each way, against the same silent socket. The + in-process path is expected to leak one descriptor per attempt — that is + the documented workaround, counted here rather than asserted — and the + helper is expected to leak none and to exit without a signal. + """ + from je_auto_control.linux_wayland import ei_client + attempts = 3 + in_process = _leak_of(attempts, lambda: _half_open_in_process(socket_path)) + exits: List[Any] = [] + with_helper = _leak_of( + attempts, lambda: exits.append(_half_open_in_helper(socket_path))) + + crashed = [code for code in exits if code is not None and code < 0] + if crashed: + print() + print(" *** REVISIT *** the libei helper process itself died") + print(f" on a signal ({[_describe_exit(c) for c in crashed]})") + print(" while giving up a half-open handshake. It is opt-in") + print(" (JE_AUTOCONTROL_WAYLAND_EI_WORKER) and must stay off") + print(" until this is understood; see Progress.md.") + print() + return (f"{attempts} half-open handshakes: in-process leaked " + f"{in_process} fd(s), the helper leaked {with_helper}; helper " + f"exits {[_describe_exit(code) for code in exits]}; " + f"{ei_client.active_worker_count()} helper(s) left running") + + def _check_connect_fails_closed(socket_path: str) -> str: """A peer that sends nothing must not be reported as a live session.""" from je_auto_control.linux_wayland import libei @@ -298,6 +461,16 @@ def main() -> int: check("press_key falls through to the ydotool CLI path", _check_keyboard_falls_back) + # --- measurements: printed for the reader, never counted -------------- + # These classify the teardown the workaround exists for. They cannot + # change this script's exit status, so a surprise here reads as a banner + # in the log rather than as a red job. + print("-" * 72) + measure("teardown of a half-open context, classified", + lambda: _classify_half_open_teardown(socket_path)) + measure("the opt-in helper process against the same half-open state", + lambda: _measure_helper_reclaims(socket_path)) + server.close() print("-" * 72) diff --git a/docker/portal_verify.py b/docker/portal_verify.py index a6f4c377b..0ec5f55ad 100644 --- a/docker/portal_verify.py +++ b/docker/portal_verify.py @@ -497,6 +497,46 @@ def _run_grant_scenario(portal: Portal, server, libei, oeffis, wayland_portal, portal)) +def _measure_classification(label: str, oeffis, expect_declined: bool, + timeout: float = GRANT_TIMEOUT) -> None: + """Print how one refusal is classified; never count it. + + ``_select_input`` stops falling back to ydotool only when the portal + *declined* — recognised from the word "denied" in liboeffis's own error + text, which no fake can vouch for. This prints the real text for every + way a request can end, so a wording change shows up here as a banner + rather than as a silent change in who is allowed to fall back. + + Nothing here touches ``_results``: it informs, it cannot fail the job. + """ + try: + oeffis.connect_eis_fd(timeout=timeout) + except oeffis.PortalConsentNotGranted as error: + found = (f"outcome={error.outcome!r} detail={error.detail!r} " + f"declined={error.declined}") + declined = error.declined + except oeffis.OeffisUnavailable as error: + found = f"a plain OeffisUnavailable ({str(error)[:80]}) declined=False" + declined = False + except Exception as error: # noqa: BLE001 # reason: a measurement that cannot run is itself a finding + print(f"note classification of {label} could not be measured: " + f"{error!r}") + return + else: + print(f"note classification of {label}: a session was granted") + return + print(f"info classification of {label} — {found}") + if declined != expect_declined: + print() + print(" *** REVISIT *** this refusal is classified as " + f"declined={declined}") + print(f" but {label} should be declined={expect_declined}. The") + print(" rule is PortalConsentNotGranted.declined in oeffis.py;") + print(" until it matches, a denied consent still falls back to") + print(" ydotool (or an absent portal no longer does).") + print() + + def _run_refusal_scenarios(socket_path: str, libei, oeffis) -> None: """Every way a portal says no, and this project's answer to each.""" with Portal(socket_path, behaviour="deny"): @@ -504,18 +544,25 @@ def _run_refusal_scenarios(socket_path: str, libei, oeffis) -> None: lambda: _check_portal_refuses(oeffis)) check("a refused portal reaches the caller as LibeiUnavailable", lambda: _check_backend_surfaces_the_refusal(libei)) + _measure_classification("a dismissed consent dialog", oeffis, True) with Portal(socket_path, behaviour="stall"): check("a consent dialog left open times out on this project's clock", lambda: _check_open_dialog_times_out(oeffis)) + _measure_classification("a dialog left open", oeffis, False, + REFUSAL_TIMEOUT) with Portal(socket_path, behaviour="no-fd"): check("a portal that withholds the descriptor fails closed", lambda: _check_portal_refuses(oeffis)) + _measure_classification("a withheld descriptor", oeffis, False) with Portal(socket_path, behaviour="close"): check("a portal that closes the session fails closed", lambda: _check_portal_refuses(oeffis)) + _measure_classification("a session the portal closed", oeffis, False) with Portal(socket_path, version=1): check("a portal too old to have ConnectToEIS fails closed", lambda: _check_portal_refuses(oeffis)) + _measure_classification("a portal without ConnectToEIS", oeffis, + False) def _run_capture_scenarios(socket_path: str, wayland_portal) -> None: @@ -564,6 +611,8 @@ def main() -> int: lambda: _check_liboeffis_binds(oeffis)) check("no portal on the bus is refused rather than waited on", lambda: _check_no_portal_is_refused(oeffis)) + _measure_classification("no portal on the bus", oeffis, False, + REFUSAL_TIMEOUT) with Portal(socket_path) as portal: _run_grant_scenario(portal, server, libei, oeffis, wayland_portal) _run_refusal_scenarios(socket_path, libei, oeffis) diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index d19029a09..be434f06a 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -292,6 +292,35 @@ dialog left open, a withheld descriptor, a closed session, a portal too old for `ConnectToEIS`, no portal at all — each of which has to fail closed on AutoControl's own clock. +What each of those outcomes *means* is now an explicit state rather than one +undifferentiated failure. `probe_capabilities()` (`AC_probe_capabilities`, the +`ac_probe_capabilities` MCP tool, and the Diagnostics tab) reports input, +capture, recording and the stop shortcut separately, each as `available`, +`not_requested`, `requesting`, `needs_permission`, `needs_setup`, +`session_closed`, `revoked` or `compositor_restarted`, with the backend +serving it and whether it reaches the whole desktop — which it does not when +the X11 backend is serving a Wayland session through XWayland. The probe asks +the desktop for nothing. Two outcomes no longer fall back to ydotool: a +consent the portal **denied**, and a session the compositor **revoked**; both +raise `WaylandAuthorisationError` until `reset_input_authorisation()`. Every +other failure falls back as before. The denial is recognised from liboeffis's +own error text, which only the `portal-verification` job can vouch for: it +prints how each of the six refusals is classified, with a `*** REVISIT ***` +banner if the dismissed dialog is not the one classified as declined. Until +that job has been read, treat "a denied consent stops the fallback" as +implemented against fakes and unverified against a real liboeffis. + +The libei session can also be moved into a helper process +(`JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`, **off by default**). The in-process +path and its deliberate one-context-one-descriptor leak on a half-open +handshake are unchanged and remain the default; the upstream `ei_unref` crash +is not fixed. `docker/libei_verify.py` now classifies that teardown — which +ways of ending a half-open context crash and which are clean — and counts +descriptors leaked in-process against through the helper, and +`docker/eis_verify.py` measures the helper's latency and what happens to a +held key when the helper is closed or killed. Those are printed as +measurements and never change either job's exit status. + What is still not covered is the consent dialog as a *dialog*: no user dismisses anything in CI, so what a real dialog looks like and how long a real one blocks stay mutter's business. The compositor also refuses global input diff --git a/docs/source/Eng/doc/record/record_doc.rst b/docs/source/Eng/doc/record/record_doc.rst index 94a136a23..60df446cc 100644 --- a/docs/source/Eng/doc/record/record_doc.rst +++ b/docs/source/Eng/doc/record/record_doc.rst @@ -71,3 +71,11 @@ The remaining helpers — ``insert_action``, ``remove_action``, ``scale_coordinates`` are also exposed over MCP as ``ac_dedupe_moves`` / ``ac_merge_sleeps`` / ``ac_trim_actions`` / ``ac_adjust_delays`` / ``ac_scale_coordinates``. + +Recording on Wayland +==================== + +``record()`` raises on a Wayland session: the compositor forbids a global +input hook. ``ActionJournal`` records the steps the program executes without +one, and the opt-in ``PhysicalRecorder`` reads your own input from kernel +devices you name. See :doc:`/Eng/doc/wayland/wayland_capabilities_doc`. diff --git a/docs/source/Eng/doc/wayland/wayland_capabilities_doc.rst b/docs/source/Eng/doc/wayland/wayland_capabilities_doc.rst new file mode 100644 index 000000000..16a788713 --- /dev/null +++ b/docs/source/Eng/doc/wayland/wayland_capabilities_doc.rst @@ -0,0 +1,213 @@ +========================================= +Capabilities and authorisation on Wayland +========================================= + +On Windows the backend either works or raises. On Wayland the interesting +answers are in between: input may go through the desktop portal or through +``/dev/uinput``; the user may have refused the consent dialog, or there may +be no portal to ask; the X11 backend may be serving a Wayland session through +XWayland and reaching only some windows. ``probe_capabilities()`` reports all +of that as data. + +Probing +======= + +.. code-block:: python + + import je_auto_control as ac + + snapshot = ac.probe_capabilities() + print(snapshot.display_server, snapshot.xwayland) + for capability in snapshot.capabilities: + print(capability.name, capability.state.value, capability.backend, + capability.desktop_wide, capability.recovery) + +**Probing has no side effect.** It reads the environment, looks names up on +``PATH``, asks the loader whether a library exists and reads the authorisation +ledger. It never opens a portal request, never shows a consent dialog, never +connects a libei sender and never sends an event. + +The same data is available everywhere: + +* JSON action: ``[["AC_probe_capabilities"]]`` +* MCP tool: ``ac_probe_capabilities`` (read-only) +* GUI: the second table of the **Diagnostics** tab + +Four capabilities are diagnosed independently, because they fail +independently: + +.. list-table:: + :header-rows: 1 + + * - Name + - What it is + * - ``input`` + - synthesising keyboard and pointer events + * - ``capture`` + - reading pixels off the screen + * - ``recording`` + - reading what the *user* types and clicks + * - ``stop_shortcut`` + - a global key that stops a running script + +States +====== + +.. list-table:: + :header-rows: 1 + + * - ``state`` + - Meaning + * - ``available`` + - usable now + * - ``not_requested`` + - usable, but the desktop has not been asked yet; it is on first use + * - ``requesting`` + - a consent request is on screen right now + * - ``needs_permission`` + - a consent was refused or left unanswered, or a device is not readable + * - ``needs_setup`` + - something has to be installed or configured + * - ``session_closed`` + - this process closed its session; the next use asks again + * - ``revoked`` + - the compositor ended a session it had granted + * - ``compositor_restarted`` + - the grant came from a compositor that is no longer the one running + * - ``unknown`` + - not determinable without side effects (macOS permissions, for example) + +Each capability also carries ``backend`` (what serves it), ``desktop_wide``, +``detail``, ``recovery`` (what to do, in English) and ``recovery_key`` (the +same advice as a GUI catalogue key). + +``restore_token`` is ``unsupported`` on the libei path: this binding does not +ask the portal for a restore token, so consent is asked once per process and +does not survive a restart. + +A refusal is not worked around +============================== + +``JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=auto`` (the default) tries libei first +and falls back to the ``ydotool`` CLI when libei *cannot be brought up*: no +libei, no RemoteDesktop portal, a handshake that never completes. That is +unchanged. + +Two outcomes no longer fall back, because both are someone with the authority +to say no, saying it: + +* the portal **denied** the request (``state == "needs_permission"``); +* the compositor **revoked** a session that had been live + (``state == "revoked"``). + +Input actions then raise ``WaylandAuthorisationError`` — which carries +``state``, ``capability`` and a ``recovery`` instruction — instead of driving +the desktop through ``/dev/uinput`` behind the refusal. To recover: + +.. code-block:: python + + ac.reset_input_authorisation() # the next input action asks again + ac.close_input_session() # end the session now (revokes the grant) + +or ``[["AC_reset_input_authorisation"]]``, or **Actions > Ask for input +permission again** in the Diagnostics tab. To use ydotool on purpose, set +``JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli`` before starting; that path never +asks the portal at all. + +A revoked session is also never written to again: the libei backend refuses +every emission after the compositor disconnects it. + +.. note:: + + A denial is recognised from liboeffis's own error text. Whether real + desktops word it that way is measured by the ``portal-verification`` CI + job, not asserted here. Where the text is not recognised the old behaviour + applies (fall back to ydotool): the authorisation is recorded as + ``failed`` and the capability shows ``ydotool`` as its backend. + +XWayland +======== + +When the X11 backend serves a Wayland session — because +``JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`` is set, or because the Wayland +backend could not load — ``snapshot.xwayland`` is ``True`` and every +capability has ``desktop_wide == False``: X11 applications are reachable, +native Wayland windows neither receive the input nor appear in captures. + +Recording +========= + +Wayland has no global input hook, so ``record()`` still raises there. Two +narrower things are available. + +**What the program executed** needs no hook on any platform: + +.. code-block:: python + + journal = ac.ActionJournal() + journal.run(actions) # the list you would give execute_action + print(journal.steps) + +**What you physically typed and clicked** can be read from kernel devices you +name. This is opt-in per device, excludes virtual (uinput) devices so that +this program's own ydotool output is never recorded as yours, and never takes +privilege: + +.. code-block:: python + + devices = [d for d in ac.list_input_devices() if not d.is_virtual] + recorder = ac.PhysicalRecorder() + recorder.start(devices[:1]) # raises InputPermissionError if denied + events = recorder.stop() # list of InputEvent(type, code, value) + +``JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES`` (comma-separated +``/dev/input/event*`` paths) records the choice for the capability probe. A +device this user may not read raises ``InputPermissionError`` with the fix: +add the user to the ``input`` group, or install a udev rule. Do not run the +program as root for this. + +The portal's *InputCapture* interface is not used: the compositor decides +when it starts, so it cannot be a recorder that starts when a script asks. + +A global stop key +================= + +``StopShortcutSession`` registers one named shortcut through the +``GlobalShortcuts`` portal and calls back when it is pressed. It reports only +that shortcut, never other keys. + +.. code-block:: python + + import threading + + stop = threading.Event() + with ac.StopShortcutSession(stop.set) as session: + session.start() + ... # run; check stop.is_set() + # leaving the block closes the portal session + +If the desktop has no GlobalShortcuts portal or the user declines, a +``ShortcutUnavailable`` (or ``ShortcutPermissionError``) names the stop +methods that still work. This path has no CI coverage; see +``test/manual_test/wayland_authorisation_checklist.md``. + +The libei helper process (opt-in) +================================= + +``JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`` runs the libei session in a helper +process. It is **off by default**. The in-process path, including its +deliberate leak of one context and one descriptor when a handshake never +completes, is unchanged and remains the default; the upstream ``ei_unref`` +crash it works around is not fixed by this. + +With the helper, a session that cannot be released safely is reclaimed by the +operating system when the helper exits. Requests are bounded (64 events per +batch, 64 KiB per frame), carry ids, honour a deadline and a cancellation, +and keys still held are released when the helper is closed. The cost is a +pipe round trip per emission; ``docker/eis_verify.py`` measures it. + +Failures split by one question — may another backend redo this? +``EiWorkerError`` (including ``EiDependencyMissing``) is a +``LibeiUnavailable``: nothing was sent, ydotool may take over. +``EiWorkerTimeout``, ``EiWorkerCancelled`` and ``EiWorkerDied`` are not: the +request was written and its fate is unknown, so they reach the caller. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e826e9bb4..ab3157f5f 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -14,6 +14,7 @@ Comprehensive guides for all AutoControl features. doc/screen/screen_doc doc/image/image_doc doc/record/record_doc + doc/wayland/wayland_capabilities_doc doc/keyword_and_executor/keyword_and_executor_doc doc/generate_report/generate_report_doc doc/callback_function/callback_function_doc diff --git a/docs/source/Zh/doc/record/record_doc.rst b/docs/source/Zh/doc/record/record_doc.rst index 7fb60e44b..013543a40 100644 --- a/docs/source/Zh/doc/record/record_doc.rst +++ b/docs/source/Zh/doc/record/record_doc.rst @@ -68,3 +68,11 @@ AutoControl 可以錄製滑鼠與鍵盤事件，並透過執行器回放。 ``adjust_delays`` 與 ``scale_coordinates`` 也透過 MCP 暴露為 ``ac_dedupe_moves`` / ``ac_merge_sleeps`` / ``ac_trim_actions`` / ``ac_adjust_delays`` / ``ac_scale_coordinates``。 + +在 Wayland 上錄製 +================= + +``record()`` 在 Wayland 工作階段會拋例外：合成器不允許全域輸入掛鉤。 +``ActionJournal`` 不需要掛鉤就能記下程式執行的步驟，而明確 opt-in 的 +``PhysicalRecorder`` 會從你指名的核心裝置讀取你自己的輸入。 +見 :doc:`/Zh/doc/wayland/wayland_capabilities_doc`。 diff --git a/docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst b/docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst new file mode 100644 index 000000000..c434759e9 --- /dev/null +++ b/docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst @@ -0,0 +1,192 @@ +========================== +Wayland 上的能力與授權狀態 +========================== + +在 Windows 上，後端不是能用就是拋例外。在 Wayland 上，有意思的答案都落在中間： +輸入可能走桌面 portal，也可能走 ``/dev/uinput``；使用者可能拒絕了同意對話框， +也可能根本沒有 portal 可以問；也可能是 X11 後端透過 XWayland 在服務一個 Wayland +工作階段，只碰得到部分視窗。``probe_capabilities()`` 把這些全部當成資料回報。 + +探測 +==== + +.. code-block:: python + + import je_auto_control as ac + + snapshot = ac.probe_capabilities() + print(snapshot.display_server, snapshot.xwayland) + for capability in snapshot.capabilities: + print(capability.name, capability.state.value, capability.backend, + capability.desktop_wide, capability.recovery) + +**探測沒有任何副作用。** 它只讀環境變數、在 ``PATH`` 上找名字、問載入器某個函式庫 +在不在、讀授權紀錄。它不會發出 portal 請求、不會跳出同意對話框、不會連上 libei +sender，也不會送出任何事件。 + +同一份資料在每個入口都拿得到： + +* JSON 動作：``[["AC_probe_capabilities"]]`` +* MCP 工具：``ac_probe_capabilities``（唯讀） +* GUI：**診斷** 分頁的第二張表 + +四項能力各自獨立診斷，因為它們各自獨立地壞： + +.. list-table:: + :header-rows: 1 + + * - 名稱 + - 是什麼 + * - ``input`` + - 合成鍵盤與指標事件 + * - ``capture`` + - 讀取螢幕像素 + * - ``recording`` + - 讀取 *使用者* 自己打的字與點的鍵 + * - ``stop_shortcut`` + - 停止執行中腳本的全域按鍵 + +狀態 +==== + +.. list-table:: + :header-rows: 1 + + * - ``state`` + - 意義 + * - ``available`` + - 現在可用 + * - ``not_requested`` + - 可用，但還沒問過桌面；第一次使用時才會問 + * - ``requesting`` + - 同意請求現在正顯示在螢幕上 + * - ``needs_permission`` + - 同意被拒絕或沒人回應，或裝置不可讀 + * - ``needs_setup`` + - 需要安裝或設定某些東西 + * - ``session_closed`` + - 本行程自己關掉了工作階段；下次使用會再問一次 + * - ``revoked`` + - 合成器收回了已經授予的工作階段 + * - ``compositor_restarted`` + - 授權來自一個已經不是現在這個的合成器 + * - ``unknown`` + - 不產生副作用就無法判定（例如 macOS 的權限） + +每項能力另外帶有 ``backend``（由誰提供）、``desktop_wide``、``detail``、 +``recovery``（英文的處理方式）與 ``recovery_key``（同一段建議在 GUI 語系表的鍵）。 + +libei 路徑上的 ``restore_token`` 是 ``unsupported``：這個綁定不向 portal 要求 +restore token，所以同意是每個行程問一次，重新啟動後不會保留。 + +拒絕不會被繞過 +============== + +``JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=auto``（預設）先試 libei，在 libei +*起不來* 的時候退回 ``ydotool`` CLI：沒有 libei、沒有 RemoteDesktop portal、 +交握一直沒完成。這一點沒有變。 + +有兩種結果不再退回，因為兩者都是有權說不的人說了不： + +* portal **拒絕** 了請求（``state == "needs_permission"``）； +* 合成器 **收回** 了一個原本存活的工作階段（``state == "revoked"``）。 + +這時輸入動作會拋出 ``WaylandAuthorisationError``——帶有 ``state``、 +``capability`` 與 ``recovery`` 指示——而不是在拒絕的背後改用 ``/dev/uinput`` +驅動桌面。復原方式： + +.. code-block:: python + + ac.reset_input_authorisation() # 下一個輸入動作會再問一次 + ac.close_input_session() # 現在就結束工作階段（撤銷授權） + +或 ``[["AC_reset_input_authorisation"]]``，或診斷分頁的 **操作 > 重新要求輸入 +授權**。若要刻意使用 ydotool，請在啟動前設定 +``JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli``；那條路徑完全不會去問 portal。 + +被收回的工作階段也不會再被寫入：合成器斷線之後，libei 後端拒絕每一次送出。 + +.. note:: + + 「拒絕」是從 liboeffis 自己的錯誤文字辨認出來的。真實桌面是否這樣措辭， + 由 ``portal-verification`` CI job 量測，不在這裡斷言。認不出來的時候沿用 + 舊行為（退回 ydotool）：授權記為 ``failed``，該能力的後端顯示為 ``ydotool``。 + +XWayland +======== + +當 X11 後端在服務一個 Wayland 工作階段時——因為設了 +``JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11``，或因為 Wayland 後端載入失敗—— +``snapshot.xwayland`` 為 ``True``，每項能力的 ``desktop_wide`` 都是 ``False``： +碰得到 X11 應用程式，原生 Wayland 視窗收不到輸入，也不會出現在擷取畫面裡。 + +錄製 +==== + +Wayland 沒有全域輸入掛鉤，所以 ``record()`` 在那裡仍然會拋例外。有兩件範圍 +較小的事可以做。 + +**程式自己執行了什麼**，在任何平台上都不需要掛鉤： + +.. code-block:: python + + journal = ac.ActionJournal() + journal.run(actions) # 就是你會交給 execute_action 的那份清單 + print(journal.steps) + +**你實際打的字與點的鍵**，可以從你指名的核心裝置讀取。這是逐裝置的明確 +opt-in，會排除虛擬（uinput）裝置，讓本程式自己的 ydotool 輸出不會被錄成你的 +輸入，而且絕不取得額外權限： + +.. code-block:: python + + devices = [d for d in ac.list_input_devices() if not d.is_virtual] + recorder = ac.PhysicalRecorder() + recorder.start(devices[:1]) # 沒有權限時拋出 InputPermissionError + events = recorder.stop() # InputEvent(type, code, value) 的清單 + +``JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES``（以逗號分隔的 +``/dev/input/event*`` 路徑）把這個選擇記下來，供能力探測使用。使用者讀不到的 +裝置會拋出 ``InputPermissionError`` 並附上處理方式：把使用者加入 ``input`` +群組，或安裝 udev 規則。不要為此以 root 執行整個程式。 + +不使用 portal 的 *InputCapture* 介面：它何時開始由合成器決定，所以不能當成 +腳本要求時才啟動的錄製器。 + +全域停止鍵 +========== + +``StopShortcutSession`` 透過 ``GlobalShortcuts`` portal 註冊一個具名的快捷鍵， +在它被按下時回呼。它只回報那一個快捷鍵，不會回報其他按鍵。 + +.. code-block:: python + + import threading + + stop = threading.Event() + with ac.StopShortcutSession(stop.set) as session: + session.start() + ... # 執行；檢查 stop.is_set() + # 離開區塊就會關閉 portal 工作階段 + +如果桌面沒有 GlobalShortcuts portal，或使用者拒絕，``ShortcutUnavailable`` +（或 ``ShortcutPermissionError``）會指出仍然可用的停止方式。這條路徑沒有 CI +覆蓋，見 ``test/manual_test/wayland_authorisation_checklist.md``。 + +libei helper 行程（opt-in） +=========================== + +``JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`` 會把 libei 工作階段放進 helper 行程。 +它 **預設關閉**。行程內的路徑——包含交握一直沒完成時刻意漏掉的一個 context 與 +一個 fd——沒有改變，仍然是預設；它所迴避的上游 ``ei_unref`` crash 並沒有因此 +被修好。 + +使用 helper 時，無法安全釋放的工作階段會在 helper 結束時由作業系統回收。 +請求有上限（每批 64 個事件、每個 frame 64 KiB）、帶有 id、遵守時限與取消， +helper 被關閉時會放開仍被按住的鍵。代價是每次送出多一趟 pipe 往返； +``docker/eis_verify.py`` 會量測它。 + +失敗依一個問題分成兩類——別的後端可以重做這件事嗎？``EiWorkerError``（含 +``EiDependencyMissing``）是 ``LibeiUnavailable``：什麼都沒送出去，ydotool 可以 +接手。``EiWorkerTimeout``、``EiWorkerCancelled`` 與 ``EiWorkerDied`` 不是：請求 +已經寫出去、結果未知，所以它們會傳到呼叫端。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index eb1f7a49f..33dd7beb0 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -14,6 +14,7 @@ AutoControl 所有功能的完整使用指南。 doc/screen/screen_doc doc/image/image_doc doc/record/record_doc + doc/wayland/wayland_capabilities_doc doc/keyword_and_executor/keyword_and_executor_doc doc/generate_report/generate_report_doc doc/callback_function/callback_function_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 475c40ca2..0549e6efa 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1122,6 +1122,25 @@ from je_auto_control.utils.diagnostics import ( Check, DiagnosticsReport, run_diagnostics, ) +# Capability and authorisation states (side-effect-free probe) +from je_auto_control.wrapper.capabilities import ( + BackendContext, Capability, CapabilitySnapshot, CapabilityStatus, + probe_capabilities, +) +from je_auto_control.linux_wayland._select_input import ( + close_input_session, reset_input_authorisation, +) +from je_auto_control.linux_wayland.authorisation import ( + WaylandAuthorisationError, +) +# Recording without a global hook, and the Wayland stop shortcut +from je_auto_control.linux_wayland.input_events import ( + ActionJournal, InputDevice, InputEvent, InputPermissionError, + InputRecordingError, PhysicalRecorder, list_input_devices, +) +from je_auto_control.linux_wayland.global_shortcuts import ( + ShortcutPermissionError, ShortcutUnavailable, StopShortcutSession, +) # Config bundle (export / import user configuration) from je_auto_control.utils.config_bundle import ( ConfigBundleExporter, ConfigBundleImporter, ImportReport, @@ -1779,6 +1798,14 @@ def start_autocontrol_gui(*args, **kwargs): "enable_usb_passthrough", "is_usb_passthrough_enabled", # System diagnostics "Check", "DiagnosticsReport", "run_diagnostics", + # Capability and authorisation states + "BackendContext", "Capability", "CapabilitySnapshot", "CapabilityStatus", + "probe_capabilities", "close_input_session", + "reset_input_authorisation", "WaylandAuthorisationError", + # Recording without a global hook, and the Wayland stop shortcut + "ActionJournal", "InputDevice", "InputEvent", "InputPermissionError", + "InputRecordingError", "PhysicalRecorder", "list_input_devices", + "ShortcutPermissionError", "ShortcutUnavailable", "StopShortcutSession", # Config bundle "ConfigBundleExporter", "ConfigBundleImporter", "ImportReport", "export_config_bundle", "import_config_bundle", diff --git a/je_auto_control/actions.pyi b/je_auto_control/actions.pyi index 35e4e39ee..802252699 100644 --- a/je_auto_control/actions.pyi +++ b/je_auto_control/actions.pyi @@ -2112,6 +2112,9 @@ def AC_press_keyboard_key(keycode: int | str, is_shift: bool = ..., skip_record: def AC_press_mouse(mouse_keycode: int | str, x: int | None = ..., y: int | None = ...) -> tuple[Any, int, int] | None: """按下滑鼠按鍵""" +def AC_probe_capabilities() -> Dict[str, Any]: + """Executor adapter: capability and authorisation states, no side effect.""" + def AC_profile_rows(rows: Any, columns: Any = ...) -> Dict[str, Any]: """Adapter: profile a row-set into per-column statistics.""" @@ -2326,6 +2329,9 @@ def AC_replay_timeline(events: List[Dict[str, Any]], speed: float = ...) -> Dict def AC_replay_trace(trace: Any) -> Dict[str, Any]: """Adapter: replay a trajectory by running each step's action via the executor.""" +def AC_reset_input_authorisation() -> Dict[str, Any]: + """Executor adapter: forget a refused or revoked Wayland input session.""" + def AC_resize_element( width: Any, height: Any, diff --git a/je_auto_control/gui/diagnostics_tab.py b/je_auto_control/gui/diagnostics_tab.py index 89a205a92..59892120a 100644 --- a/je_auto_control/gui/diagnostics_tab.py +++ b/je_auto_control/gui/diagnostics_tab.py @@ -1,5 +1,12 @@ -"""System diagnostics tab: run subsystem checks and display results.""" -from typing import Optional +"""System diagnostics tab: subsystem checks, and what this session can do. + +Two tables. The first is the battery of subsystem checks. The second is the +capability snapshot from :func:`probe_capabilities` — input, capture, +recording and the stop key, each with its state and, where something stands +in the way, what to do about it. Both are read-only views of headless data: +the same snapshot is what ``AC_probe_capabilities`` and the MCP tool return. +""" +from typing import List, Optional from PySide6.QtGui import QBrush, QColor from PySide6.QtWidgets import ( @@ -11,7 +18,13 @@ from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( language_wrapper, ) +from je_auto_control.linux_wayland._select_input import ( + close_input_session, reset_input_authorisation, +) from je_auto_control.utils.diagnostics.diagnostics import run_diagnostics +from je_auto_control.wrapper.capabilities import ( + Capability, CapabilityStatus, probe_capabilities, +) _SEVERITY_COLOR = { @@ -20,11 +33,51 @@ "error": QColor("#c0392b"), } +#: Capability states by how much attention they want. +_STATE_SEVERITY = { + CapabilityStatus.AVAILABLE: "info", + CapabilityStatus.NOT_REQUESTED: "info", + CapabilityStatus.SESSION_CLOSED: "info", + CapabilityStatus.REQUESTING: "warn", + CapabilityStatus.NEEDS_SETUP: "warn", + CapabilityStatus.UNKNOWN: "warn", + CapabilityStatus.NEEDS_PERMISSION: "error", + CapabilityStatus.REVOKED: "error", + CapabilityStatus.COMPOSITOR_RESTARTED: "error", + CapabilityStatus.UNSUPPORTED: "error", +} + +_CAPABILITY_COLUMNS = ("diag_cap_col_name", "diag_cap_col_state", + "diag_cap_col_backend", "diag_cap_col_scope", + "diag_cap_col_detail", "diag_cap_col_fix") + def _t(key: str) -> str: return language_wrapper.translate(key, key) +def capability_cells(capability: Capability) -> List[str]: + """The six cells of one capability row, translated. + + The advice comes from the catalogue when the capability names a key and + falls back to the headless English text, so a state added without a + translation still tells the user something. + """ + advice = "" + if capability.recovery_key: + advice = language_wrapper.translate(capability.recovery_key, + capability.recovery) + return [ + _t(f"cap_name_{capability.name}"), + _t(f"cap_state_{capability.state.value}"), + capability.backend, + _t("cap_scope_desktop" if capability.desktop_wide + else "cap_scope_xwayland"), + capability.detail, + advice or capability.recovery, + ] + + class DiagnosticsTab(TranslatableMixin, QWidget): """Run :func:`run_diagnostics` and render the results.""" @@ -37,28 +90,51 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._table.horizontalHeader().setSectionResizeMode( 3, QHeaderView.ResizeMode.Stretch, ) + self._capabilities_label = QLabel() + self._capabilities = QTableWidget(0, len(_CAPABILITY_COLUMNS)) + self._capabilities.setEditTriggers( + QTableWidget.EditTrigger.NoEditTriggers) + self._capabilities.setWordWrap(True) + self._capabilities.horizontalHeader().setSectionResizeMode( + len(_CAPABILITY_COLUMNS) - 1, QHeaderView.ResizeMode.Stretch, + ) self._build_layout() self._apply_table_headers() self._refresh() + self._refresh_capabilities() def _build_layout(self) -> None: # The run command runs from the Actions menu; the tab keeps only - # the summary label and the results table. + # the summary label and the result tables. root = QVBoxLayout(self) root.addWidget(self._summary_label) - root.addWidget(self._table, stretch=1) + root.addWidget(self._table, stretch=2) + root.addWidget(self._tr(self._capabilities_label, + "diag_capabilities_title")) + root.addWidget(self._capabilities, stretch=1) def menu_actions(self) -> list: """Expose tab commands to the window-level Actions menu.""" return [ ("diag_run", self._refresh), + ("diag_capabilities_refresh", self._refresh_capabilities), + ("diag_input_reask", self._ask_input_again), + ("diag_input_close", self._end_input_session), ] + def retranslate(self) -> None: + """Re-apply the registered keys and both tables' own text.""" + super().retranslate() + self._apply_table_headers() + self._refresh_capabilities() + def _apply_table_headers(self) -> None: self._table.setHorizontalHeaderLabels([ _t("diag_col_name"), _t("diag_col_severity"), _t("diag_col_status"), _t("diag_col_detail"), ]) + self._capabilities.setHorizontalHeaderLabels( + [_t(key) for key in _CAPABILITY_COLUMNS]) def _refresh(self) -> None: report = run_diagnostics() @@ -86,5 +162,28 @@ def _refresh(self) -> None: item.setForeground(QBrush(color)) self._table.setItem(row, col, item) + def _refresh_capabilities(self) -> None: + """Show the current snapshot. Asks the desktop for nothing.""" + snapshot = probe_capabilities() + self._capabilities.setRowCount(len(snapshot.capabilities)) + for row, capability in enumerate(snapshot.capabilities): + color = _SEVERITY_COLOR[_STATE_SEVERITY[capability.state]] + for col, text in enumerate(capability_cells(capability)): + item = QTableWidgetItem(text) + if col == 1: + item.setForeground(QBrush(color)) + self._capabilities.setItem(row, col, item) + self._capabilities.resizeRowsToContents() + + def _ask_input_again(self) -> None: + """Forget a refused or revoked input session; the next action asks.""" + reset_input_authorisation() + self._refresh_capabilities() + + def _end_input_session(self) -> None: + """Close the input session now, which revokes the portal grant.""" + close_input_session() + self._refresh_capabilities() + -__all__ = ["DiagnosticsTab"] +__all__ = ["DiagnosticsTab", "capability_cells"] diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 6bc8d1ebe..a461b81f5 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -178,6 +178,57 @@ "diag_col_detail": "Detail", "diag_status_ok": "OK", "diag_status_fail": "FAIL", + # Diagnostics tab: capability and authorisation states + "diag_capabilities_title": "What this session can do", + "diag_capabilities_refresh": "Refresh capabilities", + "diag_input_reask": "Ask for input permission again", + "diag_input_close": "End the input session", + "diag_cap_col_name": "Capability", + "diag_cap_col_state": "State", + "diag_cap_col_backend": "Backend", + "diag_cap_col_scope": "Reaches", + "diag_cap_col_detail": "Detail", + "diag_cap_col_fix": "What to do", + "cap_name_input": "Keyboard and mouse input", + "cap_name_capture": "Screen capture", + "cap_name_recording": "Recording your own input", + "cap_name_stop_shortcut": "Global stop key", + "cap_state_available": "Available", + "cap_state_not_requested": "Not asked yet", + "cap_state_requesting": "Waiting for your answer", + "cap_state_needs_permission": "Needs permission", + "cap_state_needs_setup": "Needs setup", + "cap_state_session_closed": "Session closed", + "cap_state_revoked": "Revoked by the desktop", + "cap_state_compositor_restarted": "Desktop was restarted", + "cap_state_unsupported": "Not supported", + "cap_state_unknown": "Not checked", + "cap_scope_desktop": "Whole desktop", + "cap_scope_xwayland": "X11 applications only (XWayland)", + "cap_fix_input_consent": ( + "Choose Actions > Ask for input permission again, repeat the action, " + "and allow the request. To use ydotool instead, set " + "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli before starting."), + "cap_fix_answer_dialog": "Answer the permission dialog on the desktop.", + "cap_fix_ydotool": ( + "Install ydotool 1.0 or newer and run ydotoold with access to " + "/dev/uinput, or install libei and liboeffis."), + "cap_fix_xwayland": ( + "Install the Wayland tools (ydotool or libei, and grim, " + "gnome-screenshot or spectacle) so the Wayland backend loads."), + "cap_fix_display": "Set DISPLAY to a running X server.", + "cap_fix_capture": ( + "Allow the screenshot request, install grim, gnome-screenshot or " + "spectacle, or set JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND."), + "cap_fix_record_setup": ( + "Name the devices to read in JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES " + "(comma-separated /dev/input/event* paths)."), + "cap_fix_record_permission": ( + "Add your user to the input group and log in again, or add a udev " + "rule. Do not run the program as root for this."), + "cap_fix_stop_manual": ( + "Stop a running script from the Actions menu or by ending the " + "process."), # USB devices tab "usb_backend_label": "Backend:", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 0740bfc51..50730893f 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -67,6 +67,59 @@ "diag_col_detail": "詳細", "diag_status_ok": "OK", "diag_status_fail": "失敗", + # 診断タブ: 機能と許可の状態 + "diag_capabilities_title": "このセッションでできること", + "diag_capabilities_refresh": "機能の状態を更新", + "diag_input_reask": "入力の許可をもう一度求める", + "diag_input_close": "入力セッションを終了", + "diag_cap_col_name": "機能", + "diag_cap_col_state": "状態", + "diag_cap_col_backend": "バックエンド", + "diag_cap_col_scope": "届く範囲", + "diag_cap_col_detail": "詳細", + "diag_cap_col_fix": "対処方法", + "cap_name_input": "キーボードとマウスの入力", + "cap_name_capture": "画面キャプチャ", + "cap_name_recording": "自分の入力の記録", + "cap_name_stop_shortcut": "グローバル停止キー", + "cap_state_available": "利用可能", + "cap_state_not_requested": "まだ確認していません", + "cap_state_requesting": "回答を待っています", + "cap_state_needs_permission": "許可が必要", + "cap_state_needs_setup": "設定が必要", + "cap_state_session_closed": "セッション終了", + "cap_state_revoked": "デスクトップにより取り消し", + "cap_state_compositor_restarted": "デスクトップが再起動されました", + "cap_state_unsupported": "非対応", + "cap_state_unknown": "未確認", + "cap_scope_desktop": "デスクトップ全体", + "cap_scope_xwayland": "X11 アプリケーションのみ (XWayland)", + "cap_fix_input_consent": ( + "「アクション」>「入力の許可をもう一度求める」を選び、操作をやり直して" + "要求を許可してください。代わりに ydotool を使う場合は、起動前に " + "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli を設定します。"), + "cap_fix_answer_dialog": "デスクトップの許可ダイアログに回答してください。", + "cap_fix_ydotool": ( + "ydotool 1.0 以降をインストールし、/dev/uinput にアクセスできる " + "ydotoold を実行するか、libei と liboeffis をインストールしてください。"), + "cap_fix_xwayland": ( + "Wayland バックエンドが読み込まれるよう、Wayland 用ツール (ydotool " + "または libei と、grim・gnome-screenshot・spectacle のいずれか) を" + "インストールしてください。"), + "cap_fix_display": "DISPLAY を実行中の X サーバーに設定してください。", + "cap_fix_capture": ( + "スクリーンショットの要求を許可するか、grim・gnome-screenshot・" + "spectacle のいずれかをインストールするか、" + "JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND を設定してください。"), + "cap_fix_record_setup": ( + "読み取るデバイスを JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES に指定して" + "ください (/dev/input/event* のパスをカンマ区切り)。"), + "cap_fix_record_permission": ( + "ユーザーを input グループに追加してログインし直すか、udev ルールを" + "追加してください。このためにプログラムを root で実行しないでください。"), + "cap_fix_stop_manual": ( + "実行中のスクリプトは「アクション」メニューから、またはプロセスを終了して" + "停止してください。"), # USB デバイスタブ "usb_backend_label": "バックエンド:", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index d002ab239..23f5edc40 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -58,6 +58,55 @@ "diag_col_detail": "详情", "diag_status_ok": "正常", "diag_status_fail": "失败", + # 诊断分页:能力与授权状态 + "diag_capabilities_title": "这个会话现在能做什么", + "diag_capabilities_refresh": "刷新能力状态", + "diag_input_reask": "重新请求输入授权", + "diag_input_close": "结束输入会话", + "diag_cap_col_name": "能力", + "diag_cap_col_state": "状态", + "diag_cap_col_backend": "后端", + "diag_cap_col_scope": "作用范围", + "diag_cap_col_detail": "详情", + "diag_cap_col_fix": "怎么处理", + "cap_name_input": "键盘与鼠标输入", + "cap_name_capture": "屏幕截取", + "cap_name_recording": "录制自己的输入", + "cap_name_stop_shortcut": "全局停止键", + "cap_state_available": "可用", + "cap_state_not_requested": "尚未请求", + "cap_state_requesting": "等待您回应", + "cap_state_needs_permission": "需要授权", + "cap_state_needs_setup": "需要设置", + "cap_state_session_closed": "会话已关闭", + "cap_state_revoked": "已被桌面撤销", + "cap_state_compositor_restarted": "桌面已重启", + "cap_state_unsupported": "不支持", + "cap_state_unknown": "未检查", + "cap_scope_desktop": "整个桌面", + "cap_scope_xwayland": "仅 X11 应用程序 (XWayland)", + "cap_fix_input_consent": ( + "选择「操作」>「重新请求输入授权」,重做一次动作并允许请求。" + "若要改用 ydotool,请在启动前设置 " + "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli。"), + "cap_fix_answer_dialog": "请回应桌面上的授权对话框。", + "cap_fix_ydotool": ( + "安装 ydotool 1.0 或更新版本,并以可访问 /dev/uinput 的权限运行 " + "ydotoold;或安装 libei 与 liboeffis。"), + "cap_fix_xwayland": ( + "安装 Wayland 工具(ydotool 或 libei,以及 grim、gnome-screenshot " + "或 spectacle),让 Wayland 后端可以加载。"), + "cap_fix_display": "请把 DISPLAY 设为运行中的 X 服务器。", + "cap_fix_capture": ( + "允许截图请求,安装 grim、gnome-screenshot 或 spectacle,或设置 " + "JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND。"), + "cap_fix_record_setup": ( + "在 JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES 指定要读取的设备" + "(以逗号分隔的 /dev/input/event* 路径)。"), + "cap_fix_record_permission": ( + "把用户加入 input 组后重新登录,或添加 udev 规则。" + "不要为此以 root 运行整个程序。"), + "cap_fix_stop_manual": "请从「操作」菜单停止运行中的脚本,或结束进程。", # USB 设备分页 "usb_backend_label": "后端:", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 046af99cd..89aadf2ad 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -59,6 +59,55 @@ "diag_col_detail": "詳情", "diag_status_ok": "正常", "diag_status_fail": "失敗", + # 診斷分頁:能力與授權狀態 + "diag_capabilities_title": "這個工作階段現在能做什麼", + "diag_capabilities_refresh": "重新整理能力狀態", + "diag_input_reask": "重新要求輸入授權", + "diag_input_close": "結束輸入工作階段", + "diag_cap_col_name": "能力", + "diag_cap_col_state": "狀態", + "diag_cap_col_backend": "後端", + "diag_cap_col_scope": "作用範圍", + "diag_cap_col_detail": "詳情", + "diag_cap_col_fix": "怎麼處理", + "cap_name_input": "鍵盤與滑鼠輸入", + "cap_name_capture": "螢幕擷取", + "cap_name_recording": "錄製自己的輸入", + "cap_name_stop_shortcut": "全域停止鍵", + "cap_state_available": "可用", + "cap_state_not_requested": "尚未要求", + "cap_state_requesting": "等待您回應", + "cap_state_needs_permission": "需要授權", + "cap_state_needs_setup": "需要設定", + "cap_state_session_closed": "工作階段已關閉", + "cap_state_revoked": "已被桌面撤銷", + "cap_state_compositor_restarted": "桌面已重新啟動", + "cap_state_unsupported": "不支援", + "cap_state_unknown": "未檢查", + "cap_scope_desktop": "整個桌面", + "cap_scope_xwayland": "僅 X11 應用程式 (XWayland)", + "cap_fix_input_consent": ( + "選擇「操作」>「重新要求輸入授權」,重做一次動作並允許要求。" + "若要改用 ydotool,請在啟動前設定 " + "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli。"), + "cap_fix_answer_dialog": "請回應桌面上的授權對話框。", + "cap_fix_ydotool": ( + "安裝 ydotool 1.0 或更新版本,並以可存取 /dev/uinput 的權限執行 " + "ydotoold;或安裝 libei 與 liboeffis。"), + "cap_fix_xwayland": ( + "安裝 Wayland 工具(ydotool 或 libei,以及 grim、gnome-screenshot " + "或 spectacle),讓 Wayland 後端可以載入。"), + "cap_fix_display": "請把 DISPLAY 設為執行中的 X 伺服器。", + "cap_fix_capture": ( + "允許截圖要求,安裝 grim、gnome-screenshot 或 spectacle,或設定 " + "JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND。"), + "cap_fix_record_setup": ( + "在 JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES 指定要讀取的裝置" + "(以逗號分隔的 /dev/input/event* 路徑)。"), + "cap_fix_record_permission": ( + "把使用者加入 input 群組後重新登入,或新增 udev 規則。" + "不要為此以 root 執行整個程式。"), + "cap_fix_stop_manual": "請從「操作」選單停止執行中的腳本,或結束行程。", # USB 裝置分頁 "usb_backend_label": "後端:", diff --git a/je_auto_control/linux_wayland/_select_input.py b/je_auto_control/linux_wayland/_select_input.py index 60bff5ea5..8a02ea330 100644 --- a/je_auto_control/linux_wayland/_select_input.py +++ b/je_auto_control/linux_wayland/_select_input.py @@ -4,7 +4,21 @@ * ``libei`` — force the native binding; raise if libei isn't installed. * ``cli`` — force ``wtype`` / ``ydotool`` shims. -* ``auto`` (default) — try libei first; fall back to CLI on failure. +* ``auto`` (default) — try libei first; fall back to CLI when libei cannot + be brought up. + +**"Cannot be brought up" is not "was refused".** A host without libei, without +a RemoteDesktop portal, or with a handshake that never completes falls back to +the CLI, as it always has. A consent the portal *denied*, and a session the +compositor *took away*, do not: both are someone with the authority to say no +saying it, and sending the same input through ``/dev/uinput`` instead would +make the answer meaningless. Those two raise +:class:`~je_auto_control.linux_wayland.authorisation.WaylandAuthorisationError` +until :func:`reset_input_authorisation` is called. ``cli`` remains the way to +choose ydotool on purpose — it never asks the portal at all. + +``JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`` (off by default) moves the libei +session into a helper process; see :mod:`ei_worker` for why and what it costs. Kept separate from :mod:`_detect` so the display-server choice and the input-pipeline choice can evolve independently. @@ -12,12 +26,23 @@ from __future__ import annotations import os -from typing import Optional +import sys +from typing import Any, Mapping, Optional + +from je_auto_control.linux_wayland.authorisation import ( + INPUT, INPUT_RECOVERY, AuthorisationState, WaylandAuthorisationError, + ledger, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger _ENV_OVERRIDE = "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND" _VALID = frozenset({"auto", "libei", "cli"}) +#: Opt-in switch for the helper-process libei session. Off unless set. +WORKER_ENV = "JE_AUTOCONTROL_WAYLAND_EI_WORKER" +_TRUTHY = frozenset({"1", "true", "yes", "on"}) + def select_input_backend(environ: Optional[dict] = None) -> str: """Return one of ``"libei"`` or ``"cli"`` based on env + libei probe.""" @@ -46,40 +71,154 @@ def _libei_loadable() -> bool: return False +def worker_enabled(environ: Optional[Mapping[str, str]] = None) -> bool: + """Whether the operator asked for the helper-process libei session.""" + env = environ if environ is not None else os.environ + return (env.get(WORKER_ENV) or "").strip().lower() in _TRUTHY + + +def _backend_source() -> Any: + """The module that owns the libei session: in-process, or the worker. + + Both expose ``connected_backend`` and ``last_probe_error``, so the rest of + this module does not care which one it is talking to. + """ + if worker_enabled(): + from je_auto_control.linux_wayland import ei_client + return ei_client + from je_auto_control.linux_wayland import libei + return libei + + def active_backend(): """Return a connected libei backend, or None to use the ydotool CLI. The single entry point ``keyboard`` and ``mouse`` use. It lives here rather than in :mod:`libei` so the dependency runs one way — this module decides which input path is wanted, :mod:`libei` only knows how to bring - one up. None is the answer on any host where libei is absent, the - portal declines, or the handshake does not complete. + one up. None is the answer on any host where libei is absent, there is no + portal to ask, or the handshake does not complete. + + :raises WaylandAuthorisationError: the portal denied the request, or the + compositor revoked a live session. Not turned into None on purpose — + see the module docstring. """ try: if select_input_backend() != "libei": return None - from je_auto_control.linux_wayland.libei import connected_backend - return connected_backend() + source = _backend_source() except (ImportError, OSError, RuntimeError): return None + record = ledger.get(INPUT) + if record.blocks_fallback: + raise WaylandAuthorisationError(record, INPUT_RECOVERY) + if record.state in _UNASKED: + # Visible to a diagnostics screen on another thread for as long as + # the consent dialog is up, which can be most of a minute. + ledger.transition(INPUT, AuthorisationState.REQUESTING, + "waiting for the desktop to answer") + try: + backend = source.connected_backend() + failure = source.last_probe_error() + except (ImportError, OSError, RuntimeError) as error: + _move_to(AuthorisationState.FAILED, repr(error)) + return None + _report_probe(backend, failure) + return backend + + +#: States from which the next input action asks the desktop again. +_UNASKED = frozenset({AuthorisationState.NOT_REQUESTED, + AuthorisationState.CLOSED}) + + +def _report_probe(backend: Any, failure: Optional[BaseException]) -> None: + """Write the probe's outcome into the ledger; raise if it was a refusal.""" + if backend is not None: + _move_to(AuthorisationState.GRANTED, "a live libei session") + return + if failure is None: + _move_to(AuthorisationState.FAILED, "libei did not come up") + return + if getattr(failure, "declined", False): + state = AuthorisationState.DECLINED + elif getattr(failure, "outcome", None) == "timeout": + state = AuthorisationState.TIMED_OUT + else: + state = AuthorisationState.FAILED + if _move_to(state, str(failure)): + autocontrol_logger.warning( + "Wayland input: libei session not available (%s): %s", + state.value, failure) + ledger.require_usable(INPUT, INPUT_RECOVERY) + + +def _move_to(state: AuthorisationState, detail: str) -> bool: + """Record ``state`` for input; False when it was already there.""" + if ledger.get(INPUT).state is state: + return False + ledger.transition(INPUT, state, detail) + return True def emitted(backend, send) -> bool: """Run one emission on ``backend``; False means the CLI has to take it. A backend that finished its handshake can still refuse a single - emission: the compositor pauses a device, or the session ends between - two calls. :mod:`libei` is documented as the fast path and never the - only one, so a refusal falls through to the ``ydotool`` / ``wtype`` - shims here rather than reaching the caller — and if those are missing - too, *they* raise. Nothing is swallowed; the failure just changes hands. + emission: the compositor pauses a device, or a move lands outside every + region. :mod:`libei` is documented as the fast path and never the only + one, so such a refusal falls through to the ``ydotool`` / ``wtype`` shims + here rather than reaching the caller — and if those are missing too, + *they* raise. Nothing is swallowed; the failure just changes hands. + + A *revoked* session is the exception. The compositor ended it, so the + emission is refused outright and so is every later one, until + :func:`reset_input_authorisation`. """ - from je_auto_control.linux_wayland.libei import LibeiUnavailable + from je_auto_control.linux_wayland.libei import ( + LibeiSessionRevoked, LibeiUnavailable, + ) try: send(backend) + except LibeiSessionRevoked as error: + record = ledger.transition(INPUT, AuthorisationState.REVOKED, + str(error)) + autocontrol_logger.warning("Wayland input: session revoked: %s", error) + raise WaylandAuthorisationError(record, INPUT_RECOVERY) from error except LibeiUnavailable: return False return True -__all__ = ["active_backend", "emitted", "select_input_backend"] +def _drop_sessions() -> None: + """Disconnect whichever libei sessions this process has brought up.""" + for name in ("libei", "ei_client"): + module = sys.modules.get(f"je_auto_control.linux_wayland.{name}") + if module is not None: + module.reset_default_backend() + + +def close_input_session() -> None: + """End the libei session now, which is what revokes the portal grant. + + The next input action asks the portal again. Recorded as ``closed`` so a + diagnosis can tell "we ended it" from "it was taken away". + """ + _drop_sessions() + ledger.transition(INPUT, AuthorisationState.CLOSED, + "closed by this process") + + +def reset_input_authorisation() -> None: + """Forget a refused or revoked input session so it can be asked again. + + Nothing is requested here: the consent dialog appears on the next input + action, not on this call. + """ + _drop_sessions() + ledger.reset(INPUT) + + +__all__ = ["WORKER_ENV", "active_backend", "close_input_session", "emitted", + "reset_input_authorisation", "select_input_backend", + "worker_enabled"] diff --git a/je_auto_control/linux_wayland/authorisation.py b/je_auto_control/linux_wayland/authorisation.py new file mode 100644 index 000000000..f1c3371b8 --- /dev/null +++ b/je_auto_control/linux_wayland/authorisation.py @@ -0,0 +1,207 @@ +"""Where each Wayland authorisation stands, as an explicit state. + +Input and capture on Wayland both hang off something the user or the +compositor can take away: a consent dialog, a portal session, the compositor +process itself. Until now none of that was written down anywhere. A dismissed +consent dialog became ``LibeiUnavailable``, which ``keyboard`` and ``mouse`` +read as "use ydotool" — so the user said no to remote control and the desktop +was driven through ``/dev/uinput`` anyway, and nothing an operator could look +at said which of the two had happened. + +This module is the record. It holds no handles and performs no I/O beyond one +``os.stat`` of the compositor's socket: the modules that own the sessions +report transitions here, and :mod:`je_auto_control.wrapper.capabilities` reads +them back without asking anyone for anything. + +Two channels, kept apart on purpose — a host can have working capture and no +input, or the reverse, and one diagnosis for both hides which: + +``input`` the RemoteDesktop portal session libei emits through +``capture`` the Screenshot portal request (the other capture tiers need none) + +Pure stdlib, importable on every platform. +""" +from __future__ import annotations + +import os +import threading +from dataclasses import dataclass, replace +from enum import Enum +from typing import Dict, Mapping, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlException + +INPUT = "input" +CAPTURE = "capture" +_CHANNELS = (INPUT, CAPTURE) + +#: What an operator can do when the input session was refused or taken away. +INPUT_RECOVERY = ( + "Allow the remote-control request when the desktop asks again: call " + "reset_input_authorisation() (AC_reset_input_authorisation, or Actions > " + "Ask for input permission again in the Diagnostics tab), then repeat the " + "action. To use ydotool instead of the portal, set " + "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli before starting — that choice " + "is never made for you after a refusal." +) + +#: What an operator can do when the screenshot request was dismissed. +CAPTURE_RECOVERY = ( + "Allow the screenshot request when the desktop asks again, install a " + "capture tool for your compositor (grim, gnome-screenshot or spectacle), " + "or name your own command in JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND." +) + + +class AuthorisationState(str, Enum): + """One step in the life of a portal request or session.""" + + NOT_REQUESTED = "not_requested" + REQUESTING = "requesting" + GRANTED = "granted" + #: The user (or the portal on their behalf) answered no. + DECLINED = "declined" + #: Nobody answered within this project's own deadline. + TIMED_OUT = "timed_out" + #: The request failed for a reason that is not a refusal: no portal on + #: the bus, a portal without the interface, a broken handshake. + FAILED = "failed" + #: This process ended the session itself. + CLOSED = "closed" + #: The compositor or portal ended a session that had been granted. + REVOKED = "revoked" + + +#: States after which input must not be handed to another backend. Both mean +#: "someone with the authority to say no, said no"; routing the same input +#: through ``/dev/uinput`` would make that answer meaningless. +BLOCKING_STATES = frozenset({AuthorisationState.DECLINED, + AuthorisationState.REVOKED}) + + +@dataclass(frozen=True) +class AuthorisationRecord: + """The current state of one channel, and why.""" + + channel: str + state: AuthorisationState = AuthorisationState.NOT_REQUESTED + detail: str = "" + #: Which compositor instance granted it — see :func:`compositor_identity`. + compositor: Optional[str] = None + + @property + def blocks_fallback(self) -> bool: + """Whether another backend may take over this channel's work.""" + return self.state in BLOCKING_STATES + + +class WaylandAuthorisationError(AutoControlException): + """An action was refused because its authorisation was refused or lost. + + Deliberately *not* a ``RuntimeError``: the backend probes in ``keyboard`` + and ``mouse`` catch that to mean "libei is not here, use the CLI", which + is exactly the fallback this error exists to stop. + """ + + def __init__(self, record: AuthorisationRecord, recovery: str) -> None: + self.state = record.state + self.capability = record.channel + self.recovery = recovery + reason = record.detail or record.state.value + super().__init__( + f"Wayland {record.channel} is not authorised ({record.state.value}" + f": {reason}). {recovery}") + + @property + def has_recovery_instruction(self) -> bool: + """Whether the error tells the operator what to do next.""" + return bool(self.recovery) + + +def compositor_identity(environ: Optional[Mapping[str, str]] = None + ) -> Optional[str]: + """Something that changes when the compositor is restarted, or None. + + The compositor's listening socket is created when it starts, so its + device, inode and creation time identify one compositor *process* well + enough to notice a new one. Reads metadata only; opens nothing. + """ + env = environ if environ is not None else os.environ + display = (env.get("WAYLAND_DISPLAY") or "").strip() + if not display: + return None + path = display if os.path.isabs(display) else os.path.join( + env.get("XDG_RUNTIME_DIR") or "", display) + try: + found = os.stat(path) + except (OSError, ValueError): + return None + return f"{found.st_dev}:{found.st_ino}:{found.st_ctime_ns}" + + +class AuthorisationLedger: + """Thread-safe record of every channel's authorisation state. + + Emission runs on whatever thread the caller is on — the executor, a + scheduler worker, a GUI slot — and the diagnostics tab reads from the GUI + thread, so every access goes through one lock. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._records: Dict[str, AuthorisationRecord] = { + channel: AuthorisationRecord(channel) for channel in _CHANNELS} + + def get(self, channel: str) -> AuthorisationRecord: + """The current record of ``channel``.""" + with self._lock: + return self._records[_known(channel)] + + def transition(self, channel: str, state: AuthorisationState, + detail: str = "") -> AuthorisationRecord: + """Move ``channel`` to ``state`` and return the new record. + + A grant remembers which compositor made it, so a later probe can tell + a session that outlived its compositor from one that is still good. + """ + granted = state is AuthorisationState.GRANTED + granted_by = compositor_identity() if granted else None + with self._lock: + current = self._records[_known(channel)] + updated = replace( + current, state=state, detail=detail, + compositor=granted_by if granted else current.compositor) + self._records[channel] = updated + return updated + + def reset(self, channel: Optional[str] = None) -> None: + """Forget ``channel`` (or every channel), so it may be asked again.""" + targets = _CHANNELS if channel is None else (_known(channel),) + with self._lock: + for name in targets: + self._records[name] = AuthorisationRecord(name) + + def require_usable(self, channel: str, recovery: str) -> None: + """Raise if ``channel`` was refused or revoked.""" + record = self.get(channel) + if record.blocks_fallback: + raise WaylandAuthorisationError(record, recovery) + + +def _known(channel: str) -> str: + """Reject a channel name nobody reports, rather than inventing a record.""" + if channel not in _CHANNELS: + raise ValueError(f"unknown authorisation channel {channel!r}") + return channel + + +#: The process-wide ledger every Wayland module reports to. +ledger = AuthorisationLedger() + + +__all__ = [ + "AuthorisationLedger", "AuthorisationRecord", "AuthorisationState", + "BLOCKING_STATES", "CAPTURE", "CAPTURE_RECOVERY", "INPUT", + "INPUT_RECOVERY", "WaylandAuthorisationError", "compositor_identity", + "ledger", +] diff --git a/je_auto_control/linux_wayland/ei_client.py b/je_auto_control/linux_wayland/ei_client.py new file mode 100644 index 000000000..e79d8e86f --- /dev/null +++ b/je_auto_control/linux_wayland/ei_client.py @@ -0,0 +1,518 @@ +"""The parent's side of the libei helper process. + +:mod:`ei_worker` is the helper; this is what talks to it. Kept apart because +they run in different processes and fail in different ways: nothing in here +loads libei, and nothing in there supervises a process. + +:class:`EiWorkerClient` owns one helper — starts it, sends it bounded batches +with a deadline, tells it to stop when the caller gives up, and reclaims its +pipes when it ends, however it ends. :class:`WorkerBackend` wraps a client in +the emission surface ``keyboard`` and ``mouse`` already call on +``LibeiBackend``, so :mod:`_select_input` can hand out either. + +Reached only when ``JE_AUTOCONTROL_WAYLAND_EI_WORKER=1``; see :mod:`ei_worker` +for why the helper exists and why it is off by default. +""" +from __future__ import annotations + +import atexit +import queue +import subprocess # nosec B404 # reason: argv list of this interpreter, no shell +import sys +import threading +import time +from typing import Any, BinaryIO, Callable, Dict, List, Optional, Sequence, Set, Tuple + +from je_auto_control.linux_wayland.ei_transport import ( + ABS_X, ABS_Y, BUTTON_CODES, REL_HWHEEL, REL_WHEEL, EiDependencyMissing, + EiEmitRefused, EiProtocolError, EiWorkerCancelled, EiWorkerDied, + EiWorkerError, EiWorkerTimeout, Held, InputAck, encode_batch, read_frame, + write_frame, +) +from je_auto_control.linux_wayland.input_events import ( + EV_ABS, EV_KEY, EV_REL, EV_SYN, InputEvent, +) +from je_auto_control.linux_wayland.libei import ( + HANDSHAKE_TIMEOUT, LibeiConsentNotGranted, LibeiSessionRevoked, + LibeiUnavailable, +) + +#: How long the helper may take to import, ask the portal, and handshake. +#: The consent dialog is a human in the loop, so this is human-scale. +START_TIMEOUT = 90.0 +#: Default deadline for one emission. +EMIT_TIMEOUT = 2.0 +_CLOSE_TIMEOUT = 3.0 +_CANCEL_GRACE = 1.0 +_POLL = 0.05 + +_EOF = object() + + +_ACTIVE: Set["EiWorkerClient"] = set() +_ACTIVE_LOCK = threading.Lock() + + +def active_worker_count() -> int: + """How many helper processes this process has started and not reaped.""" + with _ACTIVE_LOCK: + return len(_ACTIVE) + + +def _spawn_worker() -> Any: + """Start the helper with this interpreter; stderr is left inherited.""" + argv = [sys.executable, "-m", "je_auto_control.linux_wayland.ei_worker"] + return subprocess.Popen( # nosec B603 # nosemgrep # reason: fixed argv + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + +class EiWorkerClient: + """The parent's handle on one helper process. + + :param spawn: returns a process-like object with binary ``stdin`` / + ``stdout``, ``poll``, ``wait``, ``terminate`` and ``kill``. Tests pass + a fake; the default starts ``python -m …ei_worker``. + :param socket_path: connect to this EIS socket instead of asking the + portal — what the verification images use. + :param on_orphaned_keys: called with the evdev codes that were down when + the helper died unexpectedly. + """ + + def __init__(self, *, spawn: Optional[Callable[[], Any]] = None, + socket_path: Optional[str] = None, + handshake_timeout_s: float = HANDSHAKE_TIMEOUT, + start_timeout_s: float = START_TIMEOUT, + on_orphaned_keys: Optional[Callable[[List[int]], None]] = None, + ) -> None: + self._spawn = spawn or _spawn_worker + self._socket_path = socket_path + self._handshake_timeout_s = handshake_timeout_s + self._start_timeout_s = start_timeout_s + self._on_orphaned_keys = on_orphaned_keys + self._process: Any = None + self._reader: Optional[threading.Thread] = None + self._inbox: "queue.Queue[Any]" = queue.Queue() + self._lock = threading.Lock() + self._next_id = 0 + self._pressed: Set[Held] = set() + self._dead = True + self._closing = False + self._last_returncode: Optional[int] = None + + @property + def is_alive(self) -> bool: + """Whether the helper is running and has completed its handshake.""" + return not self._dead + + @property + def returncode(self) -> Optional[int]: + """The helper's exit status once it has ended, else None. + + Negative on POSIX when a signal ended it — the number a + verification reads to tell a clean exit from a crash. + """ + return self._returncode() + + @property + def pressed(self) -> Tuple[int, ...]: + """The evdev codes the helper is believed to be holding down.""" + return tuple(sorted(code for _kind, code in self._pressed)) + + def start(self) -> None: + """Start the helper and wait for its session to come up. + + :raises EiDependencyMissing: libei is not installed where it runs. + :raises LibeiConsentNotGranted: the portal did not grant a session. + :raises EiWorkerError: it could not be started or did not come up. + """ + with self._lock: + if not self._dead: + return + self._closing = False + try: + self._process = self._spawn() + except OSError as error: + raise EiWorkerError( + f"the libei helper could not be started: {error}", + ) from error + with _ACTIVE_LOCK: + _ACTIVE.add(self) + self._inbox = queue.Queue() + self._reader = threading.Thread( + target=self._pump, args=(self._process.stdout, self._inbox), + name="ei-worker-client", daemon=True) + self._reader.start() + try: + self._handshake() + except BaseException: + self._reap() + raise + self._dead = False + + def send(self, batch: Sequence[InputEvent], *, timeout_s: float, + cancel: Optional[threading.Event] = None) -> InputAck: + """Apply ``batch`` in the helper and wait for its acknowledgement. + + :param timeout_s: seconds to wait; on expiry the helper is told to + stop and :class:`EiWorkerTimeout` is raised. + :param cancel: set it from another thread to abandon the request. + :raises EiProtocolError: the batch is empty or over the limit. + :raises EiEmitRefused: libei refused it; nothing further was sent. + :raises LibeiSessionRevoked: the compositor ended the session. + :raises EiWorkerDied: the helper is gone. + """ + events = encode_batch(batch) + with self._lock: + if self._dead: + raise EiWorkerDied( + "the libei helper is not running; call " + "reset_input_authorisation() to start a new one", + self._returncode()) + self._next_id += 1 + request_id = self._next_id + started = time.monotonic() + self._write({"op": "batch", "id": request_id, "events": events}) + answer = self._await(request_id, timeout_s, cancel, events) + self._pressed = {(str(kind), int(code)) + for kind, code in answer.get("held", ())} + return self._settle(answer, request_id, + time.monotonic() - started) + + def close(self) -> None: + """Ask the helper to exit, wait for it, and reclaim its pipes. + + The helper releases anything it holds down on the way out. Safe to + call twice and from any thread. + """ + self._closing = True + with self._lock: + if self._process is None: + return + try: + write_frame(self._process.stdin, {"op": "close"}) + except (OSError, ValueError, EiProtocolError): + pass + self._reap() + self._pressed = set() + + # --- plumbing ---------------------------------------------------------- + + @staticmethod + def _pump(stream: BinaryIO, inbox: "queue.Queue[Any]") -> None: + """Move frames from the helper's stdout into the inbox until EOF.""" + while True: + try: + frame = read_frame(stream) + except (EiProtocolError, OSError, ValueError): + frame = None + if frame is None: + inbox.put(_EOF) + return + inbox.put(frame) + + def _handshake(self) -> None: + write_frame(self._process.stdin, { + "op": "start", "socket": self._socket_path, + "timeout": self._handshake_timeout_s}) + deadline = time.monotonic() + self._start_timeout_s + while True: + remaining = deadline - time.monotonic() + if self._closing: + raise EiWorkerError("the libei helper was closed while starting") + if remaining <= 0: + raise LibeiConsentNotGranted( + "the libei helper did not come up within " + f"{self._start_timeout_s:g}s (a consent dialog may be " + "waiting)", "timeout", False) + try: + frame = self._inbox.get(timeout=min(remaining, _POLL)) + except queue.Empty: + continue + if frame is _EOF: + raise EiWorkerError( + "the libei helper exited before its session came up " + f"(exit status {self._returncode()})") + if frame.get("op") == "ready": + return + if frame.get("op") == "failed": + raise _start_failure(frame) + + def _write(self, message: Dict[str, Any]) -> None: + try: + write_frame(self._process.stdin, message) + except (OSError, ValueError) as error: + raise self._died() from error + + def _await(self, request_id: int, timeout_s: float, + cancel: Optional[threading.Event], + events: List[List[int]]) -> Dict[str, Any]: + """Wait for the answer to ``request_id``, dropping any older one.""" + deadline = time.monotonic() + max(0.0, timeout_s) + while True: + if self._closing: + raise EiWorkerCancelled("the libei helper is being closed") + if cancel is not None and cancel.is_set(): + raise self._abandon(request_id, events, cancelled=True) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise self._abandon(request_id, events, cancelled=False) + frame = self._next_frame(min(remaining, _POLL)) + if frame is not None and frame.get("id") == request_id: + return frame + + def _next_frame(self, timeout: float) -> Optional[Dict[str, Any]]: + try: + frame = self._inbox.get(timeout=timeout) + except queue.Empty: + return None + if frame is _EOF: + raise self._died() + return frame + + def _abandon(self, request_id: int, events: List[List[int]], + cancelled: bool) -> Exception: + """Tell the helper to stop this request; build what to raise.""" + self._write({"op": "cancel", "id": request_id}) + applied = None + deadline = time.monotonic() + _CANCEL_GRACE + while time.monotonic() < deadline: + frame = self._next_frame(_POLL) + if frame is not None and frame.get("id") == request_id: + applied = int(frame.get("applied", 0)) + self._pressed = {(str(kind), int(code)) + for kind, code in frame.get("held", ())} + break + else: + # No answer: assume the worst about what is down, until a later + # acknowledgement says otherwise. + self._pressed |= _presses(events) + if cancelled: + return EiWorkerCancelled( + f"request {request_id} was cancelled", applied) + return EiWorkerTimeout( + f"the libei helper did not acknowledge request {request_id} in " + "time; it was told to stop") + + def _settle(self, answer: Dict[str, Any], request_id: int, + elapsed: float) -> InputAck: + """Turn the helper's answer into an ack or the error it describes.""" + if answer.get("op") == "error": + message = str(answer.get("message", "libei refused the request")) + if answer.get("revoked"): + raise LibeiSessionRevoked(message) + raise EiEmitRefused(message) + return InputAck(request_id, int(answer.get("applied", 0)), elapsed, + bool(answer.get("cancelled", False))) + + def _died(self) -> EiWorkerDied: + """Record the helper's death and describe it.""" + orphaned = list(self.pressed) + self._pressed = set() + self._reap() + if orphaned and self._on_orphaned_keys is not None: + self._on_orphaned_keys(orphaned) + return EiWorkerDied( + f"the libei helper exited unexpectedly (exit status " + f"{self._returncode()})" + + (f" with keys {orphaned} down" if orphaned else ""), + self._returncode(), orphaned) + + def _returncode(self) -> Optional[int]: + process = self._process + if process is None: + return self._last_returncode + return process.poll() + + def _reap(self) -> None: + """Stop the process if it is still there and close both pipes.""" + process, self._process = self._process, None + self._dead = True + if process is None: + return + _close_quietly(process.stdin) + self._last_returncode = _stop_process(process) + reader, self._reader = self._reader, None + if reader is not None and reader is not threading.current_thread(): + reader.join(timeout=_CLOSE_TIMEOUT) + _close_quietly(process.stdout) + with _ACTIVE_LOCK: + _ACTIVE.discard(self) + + +def _presses(events: Sequence[Sequence[int]]) -> Set[Held]: + """The keys and buttons a batch would leave down if all of it ran.""" + down: Set[Held] = set() + for kind, code, value in events: + if kind != EV_KEY: + continue + key = ("button" if code in BUTTON_CODES else "key", code) + if value == 1: + down.add(key) + elif value == 0: + down.discard(key) + return down + + +def _start_failure(frame: Dict[str, Any]) -> Exception: + """The typed error a ``failed`` frame stands for.""" + message = str(frame.get("message", "the libei helper could not start")) + code = frame.get("code") + if code == "missing_dependency": + return EiDependencyMissing(str(frame.get("dependency", "libei")), + message) + if code == "consent": + return LibeiConsentNotGranted( + message, str(frame.get("outcome", "")), + bool(frame.get("declined", False))) + return EiWorkerError(message) + + +def _close_quietly(stream: Any) -> None: + try: + if stream is not None: + stream.close() + except (OSError, ValueError): + pass + + +def _stop_process(process: Any) -> Optional[int]: + """Wait for a process that was asked to exit; escalate if it will not.""" + for stop in (None, process.terminate, process.kill): + if stop is not None: + try: + stop() + except OSError: + pass + try: + return process.wait(timeout=_CLOSE_TIMEOUT) + except subprocess.TimeoutExpired: + continue + return process.poll() + + +class WorkerBackend: + """The emission surface of :class:`LibeiBackend`, served by a helper. + + ``keyboard`` and ``mouse`` call the same methods on either, so which one + they hold is decided in :mod:`_select_input` and nowhere else. + """ + + def __init__(self, client: EiWorkerClient, + timeout_s: float = EMIT_TIMEOUT) -> None: + self._client = client + self._timeout_s = timeout_s + + @property + def is_available(self) -> bool: + """Always true: a backend only exists once its helper came up.""" + return True + + @property + def is_connected(self) -> bool: + """Whether the helper is still running.""" + return self._client.is_alive + + def _send(self, *events: InputEvent) -> InputAck: + return self._client.send(events, timeout_s=self._timeout_s) + + def press_key(self, keycode: int) -> None: + """Send a keydown for one evdev key code.""" + self._send(InputEvent(EV_KEY, int(keycode), 1)) + + def release_key(self, keycode: int) -> None: + """Send a keyup for one evdev key code.""" + self._send(InputEvent(EV_KEY, int(keycode), 0)) + + def press_button(self, button_code: int) -> None: + """Press one BTN_* code (272 is BTN_LEFT).""" + self._send(InputEvent(EV_KEY, int(button_code), 1)) + + def release_button(self, button_code: int) -> None: + """Release one BTN_* code.""" + self._send(InputEvent(EV_KEY, int(button_code), 0)) + + def click_button(self, button_code: int) -> None: + """Press then release one BTN_* code, as one request.""" + self._send(InputEvent(EV_KEY, int(button_code), 1), + InputEvent(EV_KEY, int(button_code), 0)) + + def set_position(self, x: int, y: int) -> None: + """Move the pointer to an absolute screen position.""" + self._send(InputEvent(EV_ABS, ABS_X, int(x)), + InputEvent(EV_ABS, ABS_Y, int(y)), + InputEvent(EV_SYN, 0, 0)) + + def scroll(self, dx: int, dy: int) -> None: + """Scroll by whole wheel clicks, in libei's frame (positive y down).""" + self._send(InputEvent(EV_REL, REL_HWHEEL, int(dx)), + InputEvent(EV_REL, REL_WHEEL, -int(dy)), + InputEvent(EV_SYN, 0, 0)) + + def disconnect(self) -> None: + """Close the helper, which ends the session.""" + self._client.close() + + +_DEFAULT_BACKEND: Optional[WorkerBackend] = None +_PROBE_FAILED = False +_PROBE_ERROR: Optional[BaseException] = None +_DEFAULT_LOCK = threading.Lock() + + +def connected_backend() -> Optional[WorkerBackend]: + """A backend served by a running helper, or None — probing at most once. + + The counterpart of :func:`libei.connected_backend`, with the same + contract: None means "use the ydotool CLI", and the reason is kept for + :func:`last_probe_error`. + """ + global _DEFAULT_BACKEND, _PROBE_FAILED, _PROBE_ERROR + with _DEFAULT_LOCK: + if _DEFAULT_BACKEND is not None: + return _DEFAULT_BACKEND + if _PROBE_FAILED: + return None + client = EiWorkerClient() + try: + client.start() + except (LibeiUnavailable, OSError, ValueError) as error: + _PROBE_FAILED = True + _PROBE_ERROR = error + return None + _DEFAULT_BACKEND = WorkerBackend(client) + return _DEFAULT_BACKEND + + +def last_probe_error() -> Optional[BaseException]: + """Why the helper did not come up, if it did not.""" + with _DEFAULT_LOCK: + return _PROBE_ERROR + + +def reset_default_backend() -> None: + """Close the cached helper, if any, so the next probe starts fresh.""" + global _DEFAULT_BACKEND, _PROBE_FAILED, _PROBE_ERROR + with _DEFAULT_LOCK: + backend, _DEFAULT_BACKEND = _DEFAULT_BACKEND, None + _PROBE_FAILED = False + _PROBE_ERROR = None + if backend is not None: + backend.disconnect() + + +def _close_all_at_exit() -> None: + """Do not leave a helper behind when the interpreter exits.""" + with _ACTIVE_LOCK: + clients = list(_ACTIVE) + for client in clients: + client.close() + + +atexit.register(_close_all_at_exit) + + +__all__ = [ + "EMIT_TIMEOUT", "EiWorkerClient", "START_TIMEOUT", "WorkerBackend", + "active_worker_count", "connected_backend", "last_probe_error", + "reset_default_backend", +] diff --git a/je_auto_control/linux_wayland/ei_transport.py b/je_auto_control/linux_wayland/ei_transport.py new file mode 100644 index 000000000..620f25e17 --- /dev/null +++ b/je_auto_control/linux_wayland/ei_transport.py @@ -0,0 +1,247 @@ +"""The wire between this process and the libei helper process. + +:mod:`ei_worker` runs the libei session in a child so that whatever libei +does on teardown, it does to a process nobody else lives in. This module is +everything the two sides have to agree on, and nothing else: how a message is +framed, how large one may be, what an event looks like in transit, and what +each failure is called. + +**Bounded on purpose.** A frame is a 4-byte big-endian length and that many +bytes of UTF-8 JSON, capped at :data:`MAX_FRAME_BYTES`; a batch carries at +most :data:`MAX_BATCH` events. Neither side will read a length it has not +checked, so a corrupted stream ends in :class:`EiProtocolError` rather than +in an allocation sized by garbage — and the child is driving someone's +desktop, so "it tried its best with a bad frame" is not an option. + +**Two families of failure**, split by the one question the caller has to +answer — may another backend redo this? + +:class:`EiWorkerError` is a ``LibeiUnavailable``: nothing reached the desktop, +so the ydotool path may take the action over, exactly as it does when the +in-process backend refuses. + +:class:`EiRequestUncertain` is *not*: the request was written and its fate is +unknown (a timeout, a cancellation, a worker that died mid-batch). Replaying +it through another backend could type the key twice, so these reach the +caller instead. +""" +from __future__ import annotations + +import json +import struct +from dataclasses import dataclass +from typing import ( + Any, BinaryIO, Dict, List, Mapping, Optional, Sequence, Tuple, +) + +from je_auto_control.linux_wayland.input_events import ( + EV_ABS, EV_KEY, EV_REL, EV_SYN, InputEvent, +) +from je_auto_control.linux_wayland.libei import LibeiUnavailable +from je_auto_control.utils.exception.exceptions import AutoControlException + +#: Most events one request may carry. A chord is a handful; a pointer move is +#: three. Anything longer should be several requests, each cancellable. +MAX_BATCH = 64 + +#: Largest frame either side will write or accept. +MAX_FRAME_BYTES = 64 * 1024 + +#: The capability that is unavailable when the worker cannot serve. +CAPABILITY = "input" + +# The evdev codes both sides have to read the same way. +ABS_X = 0x00 +ABS_Y = 0x01 +REL_HWHEEL = 0x06 +REL_WHEEL = 0x08 + +#: ``BTN_MOUSE`` .. ``BTN_TASK``: the ``EV_KEY`` codes that are pointer +#: buttons, and so go to libei's button entry point rather than its key one. +BUTTON_CODES = range(0x110, 0x118) + +#: One held key or button: ``("key" | "button", code)``. +Held = Tuple[str, int] + +_LENGTH = struct.Struct(">I") +_EVENT_TYPES = frozenset({EV_SYN, EV_KEY, EV_REL, EV_ABS}) +_INT32 = 2 ** 31 + + +class EiWorkerError(LibeiUnavailable): + """The helper could not serve, and nothing was sent to the desktop.""" + + capability = CAPABILITY + + +class EiDependencyMissing(EiWorkerError): + """The helper started but a library it needs is not installed.""" + + def __init__(self, dependency: str, message: str = "") -> None: + self.dependency = dependency + super().__init__( + message or f"{dependency} is not installed, so the libei helper " + f"cannot provide {CAPABILITY}") + + +class EiProtocolError(EiWorkerError): + """A frame or batch broke the limits or the format; it was not applied.""" + + +class EiEmitRefused(EiWorkerError): + """libei, in the helper, refused this emission (a paused device, a point + outside every region). The same refusal the in-process backend gives.""" + + +class EiRequestUncertain(AutoControlException): + """A request was written and what it did to the desktop is not known.""" + + capability = CAPABILITY + + +class EiWorkerTimeout(EiRequestUncertain): + """No acknowledgement arrived within the caller's deadline.""" + + +class EiWorkerCancelled(EiRequestUncertain): + """The caller cancelled; ``applied`` is how far the batch got, if known.""" + + def __init__(self, message: str, applied: Optional[int] = None) -> None: + self.applied = applied + super().__init__(message) + + +class EiWorkerDied(EiRequestUncertain): + """The helper process ended; ``pressed_keys`` were down when it did.""" + + def __init__(self, message: str, returncode: Optional[int] = None, + pressed_keys: Sequence[int] = ()) -> None: + self.returncode = returncode + self.pressed_keys = tuple(pressed_keys) + super().__init__(message) + + +@dataclass(frozen=True) +class InputAck: + """The helper's answer to one batch.""" + + request_id: int + #: How many of the batch's events were applied. + applied: int + #: Round trip as the caller saw it, in seconds. + elapsed_s: float = 0.0 + cancelled: bool = False + + +def write_frame(stream: BinaryIO, message: Mapping[str, Any]) -> None: + """Write one message and flush it. + + :raises EiProtocolError: the encoded message exceeds the frame limit. + """ + payload = json.dumps(message, separators=(",", ":")).encode("utf-8") + if len(payload) > MAX_FRAME_BYTES: + raise EiProtocolError( + f"a {len(payload)}-byte frame exceeds the {MAX_FRAME_BYTES}-byte " + "limit") + stream.write(_LENGTH.pack(len(payload)) + payload) + stream.flush() + + +def _read_exactly(stream: BinaryIO, count: int) -> bytes: + """Read ``count`` bytes, or fewer only if the stream ended.""" + chunks: List[bytes] = [] + remaining = count + while remaining > 0: + chunk = stream.read(remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def read_frame(stream: BinaryIO) -> Optional[Dict[str, Any]]: + """Read one message; None when the stream ended between messages. + + :raises EiProtocolError: an oversized length, a truncated frame, or a + payload that is not a JSON object. + """ + header = _read_exactly(stream, _LENGTH.size) + if not header: + return None + if len(header) < _LENGTH.size: + raise EiProtocolError("the stream ended inside a frame header") + (length,) = _LENGTH.unpack(header) + if length > MAX_FRAME_BYTES: + raise EiProtocolError( + f"a frame announced {length} bytes, over the {MAX_FRAME_BYTES}-" + "byte limit") + payload = _read_exactly(stream, length) + if len(payload) < length: + raise EiProtocolError("the stream ended inside a frame") + try: + message = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as error: + raise EiProtocolError(f"a frame was not valid JSON: {error}") from error + if not isinstance(message, dict): + raise EiProtocolError("a frame was not a JSON object") + return message + + +def encode_batch(batch: Sequence[InputEvent]) -> List[List[int]]: + """Turn a batch into its wire form, enforcing the batch limit. + + :raises EiProtocolError: the batch is empty, too long, or holds an event + that is not a valid evdev triple. + """ + if not batch: + raise EiProtocolError("an empty batch has nothing to send") + if len(batch) > MAX_BATCH: + raise EiProtocolError( + f"a batch of {len(batch)} events exceeds the limit of " + f"{MAX_BATCH}; send it as several requests") + return [list(_checked(event.type, event.code, event.value)) + for event in batch] + + +def decode_batch(raw: Any) -> List[InputEvent]: + """Rebuild a batch from its wire form, with the same checks. + + The helper validates everything before applying anything, so a bad batch + is refused whole rather than half-typed. + """ + if not isinstance(raw, list) or not raw: + raise EiProtocolError("a batch must be a non-empty list of events") + if len(raw) > MAX_BATCH: + raise EiProtocolError( + f"a batch of {len(raw)} events exceeds the limit of {MAX_BATCH}") + events = [] + for item in raw: + if not isinstance(item, list) or len(item) != 3: + raise EiProtocolError(f"not an event triple: {item!r}") + events.append(InputEvent(*_checked(*item))) + return events + + +def _checked(kind: Any, code: Any, value: Any) -> tuple: + """One event's three fields, or a refusal naming what is wrong.""" + for part in (kind, code, value): + if isinstance(part, bool) or not isinstance(part, int): + raise EiProtocolError( + f"event fields must be integers, got {part!r}") + if kind not in _EVENT_TYPES: + raise EiProtocolError(f"unsupported event type {kind}") + if not 0 <= code < 0x10000 or not -_INT32 <= value < _INT32: + raise EiProtocolError( + f"event ({kind}, {code}, {value}) is out of range") + return kind, code, value + + +__all__ = [ + "ABS_X", "ABS_Y", "BUTTON_CODES", "Held", "REL_HWHEEL", "REL_WHEEL", + "CAPABILITY", "EiDependencyMissing", "EiEmitRefused", "EiProtocolError", + "EiRequestUncertain", "EiWorkerCancelled", "EiWorkerDied", + "EiWorkerError", "EiWorkerTimeout", "InputAck", "MAX_BATCH", + "MAX_FRAME_BYTES", "decode_batch", "encode_batch", "read_frame", + "write_frame", +] diff --git a/je_auto_control/linux_wayland/ei_worker.py b/je_auto_control/linux_wayland/ei_worker.py new file mode 100644 index 000000000..7ee7482bc --- /dev/null +++ b/je_auto_control/linux_wayland/ei_worker.py @@ -0,0 +1,323 @@ +"""Run the libei session in a helper process, behind an opt-in switch. + +**Why this exists.** ``ei_unref`` segfaults on libei 1.3.901 when a context's +backend opened and its handshake never progressed. ``LibeiBackend._teardown`` +answers that by abandoning the context — a deliberate leak of one context and +one descriptor per process — because the alternative is a crash inside a +program that is driving someone's desktop. That in-process path is still the +default, and nothing here changes it. + +A helper process is the other answer. The session lives in a child; when the +child exits, the operating system reclaims the context and the descriptor +whatever state libei left them in, and if libei does crash on some teardown +this project has not measured, it takes down a process that holds nothing +else. The parent keeps a pipe and a typed error. + +**Why it is off by default.** Whether a *legitimate* cleanup still crashes on +a given libei can only be established on Linux, by the ``eis-verification`` +job. Until that says the helper is needed, routing every keystroke through a +second process is cost without a measured benefit: a pipe round trip per +emission, a child to supervise, and a second way to fail. So +``JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`` turns it on, and ``docker/eis_verify.py`` +measures what it costs and what it reclaims. + +**What the helper promises.** + +* Requests carry an id, and a late answer to a request the caller gave up on + is recognised and dropped, never mistaken for the next one's. +* A batch is bounded (:data:`ei_transport.MAX_BATCH`) and validated whole + before any of it is applied. +* A caller's deadline or cancellation reaches the child, which stops between + events and lets go of anything that batch had pressed. +* When the parent goes away — a clean close, or its pipe simply ending — the + child releases every key and button still down before it exits. +* When the *child* dies without warning, the parent reports which keys were + down in :class:`EiWorkerDied`. The compositor is expected to release them + as it removes the dead client's devices; that is the EI protocol's job and + is measured in CI rather than assumed here. + +This module is the helper: run as +``python -m je_auto_control.linux_wayland.ei_worker`` it serves one session +and exits. The parent's half — starting it, sending to it, reclaiming it — is +:mod:`ei_client`, and what the two agree on is :mod:`ei_transport`. +""" +from __future__ import annotations + +import os +import queue +import sys +import threading +from typing import Any, BinaryIO, Callable, Dict, Iterable, Optional, Set + +from je_auto_control.linux_wayland.ei_transport import ( + ABS_X, ABS_Y, BUTTON_CODES, REL_HWHEEL, REL_WHEEL, EiProtocolError, Held, + decode_batch, read_frame, write_frame, +) +from je_auto_control.linux_wayland.input_events import ( + EV_ABS, EV_KEY, EV_REL, EV_SYN, InputEvent, +) +from je_auto_control.linux_wayland.libei import ( + HANDSHAKE_TIMEOUT, LibeiBackend, LibeiConsentNotGranted, + LibeiSessionRevoked, LibeiUnavailable, +) + +_EOF = object() + + +class _Applier: + """Turn evdev-shaped events into calls on a libei backend. + + Absolute motion and wheel deltas are gathered until ``EV_SYN`` (or the + end of the batch), because that is how evdev spells "one pointer move": + an X event, a Y event, then a sync. + """ + + def __init__(self, backend: Any) -> None: + self._backend = backend + self.held: Set[Held] = set() + self._point: Dict[int, int] = {} + self._wheel: Dict[int, int] = {} + + def apply(self, event: InputEvent) -> None: + """Apply one event; libei's own refusals propagate.""" + if event.type == EV_KEY: + self._key(event.code, event.value) + elif event.type == EV_ABS: + self._point[event.code] = event.value + elif event.type == EV_REL: + self._wheel[event.code] = (self._wheel.get(event.code, 0) + + event.value) + elif event.type == EV_SYN: + self.flush() + + def _key(self, code: int, value: int) -> None: + if value not in (0, 1): + return # 2 is the kernel's auto-repeat; libei has no such event + kind = "button" if code in BUTTON_CODES else "key" + if kind == "button": + send = (self._backend.press_button if value + else self._backend.release_button) + else: + send = (self._backend.press_key if value + else self._backend.release_key) + send(code) + if value: + self.held.add((kind, code)) + else: + self.held.discard((kind, code)) + + def flush(self) -> None: + """Send whatever motion and scroll has been gathered.""" + point, self._point = self._point, {} + wheel, self._wheel = self._wheel, {} + if ABS_X in point and ABS_Y in point: + self._backend.set_position(point[ABS_X], point[ABS_Y]) + horizontal = wheel.get(REL_HWHEEL, 0) + vertical = wheel.get(REL_WHEEL, 0) + if horizontal or vertical: + # The kernel counts REL_WHEEL up as positive; libei counts down. + self._backend.scroll(horizontal, -vertical) + + def discard_pending(self) -> None: + """Forget gathered motion — a cancelled batch must not move later.""" + self._point, self._wheel = {}, {} + + def release(self, held: Iterable[Held]) -> None: + """Let go of ``held``; a refusal for one must not strand the rest.""" + for kind, code in sorted(held): + send = (self._backend.release_button if kind == "button" + else self._backend.release_key) + try: + send(code) + except LibeiUnavailable: + pass + self.held.discard((kind, code)) + + +class _Server: + """The child's side of the conversation.""" + + def __init__(self, reader: BinaryIO, writer: BinaryIO, + backend_factory: Callable[[], Any]) -> None: + self._reader = reader + self._writer = writer + self._factory = backend_factory + self._inbox: "queue.Queue[Any]" = queue.Queue() + self._cancelled: Set[int] = set() + self._lock = threading.Lock() + self._applier: Optional[_Applier] = None + self._backend: Any = None + + def run(self) -> int: + """Serve until the parent closes or goes away; return an exit code.""" + threading.Thread(target=self._pump, name="ei-worker-reader", + daemon=True).start() + try: + if not self._start(self._inbox.get()): + return 1 + self._serve() + return 0 + finally: + self._shutdown() + + def _pump(self) -> None: + """Read frames off the pipe; cancellations jump the queue.""" + while True: + try: + frame = read_frame(self._reader) + except (EiProtocolError, OSError, ValueError): + frame = None + if frame is None: + self._inbox.put(_EOF) + return + if frame.get("op") == "cancel": + with self._lock: + self._cancelled.add(int(frame.get("id", -1))) + continue + self._inbox.put(frame) + + def _start(self, frame: Any) -> bool: + if frame is _EOF or frame.get("op") != "start": + return False + try: + self._backend = self._connect(frame) + except LibeiConsentNotGranted as error: + self._reply({"op": "failed", "code": "consent", + "message": str(error), "outcome": error.outcome, + "declined": error.declined}) + return False + except (LibeiUnavailable, OSError, ValueError) as error: + self._reply({"op": "failed", "code": "unavailable", + "message": str(error)}) + return False + if self._backend is None: + self._reply({"op": "failed", "code": "missing_dependency", + "dependency": "libei", + "message": "libei.so.* not found on the loader path"}) + return False + self._applier = _Applier(self._backend) + self._reply({"op": "ready"}) + return True + + def _connect(self, frame: Dict[str, Any]) -> Any: + """A connected backend, or None when libei is not installed.""" + backend = self._factory() + if not backend.is_available: + return None + socket_path = frame.get("socket") + backend.connect( + timeout=float(frame.get("timeout") or HANDSHAKE_TIMEOUT), + socket_path=socket_path.encode("utf-8") if socket_path else None) + return backend + + def _serve(self) -> None: + while True: + frame = self._inbox.get() + if frame is _EOF or frame.get("op") == "close": + return + if frame.get("op") == "batch": + self._reply(self._batch(self._applier, frame)) + + def _is_cancelled(self, request_id: int) -> bool: + with self._lock: + return request_id in self._cancelled + + def _batch(self, applier: Any, frame: Dict[str, Any]) -> Dict[str, Any]: + """Apply one batch and describe how it went.""" + request_id = int(frame.get("id", -1)) + answer: Dict[str, Any] = {"op": "ack", "id": request_id, "applied": 0} + before = set(applier.held) + try: + for event in decode_batch(frame.get("events")): + if self._is_cancelled(request_id): + applier.discard_pending() + applier.release(applier.held - before) + answer["cancelled"] = True + break + applier.apply(event) + answer["applied"] += 1 + else: + applier.flush() + except LibeiSessionRevoked as error: + answer.update(op="error", message=str(error), revoked=True) + except LibeiUnavailable as error: + applier.discard_pending() + answer.update(op="error", message=str(error), revoked=False) + answer["held"] = sorted([kind, code] for kind, code in applier.held) + return answer + + def _reply(self, message: Dict[str, Any]) -> None: + try: + write_frame(self._writer, message) + except (OSError, ValueError): + pass # the parent is gone; the main loop sees EOF and unwinds + + def _shutdown(self) -> None: + """Release what is held, then let the session go.""" + if self._applier is not None: + self._applier.release(set(self._applier.held)) + if self._backend is not None: + # State-aware: a completed session is unreffed, a half-open one + # is abandoned — and here "abandoned" means "reclaimed at exit". + self._backend.disconnect() + + +def serve(reader: BinaryIO, writer: BinaryIO, + backend_factory: Optional[Callable[[], Any]] = None) -> int: + """Run the child's loop over two binary streams; return its exit code.""" + return _Server(reader, writer, backend_factory or LibeiBackend).run() + + +def main() -> int: + """Entry point of the helper process.""" + # Keep the frame stream private: anything else that writes to stdout — + # a library's diagnostics, a stray log handler — goes to stderr instead + # of into the middle of a frame. + frames = os.fdopen(os.dup(sys.stdout.fileno()), "wb", buffering=0) + os.dup2(sys.stderr.fileno(), sys.stdout.fileno()) + sys.stdout = sys.stderr + # Unbuffered, and not ``sys.stdin.buffer``: the reader thread spends its + # life blocked in a read, and a buffered reader holds a lock while it + # does — which the interpreter then cannot take at shutdown. + requests = os.fdopen(os.dup(sys.stdin.fileno()), "rb", buffering=0) + return serve(requests, frames) + + +def run_as_helper() -> None: + """Serve, then leave without unwinding the interpreter. Never returns. + + The reader thread is still blocked on the pipe when the loop ends, and + libei may hold a context that must not be released (the half-open state + this helper exists for). Both are the operating system's to reclaim, so + the process ends with ``os._exit`` once the log handlers are flushed — + measured, not stylistic: with an ordinary exit the interpreter aborted in + finalisation on the blocked reader, turning a clean answer into a crash. + """ + import faulthandler + import logging + import signal + + # A crash in here is a native one. Without this it is a bare exit status + # in the parent; with it, stderr carries the Python stack that led there. + faulthandler.enable() + + def _unwind(_signum: int, _frame: Any) -> None: + # Raised in the main thread, so the serve loop's ``finally`` runs and + # held keys are released before a polite termination takes effect. + raise SystemExit(0) + + signal.signal(signal.SIGTERM, _unwind) + try: + code = main() + except SystemExit as stop: + code = stop.code if isinstance(stop.code, int) else 0 + logging.shutdown() + sys.stderr.flush() + os._exit(code) + + +__all__ = ["main", "run_as_helper", "serve"] + + +if __name__ == "__main__": + run_as_helper() diff --git a/je_auto_control/linux_wayland/global_shortcuts.py b/je_auto_control/linux_wayland/global_shortcuts.py new file mode 100644 index 000000000..24fde9688 --- /dev/null +++ b/je_auto_control/linux_wayland/global_shortcuts.py @@ -0,0 +1,275 @@ +"""A global "stop" key on Wayland, through the GlobalShortcuts portal. + +Every other backend stops a runaway script by polling the keyboard: the +critical-exit watcher asks "is F7 down?" on a timer. Wayland will not answer +that question — ``listener.check_key_is_press`` reports ``False`` forever — +so on Wayland the panic key is inert, and a script that has taken the pointer +can only be stopped from the GUI or by killing the process. + +``org.freedesktop.portal.GlobalShortcuts`` is the supported way to get one key +back. The application registers a *named* shortcut; the compositor shows the +user what is being asked for, lets them choose the key, and from then on +delivers an ``Activated`` signal when it is pressed. That is all it does and +all this module asks of it: one shortcut, named ``stop``. It is not a key +logger and cannot be made into one — the application never learns about any +key it did not register, which is the point. + +**A session has to be closed.** The registration lives as long as the portal +session does, and the session lives as long as the D-Bus connection — so +:meth:`StopShortcutSession.close` both asks the portal to end the session and +drops the connection, and is safe to call from any thread, twice. + +Where the portal has no GlobalShortcuts (it is newer than RemoteDesktop and +not every desktop ships it) or the user declines, the caller gets a +:class:`ShortcutUnavailable` whose ``recovery`` names the stop methods that +still work. Nothing here falls back to reading devices. +""" +from __future__ import annotations + +import threading +from typing import Any, Callable, Dict, List, Optional + +from je_auto_control.linux_wayland import _dbus_client +from je_auto_control.linux_wayland.portal import ( + PORTAL_BUS, PORTAL_PATH, REQUEST_INTERFACE, _handle_token, _match_rule, + _request_path, +) +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +SHORTCUTS_INTERFACE = "org.freedesktop.portal.GlobalShortcuts" +SESSION_INTERFACE = "org.freedesktop.portal.Session" +STOP_SHORTCUT_ID = "stop" + +#: The consent dialog is a human in the loop, so this is a human-scale wait. +DEFAULT_TIMEOUT = 30.0 +_CALL_TIMEOUT = 10.0 + +STOP_RECOVERY = ( + "A running script can still be stopped from the GUI's stop control " + "(Actions menu) or by ending the process. To get a stop key, allow the " + "shortcut when the desktop asks, or bind one to the stop command in your " + "desktop's keyboard settings.") + +_ACTIVATED_RULE = (f"type='signal',sender='{PORTAL_BUS}'," + f"interface='{SHORTCUTS_INTERFACE}',member='Activated'") + + +class ShortcutUnavailable(AutoControlException): + """No stop shortcut could be registered; ``recovery`` says what still works.""" + + def __init__(self, message: str) -> None: + self.recovery = STOP_RECOVERY + super().__init__(f"{message}. {STOP_RECOVERY}") + + @property + def has_recovery_instruction(self) -> bool: + """Whether the error tells the operator what to do next.""" + return bool(self.recovery) + + +class ShortcutPermissionError(ShortcutUnavailable): + """The user, or the desktop on their behalf, declined the shortcut.""" + + +class StopShortcutSession: + """One registered stop shortcut, alive until :meth:`close`. + + :param on_stop: called, on the session's own thread, each time the + shortcut is pressed. Keep it short: set an event, do not do the work. + :param bus_factory: returns a connected session bus; injected by tests. + :param preferred_trigger: a hint such as ``"CTRL+ALT+F12"``. The desktop + may ignore it and the user may change it; it is never a guarantee. + """ + + def __init__(self, on_stop: Callable[[], None], *, + bus_factory: Optional[Callable[[], Any]] = None, + preferred_trigger: Optional[str] = None, + poll_s: float = 0.25) -> None: + self._on_stop = on_stop + self._bus_factory = bus_factory or _connected_bus + self._trigger = preferred_trigger + self._poll_s = max(0.01, float(poll_s)) + self._bus: Any = None + self._session_handle = "" + self._closing = threading.Event() + self._thread: Optional[threading.Thread] = None + self._lock = threading.Lock() + + @property + def is_open(self) -> bool: + """Whether the shortcut is registered and the session is live.""" + return bool(self._session_handle) and not self._closing.is_set() + + def __enter__(self) -> "StopShortcutSession": + self.open() + return self + + def __exit__(self, *_exception: Any) -> None: + self.close() + + def open(self, timeout: float = DEFAULT_TIMEOUT) -> None: + """Create the portal session and register the stop shortcut. + + :param timeout: seconds to wait for each portal answer, including the + time the user spends on the dialog. + :raises ShortcutPermissionError: the request was declined. + :raises ShortcutUnavailable: no portal, no GlobalShortcuts interface, + or no answer in time. + """ + if self._session_handle: + return + self._closing.clear() + try: + self._bus = self._bus_factory() + self._session_handle = self._create_session(timeout) + self._bind(timeout) + self._bus.add_match(_ACTIVATED_RULE) + except _dbus_client.DBusError as error: + self.close() + raise ShortcutUnavailable( + f"the GlobalShortcuts portal could not be used: {error}", + ) from error + except BaseException: + self.close() + raise + + def wait(self, timeout: float) -> bool: + """Wait up to ``timeout`` seconds for one press; True if it came. + + ``on_stop`` has already been called when this returns True. + """ + if not self.is_open: + return False + try: + body = self._bus.wait_for_signal( + [PORTAL_PATH], SHORTCUTS_INTERFACE, "Activated", timeout) + except _dbus_client.DBusError as error: + if not _is_timeout(error): + autocontrol_logger.warning( + "stop shortcut: the portal connection ended: %s", error) + self._closing.set() + return False + if not _is_our_stop(body, self._session_handle): + return False + self._on_stop() + return True + + def start(self) -> None: + """Listen on a background thread until :meth:`close`.""" + if self._thread is not None or not self.is_open: + return + self._thread = threading.Thread( + target=self._listen, name="wayland-stop-shortcut", daemon=True) + self._thread.start() + + def close(self) -> None: + """End the portal session and drop the connection. Idempotent.""" + self._closing.set() + thread, self._thread = self._thread, None + if thread is not None and thread is not threading.current_thread(): + thread.join(timeout=self._poll_s + _CALL_TIMEOUT) + with self._lock: + bus, self._bus = self._bus, None + handle, self._session_handle = self._session_handle, "" + if bus is None: + return + if handle: + try: + bus.call(PORTAL_BUS, handle, SESSION_INTERFACE, "Close", "", + [], timeout=_CALL_TIMEOUT) + except _dbus_client.DBusError as error: + # Dropping the connection below ends the session regardless. + autocontrol_logger.info( + "stop shortcut: Session.Close failed: %s", error) + bus.close() + + # --- the portal conversation ------------------------------------------ + + def _listen(self) -> None: + while not self._closing.is_set(): + self.wait(self._poll_s) + + def _create_session(self, timeout: float) -> str: + token = _handle_token() + options = { + "handle_token": _dbus_client.Variant("s", token), + "session_handle_token": _dbus_client.Variant("s", token + "_s"), + } + results = self._request("CreateSession", "a{sv}", [options], token, + timeout) + handle = str(results.get("session_handle", "")) + if not handle: + raise ShortcutUnavailable( + "the portal created a session but returned no handle") + return handle + + def _bind(self, timeout: float) -> None: + token = _handle_token() + shortcut: Dict[str, Any] = { + "description": _dbus_client.Variant( + "s", "Stop the running AutoControl script"), + } + if self._trigger: + shortcut["preferred_trigger"] = _dbus_client.Variant( + "s", self._trigger) + options = {"handle_token": _dbus_client.Variant("s", token)} + self._request( + "BindShortcuts", "oa(sa{sv})sa{sv}", + [self._session_handle, [(STOP_SHORTCUT_ID, shortcut)], "", + options], token, timeout) + + def _request(self, member: str, signature: str, body: List[Any], + token: str, timeout: float) -> Dict[str, Any]: + """One portal request: subscribe, call, wait for its ``Response``.""" + predicted = _request_path(self._bus.sender_token, token) + self._bus.add_match(_match_rule(predicted)) + reply = self._bus.call(PORTAL_BUS, PORTAL_PATH, SHORTCUTS_INTERFACE, + member, signature, body, timeout=_CALL_TIMEOUT) + paths = [predicted] + handle = str(reply[0]) if reply else "" + if handle and handle != predicted: + self._bus.add_match(_match_rule(handle)) + paths.append(handle) + answer = self._bus.wait_for_signal(paths, REQUEST_INTERFACE, + "Response", timeout) + return _results(member, answer) + + +def _connected_bus() -> Any: + bus = _dbus_client.SessionBus() + bus.connect() + return bus + + +def _results(member: str, body: List[Any]) -> Dict[str, Any]: + """Read a ``(u, a{sv})`` Response; anything but success is an error.""" + if len(body) < 2 or not isinstance(body[1], dict): + raise ShortcutUnavailable( + f"the portal answered {member} with something unreadable") + code = int(body[0]) + if code == 1: + raise ShortcutPermissionError( + f"the stop shortcut was declined ({member})") + if code != 0: + raise ShortcutUnavailable( + f"the portal ended {member} with response {code}") + return body[1] + + +def _is_our_stop(body: List[Any], session_handle: str) -> bool: + """Whether an ``Activated`` signal is this session's stop shortcut.""" + return (len(body) >= 2 and str(body[0]) == session_handle + and str(body[1]) == STOP_SHORTCUT_ID) + + +def _is_timeout(error: BaseException) -> bool: + """A quiet poll interval, as opposed to a connection that has ended.""" + return "did not answer in time" in str(error) + + +__all__ = [ + "DEFAULT_TIMEOUT", "SESSION_INTERFACE", "SHORTCUTS_INTERFACE", + "STOP_RECOVERY", "STOP_SHORTCUT_ID", "ShortcutPermissionError", + "ShortcutUnavailable", "StopShortcutSession", +] diff --git a/je_auto_control/linux_wayland/input_events.py b/je_auto_control/linux_wayland/input_events.py new file mode 100644 index 000000000..74a85628f --- /dev/null +++ b/je_auto_control/linux_wayland/input_events.py @@ -0,0 +1,395 @@ +"""Recording on Wayland: what this program did, and what the user did. + +X11 lets any client watch every key and click, so "record" there is one +feature. Wayland forbids that on purpose, and the answer is not to find a way +round it — it is to notice that "record" was always two different things: + +**What this program executed.** No hook is needed for that; the executor +already knows. :class:`ActionJournal` collects the steps as they run, on every +backend, with nothing read from the desktop at all. + +**What the user physically typed and clicked.** That needs the kernel's event +devices, and it is offered here as :class:`PhysicalRecorder` under three +conditions that are the whole design: + +* **opt-in, device by device.** Nothing is opened unless the caller names it. + There is no "all keyboards" default, because the keyboard is also where + passwords are typed. +* **virtual devices are excluded.** ``ydotool`` injects through + ``/dev/uinput``, which shows up as one more event device; reading it would + record this program's own output as though the user had typed it. A device + whose origin cannot be established is excluded too. +* **no privilege is taken.** Opening a device this user may not read raises + :class:`InputPermissionError` saying how to grant access. It never retries + as root and never suggests running the program as root. + +The desktop portal's *InputCapture* interface is deliberately not used. It +exists to hand the pointer to another machine once it crosses a screen edge +the compositor chose — the compositor decides when it starts, so it cannot be +a recorder that starts when a script asks. + +Pure stdlib and importable everywhere; only :class:`PhysicalRecorder`'s +default device source touches Linux-only calls, and only once started. +""" +from __future__ import annotations + +import copy +import os +import struct +import threading +from dataclasses import dataclass +from typing import Any, Callable, List, Mapping, Optional, Sequence, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + +EV_SYN = 0x00 +EV_KEY = 0x01 +EV_REL = 0x02 +EV_ABS = 0x03 + +#: ``BUS_VIRTUAL`` from ``linux/input.h`` — what a uinput device may declare. +BUS_VIRTUAL = 0x06 + +#: Where the kernel parents every uinput device in sysfs. A Bluetooth +#: keyboard also lives under ``/devices/virtual`` (``…/misc/uhid/…``) and is +#: a real keyboard, which is why the match is this narrow. +_UINPUT_SYSFS = "/devices/virtual/input/" + +#: Comma-separated ``/dev/input/event*`` paths the operator opted in to. +RECORD_DEVICES_ENV = "JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES" + +#: ``struct input_event`` in this interpreter's native layout. +_EVENT = struct.Struct("@llHHi") + +DEFAULT_MAX_EVENTS = 100_000 + +PERMISSION_RECOVERY = ( + "Give this user read access to the device: add the user to the `input` " + "group and log in again, or install a udev rule for it. Do not run the " + "whole program as root for this. To record what this program executes " + "instead, use ActionJournal — it needs no device at all.") + + +class InputRecordingError(AutoControlException): + """Physical input recording could not start, or was misused.""" + + +class InputPermissionError(InputRecordingError): + """This user may not read an input device; ``recovery`` says what to do.""" + + def __init__(self, path: str) -> None: + self.path = path + self.recovery = PERMISSION_RECOVERY + super().__init__( + f"permission denied reading {path}. {PERMISSION_RECOVERY}") + + @property + def has_recovery_instruction(self) -> bool: + """Whether the error tells the operator what to do next.""" + return bool(self.recovery) + + +@dataclass(frozen=True) +class InputDevice: + """One kernel event device and where it comes from.""" + + path: str + name: str = "" + phys: str = "" + bustype: int = 0 + #: The resolved sysfs path; empty when it could not be read. + sysfs: str = "" + + @property + def origin_known(self) -> bool: + """Whether sysfs said where this device hangs.""" + return bool(self.sysfs) + + @property + def is_virtual(self) -> bool: + """Whether this is a uinput device rather than hardware.""" + return (self.bustype == BUS_VIRTUAL + or _UINPUT_SYSFS in self.sysfs.replace(os.sep, "/") + "/") + + +@dataclass(frozen=True) +class InputEvent: + """One evdev event: ``type`` / ``code`` / ``value`` as the kernel has them. + + The same shape is what :mod:`ei_worker` accepts for emission, so a + recorded stream and a stream to send are one vocabulary. + """ + + type: int + code: int + value: int + time_s: float = 0.0 + device: str = "" + + +def configured_record_devices(environ: Optional[Mapping[str, str]] = None + ) -> Tuple[str, ...]: + """The device paths the operator opted in to reading, if any.""" + env = environ if environ is not None else os.environ + raw = env.get(RECORD_DEVICES_ENV) or "" + return tuple(part.strip() for part in raw.split(",") if part.strip()) + + +def _read_text(path: str) -> str: + try: + with open(path, encoding="utf-8", errors="replace") as handle: + return handle.read().strip() + except OSError: + return "" + + +def describe_device(path: str, sys_root: str = "/sys/class/input" + ) -> InputDevice: + """Look one ``/dev/input/event*`` node up in sysfs. + + Reads metadata only — the device itself is not opened, so this needs no + permission and tells nobody anything. + """ + node = os.path.join(sys_root, os.path.basename(path)) + if not os.path.exists(node): + return InputDevice(path) + described = os.path.join(node, "device") + try: + bustype = int(_read_text(os.path.join(described, "id", "bustype")), 16) + except ValueError: + bustype = 0 + return InputDevice( + path=path, name=_read_text(os.path.join(described, "name")), + phys=_read_text(os.path.join(described, "phys")), bustype=bustype, + sysfs=os.path.realpath(node)) + + +def list_input_devices(sys_root: str = "/sys/class/input", + dev_root: str = "/dev/input") -> List[InputDevice]: + """Every event device sysfs knows about, virtual ones included. + + For choosing what to name in :data:`RECORD_DEVICES_ENV`; check + :attr:`InputDevice.is_virtual` before recording from one. + """ + try: + names = sorted(name for name in os.listdir(sys_root) + if name.startswith("event")) + except OSError: + return [] + return [describe_device(f"{dev_root.rstrip('/')}/{name}", sys_root) + for name in names] + + +def decode_events(data: bytes, device: str = "") -> List[InputEvent]: + """Split a buffer of ``struct input_event`` records; a tail is dropped.""" + events = [] + for offset in range(0, len(data) - _EVENT.size + 1, _EVENT.size): + seconds, micros, kind, code, value = _EVENT.unpack_from(data, offset) + events.append(InputEvent(kind, code, value, + seconds + micros / 1_000_000, device)) + return events + + +class _EvdevSource: + """One opened event device, read without blocking past ``timeout``.""" + + def __init__(self, path: str) -> None: + self.path = path + self._fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) + + def read_events(self, timeout: float) -> List[InputEvent]: + """Whatever arrived within ``timeout`` seconds, possibly nothing.""" + import select + ready, _, _ = select.select([self._fd], [], [], timeout) + if not ready: + return [] + try: + data = os.read(self._fd, _EVENT.size * 64) + except BlockingIOError: + return [] + return decode_events(data, self.path) + + def close(self) -> None: + """Release the descriptor; safe to call twice.""" + if self._fd >= 0: + os.close(self._fd) + self._fd = -1 + + +class PhysicalRecorder: + """Read the user's own input from kernel devices they named. + + ``opener`` turns a device path into something with ``read_events(timeout)`` + and ``close()``; tests pass a fake, the default opens the real node. + """ + + def __init__(self, *, opener: Optional[Callable[[str], Any]] = None, + max_events: int = DEFAULT_MAX_EVENTS, + poll_s: float = 0.1) -> None: + self._opener = opener if opener is not None else _EvdevSource + self._max_events = max(1, int(max_events)) + self._poll_s = max(0.001, float(poll_s)) + self._lock = threading.Lock() + self._events: List[InputEvent] = [] + self._sources: List[Any] = [] + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + #: Devices :meth:`start` refused, and so did not read. + self.excluded: List[InputDevice] = [] + #: True when the buffer filled and later events were dropped. + self.truncated = False + + @property + def is_recording(self) -> bool: + """Whether devices are open and being read.""" + return self._thread is not None + + def start(self, devices: Sequence[InputDevice]) -> None: + """Open ``devices`` and begin reading them. + + :param devices: the devices to read — named explicitly; an empty + sequence is refused rather than read as "everything". + :raises InputPermissionError: a device is not readable by this user. + :raises InputRecordingError: nothing was named, nothing named is + physical, or a recording is already running. + """ + if self.is_recording: + raise InputRecordingError("a physical recording is already running") + if not devices: + raise InputRecordingError( + "name the devices to record: physical input recording is " + "opt-in per device (see list_input_devices() and " + f"{RECORD_DEVICES_ENV})") + accepted = [d for d in devices if d.origin_known and not d.is_virtual] + self.excluded = [d for d in devices if d not in accepted] + if not accepted: + raise InputRecordingError( + "every named device is virtual or of unknown origin, so " + "there is no physical input to record: " + + ", ".join(d.path for d in devices)) + self._sources = self._open_all(accepted) + with self._lock: + self._events = [] + self.truncated = False + self._stop.clear() + self._thread = threading.Thread( + target=self._read_loop, name="wayland-physical-recorder", + daemon=True) + self._thread.start() + + def stop(self) -> List[InputEvent]: + """Stop reading, close every device, and return what was read.""" + thread, self._thread = self._thread, None + self._stop.set() + if thread is not None: + thread.join(timeout=self._poll_s * (len(self._sources) + 1) + 2.0) + self._close_all(self._sources) + self._sources = [] + with self._lock: + events, self._events = self._events, [] + return events + + def _open_all(self, devices: Sequence[InputDevice]) -> List[Any]: + """Open every device or none: a partial recording misleads.""" + opened: List[Any] = [] + try: + for device in devices: + opened.append(self._open_one(device.path)) + except BaseException: + self._close_all(opened) + raise + return opened + + def _open_one(self, path: str) -> Any: + try: + return self._opener(path) + except PermissionError as error: + raise InputPermissionError(path) from error + except OSError as error: + raise InputRecordingError( + f"could not open {path}: {error}") from error + + @staticmethod + def _close_all(sources: Sequence[Any]) -> None: + for source in sources: + try: + source.close() + except OSError: + pass + + def _read_loop(self) -> None: + while not self._stop.is_set(): + for source in self._sources: + try: + events = source.read_events(self._poll_s) + except OSError: + # The device went away (unplugged). What was read stands. + self._stop.set() + return + self._keep(events) + + def _keep(self, events: Sequence[InputEvent]) -> None: + if not events: + return + with self._lock: + room = self._max_events - len(self._events) + if room < len(events): + self.truncated = True + self._events.extend(events[:max(0, room)]) + + +class ActionJournal: + """The steps this program executed, collected without any input hook. + + Hand :meth:`note` to the executor as its ``step_callback`` — or call + :meth:`run` — and every top-level action is appended as it starts. It + works identically on every backend because it reads nothing from the + desktop: it is the program writing down what it was told to do. + + Steps are kept verbatim so they can be replayed, which means anything + secret in an action's arguments is in here too. Treat a journal like the + action file it came from. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._steps: List[Any] = [] + + def note(self, action: Any) -> None: + """Record one action; a copy, so later mutation cannot rewrite it.""" + with self._lock: + self._steps.append(copy.deepcopy(action)) + + @property + def steps(self) -> List[Any]: + """Every step noted so far, oldest first.""" + with self._lock: + return copy.deepcopy(self._steps) + + def clear(self) -> None: + """Forget every step.""" + with self._lock: + self._steps = [] + + def run(self, action_list: Any, *, + execute: Optional[Callable[..., Any]] = None, + **options: Any) -> Any: + """Execute ``action_list`` and journal each step as it starts. + + :param execute: the executor entry point; the framework's own + ``executor.execute_action`` when omitted. + :param options: passed through (``raise_on_error``, ``dry_run``). + """ + if execute is None: + from je_auto_control.utils.executor.action_executor import executor + execute = executor.execute_action + return execute(action_list, step_callback=self.note, **options) + + +__all__ = [ + "ActionJournal", "BUS_VIRTUAL", "DEFAULT_MAX_EVENTS", "EV_ABS", "EV_KEY", + "EV_REL", "EV_SYN", "InputDevice", "InputEvent", "InputPermissionError", + "InputRecordingError", "PERMISSION_RECOVERY", "PhysicalRecorder", + "RECORD_DEVICES_ENV", "configured_record_devices", "decode_events", + "describe_device", "list_input_devices", +] diff --git a/je_auto_control/linux_wayland/libei.py b/je_auto_control/linux_wayland/libei.py index 4a0bbb28e..e798a2dd7 100644 --- a/je_auto_control/linux_wayland/libei.py +++ b/je_auto_control/linux_wayland/libei.py @@ -142,6 +142,29 @@ class LibeiUnavailable(AutoControlException, RuntimeError): """ +class LibeiConsentNotGranted(LibeiUnavailable): + """The portal did not hand over a session; ``declined`` says if it refused. + + Carries what :class:`oeffis.PortalConsentNotGranted` learned, so the + selector can tell "the user said no" from "there is no portal here" + without parsing a message. + """ + + def __init__(self, message: str, outcome: str, declined: bool) -> None: + super().__init__(message) + self.outcome = outcome + self.declined = declined + + +class LibeiSessionRevoked(LibeiUnavailable): + """The compositor ended a session that had been live. + + Unlike a paused device this does not come back, and unlike a missing + library it is somebody's decision — so it is its own type, and nothing + further is written to the context once it has been seen. + """ + + def _load_symbols() -> Optional[BoundSymbols]: """Resolve every libei entry point, or None if one is missing. @@ -186,6 +209,7 @@ def __init__(self, *, symbols: Optional[BoundSymbols] = None, self._ei: Optional[int] = None self._backend_open = False self._handshake_complete = False + self._revoked = False self._session = None self._devices: Dict[int, int] = {} # Every device this sender holds a reference to, whichever @@ -233,6 +257,10 @@ def connect(self, *, timeout: float = HANDSHAKE_TIMEOUT, with self._lock: if self.is_connected: return + if self._revoked: + raise LibeiSessionRevoked( + "this sender was disconnected by the compositor; call " + "disconnect() before connecting again") sender = self._api.ei_new_sender(None) if not sender: raise LibeiUnavailable("ei_new_sender returned NULL") @@ -328,6 +356,9 @@ def _open_backend(self, socket_path: Optional[bytes]) -> None: return try: eis_fd, session = self._portal_connect() + except oeffis.PortalConsentNotGranted as error: + raise LibeiConsentNotGranted( + str(error), error.outcome, error.declined) from error except oeffis.OeffisUnavailable as error: raise LibeiUnavailable(str(error)) from error self._session = session @@ -380,7 +411,13 @@ def _on_event(self, event: int) -> None: elif event_type == EI_EVENT_DEVICE_REMOVED: self._forget_device(self._api.ei_event_get_device(event)) elif event_type == EI_EVENT_DISCONNECT: - raise LibeiUnavailable("the compositor disconnected the sender") + # Nothing may be written after this. The devices used to stay + # marked as emulating, so the next emission pumped an empty queue, + # found them "ready" and wrote into a context the compositor had + # already dropped. + self._revoked = True + self._emulating.clear() + raise LibeiSessionRevoked("the compositor disconnected the sender") def _bind_seat(self, seat: int) -> None: """Ask a seat for the capabilities this sender emits. @@ -518,6 +555,10 @@ def _emit(self, capability: int, send: Callable[[int], None]) -> None: with self._lock: if self._ei is None: raise LibeiUnavailable("libei sender is not connected") + if self._revoked: + raise LibeiSessionRevoked( + "the compositor disconnected the sender; this session " + "cannot send again") # Pause / resume arrive asynchronously; read them before deciding # the device is still usable. self._pump(0.0) @@ -574,6 +615,7 @@ def _teardown(self) -> None: self._ei = None self._backend_open = False self._handshake_complete = False + self._revoked = False if self._session is not None: # liboeffis is a different library and closing the portal session # is what actually revokes the grant, so this always runs. @@ -599,6 +641,7 @@ def _quietly(action: Callable[[], object]) -> None: _DEFAULT_BACKEND: Optional[LibeiBackend] = None _PROBE_FAILED = False +_PROBE_ERROR: Optional[BaseException] = None _DEFAULT_LOCK = threading.Lock() @@ -609,7 +652,7 @@ def connected_backend() -> Optional[LibeiBackend]: where libei cannot be used must pay for that discovery once rather than on every keystroke. Callers treat None as "use the ydotool CLI". """ - global _DEFAULT_BACKEND, _PROBE_FAILED + global _DEFAULT_BACKEND, _PROBE_FAILED, _PROBE_ERROR with _DEFAULT_LOCK: if _DEFAULT_BACKEND is not None: return _DEFAULT_BACKEND @@ -621,13 +664,25 @@ def connected_backend() -> Optional[LibeiBackend]: return None try: backend.connect() - except (LibeiUnavailable, OSError, ValueError, AttributeError): + except (LibeiUnavailable, OSError, ValueError, AttributeError) as error: _PROBE_FAILED = True + _PROBE_ERROR = error return None _DEFAULT_BACKEND = backend return _DEFAULT_BACKEND +def last_probe_error() -> Optional[BaseException]: + """Why the one probe :func:`connected_backend` makes failed, if it did. + + ``connected_backend`` answers None for every reason there is, which is + right for "which path do I emit on" and useless for "why". The selector + reads this to tell a refused consent from an absent portal. + """ + with _DEFAULT_LOCK: + return _PROBE_ERROR + + def get_default_backend() -> Optional[LibeiBackend]: """Return the cached backend if libei resolved, without connecting.""" with _DEFAULT_LOCK: @@ -639,17 +694,19 @@ def get_default_backend() -> Optional[LibeiBackend]: def reset_default_backend() -> None: """Test hook — drop the cached backend so the probe runs fresh.""" - global _DEFAULT_BACKEND, _PROBE_FAILED + global _DEFAULT_BACKEND, _PROBE_FAILED, _PROBE_ERROR with _DEFAULT_LOCK: if _DEFAULT_BACKEND is not None: _quietly(_DEFAULT_BACKEND.disconnect) _DEFAULT_BACKEND = None _PROBE_FAILED = False + _PROBE_ERROR = None __all__ = [ "EI_DEVICE_CAP_BUTTON", "EI_DEVICE_CAP_KEYBOARD", "EI_DEVICE_CAP_POINTER_ABSOLUTE", "EI_DEVICE_CAP_SCROLL", - "HANDSHAKE_TIMEOUT", "LibeiBackend", "LibeiUnavailable", - "connected_backend", "get_default_backend", "reset_default_backend", + "HANDSHAKE_TIMEOUT", "LibeiBackend", "LibeiConsentNotGranted", + "LibeiSessionRevoked", "LibeiUnavailable", "connected_backend", + "get_default_backend", "last_probe_error", "reset_default_backend", ] diff --git a/je_auto_control/linux_wayland/listener.py b/je_auto_control/linux_wayland/listener.py index 1c44dfa12..d2e77b260 100644 --- a/je_auto_control/linux_wayland/listener.py +++ b/je_auto_control/linux_wayland/listener.py @@ -1,11 +1,19 @@ """Wayland keyboard listener stub. Wayland deliberately forbids reading the global key state from an -unprivileged client. Hooking would require either the libei input -capture protocol (not yet stable across compositors) or a kernel-level -listener via ``/dev/input/event*`` (root-only). For now, raise a -specific NotImplementedError so callers can fall back to the X11 -backend if they need key listening. +unprivileged client, so the two hook entry points raise a specific +NotImplementedError rather than pretending. + +The supported replacements are narrower than a hook, and live elsewhere: + +* a single global *stop* key — :mod:`global_shortcuts`, through the + GlobalShortcuts portal, which reports only the shortcut that was registered; +* the user's own input, from kernel devices they name — the opt-in + ``PhysicalRecorder`` in :mod:`input_events`. + +The portal's InputCapture interface is not one of them: the compositor decides +when it starts (a pointer crossing a screen edge), so it cannot be a listener +that starts when a script asks. """ from __future__ import annotations @@ -14,9 +22,11 @@ def check_key_press(*_args, **_kwargs): """Wayland clients cannot read the global key state. Raise explicitly.""" raise NotImplementedError( "Wayland forbids global key-state queries from unprivileged " - "clients. Use the X11 backend " - "(JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11), libei capture, or " - "an evdev reader (requires root).", + "clients. For a stop key use StopShortcutSession (the " + "GlobalShortcuts portal); to read your own devices use the opt-in " + "PhysicalRecorder. The X11 backend " + "(JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11) sees X11 applications " + "only under XWayland.", ) diff --git a/je_auto_control/linux_wayland/oeffis.py b/je_auto_control/linux_wayland/oeffis.py index cb4ffda23..548bc3896 100644 --- a/je_auto_control/linux_wayland/oeffis.py +++ b/je_auto_control/linux_wayland/oeffis.py @@ -68,6 +68,35 @@ class OeffisUnavailable(AutoControlException, RuntimeError): """liboeffis is missing, or the portal refused to hand over an EIS fd.""" +class PortalConsentNotGranted(OeffisUnavailable): + """The portal was asked for a session and did not hand one over. + + Still an :class:`OeffisUnavailable`, so every existing handler keeps + working. What it adds is *which way* the request ended, because the three + are not the same thing to an operator: + + ``timeout`` nobody answered within this project's deadline + ``closed`` the portal closed the session + ``disconnected`` liboeffis gave up, with ``detail`` saying why + + :attr:`declined` is true only where the library's own text says the + portal denied the request. That is the one outcome after which input must + not be routed through another backend, so it is recognised narrowly: a + host with no RemoteDesktop portal at all also ends in ``disconnected``, + and it must keep falling back to ydotool as it always has. + """ + + def __init__(self, message: str, outcome: str, detail: str = "") -> None: + super().__init__(message) + self.outcome = outcome + self.detail = detail + + @property + def declined(self) -> bool: + """Whether the portal answered the request with a refusal.""" + return "denied" in self.detail.lower() + + def load_symbols() -> Optional[BoundSymbols]: """Resolve liboeffis, or None when it is not installed.""" return bind(_LIBRARY_CANDIDATES, _PROTOTYPES) @@ -150,15 +179,18 @@ def _require_connected(symbols: BoundSymbols, handle: int, event: int, if event == OEFFIS_EVENT_CONNECTED_TO_EIS: return if event == OEFFIS_EVENT_NONE: - raise OeffisUnavailable( + raise PortalConsentNotGranted( f"the desktop portal did not answer within {timeout:g}s " - f"(a consent dialog may be waiting)", + f"(a consent dialog may be waiting)", "timeout", ) detail = _describe(symbols, handle) + closed = event == OEFFIS_EVENT_CLOSED reason = ("the desktop portal closed the remote-desktop session" - if event == OEFFIS_EVENT_CLOSED + if closed else "the desktop portal disconnected the remote-desktop session") - raise OeffisUnavailable(f"{reason}{': ' + detail if detail else ''}") + raise PortalConsentNotGranted( + f"{reason}{': ' + detail if detail else ''}", + "closed" if closed else "disconnected", detail) def _release(symbols: BoundSymbols, handle: int) -> None: @@ -192,6 +224,7 @@ def __del__(self) -> None: "OEFFIS_DEVICE_KEYBOARD", "OEFFIS_DEVICE_POINTER", "OEFFIS_DEVICE_TOUCHSCREEN", "OEFFIS_EVENT_CLOSED", "OEFFIS_EVENT_CONNECTED_TO_EIS", "OEFFIS_EVENT_DISCONNECTED", - "OEFFIS_EVENT_NONE", "OeffisUnavailable", "connect_eis_fd", + "OEFFIS_EVENT_NONE", "OeffisUnavailable", "PortalConsentNotGranted", + "connect_eis_fd", "is_available", "load_symbols", ] diff --git a/je_auto_control/linux_wayland/portal.py b/je_auto_control/linux_wayland/portal.py index bc4e446c9..644d32eb2 100644 --- a/je_auto_control/linux_wayland/portal.py +++ b/je_auto_control/linux_wayland/portal.py @@ -39,6 +39,9 @@ from typing import Any, Dict, List, Tuple from je_auto_control.linux_wayland import _dbus_client +from je_auto_control.linux_wayland.authorisation import ( + CAPTURE, AuthorisationState, ledger, +) from je_auto_control.utils.exception.exceptions import AutoControlScreenException @@ -59,6 +62,15 @@ 2: "the desktop portal ended the screenshot request", } +#: What each Response code means for the capture authorisation. Recorded so a +#: dismissed dialog reads as "needs permission" in the diagnostics rather than +#: as one more capture failure. There is no tier after the portal to fall back +#: to, so nothing is ever captured another way behind a refusal. +_RESPONSE_STATE = { + 0: AuthorisationState.GRANTED, + 1: AuthorisationState.DECLINED, +} + def is_available() -> bool: """Whether a session bus exists, so the portal tier is worth trying.""" @@ -100,6 +112,8 @@ def _request_and_await(bus: "_dbus_client.SessionBus", timeout: float) -> str: body = bus.wait_for_signal(paths, REQUEST_INTERFACE, "Response", timeout) except _dbus_client.DBusError as error: + ledger.transition(CAPTURE, AuthorisationState.TIMED_OUT, + f"no answer within {timeout:g}s") raise AutoControlScreenException( f"desktop portal did not answer within {timeout:g}s " f"(a consent dialog may be waiting)", @@ -154,6 +168,9 @@ def _match_rule(path: str) -> str: def _uri_from_response(body: List[Any]) -> str: """Read the screenshot URI out of one ``Response`` signal body.""" code, results = _response_parts(body) + ledger.transition( + CAPTURE, _RESPONSE_STATE.get(code, AuthorisationState.FAILED), + _RESPONSE_MEANING.get(code, "")) if code != 0: raise AutoControlScreenException( _RESPONSE_MEANING.get(code, f"desktop portal returned {code}"), diff --git a/je_auto_control/linux_wayland/record.py b/je_auto_control/linux_wayland/record.py index 7eeb50250..74ed4747f 100644 --- a/je_auto_control/linux_wayland/record.py +++ b/je_auto_control/linux_wayland/record.py @@ -1,10 +1,17 @@ """Wayland record stub. -Recording requires hooking global mouse + key events; Wayland forbids -that for unprivileged clients (see ``listener``). The Wayland backend -exposes the same module surface as the X11 backend so the wrapper -can swap them out, but every entry point raises a clear -NotImplementedError pointing at the X11 fallback. +Recording, as the platform seam means it, is hooking every mouse and key +event on the desktop; Wayland forbids that for unprivileged clients (see +``listener``). The Wayland backend exposes the same module surface as the X11 +backend so the wrapper can swap them out, and every entry point here still +raises a clear NotImplementedError. + +What Wayland *does* allow lives in :mod:`input_events`, and the error now +says so instead of only pointing at X11: the steps this program executes are +journalled with no hook at all (``ActionJournal``), and the user's own input +can be read from kernel devices they name (``PhysicalRecorder``, opt-in). +Neither is wired in here, on purpose — the seam's ``record()`` promises "start +capturing everything", and neither of those is that. """ from __future__ import annotations @@ -17,8 +24,12 @@ class _WaylandRecorder: def __init__(self) -> None: self._reason = ( "Wayland forbids global input recording from unprivileged " - "clients. Set JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 to " - "use the X11 backend, or run on an X11 session." + "clients. To record what this program executes, use " + "ActionJournal (no hook needed); to record your own keyboard " + "and mouse, use PhysicalRecorder with devices you name in " + "JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES. Setting " + "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 uses the X11 backend, " + "which under XWayland sees X11 applications only." ) def record(self) -> None: diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cc475cf85..5b661e4c3 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -1481,6 +1481,21 @@ def _diagnose() -> Dict[str, Any]: return run_diagnostics().to_dict() +def _probe_capabilities() -> Dict[str, Any]: + """Executor adapter: capability and authorisation states, no side effect.""" + from je_auto_control.wrapper.capabilities import probe_capabilities + return probe_capabilities().to_dict() + + +def _reset_input_authorisation() -> Dict[str, Any]: + """Executor adapter: forget a refused or revoked Wayland input session.""" + from je_auto_control.linux_wayland._select_input import ( + reset_input_authorisation, + ) + reset_input_authorisation() + return _probe_capabilities() + + def _config_export() -> Dict[str, Any]: """Executor adapter: build the config bundle dict in-memory.""" from je_auto_control.utils.config_bundle import export_config_bundle @@ -8061,6 +8076,8 @@ def __init__(self): # System diagnostics "AC_diagnose": _diagnose, + "AC_probe_capabilities": _probe_capabilities, + "AC_reset_input_authorisation": _reset_input_authorisation, # Config bundle export / import "AC_config_export": _config_export, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fecbc25ae..4b5a5df4d 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -480,6 +480,19 @@ def window_tools() -> List[MCPTool]: def system_tools() -> List[MCPTool]: return [ + MCPTool( + name="ac_probe_capabilities", + description=("Report what this session can do right now — input, " + "capture, recording and the stop shortcut — each " + "with its state (available, needs_permission, " + "needs_setup, revoked, ...), the backend serving " + "it, whether it reaches the whole desktop (false " + "under XWayland) and what to do about it. Asks the " + "desktop for nothing: no consent dialog, no input."), + input_schema=schema({}), + handler=h_system.probe_capabilities, + annotations=READ_ONLY, + ), MCPTool( name="ac_get_clipboard", description="Return the current text clipboard contents.", diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_system.py b/je_auto_control/utils/mcp_server/tools/_handlers_system.py index b995e26ff..0e56bca16 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_system.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_system.py @@ -544,6 +544,12 @@ def file_association(target): return _file_association(target) +def probe_capabilities() -> Dict[str, Any]: + """Capability and authorisation states; the same data ``AC_`` returns.""" + from je_auto_control.wrapper.capabilities import probe_capabilities as _probe + return _probe().to_dict() + + def get_clipboard() -> str: from je_auto_control.utils.clipboard.clipboard import get_clipboard as _get return _get() diff --git a/je_auto_control/wrapper/capabilities.py b/je_auto_control/wrapper/capabilities.py new file mode 100644 index 000000000..b54b9a383 --- /dev/null +++ b/je_auto_control/wrapper/capabilities.py @@ -0,0 +1,441 @@ +"""What this session can do right now, and what stands in the way if it cannot. + +``platform_wrapper`` picks a backend at import and says nothing more. That is +enough on Windows; on Wayland it leaves every interesting question open. Is +input going through the portal or through ``/dev/uinput``? Did the user refuse +the consent dialog, or is there no portal to ask? Is this the Wayland backend +at all, or the X11 one talking to XWayland and reaching only half the windows? + +:func:`probe_capabilities` answers those as data. Four capabilities, each +diagnosed on its own because they fail independently — a host with working +capture and no input is ordinary, and one verdict for both hides which: + +``input`` synthesising keyboard and pointer events +``capture`` reading pixels off the screen +``recording`` reading what the *user* types and clicks +``stop_shortcut`` a global key that stops a running script + +**Probing has no side effect.** It reads the environment, looks names up on +``PATH``, asks the loader whether a library exists, and reads the +authorisation ledger. It never opens a portal request, never shows a consent +dialog, never connects a libei sender and never sends an event — so it is safe +to call from a diagnostics screen, a health check, or an MCP client that has +no business moving the pointer. + +Everything it consults comes in through :class:`BackendContext`, which is what +makes a GNOME session describable from a Windows test run. +""" +from __future__ import annotations + +import ctypes.util +import os +import shutil +import sys +from dataclasses import asdict, dataclass, field +from enum import Enum +from typing import Any, Callable, Dict, Mapping, Optional, Tuple + +from je_auto_control.linux_wayland._detect import ( + WAYLAND_GNOME_SCREENSHOT, WAYLAND_GRIM, WAYLAND_SPECTACLE, + WAYLAND_YDOTOOL, is_wayland_session, select_display_server, +) +from je_auto_control.linux_wayland.authorisation import ( + CAPTURE, CAPTURE_RECOVERY, INPUT, INPUT_RECOVERY, AuthorisationLedger, + AuthorisationRecord, AuthorisationState, compositor_identity, ledger, +) +from je_auto_control.linux_wayland.input_events import ( + RECORD_DEVICES_ENV, configured_record_devices, +) + +RECORDING = "recording" +STOP_SHORTCUT = "stop_shortcut" + +_INPUT_BACKEND_ENV = "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND" +_CAPTURE_COMMAND_ENV = "JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND" + + +class CapabilityStatus(str, Enum): + """Where one capability stands. Compares equal to its string value.""" + + AVAILABLE = "available" + #: Usable, but the desktop has not been asked yet — it will be on first use. + NOT_REQUESTED = "not_requested" + #: A consent request is on screen right now. + REQUESTING = "requesting" + #: Someone has to say yes: a refused or unanswered consent, a device node + #: this user may not read. + NEEDS_PERMISSION = "needs_permission" + #: Something has to be installed or configured. + NEEDS_SETUP = "needs_setup" + #: This process closed its session; the next use asks again. + SESSION_CLOSED = "session_closed" + #: The compositor took a granted session away. + REVOKED = "revoked" + #: The grant came from a compositor that is no longer the one running. + COMPOSITOR_RESTARTED = "compositor_restarted" + UNSUPPORTED = "unsupported" + #: Not something this probe can determine without side effects. + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class Capability: + """One capability's diagnosis.""" + + name: str + state: CapabilityStatus + #: The backend that serves it, or would once the obstacle is cleared. + backend: str + #: False when only part of the desktop is reachable — XWayland. + desktop_wide: bool = True + detail: str = "" + #: What the operator can do about it, in English; empty when nothing is + #: wrong. ``recovery_key`` is the same advice as a GUI catalogue key. + recovery: str = "" + recovery_key: str = "" + #: The raw authorisation state behind ``state``, where there is one. + authorisation: str = "" + #: Whether consent survives a restart: ``unsupported`` on the libei path, + #: because liboeffis is never asked for a restore token by this binding. + restore_token: str = "not_applicable" + + @property + def usable(self) -> bool: + """Whether an action using this capability can be attempted now.""" + return self.state in (CapabilityStatus.AVAILABLE, + CapabilityStatus.NOT_REQUESTED, + CapabilityStatus.SESSION_CLOSED) + + def to_dict(self) -> Dict[str, Any]: + """JSON-ready form; the state becomes its plain string.""" + data = asdict(self) + data["state"] = self.state.value + data["usable"] = self.usable + return data + + +@dataclass(frozen=True) +class CapabilitySnapshot: + """Every capability of one session, at one moment.""" + + platform: str + #: ``wayland``, ``x11``, or the platform name where the question is moot. + display_server: str + #: True when the X11 backend is serving a Wayland session. + xwayland: bool + capabilities: Tuple[Capability, ...] + + def get(self, name: str) -> Capability: + """The capability called ``name``.""" + for capability in self.capabilities: + if capability.name == name: + return capability + raise KeyError(name) + + @property + def input(self) -> Capability: + """Keyboard and pointer synthesis.""" + return self.get(INPUT) + + @property + def capture(self) -> Capability: + """Screen capture.""" + return self.get(CAPTURE) + + def to_dict(self) -> Dict[str, Any]: + """JSON-ready form, shared by the executor, MCP and the GUI.""" + return { + "platform": self.platform, + "display_server": self.display_server, + "xwayland": self.xwayland, + "capabilities": [item.to_dict() for item in self.capabilities], + } + + +def _library_present(name: str) -> bool: + """Whether the loader can find ``lib``; loads nothing.""" + try: + return ctypes.util.find_library(name) is not None + except (OSError, ValueError): + return False + + +def _session_bus_present() -> bool: + """Whether a D-Bus session bus address is known; connects to nothing.""" + from je_auto_control.linux_wayland import _dbus_client + return _dbus_client.is_available() + + +def _readable(path: str) -> bool: + return os.access(path, os.R_OK) + + +def _loaded_backend() -> Optional[str]: + """Which Linux backend ``platform_wrapper`` actually imported, if it has. + + Read out of ``sys.modules`` rather than imported: importing the wrapper + is what *selects* a backend, and a probe must not be the thing that does. + """ + wrapper = sys.modules.get("je_auto_control.wrapper.platform_wrapper") + screen = getattr(wrapper, "screen", None) + module = getattr(screen, "__name__", "") + if "linux_wayland" in module: + return "wayland" + return "x11" if module else None + + +@dataclass(frozen=True) +class BackendContext: + """Everything a probe is allowed to look at. + + The defaults read the live process. A test — or a support engineer + reproducing someone else's desktop — replaces any of them. + """ + + platform: str = sys.platform + environ: Mapping[str, str] = field(default_factory=lambda: os.environ) + which: Callable[[str], Optional[str]] = shutil.which + library_present: Callable[[str], bool] = _library_present + session_bus_present: Callable[[], bool] = _session_bus_present + readable: Callable[[str], bool] = _readable + compositor: Callable[[Mapping[str, str]], Optional[str]] = ( + compositor_identity) + authorisations: AuthorisationLedger = ledger + #: ``wayland`` / ``x11`` once the wrapper has chosen, else None. + loaded_backend: Optional[str] = None + + @classmethod + def current(cls) -> "BackendContext": + """The context of the running process.""" + return cls(loaded_backend=_loaded_backend()) + + +def probe_capabilities(context: Optional[BackendContext] = None + ) -> CapabilitySnapshot: + """Diagnose input, capture, recording and the stop shortcut. + + :param context: what to inspect; the live process when omitted. + :return: one :class:`Capability` per capability. Never raises for a + capability that is merely unavailable — that is a state, not an error. + """ + ctx = context if context is not None else BackendContext.current() + if not ctx.platform.startswith("linux"): + return _other_platform(ctx) + wanted = select_display_server(ctx.environ) + serving = ctx.loaded_backend or wanted + if serving == "wayland": + return CapabilitySnapshot(ctx.platform, "wayland", False, ( + _wayland_input(ctx), _wayland_capture(ctx), + _wayland_recording(ctx), _wayland_stop_shortcut(ctx))) + return _x11(ctx, is_wayland_session(ctx.environ)) + + +# --- platforms with no session authorisation to model ---------------------- + +_OTHER_BACKENDS = {"win32": "win32", "darwin": "quartz"} + + +def _other_platform(ctx: BackendContext) -> CapabilitySnapshot: + backend = _OTHER_BACKENDS.get(ctx.platform, "x11") + if ctx.platform == "win32": + state, detail = CapabilityStatus.AVAILABLE, "" + else: + state = CapabilityStatus.UNKNOWN + detail = ("permission for this platform is granted outside the " + "process and is not probed here") + return CapabilitySnapshot(ctx.platform, ctx.platform, False, tuple( + Capability(name, state, backend, detail=detail) + for name in (INPUT, CAPTURE, RECORDING, STOP_SHORTCUT))) + + +# --- X11, including X11 serving a Wayland session -------------------------- + +_XWAYLAND_SCOPE = ( + "the X11 backend is running against XWayland: it reaches X11 " + "applications only, and native Wayland windows neither receive its " + "input nor appear in its captures") +_XWAYLAND_RECOVERY = ( + "Install the Wayland tools (ydotool 1.0+ or libei, and grim, " + "gnome-screenshot or spectacle) so the Wayland backend loads, or keep " + "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 only for X11 applications.") + + +def _x11(ctx: BackendContext, xwayland: bool) -> CapabilitySnapshot: + has_display = bool((ctx.environ.get("DISPLAY") or "").strip()) + state = (CapabilityStatus.AVAILABLE if has_display + else CapabilityStatus.NEEDS_SETUP) + detail = _XWAYLAND_SCOPE if xwayland else "" + recovery = _XWAYLAND_RECOVERY if xwayland else ( + "" if has_display else "Set DISPLAY to a running X server.") + key = "cap_fix_xwayland" if xwayland else ( + "" if has_display else "cap_fix_display") + return CapabilitySnapshot(ctx.platform, "x11", xwayland, tuple( + Capability(name, state, "xwayland" if xwayland else "x11", + desktop_wide=not xwayland, detail=detail, + recovery=recovery, recovery_key=key) + for name in (INPUT, CAPTURE, RECORDING, STOP_SHORTCUT))) + + +# --- Wayland: input --------------------------------------------------------- + +#: liboeffis is never asked for a restore token by this binding, so consent +#: is per process. A name, not a literal at the call site: it is a state, +#: and a literal there reads to a secret scanner as a hardcoded credential. +_RESTORE_UNSUPPORTED = "unsupported" + +_YDOTOOL_RECOVERY = ( + "Install ydotool 1.0 or newer and run ydotoold with access to " + "/dev/uinput, or install libei and liboeffis to use the desktop portal.") + +#: authorisation state -> (capability state, catalogue key of the advice) +_INPUT_STATES = { + AuthorisationState.NOT_REQUESTED: (CapabilityStatus.NOT_REQUESTED, ""), + AuthorisationState.REQUESTING: (CapabilityStatus.REQUESTING, + "cap_fix_answer_dialog"), + AuthorisationState.GRANTED: (CapabilityStatus.AVAILABLE, ""), + AuthorisationState.DECLINED: (CapabilityStatus.NEEDS_PERMISSION, + "cap_fix_input_consent"), + AuthorisationState.TIMED_OUT: (CapabilityStatus.NEEDS_PERMISSION, + "cap_fix_input_consent"), + AuthorisationState.CLOSED: (CapabilityStatus.SESSION_CLOSED, ""), + AuthorisationState.REVOKED: (CapabilityStatus.REVOKED, + "cap_fix_input_consent"), +} + + +def _wayland_input(ctx: BackendContext) -> Capability: + forced = (ctx.environ.get(_INPUT_BACKEND_ENV) or "auto").strip().lower() + if forced == "cli" or not ctx.library_present("ei"): + return _ydotool_input(ctx, "") + record = ctx.authorisations.get(INPUT) + if record.state is AuthorisationState.FAILED: + # Not a refusal, so the CLI really is what serves input now. + return _ydotool_input(ctx, f"libei could not start: {record.detail}") + if _compositor_changed(ctx, record): + return Capability( + INPUT, CapabilityStatus.COMPOSITOR_RESTARTED, "libei", + detail="the session was granted by a compositor that has since " + "been restarted", recovery=INPUT_RECOVERY, + recovery_key="cap_fix_input_consent", + authorisation=record.state.value, restore_token=_RESTORE_UNSUPPORTED) + state, key = _INPUT_STATES[record.state] + return Capability( + INPUT, state, "libei", detail=record.detail, + recovery=INPUT_RECOVERY if key == "cap_fix_input_consent" else ( + "Answer the consent dialog on the desktop." if key else ""), + recovery_key=key, authorisation=record.state.value, + restore_token=_RESTORE_UNSUPPORTED) + + +def _ydotool_input(ctx: BackendContext, detail: str) -> Capability: + """Input through the ``ydotool`` CLI: no consent, but a daemon to run.""" + if ctx.which(WAYLAND_YDOTOOL): + return Capability(INPUT, CapabilityStatus.AVAILABLE, "ydotool", + detail=detail) + return Capability(INPUT, CapabilityStatus.NEEDS_SETUP, "ydotool", + detail=detail or "ydotool is not on PATH", + recovery=_YDOTOOL_RECOVERY, + recovery_key="cap_fix_ydotool") + + +def _compositor_changed(ctx: BackendContext, + record: AuthorisationRecord) -> bool: + """Whether a granted session has outlived the compositor that granted it.""" + if record.state is not AuthorisationState.GRANTED or not record.compositor: + return False + now = ctx.compositor(ctx.environ) + return now is not None and now != record.compositor + + +# --- Wayland: capture ------------------------------------------------------- + +_CAPTURE_TOOLS = (WAYLAND_GRIM, WAYLAND_GNOME_SCREENSHOT, WAYLAND_SPECTACLE) + +_CAPTURE_STATES = { + AuthorisationState.NOT_REQUESTED: CapabilityStatus.NOT_REQUESTED, + AuthorisationState.REQUESTING: CapabilityStatus.REQUESTING, + AuthorisationState.GRANTED: CapabilityStatus.AVAILABLE, + AuthorisationState.DECLINED: CapabilityStatus.NEEDS_PERMISSION, + AuthorisationState.TIMED_OUT: CapabilityStatus.NEEDS_PERMISSION, + AuthorisationState.FAILED: CapabilityStatus.NEEDS_SETUP, + AuthorisationState.CLOSED: CapabilityStatus.NOT_REQUESTED, + AuthorisationState.REVOKED: CapabilityStatus.NEEDS_PERMISSION, +} + + +def _wayland_capture(ctx: BackendContext) -> Capability: + if (ctx.environ.get(_CAPTURE_COMMAND_ENV) or "").strip(): + return Capability(CAPTURE, CapabilityStatus.AVAILABLE, + f"${_CAPTURE_COMMAND_ENV}") + for tool in _CAPTURE_TOOLS: + if ctx.which(tool): + return Capability(CAPTURE, CapabilityStatus.AVAILABLE, tool) + if not ctx.session_bus_present(): + return Capability( + CAPTURE, CapabilityStatus.NEEDS_SETUP, "none", + detail="no capture tool on PATH and no session bus for the portal", + recovery=CAPTURE_RECOVERY, recovery_key="cap_fix_capture") + record = ctx.authorisations.get(CAPTURE) + state = _CAPTURE_STATES[record.state] + blocked = state in (CapabilityStatus.NEEDS_PERMISSION, + CapabilityStatus.NEEDS_SETUP) + return Capability( + CAPTURE, state, "xdg-desktop-portal", detail=record.detail, + recovery=CAPTURE_RECOVERY if blocked else "", + recovery_key="cap_fix_capture" if blocked else "", + authorisation=record.state.value) + + +# --- Wayland: recording and the stop shortcut ------------------------------- + +_RECORDING_SCOPE = ( + "Wayland has no global input hook. Actions this program executes are " + "journalled without one; reading the user's own keyboard and mouse " + "needs the evdev reader, which is opt-in") +_RECORDING_SETUP = ( + f"Name the devices to read in {RECORD_DEVICES_ENV} (comma-separated " + "/dev/input/event* paths); list candidates with list_input_devices().") +_RECORDING_PERMISSION = ( + "Give this user read access to the named devices: add it to the " + "`input` group (then log in again) or install a udev rule. Do not run " + "the whole program as root for this.") + + +def _wayland_recording(ctx: BackendContext) -> Capability: + devices = configured_record_devices(ctx.environ) + if not devices: + return Capability(RECORDING, CapabilityStatus.NEEDS_SETUP, "evdev", + detail=_RECORDING_SCOPE, recovery=_RECORDING_SETUP, + recovery_key="cap_fix_record_setup") + denied = [path for path in devices if not ctx.readable(path)] + if denied: + return Capability( + RECORDING, CapabilityStatus.NEEDS_PERMISSION, "evdev", + detail="not readable: " + ", ".join(denied), + recovery=_RECORDING_PERMISSION, + recovery_key="cap_fix_record_permission") + return Capability(RECORDING, CapabilityStatus.AVAILABLE, "evdev", + detail=", ".join(devices)) + + +def _wayland_stop_shortcut(ctx: BackendContext) -> Capability: + if not ctx.session_bus_present(): + return Capability( + STOP_SHORTCUT, CapabilityStatus.NEEDS_SETUP, "none", + detail="no session bus, so the GlobalShortcuts portal cannot be " + "reached", + recovery="Stop a running script from the GUI's stop control or " + "by ending the process.", + recovery_key="cap_fix_stop_manual") + return Capability( + STOP_SHORTCUT, CapabilityStatus.NOT_REQUESTED, + "xdg-desktop-portal GlobalShortcuts", + detail="whether this desktop's portal implements GlobalShortcuts is " + "only known once a stop-shortcut session is opened; the GUI's " + "stop control works regardless") + + +__all__ = [ + "BackendContext", "Capability", "CapabilitySnapshot", "CapabilityStatus", + "RECORDING", "STOP_SHORTCUT", "probe_capabilities", +] diff --git a/test/manual_test/wayland_authorisation_checklist.md b/test/manual_test/wayland_authorisation_checklist.md new file mode 100644 index 000000000..f55410476 --- /dev/null +++ b/test/manual_test/wayland_authorisation_checklist.md @@ -0,0 +1,168 @@ +# Wayland authorisation — manual checklist (GNOME and KDE) + +**Status: NOT EXECUTED.** Nobody has run these steps. They were written on a +Windows machine, against the code and against fakes, and exist because the +things they check have no CI job: a real consent dialog, a real person +answering it, and a real GlobalShortcuts portal. Do not cite this file as +evidence that any of it works. When someone runs it, record the desktop, +versions, date and result in `docs/updates/` and change this line. + +What CI already covers, so it is not repeated here: the portal handshake +against a real `liboeffis` and a scripted portal (`portal-verification`), the +libei sender against a real `libeis` (`eis-verification`), capture against +headless sway (`wayland-verification`). + +## Before you start + +- A Wayland session: GNOME 45+ (mutter) for sections A–D, KDE Plasma 6 for a + second pass. Record `echo $XDG_SESSION_TYPE $XDG_CURRENT_DESKTOP` and the + versions of `libei`, `liboeffis`, `xdg-desktop-portal` and its backend. +- `libei` and `liboeffis` installed (`python -c "from je_auto_control.linux_wayland import oeffis; print(oeffis.is_available())"` prints `True`). +- `ydotool` 1.0+ installed **and `ydotoold` running** — the point of several + steps is that it is available and must still not be used. +- `JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND` unset. +- A text editor focused, so a stray keystroke is visible. +- In every step, "probe" means: + + ```bash + python -c "import json, je_auto_control as ac; print(json.dumps(ac.probe_capabilities().to_dict()['capabilities'][0], indent=1))" + ``` + +## A. Probing asks for nothing + +- [ ] A1. Fresh terminal. Run the probe. **Expect:** no dialog appears, no + key is typed; `state` is `not_requested`, `backend` is `libei`, + `restore_token` is `unsupported`. +- [ ] A2. Open the GUI, Diagnostics tab. **Expect:** no dialog; the second + table shows four rows; the input row reads "Not asked yet". + +## B. Allow + +- [ ] B1. `python -c "import je_auto_control as ac; ac.type_keyboard('a')"`. + **Expect:** the desktop's remote-control dialog appears. +- [ ] B2. While the dialog is up, run the probe in a second terminal. + **Expect:** `state` is `not_requested` there (the ledger is per + process) — note this; the `requesting` state is only visible from + inside the asking process, e.g. the GUI's Diagnostics tab. +- [ ] B3. Click **Share / Allow**. **Expect:** one `a` is typed, exactly once. +- [ ] B4. In one Python session: type a key (allow), then + `ac.probe_capabilities().input.state`. **Expect:** `available`. + +## C. Deny — the step this change exists for + +- [ ] C1. New Python session: + `import je_auto_control as ac; ac.type_keyboard('a')`, and **dismiss / + deny** the dialog. + **Expect:** `WaylandAuthorisationError` is raised; **no `a` is typed** + (ydotoold is running and must not have been used). +- [ ] C2. Read the message. **Expect:** it names + `reset_input_authorisation()` and `JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli`. +- [ ] C3. `ac.probe_capabilities().input` — **Expect:** `state` is + `needs_permission`, `authorisation` is `declined`. + **If instead an `a` was typed and `authorisation` is `failed`:** this + desktop's liboeffis words the refusal without "denied". Record + `ac.probe_capabilities().input.detail` verbatim — that string is what + `PortalConsentNotGranted.declined` in `linux_wayland/oeffis.py` has to + recognise — and file it. This is the single most useful result this + checklist can produce. +- [ ] C4. `ac.type_keyboard('a')` again. **Expect:** raises again, and **no + second dialog** appears. +- [ ] C5. `ac.reset_input_authorisation(); ac.type_keyboard('a')`. + **Expect:** the dialog appears again; allowing it types one `a`. + +## D. Leave the dialog open, revoke, restart + +- [ ] D1. New session, `ac.type_keyboard('a')`, and do **not** answer for 35 + seconds. **Expect:** the call returns or raises on its own after ~30 s + (it must not hang). Record which, whether an `a` was typed through + ydotool, and the probe's `state` (`needs_permission`, authorisation + `timed_out`). Note whether the dialog is still on screen afterwards. +- [ ] D2. After an allowed session (B4), revoke it from the desktop: on GNOME + click the screen-sharing indicator in the top bar and stop sharing; on + KDE use the system-tray remote-control item. Then `ac.type_keyboard('a')`. + **Expect:** `WaylandAuthorisationError`, no `a` typed, probe `state` is + `revoked`. Record whether the desktop actually offers a way to revoke. +- [ ] D3. After an allowed session, `ac.close_input_session()`. + **Expect:** the sharing indicator disappears; probe `state` is + `session_closed`; the next `ac.type_keyboard('a')` asks again. +- [ ] D4. (GNOME on Wayland cannot restart mutter without ending the session; + do this on KDE with `kwin_wayland --replace &`, or on sway.) After an + allowed session, restart the compositor, then probe from the *same* + Python session. **Expect:** `compositor_restarted`. Record what + actually happens — the Python session may not survive. + +## E. Explicit ydotool, and XWayland + +- [ ] E1. `JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli python -c "import je_auto_control as ac; ac.type_keyboard('a')"`. + **Expect:** no dialog at all, one `a` typed; probe `backend` is `ydotool`. +- [ ] E2. `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` and run the probe. + **Expect:** `xwayland` is true in the snapshot; every capability has + `desktop_wide: false`; the Diagnostics tab's "Reaches" column reads + "X11 applications only (XWayland)". +- [ ] E3. Still under E2, focus a native Wayland window (GNOME Text Editor) + and type a key through AutoControl. **Expect:** nothing arrives. Focus + an XWayland window (`xterm`) and repeat. **Expect:** it arrives. This + confirms the stated scope is the real one. + +## F. Screenshot portal + +- [ ] F1. With no `grim`, `gnome-screenshot` or `spectacle` on `PATH`, take a + screenshot and **deny** the dialog. **Expect:** a capture error naming + the dismissal; probe capture `state` is `needs_permission`. +- [ ] F2. Repeat and **allow**. **Expect:** an image; `state` is `available`. + +## G. Recording your own input + +- [ ] G1. `python -c "import je_auto_control as ac; [print(d) for d in ac.list_input_devices()]"`. + **Expect:** your keyboard has `is_virtual` false; with `ydotoold` + running, its device is listed with `is_virtual` true (check + `d.sysfs` contains `/devices/virtual/input/`). **If a Bluetooth + keyboard is connected, expect `is_virtual` false for it** — record its + `sysfs` path either way. +- [ ] G2. As a user **not** in the `input` group, start a `PhysicalRecorder` + on your keyboard. **Expect:** `InputPermissionError` whose text names + the `input` group and says not to run as root. +- [ ] G3. As a user in the `input` group: start on `[keyboard, ydotoold]`, + type `abc` by hand, run `ydotool type xyz`, stop. **Expect:** the events + contain your `abc` key codes and **none** of `xyz`; + `recorder.excluded` lists the ydotoold device. + +## H. Global stop shortcut + +Not covered by any CI job; the fake in the unit tests is the only thing this +has ever talked to. + +- [ ] H1. ```python + import threading, je_auto_control as ac + stop = threading.Event() + s = ac.StopShortcutSession(stop.set, preferred_trigger="CTRL+ALT+F12") + s.open(); s.start() + ``` + **Expect:** either the desktop shows a dialog to confirm / assign the + shortcut, or — where the desktop's portal has no GlobalShortcuts + interface — `ShortcutUnavailable` naming the GUI stop control. Which + desktops and versions implement it was not established when this was + written; record the desktop, its version, which of the two happened, + and the exact message. +- [ ] H2. Allow it, press the shortcut. **Expect:** `stop.is_set()` is true. +- [ ] H3. Press other keys. **Expect:** nothing is reported for them. +- [ ] H4. `s.close()`, press the shortcut. **Expect:** nothing; the shortcut + no longer appears in the desktop's shortcut settings for this app. +- [ ] H5. Repeat H1 and **deny**. **Expect:** `ShortcutPermissionError` + whose `recovery` names the stop methods that still work; nothing is + left registered. + +## I. The helper process (opt-in) + +- [ ] I1. `JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`, then B1–B3. **Expect:** the + same single dialog and the same single `a`; `pgrep -f ei_worker` shows + one helper. +- [ ] I2. Hold a key through the helper + (`ac.press_keyboard_key('shift')`), then `kill -9` the helper. + **Expect:** the key does not stay stuck on the desktop (type a letter — + it should be lower-case). The next action raises `EiWorkerDied` listing + the key. Record whether the key was stuck: releasing it is the + compositor's job, and this is the only place that is checked on a real + one. +- [ ] I3. Exit Python normally with the helper running. **Expect:** + `pgrep -f ei_worker` shows nothing. diff --git a/test/unit_test/headless/test_diagnostics_capabilities_gui.py b/test/unit_test/headless/test_diagnostics_capabilities_gui.py new file mode 100644 index 000000000..f5e173331 --- /dev/null +++ b/test/unit_test/headless/test_diagnostics_capabilities_gui.py @@ -0,0 +1,123 @@ +"""The Diagnostics tab shows capability states, in every language.""" +import os + +import pytest + +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import QApplication # noqa: E402 + +from je_auto_control.gui import diagnostics_tab as tab_mod # noqa: E402 +from je_auto_control.gui.language_wrapper.english import ( # noqa: E402 + english_word_dict, +) +from je_auto_control.gui.language_wrapper.japanese import ( # noqa: E402 + japanese_word_dict, +) +from je_auto_control.gui.language_wrapper.simplified_chinese import ( # noqa: E402 + simplified_chinese_word_dict, +) +from je_auto_control.gui.language_wrapper.traditional_chinese import ( # noqa: E402 + traditional_chinese_word_dict, +) +from je_auto_control.linux_wayland import authorisation as auth # noqa: E402 +from je_auto_control.utils.diagnostics.diagnostics import ( # noqa: E402 + DiagnosticsReport, +) +from je_auto_control.wrapper import capabilities as caps # noqa: E402 + +_CATALOGUES = { + "english": english_word_dict, "japanese": japanese_word_dict, + "simplified_chinese": simplified_chinese_word_dict, + "traditional_chinese": traditional_chinese_word_dict, +} + + +@pytest.fixture(scope="module") +def app(): + return QApplication.instance() or QApplication([]) + + +def _declined_wayland(): + ledger = auth.AuthorisationLedger() + ledger.transition(auth.INPUT, auth.AuthorisationState.DECLINED, + "Portal denied Start") + return caps.probe_capabilities(caps.BackendContext( + platform="linux", + environ={"XDG_SESSION_TYPE": "wayland"}, + which=lambda name: None, library_present=lambda _name: True, + session_bus_present=lambda: False, readable=lambda _path: False, + compositor=lambda _env: None, authorisations=ledger)) + + +def test_every_state_and_every_fix_is_in_all_four_catalogues(): + """A state with no translation would show its raw key to the user.""" + source = open(caps.__file__, encoding="utf-8").read() + wanted = {f"cap_state_{state.value}" for state in caps.CapabilityStatus} + wanted |= {f"cap_name_{name}" for name in ( + caps.INPUT, caps.CAPTURE, caps.RECORDING, caps.STOP_SHORTCUT)} + wanted |= {key for key in english_word_dict + if key.startswith("cap_fix_")} + wanted |= set(tab_mod._CAPABILITY_COLUMNS) + wanted |= {"cap_scope_desktop", "cap_scope_xwayland", + "diag_capabilities_title", "diag_capabilities_refresh", + "diag_input_reask", "diag_input_close"} + for language, catalogue in _CATALOGUES.items(): + missing = sorted(key for key in wanted if not catalogue.get(key)) + assert missing == [], f"{language} lacks {missing}" + # Every advice key the headless module names exists, and none is unused. + for key in (k for k in english_word_dict if k.startswith("cap_fix_")): + assert f'"{key}"' in source, f"{key} is not used by capabilities.py" + + +def test_the_tab_shows_states_with_actionable_text(app, monkeypatch): + monkeypatch.setattr(tab_mod, "run_diagnostics", + lambda: DiagnosticsReport(checks=[])) + monkeypatch.setattr(tab_mod, "probe_capabilities", _declined_wayland) + tab = tab_mod.DiagnosticsTab() + + table = tab._capabilities + assert table.rowCount() == 4 + row = {table.horizontalHeaderItem(col).text(): table.item(0, col).text() + for col in range(table.columnCount())} + assert row[tab_mod._t("diag_cap_col_state")] == tab_mod._t( + "cap_state_needs_permission") + assert row[tab_mod._t("diag_cap_col_backend")] == "libei" + assert row[tab_mod._t("diag_cap_col_detail")] == "Portal denied Start" + assert row[tab_mod._t("diag_cap_col_fix")] == tab_mod._t( + "cap_fix_input_consent") + # Every row that is not usable says what to do. + for index, capability in enumerate(_declined_wayland().capabilities): + if capability.state is not caps.CapabilityStatus.AVAILABLE: + assert table.item(index, 5).text(), capability.name + + +def test_the_tab_commands_live_in_the_actions_menu(app, monkeypatch): + monkeypatch.setattr(tab_mod, "run_diagnostics", + lambda: DiagnosticsReport(checks=[])) + calls = [] + monkeypatch.setattr(tab_mod, "reset_input_authorisation", + lambda: calls.append("reset")) + monkeypatch.setattr(tab_mod, "close_input_session", + lambda: calls.append("close")) + tab = tab_mod.DiagnosticsTab() + actions = dict(tab.menu_actions()) + assert list(actions) == ["diag_run", "diag_capabilities_refresh", + "diag_input_reask", "diag_input_close"] + actions["diag_input_reask"]() + actions["diag_input_close"]() + actions["diag_capabilities_refresh"]() + assert calls == ["reset", "close"] + tab.retranslate() + assert tab._capabilities.rowCount() == 4 + + +def test_xwayland_is_shown_as_a_narrower_scope(): + snapshot = caps.probe_capabilities(caps.BackendContext( + platform="linux", loaded_backend="x11", + environ={"XDG_SESSION_TYPE": "wayland", "DISPLAY": ":0"})) + cells = tab_mod.capability_cells(snapshot.input) + assert cells[3] == tab_mod._t("cap_scope_xwayland") + assert cells[5] == tab_mod._t("cap_fix_xwayland") diff --git a/test/unit_test/headless/test_wayland_capability_states.py b/test/unit_test/headless/test_wayland_capability_states.py new file mode 100644 index 000000000..fbe3d58bf --- /dev/null +++ b/test/unit_test/headless/test_wayland_capability_states.py @@ -0,0 +1,511 @@ +"""Wayland capability and authorisation states, against fakes. + +Nothing under ``linux_wayland`` can run on the machine these tests were +written on, so every one drives the real code against a fake libei, a fake +liboeffis and a described environment. What they pin is behaviour that is +independent of the host: + +* a consent the portal *denied* is never worked around through ydotool; +* a session the compositor ended is never written to again; +* the X11 backend serving a Wayland session says how little it reaches; +* probing asks nobody for anything. + +Whether real ``liboeffis`` words a refusal the way the fake does is the one +thing only Linux can settle; ``docker/portal_verify.py`` measures it. +""" +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import _select_input as select_mod +from je_auto_control.linux_wayland import authorisation as auth +from je_auto_control.linux_wayland import keyboard as wl_keyboard +from je_auto_control.linux_wayland import libei as libei_mod +from je_auto_control.linux_wayland import oeffis as oeffis_mod +from je_auto_control.linux_wayland import portal as portal_mod +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlScreenException, +) +from je_auto_control.wrapper.capabilities import ( + BackendContext, CapabilityStatus, probe_capabilities, +) + +SEAT = 0x5EA7 +KEYBOARD = 0xD0001 +POINTER = 0xD0002 +KEY_A = 30 + + +class FakeLibei: + """A libei that plays back scripted events and counts what it is sent.""" + + def __init__(self): + self.pending = [] + self._queue = [] + self.writes = [] + self.rearm() + self._caps = { + KEYBOARD: {libei_mod.EI_DEVICE_CAP_KEYBOARD}, + POINTER: {libei_mod.EI_DEVICE_CAP_POINTER_ABSOLUTE, + libei_mod.EI_DEVICE_CAP_BUTTON, + libei_mod.EI_DEVICE_CAP_SCROLL}, + } + + def rearm(self): + """Queue the event stream a cooperating compositor produces.""" + self._queue = [] + self.pending = [ + (libei_mod.EI_EVENT_CONNECT, None), + (libei_mod.EI_EVENT_SEAT_ADDED, SEAT), + (libei_mod.EI_EVENT_DEVICE_ADDED, KEYBOARD), + (libei_mod.EI_EVENT_DEVICE_ADDED, POINTER), + (libei_mod.EI_EVENT_DEVICE_RESUMED, KEYBOARD), + (libei_mod.EI_EVENT_DEVICE_RESUMED, POINTER), + ] + + def __getattr__(self, name): + # Everything not spelled out below is a call this test does not + # care about: accept it and return nothing. + if name.startswith("ei_"): + return lambda *_args: None + raise AttributeError(name) + + def ei_new_sender(self, _user_data): + return 0xEEEE + + def ei_setup_backend_fd(self, _ei, _fd): + return 0 + + def ei_get_fd(self, _ei): + return 0 + + def ei_dispatch(self, _ei): + self._queue.extend(self.pending) + self.pending = [] + + def ei_get_event(self, _ei): + return self._queue.pop(0) if self._queue else None + + def ei_event_get_type(self, event): + return event[0] + + def ei_event_get_seat(self, event): + return event[1] + + def ei_event_get_device(self, event): + return event[1] + + def ei_device_has_capability(self, device, cap): + return cap in self._caps.get(device, set()) + + def ei_device_get_region(self, _device, _index): + return None + + def ei_device_keyboard_key(self, device, keycode, is_press): + self.writes.append(("key", device, keycode, is_press)) + + def ei_device_frame(self, device, _when): + self.writes.append(("frame", device)) + + +class FakeOeffis: + """A liboeffis whose portal answers the way ``outcome`` says.""" + + def __init__(self, event, message=b""): + self._event = event + self._message = message + self.sessions = 0 + + def oeffis_new(self, _user_data): + return 0x0EFF + + def oeffis_unref(self, _handle): + return None + + def oeffis_create_session(self, _handle, _devices): + self.sessions += 1 + + def oeffis_get_fd(self, _handle): + return 0 + + def oeffis_dispatch(self, _handle): + return None + + def oeffis_get_event(self, _handle): + return self._event + + def oeffis_get_eis_fd(self, _handle): + return 7 + + def oeffis_get_error_message(self, _handle): + return self._message + + +def _ready(readers, _writers, _errors, _timeout): + """A ``select`` that reports every descriptor readable at once.""" + return list(readers), [], [] + + +def _quiet(_readers, _writers, _errors, _timeout): + """A ``select`` on which nothing ever arrives.""" + return [], [], [] + + +@pytest.fixture(autouse=True) +def _clean_state(monkeypatch): + """Every test starts unasked and leaves nothing behind for the next.""" + monkeypatch.delenv("JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND", raising=False) + monkeypatch.delenv("JE_AUTOCONTROL_WAYLAND_EI_WORKER", raising=False) + auth.ledger.reset() + libei_mod.reset_default_backend() + with patch.object(libei_mod.select, "select", side_effect=_ready), \ + patch.object(oeffis_mod.select, "select", side_effect=_ready): + yield + libei_mod.reset_default_backend() + auth.ledger.reset() + + +def _wayland(**overrides): + """A GNOME-shaped Wayland session with libei and ydotool installed.""" + settings = { + "platform": "linux", + "environ": {"XDG_SESSION_TYPE": "wayland", + "WAYLAND_DISPLAY": "wayland-0", + "XDG_RUNTIME_DIR": "/run/user/1000"}, + "which": lambda name: "/usr/bin/" + name, + "library_present": lambda _name: True, + "session_bus_present": lambda: True, + "readable": lambda _path: True, + "compositor": lambda _env: "8:1:100", + "authorisations": auth.ledger, + } + settings.update(overrides) + return BackendContext(**settings) + + +def _portal_session(monkeypatch, fake_oeffis, fake_libei=None): + """Route the one libei probe through a fake portal and a fake libei.""" + fake_libei = fake_libei or FakeLibei() + real_backend = libei_mod.LibeiBackend + + def build(): + fake_libei.rearm() + return real_backend( + symbols=fake_libei, + portal_connect=lambda: oeffis_mod.connect_eis_fd( + symbols=fake_oeffis, timeout=0.5)) + + monkeypatch.setattr(libei_mod, "LibeiBackend", build) + monkeypatch.setattr(libei_mod.oeffis, "is_available", lambda: True) + monkeypatch.setattr(select_mod, "_libei_loadable", lambda: True) + return fake_libei + + +def _cli_calls(monkeypatch): + """Capture what would have been handed to ydotool.""" + calls = [] + monkeypatch.setattr(wl_keyboard, "_require_ydotool", + lambda: "/usr/bin/ydotool") + monkeypatch.setattr(wl_keyboard, "_run", + lambda argv, **_kw: calls.append(argv)) + return calls + + +# === the three behaviours the plan names ==================================== + +def test_cancel_does_not_fallback_silently(monkeypatch): + """A denied consent stops the action; ydotool is not used behind it.""" + portal = FakeOeffis(oeffis_mod.OEFFIS_EVENT_DISCONNECTED, + b"Portal denied Start") + _portal_session(monkeypatch, portal) + cli = _cli_calls(monkeypatch) + + with pytest.raises(auth.WaylandAuthorisationError) as refused: + wl_keyboard.press_key(KEY_A) + + canceled = probe_capabilities(_wayland()).input + assert canceled.state == 'needs_permission' + assert canceled.backend == "libei" + assert canceled.recovery + assert cli == [] + assert refused.value.has_recovery_instruction + assert refused.value.capability == "input" + assert isinstance(refused.value, AutoControlException) + # The fallback the keyboard module's own probe would take is a + # RuntimeError; this must not be one, or it would be swallowed there. + assert not isinstance(refused.value, RuntimeError) + + # It stays refused, and the portal is not asked again behind the user. + with pytest.raises(auth.WaylandAuthorisationError): + wl_keyboard.press_key(KEY_A) + assert portal.sessions == 1 + assert cli == [] + + +def test_revoked_session_cannot_send(monkeypatch): + """After the compositor disconnects, nothing more is written anywhere.""" + granted = FakeOeffis(oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS) + fake = _portal_session(monkeypatch, granted) + cli = _cli_calls(monkeypatch) + + wl_keyboard.press_key(KEY_A) + assert ("key", KEYBOARD, KEY_A, True) in fake.writes + assert probe_capabilities(_wayland()).input.state == "available" + + fake.writes.clear() + fake.pending = [(libei_mod.EI_EVENT_DISCONNECT, None)] + with pytest.raises(auth.WaylandAuthorisationError): + wl_keyboard.release_key(KEY_A) + with pytest.raises(auth.WaylandAuthorisationError): + wl_keyboard.press_key(KEY_A) + # And the backend itself refuses, whoever holds a reference to it. + backend = libei_mod.connected_backend() + with pytest.raises(libei_mod.LibeiSessionRevoked): + backend.press_key(KEY_A) + + writes_after_revoke = len(fake.writes) + assert writes_after_revoke == 0 + assert cli == [] + assert probe_capabilities(_wayland()).input.state == "revoked" + + +def test_xwayland_scope_is_explicit(): + """X11 serving a Wayland session reaches X11 clients only, and says so.""" + context = _wayland( + environ={"XDG_SESSION_TYPE": "wayland", "WAYLAND_DISPLAY": "wayland-0", + "DISPLAY": ":0", + "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER": "x11"}) + snapshot = probe_capabilities(context) + xwayland = snapshot.input + + assert snapshot.xwayland is True + assert xwayland.desktop_wide is False + assert snapshot.capture.desktop_wide is False + assert xwayland.backend == "xwayland" + assert "native Wayland windows" in xwayland.detail + assert xwayland.recovery_key == "cap_fix_xwayland" + + +# === the rest of the state machine ========================================== + +def test_wrapper_fallback_to_x11_is_reported_as_xwayland(): + """The wrapper can fall back without the override; the probe follows it.""" + snapshot = probe_capabilities(_wayland(loaded_backend="x11")) + assert snapshot.xwayland is True + assert snapshot.display_server == "x11" + assert snapshot.input.desktop_wide is False + + +def test_a_real_x11_session_is_desktop_wide(): + context = _wayland(environ={"XDG_SESSION_TYPE": "x11", "DISPLAY": ":0"}) + snapshot = probe_capabilities(context) + assert snapshot.xwayland is False + assert snapshot.input.desktop_wide is True + assert snapshot.input.state == "available" + + +def test_no_portal_still_falls_back_to_the_cli(monkeypatch): + """Not a refusal: a host with no RemoteDesktop portal keeps working.""" + portal = FakeOeffis(oeffis_mod.OEFFIS_EVENT_DISCONNECTED, + b"The name org.freedesktop.portal.Desktop was not " + b"provided by any .service files") + _portal_session(monkeypatch, portal) + cli = _cli_calls(monkeypatch) + + wl_keyboard.press_key(KEY_A) + + assert cli == [["/usr/bin/ydotool", "key", "30:1"]] + assert auth.ledger.get(auth.INPUT).state is auth.AuthorisationState.FAILED + serving = probe_capabilities(_wayland()).input + assert serving.backend == "ydotool" + assert serving.state == "available" + assert "libei could not start" in serving.detail + + +def test_an_unanswered_dialog_is_needs_permission(monkeypatch): + portal = FakeOeffis(oeffis_mod.OEFFIS_EVENT_NONE) + _portal_session(monkeypatch, portal) + _cli_calls(monkeypatch) + with patch.object(oeffis_mod.select, "select", side_effect=_quiet): + wl_keyboard.press_key(KEY_A) + state = auth.ledger.get(auth.INPUT).state + assert state is auth.AuthorisationState.TIMED_OUT + assert probe_capabilities(_wayland()).input.state == "needs_permission" + + +def test_only_a_stated_denial_counts_as_declined(): + """``declined`` is recognised narrowly, from the library's own text.""" + denied = oeffis_mod.PortalConsentNotGranted( + "x", "disconnected", "Portal denied CreateSession") + absent = oeffis_mod.PortalConsentNotGranted( + "x", "disconnected", "no such interface") + silent = oeffis_mod.PortalConsentNotGranted("x", "timeout") + assert denied.declined is True + assert absent.declined is False + assert silent.declined is False + assert isinstance(denied, oeffis_mod.OeffisUnavailable) + + +def test_reset_lets_the_desktop_be_asked_again(monkeypatch): + portal = FakeOeffis(oeffis_mod.OEFFIS_EVENT_DISCONNECTED, + b"Portal denied Start") + _portal_session(monkeypatch, portal) + _cli_calls(monkeypatch) + with pytest.raises(auth.WaylandAuthorisationError): + wl_keyboard.press_key(KEY_A) + + select_mod.reset_input_authorisation() + + assert probe_capabilities(_wayland()).input.state == "not_requested" + with pytest.raises(auth.WaylandAuthorisationError): + wl_keyboard.press_key(KEY_A) + assert portal.sessions == 2 + + +def test_closing_the_session_is_its_own_state(monkeypatch): + granted = FakeOeffis(oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS) + _portal_session(monkeypatch, granted) + _cli_calls(monkeypatch) + wl_keyboard.press_key(KEY_A) + + select_mod.close_input_session() + + closed = probe_capabilities(_wayland()).input + assert closed.state == "session_closed" + assert closed.usable is True + wl_keyboard.press_key(KEY_A) # asks again, and is granted again + assert granted.sessions == 2 + assert probe_capabilities(_wayland()).input.state == "available" + + +def test_forcing_the_cli_never_asks_the_portal(monkeypatch): + """``cli`` is the explicit choice of ydotool; a past refusal is moot.""" + portal = FakeOeffis(oeffis_mod.OEFFIS_EVENT_DISCONNECTED, + b"Portal denied Start") + _portal_session(monkeypatch, portal) + cli = _cli_calls(monkeypatch) + auth.ledger.transition(auth.INPUT, auth.AuthorisationState.DECLINED, "no") + monkeypatch.setenv("JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND", "cli") + + wl_keyboard.press_key(KEY_A) + + assert cli == [["/usr/bin/ydotool", "key", "30:1"]] + assert portal.sessions == 0 + + +def test_a_compositor_restart_is_its_own_state(): + with patch.object(auth, "compositor_identity", return_value="8:1:100"): + auth.ledger.transition(auth.INPUT, auth.AuthorisationState.GRANTED) + same = probe_capabilities(_wayland(compositor=lambda _env: "8:1:100")) + restarted = probe_capabilities(_wayland(compositor=lambda _env: "8:9:999")) + assert same.input.state == "available" + assert restarted.input.state == "compositor_restarted" + assert restarted.input.recovery + + +def test_the_restore_token_is_stated_rather_than_implied(): + libei_input = probe_capabilities(_wayland()).input + assert libei_input.restore_token == "unsupported" + cli = probe_capabilities(_wayland(library_present=lambda _n: False)).input + assert cli.restore_token == "not_applicable" + + +def test_input_and_capture_are_diagnosed_independently(): + """Capture working says nothing about input, and the reverse.""" + only_grim = _wayland(which=lambda name: "/usr/bin/grim" + if name == "grim" else None, + library_present=lambda _name: False) + snapshot = probe_capabilities(only_grim) + assert snapshot.capture.state == "available" + assert snapshot.capture.backend == "grim" + assert snapshot.input.state == "needs_setup" + assert snapshot.input.recovery_key == "cap_fix_ydotool" + + only_input = _wayland(which=lambda name: "/usr/bin/ydotool" + if name == "ydotool" else None, + library_present=lambda _name: False, + session_bus_present=lambda: False) + snapshot = probe_capabilities(only_input) + assert snapshot.input.state == "available" + assert snapshot.capture.state == "needs_setup" + + +def test_a_dismissed_screenshot_dialog_is_needs_permission(): + with pytest.raises(AutoControlScreenException): + portal_mod._uri_from_response([1, {}]) + portal_only = _wayland(which=lambda _name: None) + capture = probe_capabilities(portal_only).capture + assert capture.backend == "xdg-desktop-portal" + assert capture.state == "needs_permission" + assert capture.recovery + + assert portal_mod._uri_from_response([0, {"uri": "file:///tmp/a.png"}]) + assert probe_capabilities(portal_only).capture.state == "available" + + +def test_recording_is_opt_in_and_names_what_is_missing(): + unset = probe_capabilities(_wayland()).get("recording") + assert unset.state == "needs_setup" + assert "opt-in" in unset.detail + + env = {"XDG_SESSION_TYPE": "wayland", + "JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES": "/dev/input/event3"} + denied = probe_capabilities( + _wayland(environ=env, readable=lambda _p: False)).get("recording") + assert denied.state == "needs_permission" + assert "root" in denied.recovery + allowed = probe_capabilities(_wayland(environ=env)).get("recording") + assert allowed.state == "available" + + +def test_probing_has_no_side_effect(monkeypatch): + """A probe asks nobody for anything and changes nothing.""" + def forbidden(*_args, **_kwargs): + raise AssertionError("a probe must not do this") + + monkeypatch.setattr(libei_mod, "connected_backend", forbidden) + monkeypatch.setattr(libei_mod, "LibeiBackend", forbidden) + monkeypatch.setattr(oeffis_mod, "connect_eis_fd", forbidden) + monkeypatch.setattr(portal_mod, "capture_png", forbidden) + monkeypatch.setattr(portal_mod._dbus_client, "SessionBus", forbidden) + before = {name: auth.ledger.get(name) for name in (auth.INPUT, auth.CAPTURE)} + + for context in (_wayland(), _wayland(which=lambda _n: None), + _wayland(loaded_backend="x11"), BackendContext.current()): + probe_capabilities(context) + + after = {name: auth.ledger.get(name) for name in (auth.INPUT, auth.CAPTURE)} + assert after == before + + +def test_other_platforms_are_described_without_guessing(): + windows = probe_capabilities(BackendContext(platform="win32")) + assert windows.input.state is CapabilityStatus.AVAILABLE + assert windows.input.backend == "win32" + mac = probe_capabilities(BackendContext(platform="darwin")) + assert mac.input.state is CapabilityStatus.UNKNOWN + assert mac.xwayland is False + + +def test_every_surface_reads_the_same_data(): + """Headless, the executor and the snapshot's dict are one description.""" + import je_auto_control as ac + from je_auto_control.utils.executor.action_executor import executor + + expected = probe_capabilities().to_dict() + assert ac.probe_capabilities().to_dict() == expected + assert executor.event_dict["AC_probe_capabilities"]() == expected + names = [item["name"] for item in expected["capabilities"]] + assert names == ["input", "capture", "recording", "stop_shortcut"] + for item in expected["capabilities"]: + assert isinstance(item["state"], str) + + +def test_the_mcp_tool_reads_the_same_data(): + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry, + ) + tools = {tool.name: tool for tool in build_default_tool_registry()} + tool = tools["ac_probe_capabilities"] + assert tool.handler() == probe_capabilities().to_dict() + assert tool.annotations.read_only is True diff --git a/test/unit_test/headless/test_wayland_recording_paths.py b/test/unit_test/headless/test_wayland_recording_paths.py new file mode 100644 index 000000000..ebd4b39c3 --- /dev/null +++ b/test/unit_test/headless/test_wayland_recording_paths.py @@ -0,0 +1,398 @@ +"""Recording and the stop shortcut on Wayland, against fakes. + +Two different things were always called "record": what this program executed, +and what the user physically did. The first needs no hook on any platform; the +second needs kernel devices and is opt-in. These tests keep the two apart, and +pin the three rules the physical reader lives by — it reads only what it was +told to, never a virtual device, and never with borrowed privilege. + +Nothing here opens a real device or a real bus. Whether ``/sys`` really +parents uinput devices where :class:`InputDevice` looks, and whether a real +portal implements GlobalShortcuts, are Linux questions; the manual checklist +in ``test/manual_test/wayland_authorisation_checklist.md`` covers them. +""" +import threading + +import pytest + +from je_auto_control.linux_wayland import _dbus_client +from je_auto_control.linux_wayland import global_shortcuts as shortcuts +from je_auto_control.linux_wayland import input_events as events_mod +from je_auto_control.linux_wayland import listener as wl_listener +from je_auto_control.linux_wayland.input_events import ( + ActionJournal, EV_KEY, InputDevice, InputEvent, InputPermissionError, + InputRecordingError, PhysicalRecorder, +) +from je_auto_control.linux_wayland.record import wayland_recorder +from je_auto_control.utils.exception.exceptions import AutoControlException + +KEYBOARD = InputDevice( + "/dev/input/event3", name="AT Translated Set 2 keyboard", + phys="isa0060/serio0/input0", bustype=0x11, + sysfs="/sys/devices/platform/i8042/serio0/input/input3/event3") +YDOTOOL = InputDevice( + "/dev/input/event19", name="ydotoold virtual device", bustype=0x06, + sysfs="/sys/devices/virtual/input/input23/event19") +BLUETOOTH = InputDevice( + "/dev/input/event21", name="Keychron K2", bustype=0x05, + sysfs="/sys/devices/virtual/misc/uhid/0005:05AC:024F.0003/input/input25/" + "event21") +KEY_A = 30 + + +class FakeSource: + """An opened device that hands out one scripted burst, then nothing.""" + + def __init__(self, path, burst): + self.path = path + self._burst = list(burst) + self.closed = False + self.delivered = threading.Event() + + def read_events(self, _timeout): + if self._burst: + burst, self._burst = self._burst, [] + self.delivered.set() + return burst + self.delivered.set() + return [] + + def close(self): + self.closed = True + + +def _opener(bursts, opened): + def open_device(path): + opened.append(FakeSource(path, bursts.get(path, ()))) + return opened[-1] + return open_device + + +# === the four behaviours the plan names ===================================== + +def test_action_recording_needs_no_global_hook(monkeypatch): + """What the program executed is journalled with nothing read from input.""" + from je_auto_control.utils.executor.action_executor import executor + + def no_hook(*_args, **_kwargs): + raise AssertionError("journalling must not install or use a hook") + + monkeypatch.setattr(wl_listener, "hook_keyboard", no_hook) + monkeypatch.setattr(wl_listener, "check_key_press", no_hook) + monkeypatch.setattr(PhysicalRecorder, "start", no_hook) + done = [] + monkeypatch.setitem(executor.event_dict, "AC_journal_probe", + lambda **params: done.append(params)) + expected_steps = [["AC_journal_probe", {"step": 1}], + ["AC_journal_probe", {"step": 2}]] + + journal = ActionJournal() + journal.run([list(step) for step in expected_steps]) + + executor_journal_steps = journal.steps + assert executor_journal_steps == expected_steps + assert done == [{"step": 1}, {"step": 2}] + # A copy: the caller cannot rewrite history by mutating what came back. + executor_journal_steps[0][1]["step"] = 99 + assert journal.steps == expected_steps + journal.clear() + assert journal.steps == [] + + +def test_physical_reader_excludes_virtual_device(): + """This program's own ydotool output is never recorded as the user's.""" + typed = [InputEvent(EV_KEY, KEY_A, 1, 1.0, KEYBOARD.path), + InputEvent(EV_KEY, KEY_A, 0, 1.1, KEYBOARD.path)] + injected = [InputEvent(EV_KEY, 48, 1, 1.2, YDOTOOL.path)] + opened = [] + recorder = PhysicalRecorder( + opener=_opener({KEYBOARD.path: typed, YDOTOOL.path: injected}, opened), + poll_s=0.01) + + recorder.start([KEYBOARD, YDOTOOL]) + assert opened[0].delivered.wait(5) + recorded = recorder.stop() + + injected_events = [event for event in recorded + if event.device == YDOTOOL.path] + assert injected_events == [] + assert recorded == typed + assert [source.path for source in opened] == [KEYBOARD.path] + assert recorder.excluded == [YDOTOOL] + assert opened[0].closed is True + assert recorder.is_recording is False + + assert YDOTOOL.is_virtual and not KEYBOARD.is_virtual + # A Bluetooth keyboard sits under /devices/virtual too, and is real. + assert BLUETOOTH.is_virtual is False + # A uinput device that does not admit to BUS_VIRTUAL is still caught. + assert InputDevice("/dev/input/event9", bustype=0x03, + sysfs=YDOTOOL.sysfs).is_virtual + + +def test_permission_denial_is_actionable(): + """No access is a typed error that says what to do — and not "use root".""" + opened = [] + + def deny_second(path): + if path == BLUETOOTH.path: + raise PermissionError(13, "Permission denied", path) + opened.append(FakeSource(path, ())) + return opened[-1] + + recorder = PhysicalRecorder(opener=deny_second, poll_s=0.01) + with pytest.raises(InputPermissionError) as denied: + recorder.start([KEYBOARD, BLUETOOTH]) + + permission_error = denied.value + assert permission_error.has_recovery_instruction + assert permission_error.path == BLUETOOTH.path + assert "`input` group" in permission_error.recovery + assert "Do not run the whole program as root" in str(permission_error) + assert isinstance(permission_error, AutoControlException) + # All or nothing: the device that did open was closed again. + assert [source.closed for source in opened] == [True] + assert recorder.is_recording is False + assert recorder.stop() == [] + + +def test_shortcut_session_closes(): + """The registration ends when the session is closed, exactly once.""" + bus = FakeBus() + stops = [] + session = shortcuts.StopShortcutSession( + lambda: stops.append("stop"), bus_factory=lambda: bus, + preferred_trigger="CTRL+ALT+F12") + + session.open(timeout=1.0) + assert session.is_open + assert [call[2] for call in bus.calls] == ["CreateSession", + "BindShortcuts"] + bound = bus.calls[1][4] + assert bound[0] == bus.session_handle + assert [item[0] for item in bound[1]] == [shortcuts.STOP_SHORTCUT_ID] + assert bound[1][0][1]["preferred_trigger"].value == "CTRL+ALT+F12" + + assert session.wait(0.01) is False + bus.signals.append([bus.session_handle, "something-else", 1, {}]) + assert session.wait(0.01) is False + bus.signals.append(["/another/session", shortcuts.STOP_SHORTCUT_ID, 1, {}]) + assert session.wait(0.01) is False + bus.signals.append([bus.session_handle, shortcuts.STOP_SHORTCUT_ID, 2, {}]) + assert session.wait(0.01) is True + assert stops == ["stop"] + + session.close() + session.close() + closes = [call for call in bus.calls if call[2] == "Close"] + assert len(closes) == 1 + assert closes[0][0] == bus.session_handle + assert closes[0][1] == shortcuts.SESSION_INTERFACE + assert bus.closed == 1 + assert session.is_open is False + assert session.wait(0.01) is False + + +# === the physical reader's other rules ====================================== + +def test_recording_is_opt_in_per_device(): + recorder = PhysicalRecorder(opener=_opener({}, []), poll_s=0.01) + with pytest.raises(InputRecordingError) as unnamed: + recorder.start([]) + assert "opt-in" in str(unnamed.value) + with pytest.raises(InputRecordingError) as nothing_physical: + recorder.start([YDOTOOL, InputDevice("/dev/input/event7")]) + assert "virtual or of unknown origin" in str(nothing_physical.value) + assert recorder.is_recording is False + + +def test_a_second_recording_is_refused_and_the_buffer_is_bounded(): + burst = [InputEvent(EV_KEY, KEY_A, index % 2, 0.0, KEYBOARD.path) + for index in range(10)] + opened = [] + recorder = PhysicalRecorder(opener=_opener({KEYBOARD.path: burst}, opened), + max_events=4, poll_s=0.01) + recorder.start([KEYBOARD]) + with pytest.raises(InputRecordingError): + recorder.start([KEYBOARD]) + assert opened[0].delivered.wait(5) + recorded = recorder.stop() + assert recorded == burst[:4] + assert recorder.truncated is True + + +def test_events_are_decoded_in_the_native_layout(): + packed = (events_mod._EVENT.pack(10, 500_000, EV_KEY, KEY_A, 1) + + events_mod._EVENT.pack(11, 0, EV_KEY, KEY_A, 0) + + b"\x00\x01") # a truncated tail is dropped + decoded = events_mod.decode_events(packed, "/dev/input/event3") + assert decoded == [ + InputEvent(EV_KEY, KEY_A, 1, 10.5, "/dev/input/event3"), + InputEvent(EV_KEY, KEY_A, 0, 11.0, "/dev/input/event3")] + + +def test_a_device_is_described_from_sysfs_without_opening_it(tmp_path): + virtual_root = tmp_path / "devices" / "virtual" / "input" + node = virtual_root / "event19" / "device" + (node / "id").mkdir(parents=True) + (node / "name").write_text("ydotoold virtual device\n", encoding="utf-8") + (node / "id" / "bustype").write_text("0006\n", encoding="utf-8") + + described = events_mod.describe_device("/dev/input/event19", + str(virtual_root)) + assert described.name == "ydotoold virtual device" + assert described.bustype == events_mod.BUS_VIRTUAL + assert described.origin_known and described.is_virtual + + listed = events_mod.list_input_devices(str(virtual_root), "/dev/input") + assert [device.path for device in listed] == ["/dev/input/event19"] + unknown = events_mod.describe_device("/dev/input/event99", + str(virtual_root)) + assert unknown.origin_known is False + assert events_mod.list_input_devices(str(tmp_path / "absent")) == [] + + +def test_the_opt_in_setting_is_a_list_of_paths(): + env = {events_mod.RECORD_DEVICES_ENV: " /dev/input/event3 ,,/dev/input/event5"} + assert events_mod.configured_record_devices(env) == ( + "/dev/input/event3", "/dev/input/event5") + assert events_mod.configured_record_devices({}) == () + + +def test_the_global_recorder_still_refuses_and_now_says_what_works(): + """The seam's recorder is unchanged in kind: it names the alternatives.""" + with pytest.raises(NotImplementedError) as refused: + wayland_recorder.record() + message = str(refused.value) + assert "ActionJournal" in message + assert "PhysicalRecorder" in message + assert wl_listener.check_key_is_press(KEY_A) is False + + +# === the stop shortcut's other paths ======================================== + +class FakeBus: + """A session bus whose portal answers GlobalShortcuts as scripted.""" + + sender_token = "1_42" + session_handle = "/org/freedesktop/portal/desktop/session/1_42/stop" + + def __init__(self, bind_response=0, create_results=None): + self.calls = [] + self.matches = [] + self.signals = [] + self.responses = [] + self.closed = 0 + self._bind_response = bind_response + self._create_results = (create_results if create_results is not None + else {"session_handle": self.session_handle}) + + def add_match(self, rule): + self.matches.append(rule) + + def call(self, destination, path, interface, member, signature, body, + timeout=25.0): + self.calls.append((path, interface, member, signature, body)) + if member == "Close": + return [] + token = body[-1]["handle_token"].value + request = (f"{shortcuts.PORTAL_PATH}/request/{self.sender_token}/" + f"{token}") + answer = ([0, self._create_results] if member == "CreateSession" + else [self._bind_response, {}]) + self.responses.append((request, answer)) + return [request] + + def wait_for_signal(self, paths, interface, member, timeout): + if member == "Response": + for index, (path, answer) in enumerate(self.responses): + if path in paths: + del self.responses[index] + return answer + elif self.signals: + return self.signals.pop(0) + raise _dbus_client.DBusError("the session bus did not answer in time") + + def close(self): + self.closed += 1 + + +def test_a_declined_shortcut_is_actionable_and_leaves_nothing_open(): + bus = FakeBus(bind_response=1) + session = shortcuts.StopShortcutSession(lambda: None, + bus_factory=lambda: bus) + with pytest.raises(shortcuts.ShortcutPermissionError) as declined: + session.open(timeout=1.0) + assert declined.value.has_recovery_instruction + assert "GUI's stop control" in declined.value.recovery + assert isinstance(declined.value, AutoControlException) + # The half-made session was closed, not left registered. + assert [call[2] for call in bus.calls][-1] == "Close" + assert bus.closed == 1 + assert session.is_open is False + + +def test_a_portal_without_global_shortcuts_is_unavailable_not_a_crash(): + class NoInterface(FakeBus): + def call(self, *args, **kwargs): + raise _dbus_client.DBusError( + "org.freedesktop.DBus.Error.UnknownMethod: no such interface") + + bus = NoInterface() + session = shortcuts.StopShortcutSession(lambda: None, + bus_factory=lambda: bus) + with pytest.raises(shortcuts.ShortcutUnavailable) as missing: + session.open(timeout=1.0) + assert not isinstance(missing.value, shortcuts.ShortcutPermissionError) + assert missing.value.has_recovery_instruction + assert bus.closed == 1 + + +def test_the_listener_thread_stops_when_the_session_closes(): + bus = FakeBus() + pressed = threading.Event() + session = shortcuts.StopShortcutSession(pressed.set, + bus_factory=lambda: bus, + poll_s=0.01) + with session: + session.start() + bus.signals.append([bus.session_handle, shortcuts.STOP_SHORTCUT_ID, + 1, {}]) + assert pressed.wait(5) + listener = session._thread + assert listener is not None and not listener.is_alive() + assert bus.closed == 1 + + +def test_a_dead_bus_ends_the_session_instead_of_spinning(): + class Dropped(FakeBus): + def wait_for_signal(self, paths, interface, member, timeout): + if member == "Response": + return super().wait_for_signal(paths, interface, member, + timeout) + raise _dbus_client.DBusError( + "the session bus closed the connection") + + session = shortcuts.StopShortcutSession(lambda: None, + bus_factory=Dropped) + session.open(timeout=1.0) + assert session.wait(0.01) is False + assert session.is_open is False + session.close() + + +def test_the_bind_request_marshals_with_the_real_writer(): + """The one signature here the portal tier never used: ``a(sa{sv})``.""" + from je_auto_control.utils.dbus_client import session_bus + + bus = FakeBus() + shortcuts.StopShortcutSession(lambda: None, bus_factory=lambda: bus, + preferred_trigger="CTRL+ALT+F12", + ).open(timeout=1.0) + _path, _interface, _member, signature, body = bus.calls[1] + payload = session_bus._Writer() + for code, value in session_bus.body_pairs(signature, body): + payload.value(code, value) + assert signature == "oa(sa{sv})sa{sv}" + assert b"preferred_trigger" in payload.data + assert shortcuts.STOP_SHORTCUT_ID.encode() in payload.data diff --git a/test/unit_test/headless/test_wayland_worker_cleanup.py b/test/unit_test/headless/test_wayland_worker_cleanup.py new file mode 100644 index 000000000..426b04c82 --- /dev/null +++ b/test/unit_test/headless/test_wayland_worker_cleanup.py @@ -0,0 +1,517 @@ +"""The libei helper process: its wire, its limits, and what it leaves behind. + +The helper exists so that a libei session lives in a process nothing else +shares. These tests run the helper's real loop (``ei_worker.serve``) on a +thread, over real pipes, against a fake libei backend — so the framing, the +request ids, the cancellation path and the cleanup are the code that ships, +and only libei itself is pretend. One test starts a real child process. + +What they cannot say is whether libei crashes on teardown, which is the +question the helper is for. ``docker/libei_verify.py`` and +``docker/eis_verify.py`` measure that on Linux. +""" +import io +import os +import subprocess # nosec B404 # reason: starts this interpreter with a fixed argv +import sys +import threading +import time + +import pytest + +from je_auto_control.linux_wayland import _select_input as select_mod +from je_auto_control.linux_wayland import ei_transport as transport +from je_auto_control.linux_wayland import ei_client +from je_auto_control.linux_wayland import ei_worker +from je_auto_control.linux_wayland import libei as libei_mod +from je_auto_control.linux_wayland.input_events import ( + EV_ABS, EV_KEY, EV_REL, EV_SYN, InputEvent, +) +from je_auto_control.utils.exception.exceptions import AutoControlException + +KEY_A = 30 +KEY_B = 48 +BTN_LEFT = 272 + + +class FakeBackend: + """Stands in for ``LibeiBackend`` inside the helper.""" + + def __init__(self, available=True, connect_error=None): + self.is_available = available + self._connect_error = connect_error + self.calls = [] + self.connected_with = None + #: When set, the named call waits here — a stuck emission. + self.gate = None + self.fail = {} + + def connect(self, timeout, socket_path=None): + self.connected_with = (timeout, socket_path) + if self._connect_error is not None: + raise self._connect_error + + def disconnect(self): + self.calls.append(("disconnect",)) + + def _call(self, name, *args): + if self.gate is not None and self.gate[0] == (name,) + args: + self.gate[1].wait(10) + if name in self.fail: + raise self.fail[name] + self.calls.append((name,) + args) + + def press_key(self, code): + self._call("press_key", code) + + def release_key(self, code): + self._call("release_key", code) + + def press_button(self, code): + self._call("press_button", code) + + def release_button(self, code): + self._call("release_button", code) + + def set_position(self, x, y): + self._call("set_position", x, y) + + def scroll(self, dx, dy): + self._call("scroll", dx, dy) + + +class ThreadWorker: + """A helper "process" that is really ``serve`` on a thread, over pipes.""" + + def __init__(self, backend): + to_child_r, to_child_w = os.pipe() + from_child_r, from_child_w = os.pipe() + self.stdin = os.fdopen(to_child_w, "wb", buffering=0) + self.stdout = os.fdopen(from_child_r, "rb", buffering=0) + self._child_in = os.fdopen(to_child_r, "rb", buffering=0) + self._child_out = os.fdopen(from_child_w, "wb", buffering=0) + self.returncode = None + self._thread = threading.Thread( + target=self._run, args=(backend,), daemon=True) + self._thread.start() + + def _run(self, backend): + try: + code = ei_worker.serve(self._child_in, self._child_out, + lambda: backend) + finally: + for stream in (self._child_out, self._child_in): + try: + stream.close() + except OSError: + pass + self.returncode = code + + def crash(self): + """Die the way a segfault does: the pipe just ends.""" + self._child_out.close() + + def streams(self): + return (self.stdin, self.stdout, self._child_in, self._child_out) + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + self._thread.join(timeout) + if self._thread.is_alive(): + raise subprocess.TimeoutExpired("fake-worker", timeout) + return self.returncode + + def terminate(self): + """Nothing to signal; closing its stdin already ends the loop.""" + + kill = terminate + + +def _client(backend, **options): + workers = [] + + def spawn(): + workers.append(ThreadWorker(backend)) + return workers[-1] + + client = ei_client.EiWorkerClient(spawn=spawn, start_timeout_s=5.0, + **options) + return client, workers + + +def _open_descriptors(): + """How many descriptors this process holds, where the OS will say.""" + try: + return len(os.listdir("/proc/self/fd")) + except OSError: + return None + + +@pytest.fixture(autouse=True) +def _no_worker_left_behind(): + yield + ei_client.reset_default_backend() + assert ei_client.active_worker_count() == 0 + + +# === the three behaviours the plan names ==================================== + +def test_half_open_worker_exit_reclaims_resources(): + """A handshake that never completes costs the parent nothing at all. + + In-process this state is the deliberate leak: one context and one + descriptor that ``ei_unref`` cannot be trusted with. In a helper the + child simply exits, and what the parent held was two pipes. + """ + backend = FakeBackend(connect_error=libei_mod.LibeiUnavailable( + "libei connected but no seat offered both a keyboard and an " + "absolute pointer within the handshake timeout")) + fd_count_before = _open_descriptors() + streams = [] + + for _attempt in range(5): + client, workers = _client(backend) + with pytest.raises(transport.EiWorkerError) as failed: + client.start() + assert "handshake timeout" in str(failed.value) + assert client.is_alive is False + streams.extend(workers[-1].streams()) + assert workers[-1].wait(5) == 1 + + active_workers = ei_client.active_worker_count() + fd_count_after = _open_descriptors() + assert active_workers == 0 + assert fd_count_after == fd_count_before + assert all(stream.closed for stream in streams) + # The child never released a context it could not safely release: the + # backend's own connect() failure path already dealt with it. + assert ("disconnect",) not in backend.calls + + +def test_worker_death_releases_pressed_keys(): + """Whichever way the helper goes, a held key is accounted for.""" + # 1. A clean close: the helper lets go of everything on the way out. + backend = FakeBackend() + client, workers = _client(backend) + client.start() + client.send([InputEvent(EV_KEY, KEY_A, 1), + InputEvent(EV_KEY, BTN_LEFT, 1)], timeout_s=5.0) + assert client.pressed == (KEY_A, BTN_LEFT) + client.close() + assert backend.calls[-3:] == [("release_button", BTN_LEFT), + ("release_key", KEY_A), ("disconnect",)] + assert workers[0].returncode == 0 + + # 2. The parent vanishes without a word: the pipe ending is enough. + backend = FakeBackend() + client, workers = _client(backend) + client.start() + client.send([InputEvent(EV_KEY, KEY_A, 1)], timeout_s=5.0) + workers[0].stdin.close() + assert workers[0].wait(5) == 0 + assert ("release_key", KEY_A) in backend.calls + client.close() + + # 3. The helper dies without warning: the parent says which keys were + # down, to whoever asked to be told, and in the error it raises. + backend = FakeBackend() + orphaned = [] + client, workers = _client(backend, on_orphaned_keys=orphaned.append) + client.start() + client.send([InputEvent(EV_KEY, KEY_A, 1)], timeout_s=5.0) + workers[0].crash() + with pytest.raises(transport.EiWorkerDied) as died: + client.send([InputEvent(EV_KEY, KEY_B, 1)], timeout_s=5.0) + assert died.value.pressed_keys == (KEY_A,) + assert orphaned == [[KEY_A]] + assert client.pressed == () + assert client.is_alive is False + # Not a LibeiUnavailable: the request's fate is unknown, so no other + # backend may quietly replay it. + assert not isinstance(died.value, libei_mod.LibeiUnavailable) + assert isinstance(died.value, AutoControlException) + with pytest.raises(transport.EiWorkerDied): + client.send([InputEvent(EV_KEY, KEY_B, 1)], timeout_s=5.0) + + +def test_missing_dependency_is_typed(): + """No libei where the helper runs is a named error, not a dead pipe.""" + client, _workers = _client(FakeBackend(available=False)) + with pytest.raises(transport.EiDependencyMissing) as missing: + client.start() + error = missing.value + unavailable_capability = "input" + assert error.capability == unavailable_capability + assert error.dependency == "libei" + assert isinstance(error, libei_mod.LibeiUnavailable) + assert isinstance(error, AutoControlException) + + +# === limits, ids, deadlines ================================================= + +def test_a_batch_is_bounded_before_anything_is_written(): + backend = FakeBackend() + client, _workers = _client(backend) + client.start() + too_many = [InputEvent(EV_KEY, KEY_A, 1)] * (transport.MAX_BATCH + 1) + with pytest.raises(transport.EiProtocolError): + client.send(too_many, timeout_s=1.0) + with pytest.raises(transport.EiProtocolError): + client.send([], timeout_s=1.0) + with pytest.raises(transport.EiProtocolError): + client.send([InputEvent(99, 1, 1)], timeout_s=1.0) + assert backend.calls == [] + full = [InputEvent(EV_KEY, KEY_A, index % 2 ^ 1) + for index in range(transport.MAX_BATCH)] + assert client.send(full, timeout_s=5.0).applied == transport.MAX_BATCH + client.close() + + +def test_a_frame_is_bounded_in_both_directions(): + oversized = transport._LENGTH.pack(transport.MAX_FRAME_BYTES + 1) + with pytest.raises(transport.EiProtocolError): + transport.read_frame(io.BytesIO(oversized + b"x")) + with pytest.raises(transport.EiProtocolError): + transport.write_frame(io.BytesIO(), + {"pad": "x" * transport.MAX_FRAME_BYTES}) + with pytest.raises(transport.EiProtocolError): + transport.read_frame(io.BytesIO(transport._LENGTH.pack(10) + b"abc")) + with pytest.raises(transport.EiProtocolError): + transport.read_frame(io.BytesIO(transport._LENGTH.pack(2) + b"[]")) + assert transport.read_frame(io.BytesIO(b"")) is None + + stream = io.BytesIO() + transport.write_frame(stream, {"op": "ack", "id": 7}) + stream.seek(0) + assert transport.read_frame(stream) == {"op": "ack", "id": 7} + + +def test_a_bad_batch_is_refused_whole_by_the_helper(): + with pytest.raises(transport.EiProtocolError): + transport.decode_batch([[EV_KEY, KEY_A, 1], [EV_KEY, "x", 1]]) + with pytest.raises(transport.EiProtocolError): + transport.decode_batch([[EV_KEY, KEY_A, True]]) + with pytest.raises(transport.EiProtocolError): + transport.decode_batch([[EV_KEY, KEY_A]]) + with pytest.raises(transport.EiProtocolError): + transport.decode_batch("not a list") + + +def test_a_deadline_stops_the_batch_and_a_late_answer_is_dropped(): + """The request that timed out must not be mistaken for the next one.""" + backend = FakeBackend() + release_stuck = threading.Event() + backend.gate = (("press_key", KEY_A), release_stuck) + client, _workers = _client(backend) + client.start() + + started = time.monotonic() + with pytest.raises(transport.EiWorkerTimeout) as late: + client.send([InputEvent(EV_KEY, KEY_A, 1), + InputEvent(EV_KEY, KEY_B, 1)], timeout_s=0.2) + assert time.monotonic() - started < 5.0 + assert not isinstance(late.value, libei_mod.LibeiUnavailable) + # Nothing came back, so the worst is assumed about what is down. + assert client.pressed == (KEY_A, KEY_B) + + backend.gate = None + release_stuck.set() + ack = client.send([InputEvent(EV_KEY, BTN_LEFT, 1)], timeout_s=5.0) + + assert ack.request_id == 2 + assert ack.applied == 1 + # The helper stopped between events and let go of what that batch held; + # the second key of the abandoned batch was never pressed. + assert backend.calls == [("press_key", KEY_A), ("release_key", KEY_A), + ("press_button", BTN_LEFT)] + assert client.pressed == (BTN_LEFT,) + client.close() + + +def test_a_caller_can_cancel_a_request(): + backend = FakeBackend() + release_stuck = threading.Event() + backend.gate = (("press_key", KEY_A), release_stuck) + client, _workers = _client(backend) + client.start() + cancel = threading.Event() + threading.Timer(0.1, cancel.set).start() + + with pytest.raises(transport.EiWorkerCancelled): + client.send([InputEvent(EV_KEY, KEY_A, 1), + InputEvent(EV_KEY, KEY_B, 1)], timeout_s=30.0, + cancel=cancel) + + release_stuck.set() + client.close() + assert ("press_key", KEY_B) not in backend.calls + + +# === what libei says, said again in the parent ============================== + +def test_a_refusal_in_the_helper_is_a_refusal_in_the_parent(): + backend = FakeBackend() + backend.fail["set_position"] = libei_mod.LibeiUnavailable( + "(9000, 9000) lies outside every region this pointer accepts") + client, _workers = _client(backend) + client.start() + served = ei_client.WorkerBackend(client, timeout_s=5.0) + with pytest.raises(transport.EiEmitRefused) as refused: + served.set_position(9000, 9000) + assert "outside every region" in str(refused.value) + # The same type the in-process backend raises, so `emitted` hands the + # move to ydotool exactly as it does today. + assert select_mod.emitted(served, lambda b: b.set_position(1, 1)) is False + client.close() + + +def test_a_revoked_session_in_the_helper_is_revoked_in_the_parent(): + backend = FakeBackend() + backend.fail["press_key"] = libei_mod.LibeiSessionRevoked( + "the compositor disconnected the sender") + client, _workers = _client(backend) + client.start() + with pytest.raises(libei_mod.LibeiSessionRevoked): + ei_client.WorkerBackend(client, timeout_s=5.0).press_key(KEY_A) + client.close() + + +def test_a_refused_consent_keeps_its_meaning_across_the_pipe(): + backend = FakeBackend(connect_error=libei_mod.LibeiConsentNotGranted( + "the desktop portal disconnected the remote-desktop session: " + "Portal denied Start", "disconnected", True)) + client, _workers = _client(backend) + with pytest.raises(libei_mod.LibeiConsentNotGranted) as refused: + client.start() + assert refused.value.declined is True + assert refused.value.outcome == "disconnected" + + +def test_the_backend_surface_matches_what_keyboard_and_mouse_call(): + backend = FakeBackend() + client, _workers = _client(backend, socket_path="/run/user/1000/eis-0") + client.start() + assert backend.connected_with == (libei_mod.HANDSHAKE_TIMEOUT, + b"/run/user/1000/eis-0") + served = ei_client.WorkerBackend(client, timeout_s=5.0) + served.press_key(KEY_A) + served.release_key(KEY_A) + served.click_button(BTN_LEFT) + served.set_position(640, 400) + served.scroll(0, 1) + served.scroll(-2, 0) + served.disconnect() + assert backend.calls == [ + ("press_key", KEY_A), ("release_key", KEY_A), + ("press_button", BTN_LEFT), ("release_button", BTN_LEFT), + ("set_position", 640, 400), ("scroll", 0, 1), ("scroll", -2, 0), + ("disconnect",), + ] + assert served.is_connected is False + public = {name for name in dir(libei_mod.LibeiBackend) + if not name.startswith("_") and name not in {"connect"}} + assert public <= set(dir(served)) + + +def test_evdev_events_map_onto_libei_calls(): + """Auto-repeat is ignored, and a move is one call once both axes are in.""" + backend = FakeBackend() + client, _workers = _client(backend) + client.start() + client.send([ + InputEvent(EV_KEY, KEY_A, 1), InputEvent(EV_KEY, KEY_A, 2), + InputEvent(EV_ABS, transport.ABS_X, 10), + InputEvent(EV_ABS, transport.ABS_Y, 20), InputEvent(EV_SYN, 0, 0), + InputEvent(EV_REL, transport.REL_WHEEL, 1), InputEvent(EV_SYN, 0, 0), + InputEvent(EV_KEY, KEY_A, 0), + ], timeout_s=5.0) + client.close() + assert backend.calls[:-1] == [ + ("press_key", KEY_A), ("set_position", 10, 20), + # REL_WHEEL up is positive in the kernel's frame, negative in libei's. + ("scroll", 0, -1), ("release_key", KEY_A), + ] + + +# === the switch ============================================================= + +def test_the_helper_is_off_unless_asked_for(monkeypatch): + monkeypatch.delenv(select_mod.WORKER_ENV, raising=False) + assert select_mod.worker_enabled() is False + assert select_mod._backend_source() is libei_mod + for value in ("0", "", "no", "off"): + assert select_mod.worker_enabled({select_mod.WORKER_ENV: value}) is False + monkeypatch.setenv(select_mod.WORKER_ENV, "1") + assert select_mod.worker_enabled() is True + assert select_mod._backend_source() is ei_client + + +def test_the_selector_uses_the_helper_when_asked(monkeypatch): + from je_auto_control.linux_wayland import authorisation as auth + auth.ledger.reset() + backend = FakeBackend() + client, _workers = _client(backend) + monkeypatch.setenv(select_mod.WORKER_ENV, "1") + monkeypatch.setattr(select_mod, "_libei_loadable", lambda: True) + monkeypatch.setattr(ei_client, "EiWorkerClient", lambda: client) + try: + served = select_mod.active_backend() + assert isinstance(served, ei_client.WorkerBackend) + assert select_mod.emitted(served, lambda b: b.press_key(KEY_A)) + assert backend.calls == [("press_key", KEY_A)] + assert auth.ledger.get(auth.INPUT).state.value == "granted" + finally: + select_mod.reset_input_authorisation() + assert backend.calls[-2:] == [("release_key", KEY_A), ("disconnect",)] + assert auth.ledger.get(auth.INPUT).state.value == "not_requested" + + +def test_the_in_process_teardown_is_untouched(): + """The default path still abandons a half-open context — on purpose. + + The helper is an alternative behind a switch, not a replacement, so the + measured workaround in ``LibeiBackend._teardown`` must read as it did. + """ + backend = libei_mod.LibeiBackend(symbols=object()) + backend._ei = 0xEEEE + backend._backend_open = True + assert backend._safe_to_unref() is False + backend._handshake_complete = True + assert backend._safe_to_unref() is True + backend._backend_open = False + backend._handshake_complete = False + assert backend._safe_to_unref() is True + + +# === one real process ======================================================= + +_CHILD = ( + "from je_auto_control.linux_wayland import ei_worker, libei\n" + "libei._load_symbols = lambda: None\n" + "ei_worker.run_as_helper()\n" +) + + +def test_a_real_helper_process_reports_a_missing_library(): + """The same conversation across a real pipe to a real child process. + + The child is told libei is not installed, so it never reaches for a + display, a portal or a device on any host — it answers and exits. + """ + children = [] + + def spawn(): + argv = [sys.executable, "-c", _CHILD] + children.append(subprocess.Popen( # nosec B603 # nosemgrep # reason: fixed argv + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE)) + return children[-1] + + client = ei_client.EiWorkerClient(spawn=spawn, start_timeout_s=120.0) + with pytest.raises(transport.EiDependencyMissing) as missing: + client.start() + assert missing.value.capability == "input" + assert children[0].poll() == 1 + assert ei_client.active_worker_count() == 0 + assert children[0].stdin.closed and children[0].stdout.closed From 8d46db7e0df6490dc73d9ee81783df4d5e621fa5 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:11:38 +0800 Subject: [PATCH 23/44] Keep the unanswered test transfers at the reply timeout A transfer now waits its own timeout_ms on top of reply_timeout_s, so the correlation tests that let a default 1000 ms read time out each took a second longer. They ask for timeout_ms=0: what they test is the late reply, not the wait. --- test/unit_test/headless/test_usb_request_correlation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit_test/headless/test_usb_request_correlation.py b/test/unit_test/headless/test_usb_request_correlation.py index 43dcbe022..7b1664eb0 100644 --- a/test/unit_test/headless/test_usb_request_correlation.py +++ b/test/unit_test/headless/test_usb_request_correlation.py @@ -136,7 +136,9 @@ def _timed_out_bulk(wire: _Wire, handle) -> Frame: """Issue a bulk read nobody answers; return the request that timed out.""" sent_before = len(wire.sent) with pytest.raises(UsbClientTimeout): - handle.bulk_transfer(endpoint=0x81, direction="in", length=8) + # timeout_ms=0: the client waits the transfer's own timeout on top of + # reply_timeout_s, and the default 1000 ms would add a second per call. + handle.bulk_transfer(endpoint=0x81, direction="in", length=8, timeout_ms=0) return wire.sent[sent_before] @@ -258,7 +260,8 @@ def test_reply_half_received_at_the_timeout_is_still_discarded_whole(): wire = _Wire(timeout_s=0.5) handle = _opened(wire, 1) sent_before = len(wire.sent) - call = _Call(lambda: handle.bulk_transfer(endpoint=0x81, direction="in", length=8)) + call = _Call(lambda: handle.bulk_transfer( + endpoint=0x81, direction="in", length=8, timeout_ms=0)) first = wire.wait_sent(sent_before + 1) stale_frames = _big_reply(1, b"\x01" * (MAX_PAYLOAD_BYTES * 2), first) wire.client.feed_frame(stale_frames[0]) From b85ddf211de01e0e5cd26dd9ac770f38297c916a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:13:08 +0800 Subject: [PATCH 24/44] Sync scripts, locators, hotkeys, triggers and the address book without arming or leaking anything The bucket could be merged and queued but nothing connected it to what a machine actually holds. Adapters now turn local state into versioned entries and apply merged ones back: secrets and machine paths leave only as references, an arriving hotkey or trigger is created disabled, and an item that cannot be written here is not mistaken for a local deletion. Files travel by SHA-256 and replace their target atomically. The folder mirror and the clipboard channel get a guard against sending back what they just received. One session function drives the cycle for the new Config Sync tab, the AC_config_sync_* commands and the MCP tools. --- .../Eng/doc/config_sync/config_sync_doc.rst | 137 ++++ .../Zh/doc/config_sync/config_sync_doc.rst | 119 ++++ je_auto_control/__init__.py | 11 + je_auto_control/actions.pyi | 19 + je_auto_control/gui/config_sync_tab.py | 236 +++++++ .../gui/language_wrapper/english.py | 36 ++ .../gui/language_wrapper/japanese.py | 36 ++ .../language_wrapper/simplified_chinese.py | 36 ++ .../language_wrapper/traditional_chinese.py | 36 ++ .../gui/script_builder/command_schema.py | 40 ++ je_auto_control/gui/tab_registry.py | 1 + je_auto_control/utils/config_sync/__init__.py | 26 + je_auto_control/utils/config_sync/adapters.py | 531 ++++++++++++++++ je_auto_control/utils/config_sync/assets.py | 243 ++++++++ je_auto_control/utils/config_sync/outbox.py | 13 +- je_auto_control/utils/config_sync/session.py | 431 +++++++++++++ je_auto_control/utils/config_sync/versions.py | 7 +- .../utils/executor/action_executor.py | 7 + je_auto_control/utils/hotkey/hotkey_daemon.py | 11 +- .../utils/mcp_server/tools/_factories.py | 63 ++ .../utils/remote_desktop/clipboard_sync.py | 55 +- .../utils/remote_desktop/file_sync.py | 122 +++- .../headless/test_config_sync_convergence.py | 13 + .../headless/test_config_sync_tab_gui.py | 174 ++++++ test/unit_test/headless/test_sync_adapters.py | 585 ++++++++++++++++++ 25 files changed, 2954 insertions(+), 34 deletions(-) create mode 100644 je_auto_control/gui/config_sync_tab.py create mode 100644 je_auto_control/utils/config_sync/adapters.py create mode 100644 je_auto_control/utils/config_sync/assets.py create mode 100644 je_auto_control/utils/config_sync/session.py create mode 100644 test/unit_test/headless/test_config_sync_tab_gui.py create mode 100644 test/unit_test/headless/test_sync_adapters.py diff --git a/docs/source/Eng/doc/config_sync/config_sync_doc.rst b/docs/source/Eng/doc/config_sync/config_sync_doc.rst index 1e493e98e..c1cc07164 100644 --- a/docs/source/Eng/doc/config_sync/config_sync_doc.rst +++ b/docs/source/Eng/doc/config_sync/config_sync_doc.rst @@ -166,3 +166,140 @@ batch the server already has changes nothing. A failed send backs off makes at most ``max_attempts`` sends (default 5), ``wait=False`` returns instead of sleeping through a back-off, and setting ``cancel`` ends a drain even in the middle of a wait. + +Adapters: what is synced, and what stays on the machine +-------------------------------------------------------- + +``je_auto_control.utils.config_sync.adapters`` connects one bucket section to +one local store. ``SyncAdapter.snapshot()`` returns ``{key: SyncEntry}`` +measured against the state last merged, and ``apply(entries)`` writes merged +entries back and returns an ``ApplyReport`` (``written`` / ``removed`` / +``skipped`` / ``conflicts`` / ``left_disabled``). + +.. list-table:: + :header-rows: 1 + + * - Section + - Adapter + - Local store + * - ``scripts`` + - ``ScriptSyncAdapter(origin, scripts_dir)`` + - ``*.json`` action files under a folder, by relative path + * - ``locators`` + - ``LocatorSyncAdapter(origin, repository)`` + - an ``ElementRepository`` + * - ``hotkeys`` + - ``HotkeySyncAdapter(origin, daemon, scripts_dir=...)`` + - a ``HotkeyDaemon``'s bindings + * - ``triggers`` + - ``TriggerSyncAdapter(origin, engine, scripts_dir=...)`` + - a ``TriggerEngine``'s image / window / pixel / file / cron triggers + * - ``address_book`` + - ``AddressBookSyncAdapter(origin, book)`` + - the remote-desktop ``AddressBook`` + +Three rules hold for every adapter: + +* **Secrets stay local.** A field whose name marks it as a secret + (``password``, ``token``, ``api_key`` ...) leaves as + ``{"$local": "secret"}``; the receiving machine keeps its own value. A + ``${secrets.NAME}`` reference travels as it is. A script containing a + literal secret is not synced at all and is listed under ``withheld``. +* **Machine paths stay local.** A path inside ``scripts_dir`` travels as a + relative reference and is resolved against the receiver's own folder. Any + other path becomes ``{"$local": "path"}``; an item that needs it and has no + local value is reported under ``skipped`` -- and its absence here is *not* + turned into a deletion for the other machines. +* **Syncing never enables anything.** ``enabled`` is not synced. A hotkey or + trigger that arrives is created **disabled** (``HotkeyDaemon.bind(..., + enabled=False)``), an existing one keeps the state this machine gave it, + and no adapter starts an engine or runs a script. Composite triggers + (all-of / any-of / sequence) are not synced. + +Assets +------ + +A script up to 64 KiB travels inside its entry together with its SHA-256; +content that does not match the hash is not written. Larger scripts and +other files (template images) travel through an ``AssetTransport``: + +.. code-block:: python + + from je_auto_control.utils.config_sync.assets import ( + AssetManifest, DirectoryAssetTransport, publish_assets, sync_assets) + + transport = DirectoryAssetTransport("//nas/autocontrol-assets") # a folder both machines reach + manifest = AssetManifest.from_directory("scripts", ("*.png",)) + publish_assets(manifest, transport) # sender + result = sync_assets(AssetManifest(root=their_scripts, assets=manifest.assets), transport) + # AssetSyncResult(transferred, unchanged, failed, hashes, cancelled) + +``sync_assets`` checks every file against its SHA-256 and size *before* +writing, replaces the destination atomically, leaves the existing file alone +when the check fails, and refuses a path that would leave the folder. +``DirectoryAssetTransport`` is the transport that ships; implement +``fetch`` / ``store`` / ``has`` for any other blob store. + +One call, three surfaces +------------------------ + +``config_sync_run(server_url, user_id, **options)`` runs the whole cycle -- +queue local changes, drain the outbox, merge, apply -- and returns +``{state, revision, pending, conflicts, applied, withheld, assets, error}`` +with ``state`` one of ``synced`` / ``pending`` / ``conflict`` / ``offline`` / +``cancelled`` / ``resync_required``. Being unable to reach the server is not +an exception: the changes stay queued and the state is ``offline``. + +Options: ``device_id`` (default: an id created once in +``~/.je_auto_control/config_sync_device_id``), ``secret`` (default +``$AC_SIGNALING_SECRET``), ``sections`` (default ``hotkeys``, ``triggers``, +``address_book``; ``scripts`` and ``locators`` join when their path is +given), ``scripts_dir``, ``locators_path``, ``outbox_path``, ``assets_dir``, +``timeout_s``, ``wait``, ``max_attempts``. + +.. list-table:: + :header-rows: 1 + + * - Executor command + - MCP tool + - Does + * - ``AC_config_sync_run`` + - ``ac_config_sync_run`` + - sync once + * - ``AC_config_sync_status`` + - ``ac_config_sync_status`` (read-only) + - recorded state, no network, creates nothing + * - ``AC_config_sync_resolve`` + - ``ac_config_sync_resolve`` + - keep candidate ``choice`` of ``section`` / ``key`` + * - ``AC_config_sync_full_resync`` + - ``ac_config_sync_full_resync`` + - adopt the server's state after being retired; pending changes are discarded and listed + +All four are Script Builder commands under **Data**. + +.. code-block:: json + + [["AC_config_sync_run", {"server_url": "https://sync.example", "user_id": "alice", + "secret": "${secrets.sync}", "scripts_dir": "scripts"}]] + +GUI +--- + +The **Config Sync** tab (category *system*) shows the state, the last merged +revision, the number of pending changes, the last successful sync and the +last error, and lists every conflict with its candidates. Its commands -- +*Sync now*, *Cancel sync*, *Refresh sync status*, *Keep selected candidate*, +*Full resync* -- are in the Actions menu. A sync runs on a worker thread; +cancelling, or closing the tab, releases it. + +Folder mirror and clipboard: no echo +------------------------------------ + +``FolderSyncEngine.note_received(remote_name, sha256=...)`` marks a file in +the watched folder as having come from the peer; the engine does not push it +back until its content changes locally. ``FolderSyncEngine.poll_once()`` runs +one diff pass on demand. ``ClipboardEchoGuard`` (``note_remote`` / +``should_send`` / ``reset``) does the same for code that forwards clipboard +changes automatically: content that just arrived from the peer, or was +already sent, is not sent again. diff --git a/docs/source/Zh/doc/config_sync/config_sync_doc.rst b/docs/source/Zh/doc/config_sync/config_sync_doc.rst index 86c355471..7e882b1b6 100644 --- a/docs/source/Zh/doc/config_sync/config_sync_doc.rst +++ b/docs/source/Zh/doc/config_sync/config_sync_doc.rst @@ -141,3 +141,122 @@ Client 套用 server 已有的批次不會改變任何東西。送出失敗時退避 2 秒、4 秒、8 秒……最多 300 秒 (``base_delay_s`` / ``max_delay_s``),一次 drain 最多送 ``max_attempts`` 次(預設 5), ``wait=False`` 會直接返回而不在退避期間睡眠,設定 ``cancel`` 即使在等待中途也會結束 drain。 + +Adapter:同步什麼、什麼留在本機 +-------------------------------- + +``je_auto_control.utils.config_sync.adapters`` 把一個 bucket section 接到一個本機儲存。 +``SyncAdapter.snapshot()`` 回傳以上次合併狀態為基準的 ``{key: SyncEntry}``, +``apply(entries)`` 把合併後的 entry 寫回並回傳 ``ApplyReport`` +(``written`` / ``removed`` / ``skipped`` / ``conflicts`` / ``left_disabled``)。 + +.. list-table:: + :header-rows: 1 + + * - Section + - Adapter + - 本機儲存 + * - ``scripts`` + - ``ScriptSyncAdapter(origin, scripts_dir)`` + - 資料夾下的 ``*.json`` 動作檔,以相對路徑為 key + * - ``locators`` + - ``LocatorSyncAdapter(origin, repository)`` + - 一個 ``ElementRepository`` + * - ``hotkeys`` + - ``HotkeySyncAdapter(origin, daemon, scripts_dir=...)`` + - ``HotkeyDaemon`` 的綁定 + * - ``triggers`` + - ``TriggerSyncAdapter(origin, engine, scripts_dir=...)`` + - ``TriggerEngine`` 的 image / window / pixel / file / cron 觸發器 + * - ``address_book`` + - ``AddressBookSyncAdapter(origin, book)`` + - 遠端桌面的 ``AddressBook`` + +每個 adapter 都遵守三條規則: + +* **秘密留在本機。** 名稱顯示為秘密的欄位(``password``、``token``、``api_key`` ……) + 以 ``{"$local": "secret"}`` 送出;接收端保留自己的值。``${secrets.NAME}`` 參照原樣傳送。 + 含有明文秘密的腳本完全不同步,並列在 ``withheld``。 +* **機器路徑留在本機。** ``scripts_dir`` 內的路徑以相對參照傳送,並對接收端自己的資料夾解析。 + 其他路徑變成 ``{"$local": "path"}``;需要它而本機沒有對應值的項目回報在 ``skipped`` + —— 而且它在本機不存在 *不會* 被當成對其他機器的刪除。 +* **同步絕不啟用任何東西。** ``enabled`` 不同步。送達的快捷鍵或觸發器以 **停用** 狀態建立 + (``HotkeyDaemon.bind(..., enabled=False)``),既有的保留本機給它的狀態, + 沒有任何 adapter 會啟動引擎或執行腳本。複合觸發器(all-of / any-of / sequence)不同步。 + +資產 +---- + +64 KiB 以內的腳本連同它的 SHA-256 放在 entry 內傳送;內容與 hash 不符就不寫入。 +更大的腳本與其他檔案(樣板圖)經由 ``AssetTransport`` 傳送: + +.. code-block:: python + + from je_auto_control.utils.config_sync.assets import ( + AssetManifest, DirectoryAssetTransport, publish_assets, sync_assets) + + transport = DirectoryAssetTransport("//nas/autocontrol-assets") # 兩台機器都能存取的資料夾 + manifest = AssetManifest.from_directory("scripts", ("*.png",)) + publish_assets(manifest, transport) # 傳送端 + result = sync_assets(AssetManifest(root=their_scripts, assets=manifest.assets), transport) + # AssetSyncResult(transferred, unchanged, failed, hashes, cancelled) + +``sync_assets`` 在寫入 *之前* 以 SHA-256 與大小檢查每個檔案、以原子方式取代目的檔、 +檢查失敗時不動既有檔案,並拒絕會離開資料夾的路徑。隨附的 transport 是 +``DirectoryAssetTransport``;其他 blob 儲存請實作 ``fetch`` / ``store`` / ``has``。 + +一個呼叫,三個介面 +------------------ + +``config_sync_run(server_url, user_id, **options)`` 執行整個循環 —— 把本機變更入列、 +清空 outbox、合併、套用 —— 並回傳 +``{state, revision, pending, conflicts, applied, withheld, assets, error}``, +``state`` 為 ``synced`` / ``pending`` / ``conflict`` / ``offline`` / ``cancelled`` / +``resync_required`` 之一。連不到 server 不是例外:變更留在佇列,狀態為 ``offline``。 + +選項:``device_id``(預設:在 ``~/.je_auto_control/config_sync_device_id`` 建立一次的 id)、 +``secret``(預設 ``$AC_SIGNALING_SECRET``)、``sections``(預設 ``hotkeys``、``triggers``、 +``address_book``;給了路徑時加入 ``scripts`` 與 ``locators``)、``scripts_dir``、 +``locators_path``、``outbox_path``、``assets_dir``、``timeout_s``、``wait``、``max_attempts``。 + +.. list-table:: + :header-rows: 1 + + * - Executor 指令 + - MCP 工具 + - 作用 + * - ``AC_config_sync_run`` + - ``ac_config_sync_run`` + - 同步一次 + * - ``AC_config_sync_status`` + - ``ac_config_sync_status``(唯讀) + - 已記錄的狀態;不連網、不建立任何東西 + * - ``AC_config_sync_resolve`` + - ``ac_config_sync_resolve`` + - 保留 ``section`` / ``key`` 的第 ``choice`` 個候選 + * - ``AC_config_sync_full_resync`` + - ``ac_config_sync_full_resync`` + - 被退休後採用 server 的狀態;待送變更會被捨棄並列出 + +四個都是 Script Builder **Data** 分類下的指令。 + +.. code-block:: json + + [["AC_config_sync_run", {"server_url": "https://sync.example", "user_id": "alice", + "secret": "${secrets.sync}", "scripts_dir": "scripts"}]] + +GUI +--- + +**設定同步** 分頁(分類 *system*)顯示狀態、最後合併的 revision、待送變更數、 +上次成功同步與最後的錯誤,並列出每個衝突及其候選。它的指令 —— *立即同步*、*取消同步*、 +*重新整理同步狀態*、*保留所選候選*、*完整重新同步* —— 在 Actions 選單。 +同步在 worker 執行緒上執行;取消或關閉分頁都會釋放它。 + +資料夾鏡像與剪貼簿:不回送 +-------------------------- + +``FolderSyncEngine.note_received(remote_name, sha256=...)`` 把監看資料夾中的檔案標記為 +來自對端;在內容於本機變更之前,引擎不會把它推回去。``FolderSyncEngine.poll_once()`` +可隨時執行一次差異比對。``ClipboardEchoGuard``(``note_remote`` / ``should_send`` / ``reset``) +為自動轉送剪貼簿變更的程式做同樣的事:剛從對端送達、或已經送過的內容不會再送一次。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 475c40ca2..b16581fee 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -265,6 +265,13 @@ ) # Named locator repository (object repository) for native UI from je_auto_control.utils.element_repository import ElementRepository +# Cross-machine config sync: causal merge, offline outbox, adapters, assets +from je_auto_control.utils.config_sync import ( + AssetManifest, ConfigBucket, ConfigStore, ConfigSyncClient, ConfigSyncConflict, + ConfigSyncError, FullResyncRequired, MergeDecision, SyncAdapter, SyncEntry, + SyncOperation, SyncOutbox, config_sync_full_resync, config_sync_resolve, + config_sync_run, config_sync_status, merge_entries, sync_assets, +) # Step-through debugger / tracer for action lists from je_auto_control.utils.flow_debugger import FlowDebugger, trace_actions # Persistent library of reusable action sequences (skills/playbooks) @@ -1683,6 +1690,10 @@ def start_autocontrol_gui(*args, **kwargs): "Asset", "AssetStore", "AssetValue", "active_environment", "dotenv_values", "dump_dotenv", "load_dotenv", "parse_dotenv", "LayeredConfig", "SourceTrace", "deep_merge", + "AssetManifest", "ConfigBucket", "ConfigStore", "ConfigSyncClient", "ConfigSyncConflict", + "ConfigSyncError", "FullResyncRequired", "MergeDecision", "SyncAdapter", "SyncEntry", + "SyncOperation", "SyncOutbox", "config_sync_full_resync", "config_sync_resolve", + "config_sync_run", "config_sync_status", "merge_entries", "sync_assets", "ConfigField", "ConfigSchema", "coerce", "validate_config", "PathNotAllowedError", "PathPolicy", "RefResolver", "SecretRefError", "is_ref", "resolve_ref", "resolve_refs_in", diff --git a/je_auto_control/actions.pyi b/je_auto_control/actions.pyi index 35e4e39ee..ec337387f 100644 --- a/je_auto_control/actions.pyi +++ b/je_auto_control/actions.pyi @@ -705,6 +705,25 @@ def AC_config_export() -> Dict[str, Any]: def AC_config_import(bundle: Dict[str, Any], dry_run: bool = ...) -> Dict[str, Any]: """Executor adapter: apply a config bundle dict to the user config root.""" +def AC_config_sync_full_resync(server_url: str, user_id: str, **options: Any) -> Dict[str, Any]: + """Adopt the server's state after this device was retired; see :func:`run_full_resync`.""" + +def AC_config_sync_resolve( + server_url: str, + user_id: str, + section: str, + key: str, + choice: int, + **options: Any, +) -> Dict[str, Any]: + """Pick one side of a conflict; see :func:`resolve_conflict`.""" + +def AC_config_sync_run(server_url: str, user_id: str, *, cancel: Any = ..., **options: Any) -> Dict[str, Any]: + """Sync this machine's settings with ``server_url`` once; returns the report.""" + +def AC_config_sync_status(server_url: str, user_id: str, outbox_path: Optional[str] = ...) -> Dict[str, Any]: + """The recorded sync state for this account and server; no network.""" + def AC_confusable_compare(first: str, second: str) -> Dict[str, Any]: """Adapter: whether two strings render to the same skeleton.""" diff --git a/je_auto_control/gui/config_sync_tab.py b/je_auto_control/gui/config_sync_tab.py new file mode 100644 index 000000000..4af5e6366 --- /dev/null +++ b/je_auto_control/gui/config_sync_tab.py @@ -0,0 +1,236 @@ +"""Config Sync tab: this machine's sync state, and the commands that change it. + +A thin view over :mod:`je_auto_control.utils.config_sync.session` -- every +command here is one call into it, so the same sync runs from a script +(``AC_config_sync_run``) or MCP with no GUI at all. The tab shows the last +merged revision, how many changes still wait in the outbox, whether the last +attempt reached the server, and the entries where two machines changed the +same thing and a person has to choose. + +Syncing runs on a worker thread with a cancel event; closing the tab sets +the event, so a sync waiting on the network does not outlive its view. +""" +import threading +import time +from typing import Any, Callable, Dict, List, Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QFormLayout, QHeaderView, QLabel, QLineEdit, QMessageBox, QTableWidget, + QTableWidgetItem, QVBoxLayout, QWidget, +) + +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui._worker_thread import CallWorker, WorkerHandle, start_worker +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.utils.config_sync import session +from je_auto_control.utils.exception.exceptions import AutoControlException + +_FIELDS = ("server", "user", "secret", "scripts", "assets") +_COLUMNS = ("entry", "choice", "origin", "value") + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +def _release(events: List[threading.Event]) -> None: + """Set every cancel event in ``events``.""" + for event in events: + event.set() + + +class ConfigSyncTab(TranslatableMixin, QWidget): + """Status and commands for cross-machine config sync.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tr_init() + self._inputs: Dict[str, QLineEdit] = {name: QLineEdit() for name in _FIELDS} + self._inputs["secret"].setEchoMode(QLineEdit.Password) + self._state = QLabel() + self._detail = QLabel() + self._detail.setWordWrap(True) + self._conflicts = QTableWidget(0, len(_COLUMNS)) + self._status: Dict[str, Any] = {"state": "never"} + self._worker: Optional[WorkerHandle] = None + # Shared with the destroyed-signal handler, which must not touch self. + self._cancel_slot: List[threading.Event] = [] + slot = self._cancel_slot + self.destroyed.connect(lambda *_args: _release(slot)) + self._build_layout() + self._render() + + # --- layout ------------------------------------------------------------ + + def _build_layout(self) -> None: + # Sync / cancel / refresh / resolve / full resync run from the Actions + # menu; the tab keeps the inputs, the status lines and the conflicts. + root = QVBoxLayout(self) + form = QFormLayout() + for name in _FIELDS: + form.addRow(self._tr(QLabel(), f"config_sync_{name}_label"), self._inputs[name]) + root.addLayout(form) + root.addWidget(self._state) + root.addWidget(self._detail) + root.addWidget(self._tr(QLabel(), "config_sync_conflicts_title")) + root.addWidget(self._conflicts, stretch=1) + self._conflicts.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch) + self._conflicts.setSelectionBehavior(QTableWidget.SelectRows) + self._apply_translations() + + def _apply_translations(self) -> None: + self._conflicts.setHorizontalHeaderLabels( + [_t(f"config_sync_col_{column}") for column in _COLUMNS]) + + def retranslate(self) -> None: + """Re-apply the translated texts after a language change.""" + TranslatableMixin.retranslate(self) + self._apply_translations() + self._render() + + def menu_actions(self) -> list: + """Expose tab commands to the window-level Actions menu.""" + return [ + ("config_sync_run_btn", self.sync_now), + ("config_sync_cancel_btn", self.cancel), + ("config_sync_refresh_btn", self.refresh_status), + ("config_sync_resolve_btn", self.resolve_selected), + ("config_sync_resync_btn", self.full_resync), + ] + + # --- inputs ------------------------------------------------------------ + + def _settings(self) -> Optional[Dict[str, Any]]: + """The form as session arguments, or ``None`` (with a message) if incomplete.""" + values = {name: field.text().strip() for name, field in self._inputs.items()} + if not values["server"] or not values["user"]: + self._detail.setText(_t("config_sync_required")) + return None + options = {"secret": values["secret"], "scripts_dir": values["scripts"], + "assets_dir": values["assets"]} + return {"server_url": values["server"], "user_id": values["user"], + "options": {name: value for name, value in options.items() if value}} + + def is_busy(self) -> bool: + """Whether a sync is running.""" + return self._worker is not None and self._worker.isRunning() + + # --- commands ---------------------------------------------------------- + + def sync_now(self) -> None: + """Start one sync on a worker thread.""" + settings = self._settings() + if settings is None: + return + cancel = threading.Event() + self._start(lambda: session.config_sync_run( + settings["server_url"], settings["user_id"], cancel=cancel, + **settings["options"]), cancel) + + def full_resync(self) -> None: + """Adopt the server's state, discarding this machine's pending changes.""" + settings = self._settings() + if settings is None or not self._confirm(_t("config_sync_resync_confirm")): + return + self._start(lambda: session.config_sync_full_resync( + settings["server_url"], settings["user_id"], **settings["options"]), + threading.Event()) + + def _confirm(self, question: str) -> bool: + answer = QMessageBox.question(self, _t("tab_config_sync"), question) + return answer == QMessageBox.Yes + + def _start(self, call: Callable[[], Dict[str, Any]], cancel: threading.Event) -> None: + if self.is_busy(): + self._detail.setText(_t("config_sync_busy")) + return + self._cancel_slot[:] = [cancel] + self._status = {**self._status, "state": "syncing", "error": ""} + self._render() + self._worker = start_worker( + self, CallWorker(call), on_done=self._on_report, on_fail=self._on_failed, + on_thread_done=self._on_thread_done) + + def cancel(self) -> None: + """Ask the running sync to stop; its changes stay in the outbox.""" + _release(self._cancel_slot) + + def refresh_status(self) -> None: + """Re-read the recorded state from the outbox; no network.""" + settings = self._settings() + if settings is None: + return + try: + self._status = session.config_sync_status(settings["server_url"], settings["user_id"]) + except AutoControlException as error: + self._status = {**self._status, "error": str(error)} + self._render() + + def resolve_selected(self) -> None: + """Keep the selected candidate of a conflict; the next sync sends the choice.""" + settings = self._settings() + row = self._conflicts.currentRow() + item = self._conflicts.item(row, 0) if row >= 0 else None + if settings is None: + return + if item is None: + self._detail.setText(_t("config_sync_no_conflict_selected")) + return + section, key, choice = item.data(Qt.UserRole) + try: + session.config_sync_resolve(settings["server_url"], settings["user_id"], + section, key, choice, **settings["options"]) + except AutoControlException as error: + self._detail.setText(f"{_t('config_sync_error')}: {error}") + return + self.refresh_status() + + # --- worker results ---------------------------------------------------- + + def _on_report(self, report: Dict[str, Any]) -> None: + self._status = dict(report) + self.refresh_status() + + def _on_failed(self, message: str) -> None: + self._status = {**self._status, "state": "offline", "error": message} + self._render() + + def _on_thread_done(self) -> None: + self._worker = None + self._cancel_slot[:] = [] + + # --- rendering --------------------------------------------------------- + + def _render(self) -> None: + status = self._status + state = str(status.get("state", "never")) + self._state.setText(f"{_t('config_sync_state_label')}: {_t(f'config_sync_state_{state}')}") + succeeded = float(status.get("last_success") or 0.0) + when = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(succeeded)) if succeeded else "-" + lines = [ + f"{_t('config_sync_revision')}: {int(status.get('revision') or 0)}", + f"{_t('config_sync_pending')}: {int(status.get('pending') or 0)}", + f"{_t('config_sync_last_success')}: {when}", + ] + if status.get("error"): + lines.append(f"{_t('config_sync_error')}: {status['error']}") + self._detail.setText("\n".join(lines)) + self._render_conflicts(status.get("conflict_details") or []) + + def _render_conflicts(self, details: List[Dict[str, Any]]) -> None: + rows = [(detail, index, choice) for detail in details + for index, choice in enumerate(detail.get("choices", []))] + self._conflicts.setRowCount(len(rows)) + for row, (detail, index, choice) in enumerate(rows): + value = _t("config_sync_deleted") if choice.get("deleted") else str(choice.get("value")) + cells = (f"{detail['section']}/{detail['key']}", str(index), + str(choice.get("origin", "")), value) + for column, text in enumerate(cells): + item = QTableWidgetItem(text) + item.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) + item.setData(Qt.UserRole, (detail["section"], detail["key"], index)) + self._conflicts.setItem(row, column, item) + + +__all__ = ["ConfigSyncTab"] diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 6bc8d1ebe..37e1820ad 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -1095,6 +1095,42 @@ "presence_col_cursor": "Cursor", "presence_col_last_seen_iso": "Last seen", + # Config Sync Tab + "tab_config_sync": "Config Sync", + "config_sync_server_label": "Sync server URL:", + "config_sync_user_label": "User id:", + "config_sync_secret_label": "Shared secret:", + "config_sync_scripts_label": "Scripts folder (optional):", + "config_sync_assets_label": "Shared assets folder (optional):", + "config_sync_conflicts_title": "Conflicts: entries changed on two machines at once", + "config_sync_col_entry": "Entry", + "config_sync_col_choice": "Choice", + "config_sync_col_origin": "Changed on", + "config_sync_col_value": "Value", + "config_sync_run_btn": "Sync now", + "config_sync_cancel_btn": "Cancel sync", + "config_sync_refresh_btn": "Refresh sync status", + "config_sync_resolve_btn": "Keep selected candidate", + "config_sync_resync_btn": "Full resync (discard pending changes)", + "config_sync_required": "Server URL and user id are required", + "config_sync_busy": "A sync is already running", + "config_sync_resync_confirm": "Adopt the server's state and discard this machine's pending changes?", + "config_sync_no_conflict_selected": "Select a conflict candidate first", + "config_sync_error": "Error", + "config_sync_deleted": "(deleted)", + "config_sync_state_label": "State", + "config_sync_state_never": "never synced", + "config_sync_state_syncing": "syncing...", + "config_sync_state_synced": "synced", + "config_sync_state_pending": "changes waiting to be sent", + "config_sync_state_conflict": "conflict: a choice is needed", + "config_sync_state_offline": "offline: changes are queued", + "config_sync_state_cancelled": "cancelled", + "config_sync_state_resync_required": "retired: a full resync is required", + "config_sync_revision": "Last merged revision", + "config_sync_pending": "Pending changes", + "config_sync_last_success": "Last successful sync", + # Chat-Ops Tab "chatops_root_label": "Script root:", "chatops_root_placeholder": "directory of .json scripts /run can execute", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 0740bfc51..09efe0d65 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -984,6 +984,42 @@ "presence_col_cursor": "カーソル", "presence_col_last_seen_iso": "最終時刻", + # Config Sync Tab + "tab_config_sync": "設定同期", + "config_sync_server_label": "同期サーバー URL:", + "config_sync_user_label": "ユーザー ID:", + "config_sync_secret_label": "共有シークレット:", + "config_sync_scripts_label": "スクリプトフォルダー (任意):", + "config_sync_assets_label": "共有アセットフォルダー (任意):", + "config_sync_conflicts_title": "競合: 2 台で同時に変更された項目", + "config_sync_col_entry": "項目", + "config_sync_col_choice": "候補", + "config_sync_col_origin": "変更元", + "config_sync_col_value": "値", + "config_sync_run_btn": "今すぐ同期", + "config_sync_cancel_btn": "同期をキャンセル", + "config_sync_refresh_btn": "同期状態を更新", + "config_sync_resolve_btn": "選択した候補を採用", + "config_sync_resync_btn": "完全再同期 (保留中の変更を破棄)", + "config_sync_required": "サーバー URL とユーザー ID は必須です", + "config_sync_busy": "同期はすでに実行中です", + "config_sync_resync_confirm": "サーバーの状態を採用し、このマシンの保留中の変更を破棄しますか?", + "config_sync_no_conflict_selected": "先に競合の候補を選択してください", + "config_sync_error": "エラー", + "config_sync_deleted": "(削除済み)", + "config_sync_state_label": "状態", + "config_sync_state_never": "未同期", + "config_sync_state_syncing": "同期中...", + "config_sync_state_synced": "同期済み", + "config_sync_state_pending": "送信待ちの変更あり", + "config_sync_state_conflict": "競合: 選択が必要です", + "config_sync_state_offline": "オフライン: 変更はキューに保存されています", + "config_sync_state_cancelled": "キャンセルされました", + "config_sync_state_resync_required": "退役済み: 完全再同期が必要です", + "config_sync_revision": "最後にマージしたリビジョン", + "config_sync_pending": "保留中の変更", + "config_sync_last_success": "最後に成功した同期", + # Chat-Ops Tab "chatops_root_label": "スクリプトディレクトリ:", "chatops_root_placeholder": "/run で実行可能な .json スクリプトの場所", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index d002ab239..55f29028f 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -967,6 +967,42 @@ "presence_col_cursor": "光标", "presence_col_last_seen_iso": "最后活跃", + # Config Sync Tab + "tab_config_sync": "配置同步", + "config_sync_server_label": "同步服务器 URL:", + "config_sync_user_label": "用户 ID:", + "config_sync_secret_label": "共享密钥:", + "config_sync_scripts_label": "脚本文件夹(可选):", + "config_sync_assets_label": "共享资源文件夹(可选):", + "config_sync_conflicts_title": "冲突:两台机器同时修改的条目", + "config_sync_col_entry": "条目", + "config_sync_col_choice": "候选", + "config_sync_col_origin": "修改来源", + "config_sync_col_value": "值", + "config_sync_run_btn": "立即同步", + "config_sync_cancel_btn": "取消同步", + "config_sync_refresh_btn": "刷新同步状态", + "config_sync_resolve_btn": "保留所选候选", + "config_sync_resync_btn": "完整重新同步(丢弃待发送的更改)", + "config_sync_required": "必须填写服务器 URL 和用户 ID", + "config_sync_busy": "同步正在进行中", + "config_sync_resync_confirm": "采用服务器的状态并丢弃本机待发送的更改?", + "config_sync_no_conflict_selected": "请先选择一个冲突候选", + "config_sync_error": "错误", + "config_sync_deleted": "(已删除)", + "config_sync_state_label": "状态", + "config_sync_state_never": "尚未同步", + "config_sync_state_syncing": "同步中...", + "config_sync_state_synced": "已同步", + "config_sync_state_pending": "有更改等待发送", + "config_sync_state_conflict": "冲突:需要选择", + "config_sync_state_offline": "离线:更改已排队", + "config_sync_state_cancelled": "已取消", + "config_sync_state_resync_required": "已退役:需要完整重新同步", + "config_sync_revision": "最后合并的修订版", + "config_sync_pending": "待发送的更改", + "config_sync_last_success": "上次成功同步", + # Chat-Ops Tab "chatops_root_label": "脚本目录：", "chatops_root_placeholder": "可被 /run 执行的 .json 脚本目录", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 046af99cd..1093a973d 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -968,6 +968,42 @@ "presence_col_cursor": "游標", "presence_col_last_seen_iso": "最後活躍", + # Config Sync Tab + "tab_config_sync": "設定同步", + "config_sync_server_label": "同步伺服器 URL:", + "config_sync_user_label": "使用者 ID:", + "config_sync_secret_label": "共享密鑰:", + "config_sync_scripts_label": "腳本資料夾(選填):", + "config_sync_assets_label": "共享資產資料夾(選填):", + "config_sync_conflicts_title": "衝突:兩台機器同時修改的項目", + "config_sync_col_entry": "項目", + "config_sync_col_choice": "候選", + "config_sync_col_origin": "修改來源", + "config_sync_col_value": "值", + "config_sync_run_btn": "立即同步", + "config_sync_cancel_btn": "取消同步", + "config_sync_refresh_btn": "重新整理同步狀態", + "config_sync_resolve_btn": "保留所選候選", + "config_sync_resync_btn": "完整重新同步(捨棄待送出的變更)", + "config_sync_required": "必須填寫伺服器 URL 與使用者 ID", + "config_sync_busy": "同步正在進行中", + "config_sync_resync_confirm": "採用伺服器的狀態並捨棄本機待送出的變更?", + "config_sync_no_conflict_selected": "請先選擇一個衝突候選", + "config_sync_error": "錯誤", + "config_sync_deleted": "(已刪除)", + "config_sync_state_label": "狀態", + "config_sync_state_never": "尚未同步", + "config_sync_state_syncing": "同步中...", + "config_sync_state_synced": "已同步", + "config_sync_state_pending": "有變更等待送出", + "config_sync_state_conflict": "衝突:需要選擇", + "config_sync_state_offline": "離線:變更已排入佇列", + "config_sync_state_cancelled": "已取消", + "config_sync_state_resync_required": "已退休:需要完整重新同步", + "config_sync_revision": "最後合併的修訂版", + "config_sync_pending": "待送出的變更", + "config_sync_last_success": "上次成功同步", + # Chat-Ops Tab "chatops_root_label": "腳本目錄：", "chatops_root_placeholder": "可被 /run 執行的 .json 腳本目錄", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index a9638555d..2d4ec2016 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -3429,6 +3429,46 @@ def _add_resilience_specs(specs: List[CommandSpec]) -> None: ), description="Show the value and winning layer for a dotted config key.", )) + sync_target = ( + FieldSpec("server_url", FieldType.STRING, placeholder="https://sync.example"), + FieldSpec("user_id", FieldType.STRING, placeholder="alice"), + ) + sync_options = ( + FieldSpec("secret", FieldType.STRING, optional=True, placeholder="${secrets.sync}"), + FieldSpec("device_id", FieldType.STRING, optional=True, + placeholder="(this machine's stored id)"), + FieldSpec("sections", FieldType.STRING, optional=True, + placeholder="hotkeys,triggers,address_book"), + FieldSpec("scripts_dir", FieldType.FILE_PATH, optional=True), + FieldSpec("locators_path", FieldType.FILE_PATH, optional=True), + FieldSpec("assets_dir", FieldType.FILE_PATH, optional=True), + ) + specs.append(CommandSpec( + "AC_config_sync_run", "Data", "Config Sync: Sync Now", + fields=sync_target + sync_options, + description="Sync settings with the server once. Received hotkeys and " + "triggers arrive disabled; nothing is run.", + )) + specs.append(CommandSpec( + "AC_config_sync_status", "Data", "Config Sync: Status", + fields=sync_target, + description="Recorded sync state (revision, pending, conflicts); no network.", + )) + specs.append(CommandSpec( + "AC_config_sync_resolve", "Data", "Config Sync: Resolve Conflict", + fields=sync_target + ( + FieldSpec("section", FieldType.STRING, placeholder="hotkeys"), + FieldSpec("key", FieldType.STRING), + FieldSpec("choice", FieldType.INT, default=0), + ) + sync_options, + description="Keep one candidate of a conflicted entry (0-based choice).", + )) + specs.append(CommandSpec( + "AC_config_sync_full_resync", "Data", "Config Sync: Full Resync", + fields=sync_target + sync_options, + description="Adopt the server's state after this device was retired; " + "pending local changes are discarded.", + )) specs.append(CommandSpec( "AC_detect_drift", "Data", "Data Drift: Detect (PSI + KS)", fields=( diff --git a/je_auto_control/gui/tab_registry.py b/je_auto_control/gui/tab_registry.py index dd9633464..c7345b0de 100644 --- a/je_auto_control/gui/tab_registry.py +++ b/je_auto_control/gui/tab_registry.py @@ -113,6 +113,7 @@ def widget(self) -> Any: TabSpec("trace_replay", "tab_trace_replay", "automation", f"{_GUI}.trace_replay_tab", "TraceReplayTab"), TabSpec("remote_desktop", "tab_remote_desktop", "system", default_visible=True), TabSpec("presence", "tab_presence", "system", f"{_GUI}.presence_tab", "PresenceTab"), + TabSpec("config_sync", "tab_config_sync", "system", f"{_GUI}.config_sync_tab", "ConfigSyncTab"), TabSpec("rest_api", "tab_rest_api", "system", f"{_GUI}.rest_api_tab", "RestApiTab"), TabSpec("admin_console", "tab_admin_console", "system", f"{_GUI}.admin_console_tab", "AdminConsoleTab"), TabSpec("audit_log", "tab_audit_log", "system", f"{_GUI}.audit_log_tab", "AuditLogTab"), diff --git a/je_auto_control/utils/config_sync/__init__.py b/je_auto_control/utils/config_sync/__init__.py index 14c8b0aa4..777353226 100644 --- a/je_auto_control/utils/config_sync/__init__.py +++ b/je_auto_control/utils/config_sync/__init__.py @@ -22,6 +22,13 @@ to one key made apart are *both kept* as a conflict. No clock is consulted. * :mod:`.outbox` -- :class:`SyncOutbox`, the durable queue of changes not yet on the server (resent by operation id, bounded back-off, cancellable). +* :mod:`.adapters` -- :class:`SyncAdapter` and the script, locator, hotkey, + trigger and address-book adapters. Secrets and machine paths leave only as + references, and applying synced data never enables a hotkey or trigger. +* :mod:`.assets` -- :func:`sync_assets`: files fetched by SHA-256, verified, + then replaced atomically. +* :mod:`.session` -- :func:`run_sync`, the whole cycle behind the GUI tab, + the ``AC_config_sync_*`` commands and the MCP tools. Each section maps an entry id to an entry. Section names are free-form; ``hotkeys``, ``triggers``, ``address_book`` and ``custom`` are the @@ -35,6 +42,14 @@ keep the older rule: the later ``last_modified`` wins and the loser is reported in a ``ConflictRecord``. """ +from je_auto_control.utils.config_sync.adapters import ( + AddressBookSyncAdapter, ApplyReport, HotkeySyncAdapter, LocatorSyncAdapter, + ScriptSyncAdapter, SyncAdapter, TriggerSyncAdapter, +) +from je_auto_control.utils.config_sync.assets import ( + AssetManifest, AssetRef, AssetSyncError, AssetSyncResult, AssetTransport, + DirectoryAssetTransport, publish_assets, sync_assets, +) from je_auto_control.utils.config_sync.client import ( DEFAULT_SYNC_ATTEMPTS, TOMBSTONE_RETENTION_S, WIRE_VERSION, ConflictRecord, ConfigBucket, ConfigSyncClient, ConfigSyncConflict, ConfigSyncError, FullResyncRequired, SyncResult, @@ -43,6 +58,11 @@ from je_auto_control.utils.config_sync.outbox import ( DrainReport, OutboxError, SyncOutbox, default_outbox_path, ) +from je_auto_control.utils.config_sync.session import ( + SyncRunReport, config_sync_full_resync, config_sync_resolve, config_sync_run, + config_sync_status, default_adapters, default_device_id, resolve_conflict, + run_full_resync, run_sync, sync_status, +) from je_auto_control.utils.config_sync.store import ( ConfigStore, ConfigStoreError, RevisionConflictError, StoreCapacityError, default_store_path, @@ -53,6 +73,12 @@ ) __all__ = [ + "AddressBookSyncAdapter", "ApplyReport", "AssetManifest", "AssetRef", "AssetSyncError", + "AssetSyncResult", "AssetTransport", "DirectoryAssetTransport", "HotkeySyncAdapter", + "LocatorSyncAdapter", "ScriptSyncAdapter", "SyncAdapter", "SyncRunReport", + "TriggerSyncAdapter", "config_sync_full_resync", "config_sync_resolve", "config_sync_run", + "config_sync_status", "default_adapters", "default_device_id", "publish_assets", + "resolve_conflict", "run_full_resync", "run_sync", "sync_assets", "sync_status", "ConfigBucket", "ConfigStore", "ConfigStoreError", "ConflictRecord", "ConfigSyncClient", "ConfigSyncConflict", "ConfigSyncError", "DEFAULT_SYNC_ATTEMPTS", "DrainReport", "FullResyncRequired", "MergeDecision", "OutboxError", "PeerState", "RevisionConflictError", diff --git a/je_auto_control/utils/config_sync/adapters.py b/je_auto_control/utils/config_sync/adapters.py new file mode 100644 index 000000000..467abedbf --- /dev/null +++ b/je_auto_control/utils/config_sync/adapters.py @@ -0,0 +1,531 @@ +"""Adapters between local AutoControl state and synced entries. + +Each :class:`SyncAdapter` owns one bucket section. :meth:`SyncAdapter.snapshot` +turns what this machine holds into :class:`SyncEntry` values, measured +against the *baseline* (the state last merged) so only real changes get a +new version; :meth:`SyncAdapter.apply` writes merged entries back. + +Three rules hold for every adapter: + +* **Secrets stay here.** A field whose name marks it as a secret is replaced + by a ``{"$local": "secret"}`` reference before it leaves, and a script + containing a literal secret is not synced at all. The receiving machine + keeps its own value for that field. +* **Machine paths stay here.** A path inside the adapter's script directory + travels as a relative reference and is resolved against the receiver's own + directory; any other path becomes a ``{"$local": "path"}`` reference. +* **Syncing never arms anything.** ``enabled`` is not synced. A hotkey or + trigger that arrives is created disabled, one that already exists keeps + the state this machine gave it, and nothing here starts an engine or runs + a script. + +Pure standard library; imports no ``PySide6``. +""" +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field, fields +from pathlib import Path, PurePosixPath +from typing import Any, ClassVar, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union + +from je_auto_control.utils.config_sync.client import ConfigSyncError +from je_auto_control.utils.config_sync.versions import SyncEntry, SyncOperation +from je_auto_control.utils.exception.exceptions import AutoControlException + +#: Marks a value that only means something on the machine it came from. +LOCAL_REF = "$local" +#: A script up to this size travels inside its entry; a larger one is listed +#: by hash and has to come through an asset transport. +MAX_INLINE_SCRIPT_BYTES = 64 * 1024 + + +def is_local_ref(value: Any) -> bool: + """Whether ``value`` is a reference to something kept on its own machine.""" + return isinstance(value, Mapping) and LOCAL_REF in value + + +@dataclass +class ApplyReport: + """What :meth:`SyncAdapter.apply` did to this machine. + + ``skipped`` maps a key to why it was left alone; ``conflicts`` lists the + keys still waiting for a choice between concurrent changes; + ``left_disabled`` the hotkeys / triggers that arrived and were not armed. + """ + section: str = "" + written: List[str] = field(default_factory=list) + removed: List[str] = field(default_factory=list) + skipped: Dict[str, str] = field(default_factory=dict) + conflicts: List[str] = field(default_factory=list) + left_disabled: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + """A JSON-ready copy.""" + return {"section": self.section, "written": list(self.written), + "removed": list(self.removed), "skipped": dict(self.skipped), + "conflicts": list(self.conflicts), "left_disabled": list(self.left_disabled)} + + +class SyncAdapter: + """One section of the bucket, backed by some local store. + + Subclasses provide :meth:`read_local`, :meth:`write_local` and + :meth:`delete_local`; the portable-form rules and the baseline + bookkeeping are here. + """ + + section: ClassVar[str] = "" + #: Fields holding filesystem paths, made portable through ``scripts_dir``. + path_fields: ClassVar[Tuple[str, ...]] = () + #: Fields that describe this machine only and are never sent. + local_fields: ClassVar[Tuple[str, ...]] = () + + def __init__(self, origin: str, *, scripts_dir: Union[str, Path, None] = None) -> None: + if not origin: + raise ConfigSyncError("an adapter needs this device's id") + self.origin = origin + self._scripts_dir = None if scripts_dir is None else Path(scripts_dir) + self._baseline: Dict[str, SyncEntry] = {} + self._unapplied: Set[str] = set() + + def rebase(self, baseline: Mapping[str, SyncEntry], unapplied: Iterable[str] = ()) -> None: + """Set the entries local changes are measured against. + + ``unapplied`` names baseline keys that could not be written to this + machine (a path that does not exist here, a key combination this + platform refuses). Their absence here is not a deletion. + """ + self._baseline = dict(baseline) + self._unapplied = set(unapplied) + + # --- the local store (subclass responsibility) --------------------------- + + def read_local(self) -> Dict[str, Dict[str, Any]]: + """This machine's items, by key.""" + raise NotImplementedError + + def write_local(self, key: str, value: Dict[str, Any], + existing: Optional[Dict[str, Any]]) -> None: + """Create or replace ``key``; ``existing`` is the current local item.""" + raise NotImplementedError + + def delete_local(self, key: str) -> None: + """Remove ``key`` from this machine.""" + raise NotImplementedError + + # --- portable form ------------------------------------------------------- + + def _portable_path(self, name: str, raw: Any) -> Any: + if not isinstance(raw, str) or not raw: + return raw + root = self._scripts_dir + if root is not None: + try: + relative = Path(raw).resolve().relative_to(root.resolve()) + except (OSError, ValueError): + relative = None + if relative is not None: + return {LOCAL_REF: "path", "root": "scripts", "relative": relative.as_posix()} + return {LOCAL_REF: "path", "field": name, "name": Path(raw).name} + + def _local_path(self, reference: Any) -> Optional[str]: + """The local path a reference names, or ``None`` when it has none here.""" + if not is_local_ref(reference): + return reference if isinstance(reference, str) else None + parts = _contained_parts(reference.get("relative")) + if self._scripts_dir is None or reference.get("root") != "scripts" or parts is None: + return None + return str(self._scripts_dir.joinpath(*parts)) + + def to_portable(self, value: Mapping[str, Any]) -> Dict[str, Any]: + """``value`` as it may leave this machine: no secrets, no local paths.""" + from je_auto_control.utils.secret_ref import is_ref + from je_auto_control.utils.secrets_scan.secrets_scan import is_secret_key, scan_secrets + portable: Dict[str, Any] = {} + for name, item in value.items(): + if name in self.local_fields or name.startswith("_"): + continue + if name in self.path_fields: + portable[name] = self._portable_path(name, item) + elif is_local_ref(item) or is_ref(item): + portable[name] = item + elif (scan_secrets({name: item}) if is_secret_key(name) + else isinstance(item, (dict, list)) and scan_secrets(item)): + # A secret by its field name, or one nested inside a structure. + # scan_secrets passes ${secrets.NAME}: a reference may travel. + portable[name] = {LOCAL_REF: "secret", "field": name} + else: + portable[name] = item + return portable + + def from_portable(self, value: Mapping[str, Any], + existing: Optional[Mapping[str, Any]]) -> Dict[str, Any]: + """``value`` made usable here: references filled from this machine. + + Raises :class:`ConfigSyncError` when a path the item needs does not + exist on this machine in any form. + """ + local: Dict[str, Any] = {} + for name, item in value.items(): + if not is_local_ref(item): + local[name] = item + continue + resolved = self._local_path(item) if name in self.path_fields else None + if resolved is None and existing is not None and name in existing: + resolved = existing[name] + if resolved is None and name in self.path_fields: + raise ConfigSyncError(f"{name} refers to a path kept on another machine") + if resolved is not None: + local[name] = resolved + return local + + # --- snapshot / apply ---------------------------------------------------- + + def snapshot(self) -> Mapping[str, SyncEntry]: + """This machine's items as entries, versioned against the baseline. + + An item equal to the baseline keeps its entry; a changed or new one + becomes a new version made by this device; a baseline item that is + gone here becomes a tombstone. An entry still in conflict is left as + it is -- a local edit must not settle a conflict by accident. + """ + current = {key: self.to_portable(value) for key, value in self.read_local().items()} + entries = {key: self._present(key, value) for key, value in current.items()} + for key, base in self._baseline.items(): + if key not in current: + keep = base.deleted or base.in_conflict or key in self._unapplied + entries[key] = base if keep else base.removed(self.origin) + return entries + + def _present(self, key: str, value: Dict[str, Any]) -> SyncEntry: + """The entry for an item this machine holds.""" + base = self._baseline.get(key) + if base is None: + return SyncEntry.create(key, value, self.origin) + if base.in_conflict or (not base.deleted and base.value == value): + return base + return base.edited(value, self.origin) + + def pending_operations(self) -> List[SyncOperation]: + """The changes :meth:`snapshot` holds that the baseline does not.""" + return [SyncOperation(section=self.section, entry=entry) + for key, entry in sorted(self.snapshot().items()) + if key not in self._baseline + or self._baseline[key].operation_id != entry.operation_id] + + def apply(self, entries: Mapping[str, SyncEntry]) -> ApplyReport: + """Bring this machine to ``entries``; never enables or runs anything. + + A key the baseline held that ``entries`` no longer has was deleted + elsewhere and is removed here. A key only this machine has is a + change not yet sent and is kept. + """ + report = ApplyReport(section=self.section) + local = self.read_local() + for key in sorted(entries): + self._apply_one(key, entries[key], local.get(key), report) + for key, base in sorted(self._baseline.items()): + if key not in entries and not base.deleted and key in local: + self._remove(key, report) + return report + + def _remove(self, key: str, report: ApplyReport) -> None: + try: + self.delete_local(key) + except (AutoControlException, OSError, ValueError) as error: + report.skipped[key] = str(error) or type(error).__name__ + return + report.removed.append(key) + + def _apply_one(self, key: str, entry: SyncEntry, existing: Optional[Dict[str, Any]], + report: ApplyReport) -> None: + if entry.in_conflict: + report.conflicts.append(key) + return + if entry.deleted: + if existing is not None: + self._remove(key, report) + return + value = dict(entry.value or {}) + if existing is None or self.to_portable(existing) != value: + self._write(key, value, existing, report) + + def _write(self, key: str, value: Dict[str, Any], existing: Optional[Dict[str, Any]], + report: ApplyReport) -> None: + try: + self.write_local(key, self.from_portable(value, existing), existing) + except (AutoControlException, OSError, ValueError, TypeError, KeyError) as error: + report.skipped[key] = str(error) or type(error).__name__ + return + report.written.append(key) + if self.arms_on_enable and existing is None: + report.left_disabled.append(key) + + #: Whether items of this section do something once enabled. + arms_on_enable: ClassVar[bool] = False + + +def _contained_parts(relative: Any) -> Optional[Tuple[str, ...]]: + """The parts of a relative posix path that cannot leave its folder, else ``None``. + + Refuses an absolute path, ``..``, and -- because the join happens with + the receiving machine's rules -- a drive letter or a backslash, either of + which lets a Windows join walk out of the folder. + """ + if not isinstance(relative, str) or ":" in relative or "\\" in relative: + return None + path = PurePosixPath(relative) + if not path.parts or path.is_absolute() or ".." in path.parts: + return None + return path.parts + + +def _safe_relative(root: Path, key: str) -> Path: + """``root / key`` for a relative posix key that stays inside ``root``.""" + parts = _contained_parts(key) + if parts is None: + raise ConfigSyncError(f"refusing path {key!r}: it does not stay inside the folder") + return root.joinpath(*parts) + + +class ScriptSyncAdapter(SyncAdapter): + """The ``*.json`` action scripts under one directory, by relative path. + + A script travels with its SHA-256; :meth:`write_local` refuses content + whose hash does not match and replaces the file atomically. Receiving a + script only writes the file -- nothing executes it. A script holding a + literal secret is not sent: put ``${secrets.NAME}`` in the script instead. + """ + + section = "scripts" + + def __init__(self, origin: str, scripts_dir: Union[str, Path]) -> None: + super().__init__(origin, scripts_dir=scripts_dir) + self._root = Path(scripts_dir) + #: Scripts left out of the last snapshot, and why. + self.withheld: Dict[str, str] = {} + + @property + def root(self) -> Path: + """The directory whose scripts this adapter syncs.""" + return self._root + + def read_local(self) -> Dict[str, Dict[str, Any]]: + from je_auto_control.utils.secrets_scan.secrets_scan import scan_secrets + items: Dict[str, Dict[str, Any]] = {} + self.withheld = {} + if not self._root.is_dir(): + return items + for path in sorted(self._root.rglob("*.json")): + key = path.relative_to(self._root).as_posix() + try: + data = path.read_bytes() + parsed = json.loads(data.decode("utf-8")) + except (OSError, ValueError, RecursionError) as error: + self.withheld[key] = f"unreadable: {error}" + continue + if scan_secrets(parsed): + self.withheld[key] = "contains a literal secret; use a ${secrets.NAME} reference" + continue + item: Dict[str, Any] = {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)} + if len(data) <= MAX_INLINE_SCRIPT_BYTES: + item["content"] = data.decode("utf-8") + items[key] = item + return items + + def snapshot(self) -> Mapping[str, SyncEntry]: + entries = dict(super().snapshot()) + # A withheld script is still on this machine: it must not be turned + # into a deletion for everyone else. + for key in self.withheld: + base = self._baseline.get(key) + if base is not None: + entries[key] = base + else: + entries.pop(key, None) + return entries + + def write_local(self, key: str, value: Dict[str, Any], + existing: Optional[Dict[str, Any]]) -> None: + from je_auto_control.utils.json_store.json_store import atomic_write_bytes + content = value.get("content") + if not isinstance(content, str): + raise ConfigSyncError("content travels as an asset; run the asset sync for it") + data = content.encode("utf-8") + if hashlib.sha256(data).hexdigest() != value.get("sha256"): + raise ConfigSyncError("content does not match its SHA-256; not written") + target = _safe_relative(self._root, key) + target.parent.mkdir(parents=True, exist_ok=True) + atomic_write_bytes(target, data) + + def delete_local(self, key: str) -> None: + _safe_relative(self._root, key).unlink(missing_ok=True) + + +class LocatorSyncAdapter(SyncAdapter): + """The named accessibility locators of an ``ElementRepository``.""" + + section = "locators" + + def __init__(self, origin: str, repository: Any) -> None: + super().__init__(origin) + self._repository = repository + + def read_local(self) -> Dict[str, Dict[str, Any]]: + return {key: dict(value) for key, value in self._repository.all().items()} + + def write_local(self, key: str, value: Dict[str, Any], + existing: Optional[Dict[str, Any]]) -> None: + self._repository.save(key, name=value.get("name"), role=value.get("role"), + app_name=value.get("app_name")) + + def delete_local(self, key: str) -> None: + self._repository.remove(key) + + +class HotkeySyncAdapter(SyncAdapter): + """The bindings of a ``HotkeyDaemon``; an arriving binding is never armed.""" + + section = "hotkeys" + path_fields = ("script_path",) + local_fields = ("enabled", "fired", "binding_id") + arms_on_enable = True + + def __init__(self, origin: str, daemon: Any, *, + scripts_dir: Union[str, Path, None] = None) -> None: + super().__init__(origin, scripts_dir=scripts_dir) + self._daemon = daemon + + def _bindings(self) -> Dict[str, Any]: + return {binding.binding_id: binding for binding in self._daemon.list_bindings()} + + def read_local(self) -> Dict[str, Dict[str, Any]]: + return {key: {"combo": binding.combo, "script_path": binding.script_path, + "enabled": binding.enabled} + for key, binding in self._bindings().items()} + + def write_local(self, key: str, value: Dict[str, Any], + existing: Optional[Dict[str, Any]]) -> None: + # A new binding starts disabled; an existing one keeps what this + # machine chose. The synced value has no say in either. + enabled = bool(existing.get("enabled")) if existing is not None else False + self._daemon.bind(str(value["combo"]), str(value["script_path"]), binding_id=key, + enabled=enabled) + + def delete_local(self, key: str) -> None: + self._daemon.unbind(key) + + +def _trigger_types() -> Dict[str, Any]: + from je_auto_control.utils.triggers import trigger_engine + return {cls.__name__: cls for cls in ( + trigger_engine.ImageAppearsTrigger, trigger_engine.WindowAppearsTrigger, + trigger_engine.PixelColorTrigger, trigger_engine.FilePathTrigger, + trigger_engine.CronTrigger)} + + +class TriggerSyncAdapter(SyncAdapter): + """The triggers of a ``TriggerEngine``; an arriving trigger is never armed. + + Image, window, pixel, file and cron triggers are synced. Composite + triggers (all-of / any-of / sequence) hold other trigger objects and stay + local; they are listed in :attr:`withheld`. + """ + + section = "triggers" + path_fields = ("script_path", "image_path", "watch_path") + local_fields = ("enabled", "fired", "trigger_id") + arms_on_enable = True + + def __init__(self, origin: str, engine: Any, *, + scripts_dir: Union[str, Path, None] = None) -> None: + super().__init__(origin, scripts_dir=scripts_dir) + self._engine = engine + #: Triggers left out of the last snapshot, and why. + self.withheld: Dict[str, str] = {} + + def read_local(self) -> Dict[str, Dict[str, Any]]: + known = _trigger_types() + items: Dict[str, Dict[str, Any]] = {} + self.withheld = {} + for trigger in self._engine.list_triggers(): + kind = type(trigger).__name__ + if kind not in known: + self.withheld[trigger.trigger_id] = f"{kind} is not synced" + continue + item = {spec.name: getattr(trigger, spec.name) for spec in fields(trigger) + if not spec.name.startswith("_")} + if isinstance(item.get("target_rgb"), tuple): + item["target_rgb"] = list(item["target_rgb"]) + items[trigger.trigger_id] = {"type": kind, **item} + return items + + def snapshot(self) -> Mapping[str, SyncEntry]: + entries = dict(super().snapshot()) + for key in self.withheld: + entries.pop(key, None) + return entries + + def write_local(self, key: str, value: Dict[str, Any], + existing: Optional[Dict[str, Any]]) -> None: + known = _trigger_types() + kind = value.get("type") + if kind not in known: + raise ConfigSyncError(f"unknown trigger type {kind!r}") + allowed = {spec.name for spec in fields(known[kind]) if not spec.name.startswith("_")} + arguments = {name: item for name, item in value.items() + if name in allowed and name not in self.local_fields} + if "target_rgb" in arguments: + arguments["target_rgb"] = tuple(arguments["target_rgb"]) + # Disabled unless this machine had already enabled the same trigger. + enabled = bool(existing.get("enabled")) if existing is not None else False + self._engine.add(known[kind](trigger_id=key, enabled=enabled, **arguments)) + + def delete_local(self, key: str) -> None: + self._engine.remove(key) + + +class AddressBookSyncAdapter(SyncAdapter): + """The remote-desktop address book: hosts, labels, tags and favourites. + + ``last_used`` describes this machine's own history and is not synced; + any credential-like field stays local as a reference. + """ + + section = "address_book" + local_fields = ("last_used",) + _SEPARATOR = "|" + + def __init__(self, origin: str, book: Any) -> None: + super().__init__(origin) + self._book = book + + def read_local(self) -> Dict[str, Dict[str, Any]]: + return {f"{entry.get('server_url', '')}{self._SEPARATOR}{entry.get('host_id', '')}": dict(entry) + for entry in self._book.list_entries() + if entry.get("server_url") and entry.get("host_id")} + + def write_local(self, key: str, value: Dict[str, Any], + existing: Optional[Dict[str, Any]]) -> None: + host_id, server_url = str(value["host_id"]), str(value["server_url"]) + self._book.upsert(host_id=host_id, server_url=server_url, + label=str(value.get("label", "")), + mac_address=value.get("mac_address"), + broadcast_address=value.get("broadcast_address")) + self._book.set_tags(host_id=host_id, server_url=server_url, + tags=list(value.get("tags") or [])) + favourite = bool(existing.get("favorite")) if existing is not None else False + if bool(value.get("favorite")) != favourite: + self._book.toggle_favorite(host_id=host_id, server_url=server_url) + + def delete_local(self, key: str) -> None: + server_url, _separator, host_id = key.rpartition(self._SEPARATOR) + self._book.remove(host_id=host_id, server_url=server_url) + + +__all__ = [ + "AddressBookSyncAdapter", "ApplyReport", "HotkeySyncAdapter", "LOCAL_REF", + "LocatorSyncAdapter", "MAX_INLINE_SCRIPT_BYTES", "ScriptSyncAdapter", "SyncAdapter", + "TriggerSyncAdapter", "is_local_ref", +] diff --git a/je_auto_control/utils/config_sync/assets.py b/je_auto_control/utils/config_sync/assets.py new file mode 100644 index 000000000..2b5d41b9f --- /dev/null +++ b/je_auto_control/utils/config_sync/assets.py @@ -0,0 +1,243 @@ +"""Move the files synced scripts refer to, verified by content hash. + +A script that arrives without the template images it matches against fails +on its first step. Config sync lists such files in an :class:`AssetManifest` +-- relative path, SHA-256, size -- and :func:`sync_assets` brings each one in +through an :class:`AssetTransport`: the bytes are hashed before anything is +written, and the file is replaced atomically, so a transfer that was cut +short or tampered with never becomes a half-written or wrong file. + +Pure standard library; imports no ``PySide6``. +""" +from __future__ import annotations + +import hashlib +import threading +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Iterable, List, Mapping, Optional, Protocol, Tuple, Union + +from je_auto_control.utils.config_sync.client import ConfigSyncError + +_SHA256_HEX_CHARS = 64 + + +class AssetSyncError(ConfigSyncError): + """An asset could not be read, fetched, verified or written.""" + + +def file_sha256(path: Union[str, Path]) -> str: + """The SHA-256 of a file's content, read in chunks.""" + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _checked_digest(value: Any) -> str: + text = str(value).lower() + if len(text) != _SHA256_HEX_CHARS or any(char not in "0123456789abcdef" for char in text): + raise AssetSyncError(f"not a SHA-256 hex digest: {value!r}") + return text + + +@dataclass(frozen=True) +class AssetRef: + """One file: where it belongs under the root, and what it must contain.""" + path: str + sha256: str + size: int + + def to_dict(self) -> Dict[str, Any]: + """A JSON-ready copy.""" + return {"path": self.path, "sha256": self.sha256, "size": self.size} + + +@dataclass(frozen=True) +class AssetManifest: + """The files that should exist under ``root``, by relative posix path.""" + root: Path + assets: Tuple[AssetRef, ...] = () + + @classmethod + def from_directory(cls, root: Union[str, Path], + patterns: Iterable[str] = ("*",)) -> "AssetManifest": + """List the files under ``root`` matching any of ``patterns`` (recursive).""" + base = Path(root) + found: Dict[str, AssetRef] = {} + for pattern in patterns: + for path in sorted(base.rglob(pattern)) if base.is_dir() else (): + if path.is_file(): + relative = path.relative_to(base).as_posix() + found[relative] = AssetRef(relative, file_sha256(path), path.stat().st_size) + return cls(root=base, assets=tuple(found[key] for key in sorted(found))) + + @classmethod + def from_entries(cls, root: Union[str, Path], + entries: Mapping[str, Mapping[str, Any]]) -> "AssetManifest": + """Build a manifest from ``{relative path: {"sha256": ..., "size": ...}}``.""" + refs = [] + for path, body in sorted(entries.items()): + size = body.get("size", 0) + if isinstance(size, bool) or not isinstance(size, int) or size < 0: + raise AssetSyncError(f"asset {path!r}: size must be an integer >= 0") + refs.append(AssetRef(str(path), _checked_digest(body.get("sha256")), size)) + return cls(root=Path(root), assets=tuple(refs)) + + def destination(self, asset: AssetRef) -> Path: + """Where ``asset`` goes; refuses a path that would leave the root.""" + relative = PurePosixPath(asset.path) + if relative.is_absolute() or not relative.parts or ".." in relative.parts \ + or ":" in asset.path or "\\" in asset.path: + raise AssetSyncError(f"asset path {asset.path!r} does not stay inside the folder") + target = self.root.joinpath(*relative.parts) + root = self.root.resolve() + if root != target.resolve() and root not in target.resolve().parents: + raise AssetSyncError(f"asset path {asset.path!r} resolves outside the folder") + return target + + +class AssetTransport(Protocol): + """Where asset content travels: a content-addressed blob store.""" + + def fetch(self, sha256: str) -> bytes: + """The content stored under ``sha256``; raise ``AssetSyncError`` if absent.""" + + def store(self, sha256: str, data: bytes) -> None: + """Keep ``data`` under ``sha256``.""" + + def has(self, sha256: str) -> bool: + """Whether content for ``sha256`` is already stored.""" + + +class DirectoryAssetTransport: + """An :class:`AssetTransport` over a folder both machines can reach. + + A shared drive, a mounted bucket or a folder another sync tool mirrors. + Blobs are named by their hash, so identical files are stored once. + """ + + def __init__(self, directory: Union[str, Path]) -> None: + self._directory = Path(directory) + + def _blob(self, sha256: str) -> Path: + return self._directory / _checked_digest(sha256) + + def fetch(self, sha256: str) -> bytes: + """Read the blob stored under ``sha256``.""" + try: + return self._blob(sha256).read_bytes() + except OSError as error: + raise AssetSyncError(f"asset {sha256} is not in {self._directory}: {error}") from error + + def store(self, sha256: str, data: bytes) -> None: + """Write ``data`` under ``sha256`` atomically.""" + from je_auto_control.utils.json_store.json_store import atomic_write_bytes + try: + self._directory.mkdir(parents=True, exist_ok=True) + atomic_write_bytes(self._blob(sha256), data) + except OSError as error: + raise AssetSyncError(f"cannot store asset {sha256}: {error}") from error + + def has(self, sha256: str) -> bool: + """Whether the blob exists.""" + return self._blob(sha256).is_file() + + +@dataclass +class AssetSyncResult: + """What :func:`sync_assets` or :func:`publish_assets` did. + + ``transferred`` and ``unchanged`` list relative paths, ``failed`` maps a + path to why it was not written, ``hashes`` records the verified SHA-256 + of every file now in place. + """ + transferred: List[str] = field(default_factory=list) + unchanged: List[str] = field(default_factory=list) + failed: Dict[str, str] = field(default_factory=dict) + hashes: Dict[str, str] = field(default_factory=dict) + cancelled: bool = False + + def to_dict(self) -> Dict[str, Any]: + """A JSON-ready copy.""" + return {"transferred": list(self.transferred), "unchanged": list(self.unchanged), + "failed": dict(self.failed), "hashes": dict(self.hashes), + "cancelled": self.cancelled} + + +def _receive(manifest: AssetManifest, asset: AssetRef, transport: AssetTransport) -> bool: + """Bring one asset in; whether a file was written.""" + from je_auto_control.utils.json_store.json_store import atomic_write_bytes + target = manifest.destination(asset) + wanted = _checked_digest(asset.sha256) + try: + if target.is_file() and file_sha256(target) == wanted: + return False + data = transport.fetch(wanted) + if len(data) != asset.size or hashlib.sha256(data).hexdigest() != wanted: + raise AssetSyncError("content does not match its SHA-256 and size; not written") + target.parent.mkdir(parents=True, exist_ok=True) + atomic_write_bytes(target, data) + except OSError as error: + raise AssetSyncError(str(error)) from error + return True + + +def sync_assets(manifest: AssetManifest, transport: AssetTransport, *, + cancel: Optional[threading.Event] = None) -> AssetSyncResult: + """Make the files under ``manifest.root`` match the manifest. + + A file already holding the right content is left alone. Anything else is + fetched, checked against its SHA-256 and size, and only then moved into + place in one step -- the existing file stays untouched when the check + fails. One bad asset does not stop the others; it is reported in + ``failed``. Setting ``cancel`` stops before the next file. + """ + result = AssetSyncResult() + for asset in manifest.assets: + if cancel is not None and cancel.is_set(): + result.cancelled = True + break + try: + written = _receive(manifest, asset, transport) + except AssetSyncError as error: + result.failed[asset.path] = str(error) + continue + (result.transferred if written else result.unchanged).append(asset.path) + result.hashes[asset.path] = asset.sha256 + return result + + +def publish_assets(manifest: AssetManifest, transport: AssetTransport, *, + cancel: Optional[threading.Event] = None) -> AssetSyncResult: + """Put the manifest's local files where other machines can fetch them. + + A file whose content no longer matches the manifest is reported in + ``failed`` rather than stored under the wrong hash. + """ + result = AssetSyncResult() + for asset in manifest.assets: + if cancel is not None and cancel.is_set(): + result.cancelled = True + break + try: + if transport.has(asset.sha256): + result.unchanged.append(asset.path) + else: + data = manifest.destination(asset).read_bytes() + if hashlib.sha256(data).hexdigest() != _checked_digest(asset.sha256): + raise AssetSyncError("file changed since the manifest was built") + transport.store(asset.sha256, data) + result.transferred.append(asset.path) + except (AssetSyncError, OSError) as error: + result.failed[asset.path] = str(error) + continue + result.hashes[asset.path] = asset.sha256 + return result + + +__all__ = [ + "AssetManifest", "AssetRef", "AssetSyncError", "AssetSyncResult", "AssetTransport", + "DirectoryAssetTransport", "file_sha256", "publish_assets", "sync_assets", +] diff --git a/je_auto_control/utils/config_sync/outbox.py b/je_auto_control/utils/config_sync/outbox.py index 3f382ada5..001623aaa 100644 --- a/je_auto_control/utils/config_sync/outbox.py +++ b/je_auto_control/utils/config_sync/outbox.py @@ -89,10 +89,19 @@ def __init__(self, db_path: Union[str, Path, None] = None, *, account: str, endp self._max_delay = max(self._base_delay, float(max_delay_s)) self._resolved_path: Optional[str] = None + @property + def path(self) -> Path: + """Where the database lives (resolving the default if none was given).""" + configured = self._configured_path + return Path(configured) if configured is not None else default_outbox_path() + + def exists(self) -> bool: + """Whether the database file is there; asking does not create it.""" + return self.path.is_file() + def _connection(self) -> ContextManager[Any]: if self._resolved_path is None: - configured = self._configured_path - path = Path(configured) if configured is not None else default_outbox_path() + path = self.path try: path.parent.mkdir(parents=True, exist_ok=True) except OSError as error: diff --git a/je_auto_control/utils/config_sync/session.py b/je_auto_control/utils/config_sync/session.py new file mode 100644 index 000000000..7f0929929 --- /dev/null +++ b/je_auto_control/utils/config_sync/session.py @@ -0,0 +1,431 @@ +"""Run one config sync end to end: adapters -> outbox -> server -> adapters. + +:func:`run_sync` is the whole cycle the GUI tab, the ``AC_config_sync_*`` +commands and the MCP tools share: + +1. each adapter's local changes become operations in the durable outbox; +2. the outbox is drained to the server (resent by operation id, bounded + back-off, cancellable); +3. the merged state is applied back through the adapters -- which never + enable a hotkey or trigger and never run a script; +4. the merged bucket becomes the new baseline, and the outcome is stored so + the status can be shown without touching the network. + +Pure standard library; imports no ``PySide6``. +""" +from __future__ import annotations + +import os +import socket +import threading +import time +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from je_auto_control.utils.config_sync.adapters import ( + AddressBookSyncAdapter, HotkeySyncAdapter, LocatorSyncAdapter, ScriptSyncAdapter, + SyncAdapter, TriggerSyncAdapter, +) +from je_auto_control.utils.config_sync.assets import ( + AssetManifest, AssetTransport, DirectoryAssetTransport, publish_assets, sync_assets, +) +from je_auto_control.utils.config_sync.client import ( + ConfigBucket, ConfigSyncClient, ConfigSyncError, FullResyncRequired, SyncResult, +) +from je_auto_control.utils.config_sync.outbox import DEFAULT_DRAIN_ATTEMPTS, SyncOutbox +from je_auto_control.utils.config_sync.versions import SyncOperation + +STATE_SYNCED = "synced" +STATE_PENDING = "pending" +STATE_CONFLICT = "conflict" +STATE_OFFLINE = "offline" +STATE_CANCELLED = "cancelled" +STATE_RESYNC_REQUIRED = "resync_required" + +#: Sections synced when the caller does not choose. +DEFAULT_SECTIONS = ("hotkeys", "triggers", "address_book") +_STATUS = "status" +_UNAPPLIED = "unapplied" +_OPTIONS = frozenset({ + "device_id", "secret", "sections", "scripts_dir", "locators_path", "outbox_path", + "assets_dir", "timeout_s", "wait", "max_attempts", +}) + + +@dataclass +class SyncRunReport: + """How one sync ended, in the terms the status view shows. + + ``state`` is one of ``synced`` / ``pending`` / ``conflict`` / ``offline`` + / ``cancelled`` / ``resync_required``; ``revision`` the last revision + merged from the server; ``pending`` the operations still queued; + ``conflicts`` the ``section/key`` names waiting for a choice. + """ + state: str = STATE_SYNCED + revision: int = 0 + pending: int = 0 + conflicts: List[str] = field(default_factory=list) + applied: Dict[str, Dict[str, Any]] = field(default_factory=dict) + withheld: Dict[str, str] = field(default_factory=dict) + assets: Dict[str, Any] = field(default_factory=dict) + error: str = "" + finished_at: float = 0.0 + last_success: float = 0.0 + + def to_dict(self) -> Dict[str, Any]: + """A JSON-ready copy.""" + return {"state": self.state, "revision": self.revision, "pending": self.pending, + "conflicts": list(self.conflicts), "applied": dict(self.applied), + "withheld": dict(self.withheld), "assets": dict(self.assets), + "error": self.error, "finished_at": self.finished_at, + "last_success": self.last_success} + + +def default_device_id_path() -> Path: + """``~/.je_auto_control/config_sync_device_id``, resolved at call time.""" + return Path.home() / ".je_auto_control" / "config_sync_device_id" + + +def default_device_id() -> str: + """This machine's stable sync id, created on first use. + + Version vectors count changes per device id, so two machines must never + share one; the id is the host name plus a random suffix, kept in + :func:`default_device_id_path`. + """ + from je_auto_control.utils.json_store.json_store import atomic_write_text + path = default_device_id_path() + try: + known = path.read_text(encoding="utf-8").strip() + except OSError: + known = "" + if known: + return known + device_id = f"{socket.gethostname() or 'device'}-{uuid.uuid4().hex[:12]}" + try: + path.parent.mkdir(parents=True, exist_ok=True) + atomic_write_text(path, device_id + "\n") + except OSError as error: + raise ConfigSyncError(f"cannot store this device's sync id at {path}: {error}") from error + return device_id + + +def default_adapters(device_id: str, *, sections: Optional[Sequence[str]] = None, + scripts_dir: Optional[str] = None, + locators_path: Optional[str] = None) -> List[SyncAdapter]: + """Adapters over this process's hotkey daemon, trigger engine and address book. + + ``scripts`` needs ``scripts_dir`` and ``locators`` needs ``locators_path``; + naming a section whose store was not given, or an unknown section, is an + error rather than a silent omission. + """ + wanted = list(sections) if sections else [ + *DEFAULT_SECTIONS, *(["scripts"] if scripts_dir else []), + *(["locators"] if locators_path else [])] + return [_build_adapter(name, device_id, scripts_dir, locators_path) for name in wanted] + + +def _build_adapter(name: str, device_id: str, scripts_dir: Optional[str], + locators_path: Optional[str]) -> SyncAdapter: + if name == "hotkeys": + from je_auto_control.utils.hotkey.hotkey_daemon import default_hotkey_daemon + return HotkeySyncAdapter(device_id, default_hotkey_daemon, scripts_dir=scripts_dir) + if name == "triggers": + from je_auto_control.utils.triggers.trigger_engine import default_trigger_engine + return TriggerSyncAdapter(device_id, default_trigger_engine, scripts_dir=scripts_dir) + if name == "address_book": + from je_auto_control.utils.remote_desktop.address_book import default_address_book + return AddressBookSyncAdapter(device_id, default_address_book()) + if name == "scripts" and scripts_dir: + return ScriptSyncAdapter(device_id, scripts_dir) + if name == "locators" and locators_path: + from je_auto_control.utils.element_repository import ElementRepository + return LocatorSyncAdapter(device_id, ElementRepository(locators_path)) + raise ConfigSyncError( + f"cannot sync section {name!r}: unknown, or its scripts_dir / locators_path is missing") + + +def _conflict_names(bucket: ConfigBucket) -> List[str]: + return [f"{section}/{entry.key}" for section, entry in bucket.conflicts()] + + +def _unapplied(outbox: SyncOutbox) -> Dict[str, List[str]]: + stored = outbox.get_state(_UNAPPLIED) + return stored if isinstance(stored, dict) else {} + + +def _stage(outbox: SyncOutbox, adapters: Sequence[SyncAdapter], + baseline: ConfigBucket) -> Dict[str, str]: + """Queue every adapter's local changes; returns what was withheld and why.""" + unapplied = _unapplied(outbox) + withheld: Dict[str, str] = {} + for adapter in adapters: + adapter.rebase(baseline.sync_entries(adapter.section), + unapplied=unapplied.get(adapter.section, ())) + for operation in adapter.pending_operations(): + outbox.enqueue(operation) + baseline.put_entry(operation.section, operation.entry) + for key, reason in getattr(adapter, "withheld", {}).items(): + withheld[f"{adapter.section}/{key}"] = reason + # The baseline now includes the queued changes, so the next run does not + # mint a second version of an edit that is merely waiting to be sent. + outbox.save_baseline(baseline) + return withheld + + +def _apply(outbox: SyncOutbox, adapters: Sequence[SyncAdapter], merged: ConfigBucket, + report: SyncRunReport) -> None: + unapplied: Dict[str, List[str]] = {} + for adapter in adapters: + applied = adapter.apply(merged.sync_entries(adapter.section)) + report.applied[adapter.section] = applied.to_dict() + if applied.skipped: + # Present in the bucket but not on this machine: the next snapshot + # must not mistake that for a local deletion. + unapplied[adapter.section] = sorted(applied.skipped) + outbox.set_state(_UNAPPLIED, unapplied) + outbox.save_baseline(merged) + report.revision = merged.revision + report.conflicts = _conflict_names(merged) + + +def _script_assets(adapters: Sequence[SyncAdapter], bucket: ConfigBucket, + ) -> Optional[AssetManifest]: + """The scripts too large to travel in their entry, as an asset manifest.""" + for adapter in adapters: + if isinstance(adapter, ScriptSyncAdapter): + large = {key: value for key, value in bucket.values("scripts").items() + if "content" not in value} + return AssetManifest.from_entries(adapter.root, large) + return None + + +def _present(manifest: AssetManifest) -> AssetManifest: + """The part of ``manifest`` this machine actually has on disk.""" + return AssetManifest(root=manifest.root, assets=tuple( + asset for asset in manifest.assets if manifest.destination(asset).is_file())) + + +def _finish(outbox: SyncOutbox, report: SyncRunReport) -> SyncRunReport: + previous = outbox.get_state(_STATUS) + report.finished_at = time.time() + report.pending = len(outbox.pending()) + earlier = previous.get("last_success", 0.0) if isinstance(previous, dict) else 0.0 + report.last_success = report.finished_at if report.state in ( + STATE_SYNCED, STATE_CONFLICT) else float(earlier or 0.0) + outbox.set_state(_STATUS, report.to_dict()) + return report + + +def _exchange(client: ConfigSyncClient, outbox: SyncOutbox, device_id: str, + cancel: Optional[threading.Event], wait: bool, + max_attempts: int) -> Tuple[Optional[SyncResult], str, bool]: + """Drain the outbox and pull; ``(result, error, cancelled)``.""" + results: List[SyncResult] = [] + + def send(batch: List[SyncOperation]) -> None: + results.append(client.push_operations(batch, device_id=device_id)) + + drained = outbox.drain(send, cancel=cancel, wait=wait, max_attempts=max_attempts) + if drained.cancelled or drained.offline: + return None, drained.error, drained.cancelled + if not results: + try: + results.append(client.push_operations([], device_id=device_id)) + except FullResyncRequired: + raise + except ConfigSyncError as error: + return None, str(error), False + return results[-1], "", False + + +def run_sync(client: ConfigSyncClient, outbox: SyncOutbox, adapters: Sequence[SyncAdapter], *, + device_id: str, cancel: Optional[threading.Event] = None, wait: bool = False, + max_attempts: int = DEFAULT_DRAIN_ATTEMPTS, + asset_transport: Optional[AssetTransport] = None) -> SyncRunReport: + """Sync once and report the outcome; see the module docstring for the steps. + + A failure to reach the server is not an exception: the changes stay in + the outbox and the report says ``offline``. A device the group retired + gets ``resync_required`` -- call :func:`run_full_resync`. + """ + baseline = outbox.load_baseline() or ConfigBucket(user_id=client.user_id) + report = SyncRunReport(revision=baseline.revision) + report.withheld = _stage(outbox, adapters, baseline) + manifest = _script_assets(adapters, baseline) + if asset_transport is not None and manifest is not None: + report.assets["published"] = publish_assets( + _present(manifest), asset_transport, cancel=cancel).to_dict() + try: + result, error, cancelled = _exchange(client, outbox, device_id, cancel, wait, max_attempts) + except FullResyncRequired as required: + report.state, report.error = STATE_RESYNC_REQUIRED, str(required) + return _finish(outbox, report) + if result is None: + report.state = STATE_CANCELLED if cancelled else STATE_OFFLINE + report.error = error + report.conflicts = _conflict_names(baseline) + return _finish(outbox, report) + incoming = _script_assets(adapters, result.bucket) + if asset_transport is not None and incoming is not None: + report.assets["received"] = sync_assets(incoming, asset_transport, cancel=cancel).to_dict() + _apply(outbox, adapters, result.bucket, report) + report.state = STATE_CONFLICT if report.conflicts else STATE_SYNCED + return _finish(outbox, report) + + +def run_full_resync(client: ConfigSyncClient, outbox: SyncOutbox, + adapters: Sequence[SyncAdapter], *, device_id: str) -> SyncRunReport: + """Re-admit a retired device: adopt the server's state, drop pending changes. + + The discarded operations are listed under ``withheld`` so nothing + disappears without a trace. Items this machine never synced are kept and + offered again by the next :func:`run_sync`. + """ + baseline = outbox.load_baseline() or ConfigBucket(user_id=client.user_id) + report = SyncRunReport(revision=baseline.revision) + try: + adopted = client.full_resync(device_id=device_id) + except ConfigSyncError as error: + report.state, report.error = STATE_OFFLINE, str(error) + return _finish(outbox, report) + for operation in outbox.clear(): + report.withheld[f"{operation.section}/{operation.entry.key}"] = ( + "discarded by the full resync: made against state the group has moved past") + for adapter in adapters: + adapter.rebase(baseline.sync_entries(adapter.section)) + _apply(outbox, adapters, adopted, report) + report.state = STATE_CONFLICT if report.conflicts else STATE_SYNCED + return _finish(outbox, report) + + +def _conflict_details(bucket: ConfigBucket) -> List[Dict[str, Any]]: + """Each conflicted entry with its candidates, in the order ``choice`` counts.""" + return [{"section": section, "key": entry.key, "choices": [ + {"origin": sibling.origin, "deleted": sibling.deleted, + "value": None if sibling.value is None else dict(sibling.value)} + for sibling in entry.siblings]} for section, entry in bucket.conflicts()] + + +def sync_status(outbox: SyncOutbox) -> Dict[str, Any]: + """The last recorded outcome plus the live queue length; no network. + + Read-only in full: asking about an account that never synced does not + create its outbox database. + """ + if not outbox.exists(): + return {**SyncRunReport(state="never").to_dict(), "conflict_details": []} + stored = outbox.get_state(_STATUS) + status = dict(stored) if isinstance(stored, dict) else SyncRunReport(state="never").to_dict() + baseline = outbox.load_baseline() + status["pending"] = len(outbox.pending()) + known = baseline if baseline is not None else ConfigBucket(user_id="") + status["conflicts"] = _conflict_names(known) + status["conflict_details"] = _conflict_details(known) + status["revision"] = known.revision + if status["pending"] and status["state"] == STATE_SYNCED: + status["state"] = STATE_PENDING + return status + + +def resolve_conflict(outbox: SyncOutbox, adapters: Sequence[SyncAdapter], *, device_id: str, + section: str, key: str, choice: int) -> Dict[str, Any]: + """Settle a conflict by picking sibling number ``choice`` (0-based). + + The choice is applied to this machine and queued; the next + :func:`run_sync` sends it, and it supersedes every candidate everywhere. + """ + baseline = outbox.load_baseline() or ConfigBucket(user_id="") + entry = baseline.get_entry(section, key) + if entry is None or not entry.in_conflict: + raise ConfigSyncError(f"{section}/{key} is not in conflict") + if choice not in range(len(entry.siblings)) or isinstance(choice, bool): + raise ConfigSyncError(f"choice must be 0..{len(entry.siblings) - 1}") + picked = entry.siblings[choice] + settled = entry.resolved(None if picked.deleted else picked.value, device_id, + modified_at=time.time()) + for adapter in adapters: + if adapter.section == section: + adapter.rebase(baseline.sync_entries(section)) + adapter.apply({**baseline.sync_entries(section), key: settled}) + outbox.enqueue(SyncOperation(section=section, entry=settled)) + baseline.put_entry(section, settled) + outbox.save_baseline(baseline) + return {"section": section, "key": key, "deleted": settled.deleted, + "value": None if settled.value is None else dict(settled.value)} + + +# --- one-call entry points for the executor, MCP and the GUI ------------------- + +def _session(server_url: str, user_id: str, options: Mapping[str, Any], + ) -> Tuple[ConfigSyncClient, SyncOutbox, List[SyncAdapter], str]: + unknown = sorted(set(options) - _OPTIONS) + if unknown: + raise ConfigSyncError(f"unknown config sync option(s): {', '.join(unknown)}") + chosen = {name: value for name, value in options.items() if value not in (None, "")} + device_id = str(chosen.get("device_id") or default_device_id()) + client = ConfigSyncClient( + server_url, user_id=user_id, timeout_s=float(chosen.get("timeout_s", 5.0)), + secret=chosen.get("secret") or os.environ.get("AC_SIGNALING_SECRET") or None) + outbox = SyncOutbox(chosen.get("outbox_path"), account=user_id, endpoint=client.server_url) + sections = chosen.get("sections") + if isinstance(sections, str): + sections = [name.strip() for name in sections.split(",") if name.strip()] + adapters = default_adapters(device_id, sections=sections, + scripts_dir=chosen.get("scripts_dir"), + locators_path=chosen.get("locators_path")) + return client, outbox, adapters, device_id + + +def config_sync_run(server_url: str, user_id: str, *, + cancel: Optional[threading.Event] = None, **options: Any) -> Dict[str, Any]: + """Sync this machine's settings with ``server_url`` once; returns the report. + + Options: ``device_id`` (default: this machine's stored id), ``secret`` + (default ``$AC_SIGNALING_SECRET``), ``sections``, ``scripts_dir``, + ``locators_path``, ``outbox_path``, ``assets_dir`` (a folder both + machines reach, for scripts too large to inline), ``timeout_s``, + ``wait`` (sleep through a retry back-off instead of returning) and + ``max_attempts``. + """ + if cancel is not None and not isinstance(cancel, threading.Event): + raise ConfigSyncError("cancel must be a threading.Event") + client, outbox, adapters, device_id = _session(server_url, user_id, options) + assets_dir = options.get("assets_dir") + return run_sync( + client, outbox, adapters, device_id=device_id, cancel=cancel, + wait=bool(options.get("wait", False)), + max_attempts=int(options.get("max_attempts") or DEFAULT_DRAIN_ATTEMPTS), + asset_transport=DirectoryAssetTransport(assets_dir) if assets_dir else None).to_dict() + + +def config_sync_status(server_url: str, user_id: str, + outbox_path: Optional[str] = None) -> Dict[str, Any]: + """The recorded sync state for this account and server; no network.""" + endpoint = str(server_url).rstrip("/") + return sync_status(SyncOutbox(outbox_path or None, account=user_id, endpoint=endpoint)) + + +def config_sync_full_resync(server_url: str, user_id: str, **options: Any) -> Dict[str, Any]: + """Adopt the server's state after this device was retired; see :func:`run_full_resync`.""" + client, outbox, adapters, device_id = _session(server_url, user_id, options) + return run_full_resync(client, outbox, adapters, device_id=device_id).to_dict() + + +def config_sync_resolve(server_url: str, user_id: str, section: str, key: str, + choice: int, **options: Any) -> Dict[str, Any]: + """Pick one side of a conflict; see :func:`resolve_conflict`.""" + _client, outbox, adapters, device_id = _session(server_url, user_id, options) + return resolve_conflict(outbox, adapters, device_id=device_id, section=section, + key=key, choice=choice) + + +__all__ = [ + "DEFAULT_SECTIONS", "STATE_CANCELLED", "STATE_CONFLICT", "STATE_OFFLINE", "STATE_PENDING", + "STATE_RESYNC_REQUIRED", "STATE_SYNCED", "SyncRunReport", "config_sync_full_resync", + "config_sync_resolve", "config_sync_run", "config_sync_status", "default_adapters", + "default_device_id", "default_device_id_path", "resolve_conflict", "run_full_resync", + "run_sync", "sync_status", +] diff --git a/je_auto_control/utils/config_sync/versions.py b/je_auto_control/utils/config_sync/versions.py index 1e333d4ce..91b4b5ff1 100644 --- a/je_auto_control/utils/config_sync/versions.py +++ b/je_auto_control/utils/config_sync/versions.py @@ -259,8 +259,11 @@ def _conflicted(key: str, left: SyncEntry, right: SyncEntry) -> SyncEntry: for other in pool)] survivors.sort(key=_identity) vector = join_vectors(left.vector, right.vector) - if len(survivors) == 1: - only = survivors[0] + only = survivors[0] + if all(other.value == only.value and other.deleted == only.deleted for other in survivors): + # One candidate left -- or several that say the same thing, as when + # two machines each already had the same entry before first syncing. + # There is nothing to choose between, so it is not a conflict. return SyncEntry(key=key, value=only.value, vector=vector, origin=only.origin, operation_id=only.operation_id, deleted=only.deleted, modified_at=only.modified_at, deleted_revision=only.deleted_revision) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index cc475cf85..cea698f3c 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -34,6 +34,9 @@ ) from je_auto_control.utils.executor.action_redaction import describe_action, redact_actions from je_auto_control.utils.executor.mouse_aliases import MOUSE_BUTTON_COMMANDS +from je_auto_control.utils.config_sync.session import ( + config_sync_full_resync, config_sync_resolve, config_sync_run, config_sync_status, +) from je_auto_control.utils.llm.planner import ( plan_actions as llm_plan_actions, run_from_description as llm_run_from_description, @@ -7567,6 +7570,10 @@ def __init__(self): "AC_parse_sse": _parse_sse, "AC_resolve_config": _resolve_config, "AC_explain_config": _explain_config, + "AC_config_sync_run": config_sync_run, + "AC_config_sync_status": config_sync_status, + "AC_config_sync_resolve": config_sync_resolve, + "AC_config_sync_full_resync": config_sync_full_resync, "AC_check_compatibility": _check_compatibility, "AC_ts_rate": _ts_rate, "AC_ts_downsample": _ts_downsample, diff --git a/je_auto_control/utils/hotkey/hotkey_daemon.py b/je_auto_control/utils/hotkey/hotkey_daemon.py index e2211e03e..07ffdeef8 100644 --- a/je_auto_control/utils/hotkey/hotkey_daemon.py +++ b/je_auto_control/utils/hotkey/hotkey_daemon.py @@ -145,8 +145,13 @@ def __init__(self, self._stop = threading.Event() def bind(self, combo: str, script_path: str, - binding_id: Optional[str] = None) -> HotkeyBinding: - """Register a hotkey → script binding. Safe to call before/after start.""" + binding_id: Optional[str] = None, *, enabled: bool = True) -> HotkeyBinding: + """Register a hotkey → script binding. Safe to call before/after start. + + ``enabled=False`` registers it without arming it, so it is never + live between being stored and being switched off (config sync adds + bindings that came from another machine this way). + """ split_combo(combo) # A key the platform cannot take fails here, not later on every tick. if sys.platform == "win32": @@ -156,7 +161,7 @@ def bind(self, combo: str, script_path: str, _combo_to_macos(combo) bid = binding_id or uuid.uuid4().hex[:8] binding = HotkeyBinding( - binding_id=bid, combo=combo, script_path=script_path, + binding_id=bid, combo=combo, script_path=script_path, enabled=bool(enabled), ) with self._lock: self._bindings[bid] = binding diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fecbc25ae..24b8d6eb0 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -18,6 +18,7 @@ from je_auto_control.utils.mcp_server.tools import _handlers_runs as h_runs from je_auto_control.utils.mcp_server.tools import _handlers_scheduling as h_sched from je_auto_control.utils.mcp_server.tools import _handlers_remote as h_remote +from je_auto_control.utils.config_sync import session as h_sync from je_auto_control.utils.mcp_server.tools._base import ( DESTRUCTIVE, MCPTool, MCPToolAnnotations, NON_DESTRUCTIVE, READ_ONLY, SIDE_EFFECT_ONLY, schema, @@ -8994,6 +8995,67 @@ def media_assert_tools() -> List[MCPTool]: ] +def config_sync_tools() -> List[MCPTool]: + target = {"server_url": {"type": "string"}, "user_id": {"type": "string"}} + options = { + "device_id": {"type": "string"}, "secret": {"type": "string"}, + "sections": {"type": "array", "items": {"type": "string"}}, + "scripts_dir": {"type": "string", "format": "path"}, + "locators_path": {"type": "string", "format": "path"}, + "outbox_path": {"type": "string", "format": "path"}, + "assets_dir": {"type": "string", "format": "path"}, + "timeout_s": {"type": "number"}, + } + required = ["server_url", "user_id"] + return [ + MCPTool( + name="ac_config_sync_run", + description=("Sync this machine's hotkeys, triggers, address book (and " + "scripts / locators when their paths are given) with the " + "config-sync server once. Local changes are queued durably " + "and sent on top of the server's revision; concurrent edits " + "of one entry are kept as a conflict. Received hotkeys and " + "triggers are created DISABLED and nothing is run. Returns " + "{state, revision, pending, conflicts, applied, error}."), + input_schema=schema({**target, **options, "wait": {"type": "boolean"}, + "max_attempts": {"type": "integer"}}, required), + handler=h_sync.config_sync_run, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_config_sync_status", + description=("Report the recorded config-sync state for an account and " + "server without touching the network: {state, revision, " + "pending, conflicts, conflict_details, last_success, error}."), + input_schema=schema({**target, "outbox_path": options["outbox_path"]}, required), + handler=h_sync.config_sync_status, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_config_sync_resolve", + description=("Settle a config-sync conflict by keeping candidate number " + "'choice' (0-based, as listed in conflict_details) of " + "'section'/'key'. Applied locally and queued; the next sync " + "sends it."), + input_schema=schema({**target, "section": {"type": "string"}, + "key": {"type": "string"}, "choice": {"type": "integer"}, + **options}, + required + ["section", "key", "choice"]), + handler=h_sync.config_sync_resolve, + annotations=DESTRUCTIVE, + ), + MCPTool( + name="ac_config_sync_full_resync", + description=("After this device was retired from the sync group: adopt " + "the server's state and DISCARD this machine's pending " + "changes (listed under 'withheld')."), + input_schema=schema({**target, **options}, required), + handler=h_sync.config_sync_full_resync, + annotations=DESTRUCTIVE, + ), + ] + + ALL_FACTORIES = ( mouse_tools, keyboard_tools, screen_tools, image_and_ocr_tools, window_tools, system_tools, recording_tools, drag_and_send_tools, @@ -9060,4 +9122,5 @@ def media_assert_tools() -> List[MCPTool]: visual_regression_tools, state_machine_tools, codegen_tools, flakiness_tools, suite_tools, quarantine_tools, a11y_audit_tools, device_matrix_tools, media_assert_tools, + config_sync_tools, ) diff --git a/je_auto_control/utils/remote_desktop/clipboard_sync.py b/je_auto_control/utils/remote_desktop/clipboard_sync.py index ff1976070..9a1a683f3 100644 --- a/je_auto_control/utils/remote_desktop/clipboard_sync.py +++ b/je_auto_control/utils/remote_desktop/clipboard_sync.py @@ -5,10 +5,17 @@ * ``{"kind": "text", "text": "..."}`` * ``{"kind": "image", "format": "png", "data_b64": "..."}`` + +Applying a peer's clipboard changes the local clipboard, which a watcher +then sees as a change and sends back -- and the peer does the same. +:class:`ClipboardEchoGuard` breaks that loop for any code that forwards +clipboard changes automatically. """ import base64 +import hashlib import json -from typing import Any, Dict, Tuple +import threading +from typing import Any, Dict, Optional, Tuple from je_auto_control.utils.exception.exceptions import AutoControlException @@ -17,6 +24,52 @@ class ClipboardSyncError(AutoControlException, ValueError): """Raised when a CLIPBOARD payload is malformed or unsupported.""" +def _fingerprint(kind: str, data: Any) -> str: + raw = data.encode("utf-8") if isinstance(data, str) else bytes(data) + return f"{kind}:{hashlib.sha256(raw).hexdigest()}" + + +class ClipboardEchoGuard: + """Decides whether a local clipboard change should be sent to the peer. + + Call :meth:`note_remote` with what was just applied from the peer, and + ask :meth:`should_send` before forwarding a local change. Content that + is exactly what the peer last sent is not sent back, and content already + sent is not sent again while the clipboard still holds it, so two + machines watching each other's clipboard settle after one transfer + instead of bouncing it forever. Only fingerprints are kept, never the + clipboard content. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._from_remote: Optional[str] = None + self._sent: Optional[str] = None + + def note_remote(self, kind: str, data: Any) -> None: + """Record content that arrived from the peer and was applied locally.""" + with self._lock: + self._from_remote = _fingerprint(kind, data) + + def should_send(self, kind: str, data: Any) -> bool: + """Whether this local clipboard content is news to the peer. + + Returning ``True`` records the content as sent. + """ + fingerprint = _fingerprint(kind, data) + with self._lock: + if fingerprint in (self._from_remote, self._sent): + return False + self._sent = fingerprint + return True + + def reset(self) -> None: + """Forget both sides (after a reconnect: the peer may hold anything).""" + with self._lock: + self._from_remote = None + self._sent = None + + def encode_text(text: str) -> bytes: """Encode a text-clipboard payload.""" if not isinstance(text, str): diff --git a/je_auto_control/utils/remote_desktop/file_sync.py b/je_auto_control/utils/remote_desktop/file_sync.py index f71eb9a49..b55e56ee4 100644 --- a/je_auto_control/utils/remote_desktop/file_sync.py +++ b/je_auto_control/utils/remote_desktop/file_sync.py @@ -11,17 +11,26 @@ Polling interval default 3s — enough for most edit/save workflows without burning CPU; bump it lower for tighter sync. + +When both sides mirror the same folder, a file that *arrived* from the peer +is new on disk and would be pushed straight back, and back again from the +other side. The receiving code tells the engine with +:meth:`FolderSyncEngine.note_received`; the engine then leaves that file +alone until its content changes locally. """ from __future__ import annotations +import hashlib import threading from pathlib import Path -from typing import Callable, Dict, Optional +from typing import Callable, Dict, List, Optional from je_auto_control.utils.logging.logging_instance import autocontrol_logger _DEFAULT_POLL_S = 3.0 +#: How many "this came from the peer" notes one engine keeps. +_MAX_RECEIVED_NOTES = 4096 class FolderSyncEngine: @@ -40,6 +49,8 @@ def __init__(self, *, watch_dir: Path, self._interval = max(0.5, float(poll_interval_s)) self._include_subdirs = bool(include_subdirs) self._snapshot: Dict[str, float] = {} # rel_path -> mtime + self._received: Dict[str, str] = {} # rel_path -> sha256 of what the peer sent + self._received_lock = threading.Lock() self._stop = threading.Event() self._ready = threading.Event() self._thread: Optional[threading.Thread] = None @@ -86,6 +97,88 @@ def wait_until_ready(self, timeout: float = 5.0) -> bool: """ return self._ready.wait(timeout) + def note_received(self, remote_name: str, *, sha256: Optional[str] = None) -> None: + """Record that ``remote_name`` in the watched folder came from the peer. + + The engine will not push it back while its content is still what the + peer sent. Pass the content's ``sha256`` to note a file *before* it + is written -- otherwise a poll landing between the write and this + call would already have echoed it; without it the file on disk is + hashed now. A later local edit changes the hash and is pushed as usual. + """ + rel = Path(remote_name).as_posix() + digest = sha256.lower() if sha256 else self._digest(self._watch / rel) + if digest is None: + return + with self._received_lock: + # A note may be made before its file lands, so notes cannot be + # pruned by what is on disk; the oldest go once there are many. + self._received.pop(rel, None) + self._received[rel] = digest + while len(self._received) > _MAX_RECEIVED_NOTES: + self._received.pop(next(iter(self._received))) + + @staticmethod + def _digest(path: Path) -> Optional[str]: + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + return None + + def _is_echo(self, rel: str) -> bool: + """Whether ``rel`` still holds exactly what the peer sent.""" + with self._received_lock: + noted = self._received.get(rel) + if noted is None: + return False + if self._digest(self._watch / rel) == noted: + return True + # Edited here since it arrived: it is a local change again. + with self._received_lock: + self._received.pop(rel, None) + return False + + def poll_once(self) -> List[str]: + """Run one diff pass now and return the relative paths pushed. + + The first pass only records what is already there. The background + thread calls this every interval; tests and callers that drive the + engine themselves can call it directly. + """ + if not self._ready.is_set(): + self._snapshot = self._scan() + self._ready.set() + return [] + current = self._scan() + pushed: List[str] = [] + changed = [(rel, mtime) for rel, mtime in current.items() + if self._snapshot.get(rel, float("-inf")) < mtime] + for rel, mtime in changed: + if self._is_echo(rel): + self._snapshot[rel] = mtime + elif self._push(rel, mtime): + pushed.append(rel) + # Track deletions in snapshot (don't propagate, just stop + # tracking). Do NOT blindly merge ``current`` here — that would + # mark failed sends as already-synced and break the next-tick + # retry promise made in this engine's docstring. Successful + # sends already updated ``_snapshot[rel]``. + self._snapshot = { + rel: mtime for rel, mtime in self._snapshot.items() + if rel in current + } + return pushed + + def _push(self, rel: str, mtime: float) -> bool: + try: + self._sender(str(self._watch / rel), rel) + except (RuntimeError, OSError, ValueError) as error: + autocontrol_logger.warning("folder sync push %s: %r", rel, error) + return False + self._snapshot[rel] = mtime + autocontrol_logger.info("folder sync: pushed %s", rel) + return True + def _scan(self) -> Dict[str, float]: out: Dict[str, float] = {} try: @@ -107,35 +200,12 @@ def _loop(self, stop: threading.Event) -> None: # Build initial snapshot WITHOUT sending; treat pre-existing files # as "already synced" so engaging sync mid-edit doesn't re-upload # the entire directory. - self._snapshot = self._scan() - self._ready.set() + self.poll_once() while not stop.is_set(): stop.wait(self._interval) if stop.is_set(): return - current = self._scan() - for rel, mtime in current.items(): - prev = self._snapshot.get(rel) - if prev is not None and prev >= mtime: - continue - full = self._watch / rel - try: - self._sender(str(full), rel) - self._snapshot[rel] = mtime - autocontrol_logger.info("folder sync: pushed %s", rel) - except (RuntimeError, OSError, ValueError) as error: - autocontrol_logger.warning( - "folder sync push %s: %r", rel, error, - ) - # Track deletions in snapshot (don't propagate, just stop - # tracking). Do NOT blindly merge ``current`` here — that would - # mark failed sends as already-synced and break the next-tick - # retry promise made in this engine's docstring. Successful - # sends already updated ``_snapshot[rel]`` above. - self._snapshot = { - rel: mtime for rel, mtime in self._snapshot.items() - if rel in current - } + self.poll_once() __all__ = ["FolderSyncEngine"] diff --git a/test/unit_test/headless/test_config_sync_convergence.py b/test/unit_test/headless/test_config_sync_convergence.py index b46ea2605..9ce4be240 100644 --- a/test/unit_test/headless/test_config_sync_convergence.py +++ b/test/unit_test/headless/test_config_sync_convergence.py @@ -103,6 +103,19 @@ def test_an_edit_made_apart_from_a_delete_is_a_conflict_not_a_silent_loss(): assert merge_entries(base, tombstone).entry == tombstone +def test_the_same_value_made_apart_is_not_a_conflict(): + """Two machines that each already had the entry agree; nothing to choose.""" + left = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "laptop") + right = SyncEntry.create("hk1", {"combo": "ctrl+a"}, "desktop") + decision = merge_entries(left, right) + assert decision.conflict is None and not decision.entry.in_conflict + assert decision.entry.value == {"combo": "ctrl+a"} + assert decision.entry.vector == {"laptop": 1, "desktop": 1} + assert merge_entries(right, left).entry == decision.entry + for side in (left, right): + assert merge_entries(side, decision.entry).entry == decision.entry + + def test_resolving_a_conflict_supersedes_every_sibling(): base = SyncEntry.create("hk1", {"combo": "a"}, "laptop") left, right = base.edited({"combo": "l"}, "laptop"), base.edited({"combo": "r"}, "desktop") diff --git a/test/unit_test/headless/test_config_sync_tab_gui.py b/test/unit_test/headless/test_config_sync_tab_gui.py new file mode 100644 index 000000000..dc1ff453c --- /dev/null +++ b/test/unit_test/headless/test_config_sync_tab_gui.py @@ -0,0 +1,174 @@ +"""The Config Sync tab: status view, Actions-menu commands, cancellable worker.""" +import os +import threading +import time + +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + +from PySide6.QtCore import QCoreApplication, QEvent # noqa: E402 +from PySide6.QtWidgets import QApplication # noqa: E402 + +from je_auto_control.gui import config_sync_tab as tab_module # noqa: E402 +from je_auto_control.gui.config_sync_tab import ConfigSyncTab # noqa: E402 + +_CONFLICTED = { + "state": "conflict", "revision": 7, "pending": 2, "last_success": 1_700_000_000.0, + "error": "", "conflicts": ["hotkeys/hk1"], + "conflict_details": [{"section": "hotkeys", "key": "hk1", "choices": [ + {"origin": "laptop", "deleted": False, "value": {"combo": "ctrl+l"}}, + {"origin": "desktop", "deleted": True, "value": None}]}], +} + + +@pytest.fixture(scope="module") +def qapp(): + return QApplication.instance() or QApplication([]) + + +@pytest.fixture +def tab(qapp, monkeypatch): + monkeypatch.setattr(tab_module.session, "config_sync_status", + lambda _url, _user: dict(_CONFLICTED)) + widget = ConfigSyncTab() + widget._inputs["server"].setText("https://sync.invalid") # noqa: SLF001 + widget._inputs["user"].setText("alice") # noqa: SLF001 + yield widget + widget.cancel() + widget.deleteLater() + + +def _pump(qapp, done, timeout_s=5.0): + deadline = time.monotonic() + timeout_s + while not done() and time.monotonic() < deadline: + qapp.processEvents() + time.sleep(0.01) + qapp.processEvents() + return done() + + +def test_the_tab_is_registered_once_and_exposes_its_commands(tab): + from je_auto_control.gui.tab_registry import TAB_SPECS + specs = [spec for spec in TAB_SPECS if spec.key == "config_sync"] + assert len(specs) == 1 and specs[0].class_name == "ConfigSyncTab" + actions = tab.menu_actions() + assert [key for key, _handler in actions] == [ + "config_sync_run_btn", "config_sync_cancel_btn", "config_sync_refresh_btn", + "config_sync_resolve_btn", "config_sync_resync_btn"] + assert all(callable(handler) for _key, handler in actions) + + +def test_every_text_exists_in_all_four_languages(tab): + import re + from pathlib import Path + from je_auto_control.gui.language_wrapper import ( + english, japanese, simplified_chinese, traditional_chinese) + source = Path(tab_module.__file__).read_text(encoding="utf-8") + keys = set(re.findall(r'"((?:tab_)?config_sync_[a-z_]+)"', source)) + keys |= {f"config_sync_{name}_label" for name in tab_module._FIELDS} # noqa: SLF001 + keys |= {f"config_sync_col_{name}" for name in tab_module._COLUMNS} # noqa: SLF001 + keys |= {f"config_sync_state_{state}" for state in ( + "never", "syncing", "synced", "pending", "conflict", "offline", "cancelled", + "resync_required")} + keys.discard("config_sync_state_") + for module in (english, japanese, simplified_chinese, traditional_chinese): + catalogue = next(value for value in vars(module).values() + if isinstance(value, dict) and "tab_presence" in value) + missing = sorted(key for key in keys if key not in catalogue) + assert not missing, f"{module.__name__} lacks {missing}" + + +def test_refresh_shows_revision_pending_and_conflicts(tab): + tab.refresh_status() + assert "7" in tab._detail.text() and "2" in tab._detail.text() # noqa: SLF001 + table = tab._conflicts # noqa: SLF001 + assert table.rowCount() == 2 + assert table.item(0, 0).text() == "hotkeys/hk1" and table.item(1, 2).text() == "desktop" + + +def test_server_and_user_are_required(tab, monkeypatch): + called = [] + monkeypatch.setattr(tab_module.session, "config_sync_run", + lambda *args, **kwargs: called.append(args)) + tab._inputs["server"].setText("") # noqa: SLF001 + tab.sync_now() + assert called == [] and not tab.is_busy() + + +def test_sync_runs_off_the_gui_thread_and_can_be_cancelled(tab, qapp, monkeypatch): + seen = {} + + def slow_sync(server_url, user_id, cancel=None, **options): + seen.update(server_url=server_url, user_id=user_id, options=options, + thread=threading.current_thread()) + cancelled = cancel.wait(10) + return {"state": "cancelled" if cancelled else "synced", "revision": 7, "pending": 2} + + monkeypatch.setattr(tab_module.session, "config_sync_run", slow_sync) + tab._inputs["secret"].setText("s3cret") # noqa: SLF001 + tab.sync_now() + assert tab.is_busy() + tab.sync_now() # a second request while one runs is refused + assert _pump(qapp, lambda: "thread" in seen) + assert seen["thread"] is not threading.main_thread() + assert seen["options"] == {"secret": "s3cret"} + + tab.cancel() + assert _pump(qapp, lambda: not tab.is_busy() and tab._worker is None) # noqa: SLF001 + assert tab._cancel_slot == [] # noqa: SLF001 + + +def test_a_failed_sync_is_shown_not_raised(tab, qapp, monkeypatch): + def broken(*_args, **_kwargs): + raise RuntimeError("the server predates revision-checked writes") + + monkeypatch.setattr(tab_module.session, "config_sync_run", broken) + tab.sync_now() + assert _pump(qapp, lambda: tab._worker is None) # noqa: SLF001 + assert "predates" in tab._detail.text() # noqa: SLF001 + + +def test_resolving_sends_the_selected_candidate(tab, monkeypatch): + resolved = [] + monkeypatch.setattr(tab_module.session, "config_sync_resolve", + lambda *args, **options: resolved.append(args)) + tab.resolve_selected() + assert resolved == [], "nothing selected yet" + tab.refresh_status() + tab._conflicts.selectRow(1) # noqa: SLF001 + tab.resolve_selected() + assert resolved == [("https://sync.invalid", "alice", "hotkeys", "hk1", 1)] + + +def test_a_full_resync_needs_confirmation(tab, qapp, monkeypatch): + ran = [] + monkeypatch.setattr(tab_module.session, "config_sync_full_resync", + lambda *args, **options: ran.append(args) or {"state": "synced"}) + monkeypatch.setattr(ConfigSyncTab, "_confirm", lambda _self, _question: False) + tab.full_resync() + assert ran == [] and not tab.is_busy() + monkeypatch.setattr(ConfigSyncTab, "_confirm", lambda _self, _question: True) + tab.full_resync() + assert _pump(qapp, lambda: tab._worker is None) # noqa: SLF001 + assert ran == [("https://sync.invalid", "alice")] + + +def test_closing_the_tab_releases_a_running_sync(qapp, monkeypatch): + released = threading.Event() + + def slow_sync(_url, _user, cancel=None, **_options): + if cancel.wait(10): + released.set() + return {"state": "cancelled"} + + monkeypatch.setattr(tab_module.session, "config_sync_run", slow_sync) + monkeypatch.setattr(tab_module.session, "config_sync_status", lambda _url, _user: {}) + widget = ConfigSyncTab() + widget._inputs["server"].setText("https://sync.invalid") # noqa: SLF001 + widget._inputs["user"].setText("alice") # noqa: SLF001 + widget.sync_now() + widget.deleteLater() + QCoreApplication.sendPostedEvents(None, QEvent.DeferredDelete) + assert released.wait(5), "the worker was still waiting after its tab was destroyed" diff --git a/test/unit_test/headless/test_sync_adapters.py b/test/unit_test/headless/test_sync_adapters.py new file mode 100644 index 000000000..e34b314c2 --- /dev/null +++ b/test/unit_test/headless/test_sync_adapters.py @@ -0,0 +1,585 @@ +"""Config-sync adapters: what leaves this machine, and what arriving data may do. + +Syncing settings between machines must not carry secrets or machine paths, +must not arm a hotkey or trigger on the machine that receives it, and must +not leave a script without the content its hash promises. The folder mirror +and the clipboard channel must not send back what they just received. +""" +import hashlib +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +from je_auto_control.utils.config_sync import ( + ConfigSyncClient, ConfigSyncConflict, ConfigSyncError, SyncOutbox, +) +from je_auto_control.utils.config_sync.adapters import ( + AddressBookSyncAdapter, HotkeySyncAdapter, LocatorSyncAdapter, ScriptSyncAdapter, + SyncAdapter, TriggerSyncAdapter, is_local_ref, +) +from je_auto_control.utils.config_sync.assets import ( + AssetManifest, AssetSyncError, DirectoryAssetTransport, publish_assets, sync_assets, +) +from je_auto_control.utils.config_sync.session import ( + config_sync_status, default_device_id, resolve_conflict, run_full_resync, run_sync, + sync_status, +) +from je_auto_control.utils.hotkey.hotkey_daemon import HotkeyDaemon +from je_auto_control.utils.remote_desktop.clipboard_sync import ClipboardEchoGuard +from je_auto_control.utils.remote_desktop.file_sync import FolderSyncEngine +from je_auto_control.utils.triggers.trigger_engine import ( + AllOfTrigger, CronTrigger, ImageAppearsTrigger, TriggerEngine, +) + +_URL = "https://sync.invalid" + + +class _Server: + """The revision-checked bucket endpoint, in memory.""" + + def __init__(self): + self.body = None + self.revision = 0 + self.offline = False + + def request(self, method, body=None): + if self.offline: + raise ConfigSyncError("config sync: connection refused") + if method == "GET": + return self.body + if body["base_revision"] != self.revision: + raise ConfigSyncConflict("behind", self.revision) + self.revision += 1 + self.body = {**body["bucket"], "revision": self.revision} + return {"ok": True, "revision": self.revision} + + +@pytest.fixture +def server(): + endpoint = _Server() + with patch.object(ConfigSyncClient, "_request", + new=lambda _client, method, body=None: endpoint.request(method, body)): + yield endpoint + + +class _Machine: + """One device: its own outbox, its own stores, the shared server.""" + + def __init__(self, tmp_path, name): + self.name = name + self.home = tmp_path / name + self.scripts = self.home / "scripts" + self.scripts.mkdir(parents=True) + self.outbox = SyncOutbox(self.home / "outbox.sqlite3", account="alice", endpoint=_URL, + base_delay_s=0.0) + self.client = ConfigSyncClient(_URL, user_id="alice") + self.adapters = [] + + def sync(self, **options): + return run_sync(self.client, self.outbox, self.adapters, device_id=self.name, **options) + + +class _DictAdapter(SyncAdapter): + """A section backed by a plain dict, for the rules every adapter shares.""" + + section = "custom" + path_fields = ("script_path",) + + def __init__(self, origin, store, scripts_dir=None): + super().__init__(origin, scripts_dir=scripts_dir) + self.store = store + + def read_local(self): + return {key: dict(value) for key, value in self.store.items()} + + def write_local(self, key, value, existing): + self.store[key] = dict(value) + + def delete_local(self, key): + self.store.pop(key, None) + + +def _pair(tmp_path): + return _Machine(tmp_path, "laptop"), _Machine(tmp_path, "desktop") + + +# --- scripts and assets ---------------------------------------------------- + +def test_script_asset_hash_round_trip(tmp_path, server): + laptop, desktop = _pair(tmp_path) + shared = tmp_path / "shared-assets" + small = json.dumps([["AC_click_mouse", {"mouse_keycode": "mouse_left"}]]).encode("utf-8") + large = json.dumps([["AC_write", {"write_string": "x" * 100_000}]]).encode("utf-8") + (laptop.scripts / "sub").mkdir() + (laptop.scripts / "small.json").write_bytes(small) + (laptop.scripts / "sub" / "large.json").write_bytes(large) + laptop.adapters = [ScriptSyncAdapter("laptop", laptop.scripts)] + desktop.adapters = [ScriptSyncAdapter("desktop", desktop.scripts)] + transport = DirectoryAssetTransport(shared) + + assert laptop.sync(asset_transport=transport).state == "synced" + report = desktop.sync(asset_transport=transport) + + assert report.state == "synced" + for name, source in (("small.json", small), ("sub/large.json", large)): + source_hash = hashlib.sha256(source).hexdigest() + received_hash = hashlib.sha256((desktop.scripts / name).read_bytes()).hexdigest() + assert received_hash == source_hash + # The large script went through the transport, not through the bucket. + assert "content" not in server.body["sections"]["scripts"]["sub/large.json"]["value"] + assert report.assets["received"]["transferred"] == ["sub/large.json"] + # Nothing changed, so nothing is sent again -- and nothing was deleted. + before = server.revision + assert desktop.sync(asset_transport=transport).state == "synced" + assert laptop.sync(asset_transport=transport).state == "synced" + assert server.revision == before + assert (laptop.scripts / "sub" / "large.json").read_bytes() == large + + +def test_a_large_script_without_its_content_is_not_mistaken_for_a_deletion(tmp_path, server): + laptop, desktop = _pair(tmp_path) + large = json.dumps([["AC_write", {"write_string": "x" * 100_000}]]).encode("utf-8") + (laptop.scripts / "large.json").write_bytes(large) + laptop.adapters = [ScriptSyncAdapter("laptop", laptop.scripts)] + desktop.adapters = [ScriptSyncAdapter("desktop", desktop.scripts)] + laptop.sync() + + report = desktop.sync() # no transport: the content cannot arrive + assert "large.json" in report.applied["scripts"]["skipped"] + assert not (desktop.scripts / "large.json").exists() + desktop.sync() + laptop.sync() + assert (laptop.scripts / "large.json").read_bytes() == large, "the laptop's copy survived" + assert server.body["sections"]["scripts"]["large.json"]["deleted"] is False + + +def test_a_tampered_asset_is_refused_and_the_old_file_survives(tmp_path): + root = tmp_path / "scripts" + root.mkdir() + (root / "template.png").write_bytes(b"old image") + wanted = b"new image" + manifest = AssetManifest.from_entries(root, {"template.png": { + "sha256": hashlib.sha256(wanted).hexdigest(), "size": len(wanted)}}) + + class _Tampering: + def fetch(self, sha256): + return b"evil bits" + + result = sync_assets(manifest, _Tampering()) + assert "template.png" in result.failed and result.transferred == [] + assert (root / "template.png").read_bytes() == b"old image" + assert [path.name for path in root.iterdir()] == ["template.png"], "no temp file left" + + +def test_assets_publish_fetch_and_stay_inside_the_folder(tmp_path): + source, target, shared = tmp_path / "a", tmp_path / "b", tmp_path / "shared" + (source / "img").mkdir(parents=True) + target.mkdir() + (source / "img" / "ok.png").write_bytes(b"\x89PNG data") + manifest = AssetManifest.from_directory(source, ("*.png",)) + transport = DirectoryAssetTransport(shared) + assert publish_assets(manifest, transport).transferred == ["img/ok.png"] + assert publish_assets(manifest, transport).unchanged == ["img/ok.png"] + + incoming = AssetManifest(root=target, assets=manifest.assets) + assert sync_assets(incoming, transport).transferred == ["img/ok.png"] + assert sync_assets(incoming, transport).unchanged == ["img/ok.png"] + assert (target / "img" / "ok.png").read_bytes() == b"\x89PNG data" + + digest = manifest.assets[0].sha256 + for escaping in ("../outside.png", "/abs.png", "C:/abs.png", "a\\..\\b.png"): + bad = AssetManifest.from_entries(target, {escaping: {"sha256": digest, "size": 9}}) + assert escaping in sync_assets(bad, transport).failed + assert not (tmp_path / "outside.png").exists() + with pytest.raises(AssetSyncError): + AssetManifest.from_entries(target, {"x.png": {"sha256": "not-a-hash", "size": 1}}) + + +def test_a_script_with_a_literal_secret_is_withheld(tmp_path, server): + laptop, _desktop = _pair(tmp_path) + (laptop.scripts / "login.json").write_text(json.dumps( + [["AC_write", {"password": "hunter2-very-secret-value"}]]), encoding="utf-8") + (laptop.scripts / "ok.json").write_text(json.dumps( + [["AC_write", {"password": "${secrets.login}"}]]), encoding="utf-8") + laptop.adapters = [ScriptSyncAdapter("laptop", laptop.scripts)] + report = laptop.sync() + assert "scripts/login.json" in report.withheld + assert set(server.body["sections"]["scripts"]) == {"ok.json"} + assert "hunter2" not in json.dumps(server.body) + + +# --- secrets and machine paths --------------------------------------------- + +def test_secret_is_local(tmp_path, server): + laptop, desktop = _pair(tmp_path) + laptop_store = {"vpn": {"host": "10.0.0.1", "password": "laptop-pass-123", + "api_token": "${secrets.vpn_token}"}} + desktop_store = {"vpn": {"host": "old", "password": "desktop-pass-456"}} + laptop.adapters = [_DictAdapter("laptop", laptop_store)] + desktop.adapters = [_DictAdapter("desktop", desktop_store)] + + laptop.sync() + sent = server.body["sections"]["custom"]["vpn"]["value"] + sync_payload_contains_secret = "laptop-pass-123" in json.dumps(server.body) + assert sync_payload_contains_secret is False + assert is_local_ref(sent["password"]) + assert sent["api_token"] == "${secrets.vpn_token}", "a reference is not a secret" + + # The desktop pushes its own older version of the same key: a conflict, + # not a silent overwrite of either side. + report = desktop.sync() + assert report.state == "conflict" and report.conflicts == ["custom/vpn"] + resolved = resolve_conflict(desktop.outbox, desktop.adapters, device_id="desktop", + section="custom", key="vpn", choice=_sibling(server, "laptop")) + assert resolved["value"]["host"] == "10.0.0.1" + assert desktop.sync().state == "synced" + assert desktop_store["vpn"]["host"] == "10.0.0.1" + assert desktop_store["vpn"]["password"] == "desktop-pass-456", "its own secret stayed" + assert laptop.sync().state == "synced" + assert laptop_store["vpn"]["password"] == "laptop-pass-123" + + +def _sibling(server, origin): + siblings = server.body["sections"]["custom"]["vpn"]["siblings"] + return [sibling["origin"] for sibling in siblings].index(origin) + + +def test_machine_paths_travel_as_references(tmp_path, server): + laptop, desktop = _pair(tmp_path) + outside = tmp_path / "elsewhere" / "private.json" + laptop_store = {"in": {"script_path": str(laptop.scripts / "run.json")}, + "out": {"script_path": str(outside)}} + desktop_store = {} + laptop.adapters = [_DictAdapter("laptop", laptop_store, laptop.scripts)] + desktop.adapters = [_DictAdapter("desktop", desktop_store, desktop.scripts)] + laptop.sync() + assert str(tmp_path) not in json.dumps(server.body), "no absolute path left the machine" + + report = desktop.sync() + assert Path(desktop_store["in"]["script_path"]) == desktop.scripts / "run.json" + assert "out" in report.applied["custom"]["skipped"] and "out" not in desktop_store + # Not having it here is not a deletion for the laptop. + desktop.sync() + laptop.sync() + assert "out" in laptop_store + + +def test_a_reference_cannot_escape_the_scripts_folder(tmp_path): + adapter = _DictAdapter("laptop", {}, tmp_path / "scripts") + escaping = {"$local": "path", "root": "scripts", "relative": "../../etc/passwd"} + with pytest.raises(ConfigSyncError): + adapter.from_portable({"script_path": escaping}, None) + for drive_or_backslash in ("C:/Windows/x.json", "a\\..\\..\\x.json"): + with pytest.raises(ConfigSyncError): + adapter.from_portable({"script_path": { + "$local": "path", "root": "scripts", "relative": drive_or_backslash}}, None) + good = hashlib.sha256(b"[]").hexdigest() + for key in ("../evil.json", "/abs.json", "C:/abs.json", "a\\..\\..\\evil.json"): + with pytest.raises(ConfigSyncError): + ScriptSyncAdapter("laptop", tmp_path / "s").write_local( + key, {"content": "[]", "sha256": good}, None) + assert not (tmp_path / "evil.json").exists() + with pytest.raises(ConfigSyncError): + ScriptSyncAdapter("laptop", tmp_path).write_local( + "ok.json", {"content": "[]", "sha256": "0" * 64}, None) + + +# --- hotkeys and triggers are never armed by a sync ------------------------ + +def test_sync_never_enables_trigger(tmp_path, server): + laptop, desktop = _pair(tmp_path) + ran = [] + source = TriggerEngine(executor=ran.append) + target = TriggerEngine(executor=ran.append) + source.add(ImageAppearsTrigger(trigger_id="img", script_path=str(laptop.scripts / "a.json"), + image_path=str(laptop.scripts / "t.png"), enabled=True)) + source.add(CronTrigger(trigger_id="cron", script_path=str(laptop.scripts / "a.json"), + cron="0 9 * * *", enabled=True)) + source.add(AllOfTrigger(trigger_id="both", script_path=str(laptop.scripts / "a.json"))) + laptop.adapters = [TriggerSyncAdapter("laptop", source, scripts_dir=laptop.scripts)] + desktop.adapters = [TriggerSyncAdapter("desktop", target, scripts_dir=desktop.scripts)] + + assert "triggers/both" in laptop.sync().withheld + report = desktop.sync() + + received = {trigger.trigger_id: trigger for trigger in target.list_triggers()} + assert set(received) == {"img", "cron"} + enabled_triggers = [trigger.trigger_id for trigger in received.values() if trigger.enabled] + assert enabled_triggers == [] + assert sorted(report.applied["triggers"]["left_disabled"]) == ["cron", "img"] + assert Path(received["img"].image_path) == desktop.scripts / "t.png" + assert received["cron"].cron == "0 9 * * *" + assert not target.is_running and not source.is_running and ran == [] + assert "enabled" not in json.dumps(server.body["sections"]["triggers"]) + + # The laptop edits its (enabled) trigger; the desktop's stays off. + next(t for t in source.list_triggers() if t.trigger_id == "cron").cron = "30 9 * * *" + laptop.sync() + desktop.sync() + cron = next(t for t in target.list_triggers() if t.trigger_id == "cron") + assert cron.cron == "30 9 * * *" and cron.enabled is False + # Once this machine's user enables it, a later sync leaves that alone too. + target.set_enabled("cron", True) + next(t for t in source.list_triggers() if t.trigger_id == "cron").cron = "45 9 * * *" + laptop.sync() + desktop.sync() + cron = next(t for t in target.list_triggers() if t.trigger_id == "cron") + assert cron.cron == "45 9 * * *" and cron.enabled is True + assert next(t for t in source.list_triggers() if t.trigger_id == "img").enabled is True + + +def test_sync_never_enables_a_hotkey(tmp_path, server): + laptop, desktop = _pair(tmp_path) + source, target = HotkeyDaemon(), HotkeyDaemon() + source.bind("ctrl+alt+k", str(laptop.scripts / "a.json"), binding_id="hk1") + laptop.adapters = [HotkeySyncAdapter("laptop", source, scripts_dir=laptop.scripts)] + desktop.adapters = [HotkeySyncAdapter("desktop", target, scripts_dir=desktop.scripts)] + laptop.sync() + report = desktop.sync() + + [binding] = target.list_bindings() + assert binding.combo == "ctrl+alt+k" and binding.enabled is False + assert Path(binding.script_path) == desktop.scripts / "a.json" + assert report.applied["hotkeys"]["left_disabled"] == ["hk1"] + assert not target.is_running + assert source.list_bindings()[0].enabled is True, "the sender's own state is untouched" + + source.unbind("hk1") + laptop.sync() + desktop.sync() + assert target.list_bindings() == [] + + +def test_a_binding_this_platform_refuses_is_skipped_not_deleted_for_everyone(tmp_path, server): + laptop, desktop = _pair(tmp_path) + + class _Picky(HotkeyDaemon): + def bind(self, combo, script_path, binding_id=None, *, enabled=True): + raise ValueError(f"cannot register {combo} here") + + source, target = HotkeyDaemon(), _Picky() + source.bind("ctrl+alt+k", str(laptop.scripts / "a.json"), binding_id="hk1") + laptop.adapters = [HotkeySyncAdapter("laptop", source, scripts_dir=laptop.scripts)] + desktop.adapters = [HotkeySyncAdapter("desktop", target, scripts_dir=desktop.scripts)] + laptop.sync() + assert "hk1" in desktop.sync().applied["hotkeys"]["skipped"] + desktop.sync() + laptop.sync() + assert [binding.binding_id for binding in source.list_bindings()] == ["hk1"] + + +# --- locators and the address book ----------------------------------------- + +def test_locators_and_address_book_round_trip(tmp_path, server): + from je_auto_control.utils.element_repository import ElementRepository + from je_auto_control.utils.remote_desktop.address_book import AddressBook + laptop, desktop = _pair(tmp_path) + repos = [ElementRepository(str(machine.home / "locators.json")) + for machine in (laptop, desktop)] + books = [AddressBook(machine.home / "address_book.json") for machine in (laptop, desktop)] + repos[0].save("ok_button", name="OK", role="button") + books[0].upsert(host_id="host1", server_url="https://rd.invalid", label="Office") + books[0].set_tags(host_id="host1", server_url="https://rd.invalid", tags=["work"]) + books[0].toggle_favorite(host_id="host1", server_url="https://rd.invalid") + laptop.adapters = [LocatorSyncAdapter("laptop", repos[0]), + AddressBookSyncAdapter("laptop", books[0])] + desktop.adapters = [LocatorSyncAdapter("desktop", repos[1]), + AddressBookSyncAdapter("desktop", books[1])] + laptop.sync() + desktop.sync() + + assert repos[1].get("ok_button") == {"name": "OK", "role": "button"} + [entry] = books[1].list_entries() + assert (entry["label"], entry["tags"], entry["favorite"]) == ("Office", ["work"], True) + assert "last_used" not in json.dumps(server.body["sections"]["address_book"]) + + repos[1].remove("ok_button") + books[1].remove(host_id="host1", server_url="https://rd.invalid") + desktop.sync() + laptop.sync() + assert repos[0].all() == {} and books[0].list_entries() == [] + + +# --- status, offline, resync ----------------------------------------------- + +def test_offline_changes_wait_in_the_outbox_and_the_status_says_so(tmp_path, server): + laptop, _desktop = _pair(tmp_path) + store = {"a": {"v": 1}} + laptop.adapters = [_DictAdapter("laptop", store)] + server.offline = True + report = laptop.sync(max_attempts=2) + assert (report.state, report.pending) == ("offline", 1) and report.error + assert laptop.sync(max_attempts=1).pending == 1, "the same edit is not queued twice" + status = sync_status(laptop.outbox) + assert (status["state"], status["pending"], status["revision"]) == ("offline", 1, 0) + + server.offline = False + report = laptop.sync() + assert (report.state, report.pending, report.revision) == ("synced", 0, 1) + status = config_sync_status(_URL, "alice", str(laptop.home / "outbox.sqlite3")) + assert status["state"] == "synced" and status["last_success"] > 0 + + +def test_a_cancelled_sync_stops_and_keeps_its_changes(tmp_path, server): + import threading + laptop, _desktop = _pair(tmp_path) + laptop.adapters = [_DictAdapter("laptop", {"a": {"v": 1}})] + cancel = threading.Event() + cancel.set() + report = laptop.sync(cancel=cancel) + assert report.state == "cancelled" and report.pending == 1 and server.revision == 0 + + +def test_a_retired_device_is_told_to_resync_and_adopts_the_server_state(tmp_path, server): + laptop, desktop = _pair(tmp_path) + laptop_store, desktop_store = {"a": {"v": 1}}, {} + laptop.adapters = [_DictAdapter("laptop", laptop_store)] + desktop.adapters = [_DictAdapter("desktop", desktop_store)] + laptop.sync() + desktop.sync() + laptop.client.retire_peer("desktop") + del laptop_store["a"] + laptop.sync() + laptop.sync() # the tombstone is collected: nobody waits for desktop + assert server.body["sections"]["custom"] == {} + + desktop_store["a"] = {"v": "edited while away"} + report = desktop.sync() + assert report.state == "resync_required" + assert server.body["sections"]["custom"] == {}, "the stale edit did not get in" + + report = run_full_resync(desktop.client, desktop.outbox, desktop.adapters, + device_id="desktop") + assert report.state == "synced" and "custom/a" in report.withheld + assert desktop_store == {}, "the deleted entry did not come back" + assert desktop.sync().state == "synced" + + +def test_the_device_id_is_created_once_and_kept(tmp_path, monkeypatch): + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + first = default_device_id() + assert first and default_device_id() == first + assert (tmp_path / ".je_auto_control" / "config_sync_device_id").read_text( + encoding="utf-8").strip() == first + + +# --- the mirror channels do not echo --------------------------------------- + +def test_clipboard_does_not_echo(): + host, viewer = ClipboardEchoGuard(), ClipboardEchoGuard() + wire = [] + + def copy_on(sender, receiver, text): + """A local copy on one side, forwarded and applied on the other.""" + if sender.should_send("text", text): + wire.append(text) + receiver.note_remote("text", text) + return True + return False + + assert copy_on(viewer, host, "hello") is True + # The host's watcher now sees "hello" on its clipboard: it must not go back. + assert copy_on(host, viewer, "hello") is False + # Nor does the viewer keep resending what the clipboard still holds. + assert copy_on(viewer, host, "hello") is False + assert wire == ["hello"] + # A real change on the host does travel, once. + assert copy_on(host, viewer, "reply") is True + assert copy_on(viewer, host, "reply") is False + assert wire == ["hello", "reply"] + assert host.should_send("image", b"hello") is True, "another kind is other content" + host.reset() + assert host.should_send("text", "reply") is True, "after a reconnect nothing is assumed" + + +def test_a_folder_mirror_does_not_push_back_what_it_received(tmp_path): + watch = tmp_path / "watch" + watch.mkdir() + sent = [] + engine = FolderSyncEngine(watch_dir=watch, sender=lambda _path, name: sent.append(name)) + assert engine.poll_once() == [] # baseline + + # Noted before the bytes land, so a poll in between cannot echo it. + incoming = b"from the peer" + engine.note_received("report.txt", sha256=hashlib.sha256(incoming).hexdigest()) + (watch / "report.txt").write_bytes(incoming) + (watch / "mine.txt").write_text("made here", encoding="utf-8") + assert engine.poll_once() == ["mine.txt"] + assert engine.poll_once() == [] and sent == ["mine.txt"] + + # Edited here afterwards: now it is a local change and goes out. + import os + (watch / "report.txt").write_bytes(b"edited locally") + stamp = (watch / "report.txt").stat().st_mtime + 5 + os.utime(watch / "report.txt", (stamp, stamp)) + assert engine.poll_once() == ["report.txt"] + + +def test_a_received_file_noted_after_it_landed_is_not_echoed(tmp_path): + watch = tmp_path / "watch" + watch.mkdir() + sent = [] + engine = FolderSyncEngine(watch_dir=watch, sender=lambda _path, name: sent.append(name)) + engine.poll_once() + (watch / "a.bin").write_bytes(b"payload") + engine.note_received("a.bin") + assert engine.poll_once() == [] and sent == [] + + +# --- the delivery surfaces: executor, MCP, Script Builder, facade ---------- + +_COMMANDS = ("AC_config_sync_run", "AC_config_sync_status", "AC_config_sync_resolve", + "AC_config_sync_full_resync") + + +def test_the_commands_run_from_an_action_list(tmp_path, server): + from je_auto_control.utils.executor.action_executor import executor + scripts = tmp_path / "scripts" + scripts.mkdir() + (scripts / "a.json").write_text("[]", encoding="utf-8") + target = {"server_url": _URL, "user_id": "alice"} + options = {"device_id": "laptop", "sections": "scripts", "scripts_dir": str(scripts), + "outbox_path": str(tmp_path / "outbox.sqlite3")} + record = executor.execute_action([ + ["AC_config_sync_run", {**target, **options}], + ["AC_config_sync_status", {**target, "outbox_path": options["outbox_path"]}], + ]) + run, status = list(record.values()) + assert run["state"] == "synced" and run["revision"] == 1 + assert status["state"] == "synced" and status["pending"] == 0 + assert set(server.body["sections"]["scripts"]) == {"a.json"} + json.dumps(record) + + +def test_an_unknown_option_or_section_is_refused(tmp_path, server): + from je_auto_control.utils.config_sync import config_sync_run + options = {"device_id": "laptop", "outbox_path": str(tmp_path / "outbox.sqlite3")} + with pytest.raises(ConfigSyncError, match="unknown config sync option"): + config_sync_run(_URL, "alice", enable_everything=True, **options) + with pytest.raises(ConfigSyncError, match="cannot sync section"): + config_sync_run(_URL, "alice", sections=["scripts"], **options) + assert server.revision == 0 + + +def test_every_surface_knows_the_commands(): + import je_auto_control as ac + from je_auto_control.gui.script_builder.command_schema import _build_specs + from je_auto_control.utils.executor.action_executor import executor + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + assert set(_COMMANDS) <= set(executor.known_commands()) + assert set(_COMMANDS) <= {spec.command for spec in _build_specs()} + tools = {tool.name: tool + for tool in build_default_tool_registry(read_only=False, aliases=False)} + for command in _COMMANDS: + assert "ac_" + command[3:] in tools + assert tools["ac_config_sync_status"].annotations.read_only + for acting in ("ac_config_sync_run", "ac_config_sync_resolve", "ac_config_sync_full_resync"): + assert tools[acting].annotations.destructive and not tools[acting].annotations.read_only + read_only = {tool.name + for tool in build_default_tool_registry(read_only=True, aliases=False)} + assert "ac_config_sync_run" not in read_only and "ac_config_sync_status" in read_only + for name in ("ConfigSyncClient", "SyncEntry", "SyncOutbox", "merge_entries", "sync_assets", + "config_sync_run", "SyncAdapter", "ConfigStore"): + assert name in ac.__all__ and hasattr(ac, name) From 0f03a822dfa74a6324970c7be2f28eb3c4947a0a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:13:57 +0800 Subject: [PATCH 25/44] Fix the Sphinx warnings in the pages changed since v0.0.225 The docs had not been built since about fifty pages changed. A build showed 211 warnings, 27 of them in those pages: inline literals and bold text touching a full-width bracket or a CJK character (rendered as raw markup), two malformed tables whose first column was narrower than a row, a section and a toctree entry pasted twice in each language, and title underlines shorter than their titles. All 27 are fixed; the other 184 are in pages this period did not touch. --- .../Eng/doc/new_features/new_features_doc.rst | 4 +-- .../doc/new_features/v223_features_doc.rst | 15 ---------- .../Eng/doc/new_features/v22_features_doc.rst | 4 +-- .../usb_passthrough_operator_guide.rst | 30 +++++++++---------- docs/source/Eng/eng_index.rst | 1 - docs/source/Zh/doc/cli/cli_doc.rst | 4 +-- .../Zh/doc/mcp_server/mcp_server_doc.rst | 2 +- .../Zh/doc/new_features/new_features_doc.rst | 2 +- .../Zh/doc/new_features/v116_features_doc.rst | 2 +- .../Zh/doc/new_features/v223_features_doc.rst | 16 ++-------- .../Zh/doc/new_features/v2_features_doc.rst | 6 ++-- .../Zh/doc/new_features/v4_features_doc.rst | 2 +- .../Zh/doc/new_features/v6_features_doc.rst | 4 +-- .../Zh/doc/new_features/v85_features_doc.rst | 2 +- .../usb_passthrough_design.rst | 2 +- .../usb_passthrough_operator_guide.rst | 30 +++++++++---------- docs/source/Zh/zh_index.rst | 1 - 17 files changed, 49 insertions(+), 78 deletions(-) diff --git a/docs/source/Eng/doc/new_features/new_features_doc.rst b/docs/source/Eng/doc/new_features/new_features_doc.rst index c4fc2a8a5..dd09109ec 100644 --- a/docs/source/Eng/doc/new_features/new_features_doc.rst +++ b/docs/source/Eng/doc/new_features/new_features_doc.rst @@ -1,6 +1,6 @@ -===================== +====================== New Features (2026-04) -===================== +====================== This page documents the April 2026 additions to AutoControl. Every new feature ships with a headless Python API **and** a GUI affordance, and is diff --git a/docs/source/Eng/doc/new_features/v223_features_doc.rst b/docs/source/Eng/doc/new_features/v223_features_doc.rst index 8b64c4fc4..91b68ad6a 100644 --- a/docs/source/Eng/doc/new_features/v223_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v223_features_doc.rst @@ -40,21 +40,6 @@ A tab is built the first time it is opened: the window starts with the three default tabs and the forms the main widget owns, and imports the module of any other tab only when you open it. -The navigation panel --------------------- - -Every registered tab is listed on the left of the window, grouped by the same -five categories, whether it is open or not; open tabs are shown in bold. -Click a feature to open it (or bring it to the front). The search box filters -the list as you type — by title, by key (``usb_devices``) or by category — and -**Return** opens the first match. ``Ctrl+K`` (**View → Search Features...**) -puts the cursor in the search box from anywhere, and ``Ctrl+B`` -(**View → Navigation Panel**) hides or shows the panel. - -A tab is built the first time it is opened: the window starts with the three -default tabs and the forms the main widget owns, and imports the module of -any other tab only when you open it. - The View menu ------------- diff --git a/docs/source/Eng/doc/new_features/v22_features_doc.rst b/docs/source/Eng/doc/new_features/v22_features_doc.rst index 13c53a210..dc99a901f 100644 --- a/docs/source/Eng/doc/new_features/v22_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v22_features_doc.rst @@ -16,7 +16,7 @@ dependency); wired through the full stack. Numbering and the legend -======================= +======================== :: @@ -34,7 +34,7 @@ unit-testable with synthetic elements. Live "mark then click" loop -========================== +=========================== :: diff --git a/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst b/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst index 0e7546078..faaedfa5b 100644 --- a/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst +++ b/docs/source/Eng/doc/operations_layer/usb_passthrough_operator_guide.rst @@ -294,21 +294,21 @@ Everything the GUI does is also an executor command, so JSON action files, the socket server, and the scheduler can drive USB passthrough with no GUI: -================================ ============================================ -Command Purpose -================================ ============================================ -``AC_usb_passthrough_enable`` Toggle the feature flag (``enabled`` bool) -``AC_usb_passthrough_status`` Report whether passthrough is enabled -``AC_usb_acl_list`` List ACL rules + default + integrity state -``AC_usb_acl_add`` Add a per-device rule -``AC_usb_acl_remove`` Remove a rule -``AC_usb_acl_set_default`` Set the default policy (allow/deny) -``AC_usb_acl_export`` / ``_import`` Back up / restore the ACL as JSON -``AC_usb_loopback_list`` List ACL-visible devices over loopback -``AC_usb_loopback_open`` Claim a local device + read its descriptor -``AC_usb_remote_list`` List a remote host's devices (live WebRTC) -``AC_usb_remote_open`` Claim a remote device + read its descriptor -================================ ============================================ +==================================== ============================================ +Command Purpose +==================================== ============================================ +``AC_usb_passthrough_enable`` Toggle the feature flag (``enabled`` bool) +``AC_usb_passthrough_status`` Report whether passthrough is enabled +``AC_usb_acl_list`` List ACL rules + default + integrity state +``AC_usb_acl_add`` Add a per-device rule +``AC_usb_acl_remove`` Remove a rule +``AC_usb_acl_set_default`` Set the default policy (allow/deny) +``AC_usb_acl_export`` / ``_import`` Back up / restore the ACL as JSON +``AC_usb_loopback_list`` List ACL-visible devices over loopback +``AC_usb_loopback_open`` Claim a local device + read its descriptor +``AC_usb_remote_list`` List a remote host's devices (live WebRTC) +``AC_usb_remote_open`` Claim a remote device + read its descriptor +==================================== ============================================ Example JSON action:: diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index e826e9bb4..b42f2267b 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -21,7 +21,6 @@ Comprehensive guides for all AutoControl features. doc/socket_driver/socket_driver_doc doc/mcp_server/mcp_server_doc doc/ai_agents/ai_agents_doc - doc/ai_agents/ai_agents_doc doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/create_project/create_project_doc diff --git a/docs/source/Zh/doc/cli/cli_doc.rst b/docs/source/Zh/doc/cli/cli_doc.rst index e76b14f70..0dc9ef092 100644 --- a/docs/source/Zh/doc/cli/cli_doc.rst +++ b/docs/source/Zh/doc/cli/cli_doc.rst @@ -37,9 +37,9 @@ AutoControl 可以直接從命令列執行自動化腳本。 ====================== ``AC_add_package_to_executor`` 與 ``AC_add_package_to_callback_executor`` 會拒絕沒有被放行的套件。 -從命令列放行的方式有兩種：環境變數 ``JE_AUTOCONTROL_ALLOWED_PACKAGES``（以逗號分隔的套件名稱，含子模組）， +從命令列放行的方式有兩種：環境變數 ``JE_AUTOCONTROL_ALLOWED_PACKAGES``\ （以逗號分隔的套件名稱，含子模組）， 對上面的旗標、``start-server``、``start-rest`` 與 MCP server 等所有入口都適用；以及 ``je_auto_control run`` -的 ``--allow-package NAME``（可重複），只對該次執行有效。 +的 ``--allow-package NAME``\ （可重複），只對該次執行有效。 .. code-block:: bash diff --git a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst index f8a9546a1..9e381b607 100644 --- a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst @@ -461,7 +461,7 @@ client 不用改,可以和 2026-07-28 的 client 並存。 只有名稱符合時才會解析，其餘回工具執行錯誤。沒設時和以前一樣，任何變數都讀得到—— 包括放 API 金鑰的那些。 -在程式裡，同一份設定是 ``server.argument_policy``（:class:`ArgumentPolicy`，內含 +在程式裡，同一份設定是 ``server.argument_policy``\ （:class:`ArgumentPolicy`，內含 :class:`je_auto_control.PathPolicy` 與允許清單）；自己建立的伺服器可以指派另一個。 破壞性動作確認(Elicitation) diff --git a/docs/source/Zh/doc/new_features/new_features_doc.rst b/docs/source/Zh/doc/new_features/new_features_doc.rst index d51ef710f..daad97e9b 100644 --- a/docs/source/Zh/doc/new_features/new_features_doc.rst +++ b/docs/source/Zh/doc/new_features/new_features_doc.rst @@ -288,7 +288,7 @@ Action-JSON 指令：``AC_vlm_locate``、``AC_vlm_click``。GUI： ======================= 排程器、觸發器、熱鍵守護程序、REST API 與 GUI 手動回放的每一次執行 -都會被寫入 ``~/.je_auto_control/history.db``（SQLite）。失敗時會自動 +都會被寫入 ``~/.je_auto_control/history.db``\ （SQLite）。失敗時會自動 擷取螢幕截圖並附到該筆紀錄上:: from je_auto_control import default_history_store diff --git a/docs/source/Zh/doc/new_features/v116_features_doc.rst b/docs/source/Zh/doc/new_features/v116_features_doc.rst index 8c779a5df..b2f90cbbf 100644 --- a/docs/source/Zh/doc/new_features/v116_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v116_features_doc.rst @@ -26,7 +26,7 @@ 名稱——``linear`` / ``ease_in_out_quad`` / ``ease_out_cubic`` / ``ease_in_cubic``),且不重複共用的交接點。 ``move_along_path`` 沿路徑發出移動事件;``drag_path`` 在第一個路徑點按下、移動經過整段路徑、在最後一點放開—— 用於多停靠點拖曳。兩者皆可傳入 ``sink`` 以供無頭測試。``drag_path`` 也接受 ``tween_drag`` 的節奏參數 -``step_delay_s``(每次移動後)與 ``settle_s``(按下前、按下後、放開前),而且和它一樣,任何一步丟出例外時會在 +``step_delay_s``\ (每次移動後)與 ``settle_s``\ (按下前、按下後、放開前),而且和它一樣,任何一步丟出例外時會在 游標停下的位置放開按鍵。 執行器命令 diff --git a/docs/source/Zh/doc/new_features/v223_features_doc.rst b/docs/source/Zh/doc/new_features/v223_features_doc.rst index 6f0e01764..37819630e 100644 --- a/docs/source/Zh/doc/new_features/v223_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v223_features_doc.rst @@ -28,20 +28,8 @@ Actions 選單 視窗左側依同樣五個分類列出每一個已註冊的分頁,不論是否已開啟;已開啟的以粗體 顯示。點一下就開啟該功能(已開啟則切到最前面)。搜尋框會隨輸入過濾清單——比對 標題、鍵名(``usb_devices``)或分類——按 **Return** 開啟第一個符合的項目。 -``Ctrl+K``(**View → Search Features...**)從任何地方把游標移到搜尋框, -``Ctrl+B``(**View → Navigation Panel**)隱藏或顯示面板。 - -分頁在第一次開啟時才建立:視窗啟動時只有三個預設分頁與主元件自己的表單,其他 -分頁的模組要到開啟時才匯入。 - -導覽面板 --------- - -視窗左側依同樣五個分類列出每一個已註冊的分頁,不論是否已開啟;已開啟的以粗體 -顯示。點一下就開啟該功能(已開啟則切到最前面)。搜尋框會隨輸入過濾清單——比對 -標題、鍵名(``usb_devices``)或分類——按 **Return** 開啟第一個符合的項目。 -``Ctrl+K``(**View → Search Features...**)從任何地方把游標移到搜尋框, -``Ctrl+B``(**View → Navigation Panel**)隱藏或顯示面板。 +``Ctrl+K``\ (**View → Search Features...**)從任何地方把游標移到搜尋框, +``Ctrl+B``\ (**View → Navigation Panel**)隱藏或顯示面板。 分頁在第一次開啟時才建立:視窗啟動時只有三個預設分頁與主元件自己的表單,其他 分頁的模組要到開啟時才匯入。 diff --git a/docs/source/Zh/doc/new_features/v2_features_doc.rst b/docs/source/Zh/doc/new_features/v2_features_doc.rst index df7709db9..ac950d60e 100644 --- a/docs/source/Zh/doc/new_features/v2_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v2_features_doc.rst @@ -283,7 +283,7 @@ Wayland libei native backend 對 ``libei.so.*`` 的 ctypes 綁定，繞過 CLI shim 取得微秒級延遲。 以 ``JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=libei|cli|auto`` 啟用； -``auto``（預設）在 libei 可載入時用 libei，否則用 CLI，現有 +``auto``\ （預設）在 libei 可載入時用 libei，否則用 CLI，現有 部署不會中斷。 @@ -377,7 +377,7 @@ helper（``je_auto_control.gui.flow_editor.layout_steps``）可單元 * ``backend`` — ``"anthropic"`` 或 ``"openai"``。``AC_run_agent`` 預設只提供聚焦、低風險的 computer-use allow-list，不再把完整的 ``AC_*`` 命令目錄交給模型。需要自訂工具集時， 可直接用 ``export_anthropic_tools(only=[...])`` 或 ``export_openai_tools(only=[...])`` 建立 backend。 -* ``max_steps``（預設 25）、``wall_seconds``（預設 300.0）。 +* ``max_steps``\ （預設 25）、``wall_seconds``\ （預設 300.0）。 * ``model`` / ``max_tokens`` — backend 專屬覆寫。 每次向模型發出的請求 120 秒逾時，對話裡只重送最新的三張截圖（較早的換成一行文字）， @@ -411,7 +411,7 @@ Anthropic 原生 Computer-Use 路徑（``computer_20251124``）仍透過 redacted_bytes, result = redact_png_bytes(png_bytes, policy=POLICY_STRICT) ``AC_redact_screenshot`` 與 ``ac_redact_screenshot`` 從磁碟讀取 -PNG、跑 engine、寫回 ``output_path``（未指定時覆蓋原檔），並回傳 +PNG、跑 engine、寫回 ``output_path``\ （未指定時覆蓋原檔），並回傳 合併後的 bounding box list 供稽核。 diff --git a/docs/source/Zh/doc/new_features/v4_features_doc.rst b/docs/source/Zh/doc/new_features/v4_features_doc.rst index 201592a29..8b562ccad 100644 --- a/docs/source/Zh/doc/new_features/v4_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v4_features_doc.rst @@ -168,5 +168,5 @@ GUI filter)都會快照進 undo stack;**Ctrl+Z** 與 Undo 按鈕可還原前一狀態。 * **觸發器分頁** — *Combine selected* 把選取的觸發器包成 AllOf / AnyOf / Sequence 複合觸發器;新增 **Cron** 觸發器型別。 -* **斷言分頁** — 新增 **VLM**(「畫面符合描述」)斷言型別。 +* **斷言分頁** — 新增 **VLM**\ (「畫面符合描述」)斷言型別。 * 每個新的 ``AC_*`` 指令都可在視覺化 **Script Builder** 中建構。 diff --git a/docs/source/Zh/doc/new_features/v6_features_doc.rst b/docs/source/Zh/doc/new_features/v6_features_doc.rst index 71958c7a5..176798db7 100644 --- a/docs/source/Zh/doc/new_features/v6_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v6_features_doc.rst @@ -3,7 +3,7 @@ ============================================ 兩個早已存在、卻從未接上其餘各層的 headless 核心,現在成為一級功能: -**黃金影像視覺回歸**與**宣告式有限狀態機**執行器。兩者都提供 facade +**黃金影像視覺回歸**\ 與\ **宣告式有限狀態機**\ 執行器。兩者都提供 facade re-export、``AC_*`` 執行器指令、MCP 工具與 Script Builder 項目,並有 headless 測試(PIL 影像 / spec 以注入方式提供,完全不需真實螢幕)。 @@ -35,7 +35,7 @@ headless 測試(PIL 影像 / spec 以注入方式提供,完全不需真實螢幕 * ``AC_take_golden`` — 擷取並存基準圖(可選 ``region``)。 * ``AC_assert_visual`` — 把畫面與黃金影像比對,不符即拋例外(可存 - ``diff_path``)。**首跑**(基準不存在)時會擷取基準並通過,除非 + ``diff_path``)。**首跑**\ (基準不存在)時會擷取基準並通過,除非 ``create_if_missing`` 設為 false。 diff --git a/docs/source/Zh/doc/new_features/v85_features_doc.rst b/docs/source/Zh/doc/new_features/v85_features_doc.rst index 84588f972..dd5e85a6e 100644 --- a/docs/source/Zh/doc/new_features/v85_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v85_features_doc.rst @@ -1,5 +1,5 @@ URI-Scheme 值參照 -================ +================= ``script_vars.interpolate`` 把單一間接寫死(``${secrets.NAME}`` → vault),而 ``AssetStore`` 的憑證 參照僅限 vault 名稱。沒有一個通用、可插拔的讀取時間接機制 —— 也就是現代設定模式:儲存一個*指標* diff --git a/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst b/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst index 74a48463f..ec7a1232c 100644 --- a/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst +++ b/docs/source/Zh/doc/operations_layer/usb_passthrough_design.rst @@ -388,7 +388,7 @@ macOS entitlement、Windows WinUSB 通常不需要）。README 會逐 OS 八個原始未決問題均已拍板，對應實作見上方各節： -1. **OQ1 — Channel 可靠度**：``maxRetransmits=None``（完全可靠有序）。 +1. **OQ1 — Channel 可靠度**：``maxRetransmits=None``\ （完全可靠有序）。 2. **OQ2 — frame 分片**：已實作 ``fragment_payload`` + EOF 重組。 3. **OQ3 — ``LIST`` 走 channel**：是，ACL 過濾後經 channel 回傳。 4. **OQ4 — Backpressure 顆粒度**：per-claim（per-endpoint 屬 YAGNI）。 diff --git a/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst b/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst index 4278b983d..4301f7b43 100644 --- a/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst +++ b/docs/source/Zh/doc/operations_layer/usb_passthrough_operator_guide.rst @@ -269,21 +269,21 @@ OPEN 後 host 鍵盤停止運作 Linux：HID 裝置被 claim GUI 能做的事都有對應的 executor 指令，所以 JSON action 檔、socket server 與排程器都能在沒有 GUI 的情況下驅動 USB passthrough： -================================ ============================================ -指令 用途 -================================ ============================================ -``AC_usb_passthrough_enable`` 切換 feature flag（``enabled`` 布林） -``AC_usb_passthrough_status`` 回報是否已啟用 -``AC_usb_acl_list`` 列出 ACL 規則 + 預設 + 完整性狀態 -``AC_usb_acl_add`` 新增 per-device 規則 -``AC_usb_acl_remove`` 移除規則 -``AC_usb_acl_set_default`` 設定預設政策（allow/deny） -``AC_usb_acl_export`` / ``_import`` 以 JSON 備份／還原 ACL -``AC_usb_loopback_list`` 經 loopback 列出 ACL 可見裝置 -``AC_usb_loopback_open`` claim 本機裝置並讀描述元 -``AC_usb_remote_list`` 列出遠端主機裝置（live WebRTC） -``AC_usb_remote_open`` claim 遠端裝置並讀描述元 -================================ ============================================ +==================================== ============================================ +指令 用途 +==================================== ============================================ +``AC_usb_passthrough_enable`` 切換 feature flag（``enabled`` 布林） +``AC_usb_passthrough_status`` 回報是否已啟用 +``AC_usb_acl_list`` 列出 ACL 規則 + 預設 + 完整性狀態 +``AC_usb_acl_add`` 新增 per-device 規則 +``AC_usb_acl_remove`` 移除規則 +``AC_usb_acl_set_default`` 設定預設政策（allow/deny） +``AC_usb_acl_export`` / ``_import`` 以 JSON 備份／還原 ACL +``AC_usb_loopback_list`` 經 loopback 列出 ACL 可見裝置 +``AC_usb_loopback_open`` claim 本機裝置並讀描述元 +``AC_usb_remote_list`` 列出遠端主機裝置（live WebRTC） +``AC_usb_remote_open`` claim 遠端裝置並讀描述元 +==================================== ============================================ JSON action 範例:: diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index eb1f7a49f..2610e1f66 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -21,7 +21,6 @@ AutoControl 所有功能的完整使用指南。 doc/socket_driver/socket_driver_doc doc/mcp_server/mcp_server_doc doc/ai_agents/ai_agents_doc - doc/ai_agents/ai_agents_doc doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/create_project/create_project_doc From c2d9820db3fed53fc7724da547231f50df184085 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:14:05 +0800 Subject: [PATCH 26/44] Generate every mobile surface from one command table so none can be missing The adb commands had executor entries but no Script Builder schema and no MCP tool, and the new session, gesture, locating, app and extension features had no command at all. MOBILE_COMMANDS now describes each AC_android_* / AC_ios_* command once; the executor handlers, MCP tools, builder schema and the new Mobile tab are generated from it, and a parity test fails when a surface drifts. device_setup_report gives the backend and OS versions and each capability's state for checking an emulator, device or remote WebDriverAgent setup. The setup docs say plainly that none of it was run on hardware. --- docs/source/Eng/doc/mobile/mobile_doc.rst | 119 ++++ docs/source/Zh/doc/mobile/mobile_doc.rst | 110 ++++ je_auto_control/__init__.py | 8 +- je_auto_control/actions.pyi | 354 ++++++++++- je_auto_control/android/session.py | 14 + je_auto_control/gui/device_matrix_tab.py | 7 +- .../gui/language_wrapper/english.py | 15 + .../gui/language_wrapper/japanese.py | 15 + .../language_wrapper/simplified_chinese.py | 15 + .../language_wrapper/traditional_chinese.py | 15 + je_auto_control/gui/mobile_tab.py | 186 ++++++ .../gui/script_builder/command_schema.py | 18 + je_auto_control/gui/tab_registry.py | 1 + je_auto_control/ios/session.py | 13 + .../utils/executor/action_executor.py | 9 + .../utils/mcp_server/tools/_factories.py | 17 +- je_auto_control/wrapper/device_context.py | 48 +- je_auto_control/wrapper/mobile_commands.py | 579 ++++++++++++++++++ .../headless/test_mobile_surface_parity.py | 295 +++++++++ 19 files changed, 1830 insertions(+), 8 deletions(-) create mode 100644 je_auto_control/gui/mobile_tab.py create mode 100644 je_auto_control/wrapper/mobile_commands.py create mode 100644 test/unit_test/headless/test_mobile_surface_parity.py diff --git a/docs/source/Eng/doc/mobile/mobile_doc.rst b/docs/source/Eng/doc/mobile/mobile_doc.rst index 8015a4c16..6de0e1bb2 100644 --- a/docs/source/Eng/doc/mobile/mobile_doc.rst +++ b/docs/source/Eng/doc/mobile/mobile_doc.rst @@ -343,3 +343,122 @@ WebDriverAgent covers what it does not provide:: register_mobile_extension("ios", lambda session: MyTideviceAdapter(session)) No such adapter ships with AutoControl. + +Commands, MCP tools, Script Builder, GUI +======================================== + +Every mobile command is described once, in +``je_auto_control.MOBILE_COMMANDS`` (``wrapper/mobile_commands.py``). The +executor's ``AC_android_*`` / ``AC_ios_*`` commands, the ``ac_android_*`` / +``ac_ios_*`` MCP tools, the Script Builder's **Android** and **iOS** +categories and the **Mobile** tab are all generated from that table, so a +command cannot exist on one surface and be missing from another +(``test_mobile_surface_parity.py`` fails if it does). + +Each command takes an optional address — ``serial`` and ``adb_path`` on +Android, ``url`` on iOS — plus ``device_timeout_s``. Without an address it +runs on the session bound by ``use_device`` (a device-matrix worker), else on +the backend's default device. Unknown parameters are rejected. + +.. list-table:: + :header-rows: 1 + :widths: 24 38 38 + + * - Group + - Android + - iOS + * - Device + - ``AC_android_device_info``, ``AC_android_screen_info``, + ``AC_android_list_devices`` + - ``AC_ios_device_info``, ``AC_ios_screen_info`` + * - Input + - ``AC_android_tap``, ``_swipe``, ``_long_press``, ``_drag``, + ``_pinch``, ``_key``, ``_text``, ``_type_text`` + - ``AC_ios_tap``, ``_swipe``, ``_long_press``, ``_drag``, ``_pinch``, + ``_press_key``, ``_type`` + * - Screen and locating + - ``AC_android_screenshot``, ``_find_image``, ``_find_text``, + ``_find_by_description``, ``_self_heal``, ``_find_element``, + ``_click_element``, ``_dump_hierarchy`` + - ``AC_ios_screenshot``, ``_find_image``, ``_find_text``, + ``_find_by_description``, ``_self_heal``, ``_find_element``, + ``_click_element``, ``_dump_source`` + * - Apps and alerts + - ``AC_android_launch_app``, ``_stop_app``, ``_app_state``, + ``_wait_for_app``, ``_alert_accept``, ``_alert_dismiss`` + - the same six under ``AC_ios_`` + * - Extensions + - ``AC_android_install_app``, ``_push_file``, ``_pull_file``, + ``_get_clipboard``, ``_set_clipboard``, ``_start_recording``, + ``_stop_recording`` + - the same seven under ``AC_ios_``; all but ``_set_clipboard`` raise + ``DeviceUnsupportedError`` until an adapter is registered + * - Shell + - ``AC_android_shell`` + - none: there is no shell to run on iOS + +The ``_find_*`` and ``_self_heal`` commands take ``tap`` to tap what they +find. ``AC_android_shell`` runs an arbitrary command on the device and is +deliberately not offered as an MCP tool. + +From Python the same table is reachable as +``run_mobile_command(name, params)`` and ``mobile_capability_matrix()``; the +latter also lists the desktop features with no mobile counterpart (window +management, mouse buttons and wheel, keyboard shortcuts, the desktop +accessibility tree, COM, USB host passthrough, global hotkeys, desktop +capture) with the limitation and the alternative for each. + +**Mobile tab.** Pick the platform, enter the serial or WebDriverAgent URL, +choose a command and edit its parameters as a JSON object. *Probe device*, +*Run command* and *Fill parameter template* are in the Actions menu. Probing +shows the capability table and the setup report and sends no input. Both +actions run on the GUI thread and block it until the device answers or the +timeout passes. + +Setup +===== + +.. warning:: + + Untested on hardware. These are the steps the code is written for; none of + them was carried out against a real emulator, phone or WebDriverAgent. + +Android emulator or device +-------------------------- + +1. Install `Android platform-tools + `_ and put + ``adb`` on ``PATH`` (or pass ``adb_path``). +2. Emulator: start an AVD; it appears as ``emulator-5554``. Device: enable + *Developer options* and *USB debugging*, connect it, and accept the + authorisation prompt. Over Wi-Fi: ``adb connect :5555``. +3. ``adb devices`` must show the serial in state ``device``. ``unauthorized`` + surfaces here as ``needs_permission`` / ``DevicePermissionError``. +4. Optional, for the widget tree, pinch, dialogs, clipboard and Unicode text: + ``pip install uiautomator2``. +5. Optional, for Unicode text without ``uiautomator2``: install ADBKeyBoard + and select it with ``adb shell ime set com.android.adbkeyboard/.AdbIME``. +6. Check: ``AC_android_device_info`` (or the Mobile tab's *Probe device*) + reports the adb build, the Android release and each capability. + +iOS device or simulator, local or remote WebDriverAgent +------------------------------------------------------- + +Building and signing WebDriverAgent needs a Mac with Xcode; for a physical +device it also needs an Apple developer signing identity. AutoControl does +not build or install it. + +1. On the Mac, build and run WebDriverAgent on the device or simulator + (``xcodebuild ... test`` on the ``WebDriverAgentRunner`` scheme) and make + port 8100 reachable — ``iproxy 8100 8100`` for a USB device. +2. On the host that runs AutoControl (any OS): ``pip install facebook-wda``. +3. Local: the URL is ``http://localhost:8100``. Remote: use the Mac's or the + device's address, for example ``http://192.168.1.20:8100``. WebDriverAgent + has no authentication, so keep it on a trusted network or behind a tunnel. +4. Check: ``AC_ios_device_info`` with ``url`` reports the WebDriverAgent + build and the iOS version from ``/status``; an endpoint that does not + answer is reported as ``needs_dependency`` with the connection error. + +Driving a remote WebDriverAgent from Windows or Linux is the supported way to +use iOS from those hosts. That is not a claim that Xcode builds or signing +were verified on them. diff --git a/docs/source/Zh/doc/mobile/mobile_doc.rst b/docs/source/Zh/doc/mobile/mobile_doc.rst index ff5d9ec21..9eb544ec4 100644 --- a/docs/source/Zh/doc/mobile/mobile_doc.rst +++ b/docs/source/Zh/doc/mobile/mobile_doc.rst @@ -317,3 +317,113 @@ iOS 裝置不會用到任何 ``adb`` 的東西。要補上 iOS 缺少的功能 register_mobile_extension("ios", lambda session: MyTideviceAdapter(session)) AutoControl 本身沒有附帶這樣的 adapter。 + +指令、MCP 工具、Script Builder、GUI +=================================== + +每個行動指令只描述一次，就在 ``je_auto_control.MOBILE_COMMANDS`` +（``wrapper/mobile_commands.py``\ ）。Executor 的 ``AC_android_*`` / ``AC_ios_*`` +指令、\ ``ac_android_*`` / ``ac_ios_*`` MCP 工具、Script Builder 的 **Android** 與 +**iOS** 分類，以及 **Mobile** 分頁，全部由這張表產生，所以不會有指令只出現在 +某一個介面、卻在另一個介面缺席（缺了 ``test_mobile_surface_parity.py`` 就會失敗）。 + +每個指令都接受選用的位址——Android 是 ``serial`` 與 ``adb_path``\ ，iOS 是 +``url``\ ——以及 ``device_timeout_s``\ 。沒寫位址時會跑在 ``use_device`` 綁定的 +session（device matrix 的 worker）上，否則跑在後端的預設裝置上。不認得的參數 +會被拒絕。 + +.. list-table:: + :header-rows: 1 + :widths: 24 38 38 + + * - 分組 + - Android + - iOS + * - 裝置 + - ``AC_android_device_info``\ 、\ ``AC_android_screen_info``\ 、 + ``AC_android_list_devices`` + - ``AC_ios_device_info``\ 、\ ``AC_ios_screen_info`` + * - 輸入 + - ``AC_android_tap``\ 、\ ``_swipe``\ 、\ ``_long_press``\ 、\ ``_drag``\ 、 + ``_pinch``\ 、\ ``_key``\ 、\ ``_text``\ 、\ ``_type_text`` + - ``AC_ios_tap``\ 、\ ``_swipe``\ 、\ ``_long_press``\ 、\ ``_drag``\ 、 + ``_pinch``\ 、\ ``_press_key``\ 、\ ``_type`` + * - 畫面與定位 + - ``AC_android_screenshot``\ 、\ ``_find_image``\ 、\ ``_find_text``\ 、 + ``_find_by_description``\ 、\ ``_self_heal``\ 、\ ``_find_element``\ 、 + ``_click_element``\ 、\ ``_dump_hierarchy`` + - ``AC_ios_screenshot``\ 、\ ``_find_image``\ 、\ ``_find_text``\ 、 + ``_find_by_description``\ 、\ ``_self_heal``\ 、\ ``_find_element``\ 、 + ``_click_element``\ 、\ ``_dump_source`` + * - App 與 alert + - ``AC_android_launch_app``\ 、\ ``_stop_app``\ 、\ ``_app_state``\ 、 + ``_wait_for_app``\ 、\ ``_alert_accept``\ 、\ ``_alert_dismiss`` + - 同樣六個，前綴為 ``AC_ios_`` + * - 擴充功能 + - ``AC_android_install_app``\ 、\ ``_push_file``\ 、\ ``_pull_file``\ 、 + ``_get_clipboard``\ 、\ ``_set_clipboard``\ 、\ ``_start_recording``\ 、 + ``_stop_recording`` + - 同樣七個，前綴為 ``AC_ios_``\ ；在註冊 adapter 之前，除了 + ``_set_clipboard`` 以外都會丟 ``DeviceUnsupportedError`` + * - Shell + - ``AC_android_shell`` + - 無：iOS 沒有可以執行的 shell + +``_find_*`` 與 ``_self_heal`` 指令可以帶 ``tap`` 來點擊找到的位置。 +``AC_android_shell`` 會在裝置上執行任意指令，刻意不提供成 MCP 工具。 + +在 Python 裡，同一張表可以透過 ``run_mobile_command(name, params)`` 與 +``mobile_capability_matrix()`` 取用；後者同時列出沒有行動對應的桌面功能 +（視窗管理、滑鼠按鍵與滾輪、鍵盤快速鍵、桌面無障礙樹、COM、USB 主機轉接、 +全域熱鍵、桌面擷取），以及每一項的限制與替代做法。 + +**Mobile 分頁。** 選平台、輸入 serial 或 WebDriverAgent URL、選指令，並以 JSON +物件編輯參數。「探測裝置」、「執行指令」、「填入參數範本」都在 Actions 選單。 +探測會顯示能力表與設定報告，不送出任何輸入。這兩個動作都在 GUI 執行緒上執行， +在裝置回應或逾時之前會卡住介面。 + +設定 +==== + +.. warning:: + + 尚未在實機上測試。以下是程式碼預期的步驟；沒有任何一步實際對真的模擬器、 + 手機或 WebDriverAgent 執行過。 + +Android 模擬器或實機 +-------------------- + +1. 安裝 `Android platform-tools + `_\ ，並把 + ``adb`` 放進 ``PATH``\ （或傳入 ``adb_path``\ ）。 +2. 模擬器：啟動 AVD，它會以 ``emulator-5554`` 出現。實機：開啟「開發人員選項」 + 與「USB 偵錯」，接上後在裝置上接受授權提示。走 Wi-Fi： + ``adb connect :5555``\ 。 +3. ``adb devices`` 必須顯示該 serial 的狀態為 ``device``\ 。\ ``unauthorized`` + 在這裡會呈現為 ``needs_permission`` / ``DevicePermissionError``\ 。 +4. 選用（元件樹、pinch、對話框、剪貼簿、Unicode 文字需要）： + ``pip install uiautomator2``\ 。 +5. 選用（不裝 ``uiautomator2`` 而要輸入 Unicode）：安裝 ADBKeyBoard，並以 + ``adb shell ime set com.android.adbkeyboard/.AdbIME`` 選用它。 +6. 檢查：\ ``AC_android_device_info``\ （或 Mobile 分頁的「探測裝置」）會回報 + adb 版本、Android 版本與每一項能力。 + +iOS 實機或模擬器，本機或遠端 WebDriverAgent +------------------------------------------- + +建置與簽署 WebDriverAgent 需要裝有 Xcode 的 Mac；實機還需要 Apple 開發者簽章 +身分。AutoControl 不負責建置或安裝它。 + +1. 在 Mac 上對裝置或模擬器建置並執行 WebDriverAgent（對 + ``WebDriverAgentRunner`` scheme 執行 ``xcodebuild ... test``\ ），並讓 8100 + 埠可以連到——USB 裝置用 ``iproxy 8100 8100``\ 。 +2. 在執行 AutoControl 的主機（任何作業系統）：\ ``pip install facebook-wda``\ 。 +3. 本機：URL 是 ``http://localhost:8100``\ 。遠端：使用 Mac 或裝置的位址，例如 + ``http://192.168.1.20:8100``\ 。WebDriverAgent 沒有任何驗證機制，請放在可信任 + 的網路或通道後面。 +4. 檢查：帶 ``url`` 的 ``AC_ios_device_info`` 會從 ``/status`` 回報 + WebDriverAgent 版本與 iOS 版本；沒有回應的端點會回報為 + ``needs_dependency``\ ，並附上連線錯誤。 + +從 Windows 或 Linux 操作遠端的 WebDriverAgent，是這些主機使用 iOS 的支援方式。 +這並不代表已在這些主機上驗證過 Xcode 建置或簽署。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index b03e41df5..e9a236bb9 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1040,7 +1040,7 @@ ) # Mobile device contexts (one isolated session per Android / iOS device) from je_auto_control.wrapper.device_context import ( - AlertNotPresentError, AppState, + AlertNotPresentError, AppState, DeviceSetupReport, DeviceCancelledError, DeviceCapability, DeviceClosedError, DeviceContext, DeviceError, DevicePermissionError, DeviceSession, DeviceTimeoutError, DeviceUnavailableError, DeviceUnsupportedError, Drag, LongPress, Pinch, @@ -1051,6 +1051,10 @@ MobileExtension, accept_alert, app_state, dismiss_alert, launch_app, mobile_extension, register_mobile_extension, stop_app, wait_for_app, ) +from je_auto_control.wrapper.mobile_commands import ( + MOBILE_COMMANDS, MobileCommand, device_setup_report, mobile_capability_matrix, + run_mobile_command, +) # Media assertions (audio activity, video motion) from je_auto_control.utils.media_assert import ( MediaAssertionResult, assert_audio_activity, assert_video_changes, @@ -1955,6 +1959,8 @@ def start_autocontrol_gui(*args, **kwargs): "AlertNotPresentError", "AppState", "MobileExtension", "accept_alert", "app_state", "dismiss_alert", "launch_app", "mobile_extension", "register_mobile_extension", "stop_app", "wait_for_app", + "DeviceSetupReport", "MOBILE_COMMANDS", "MobileCommand", "device_setup_report", + "mobile_capability_matrix", "run_mobile_command", # Media assertions "MediaAssertionResult", "assert_audio_activity", "assert_video_changes", "measure_audio_rms", "video_segment_motion", diff --git a/je_auto_control/actions.pyi b/je_auto_control/actions.pyi index 35e4e39ee..12e6f9e7d 100644 --- a/je_auto_control/actions.pyi +++ b/je_auto_control/actions.pyi @@ -152,6 +152,28 @@ def AC_anchor_locate_all( ) -> Dict[str, Any]: """Executor adapter: every anchor-relative match, nearest-first.""" +def AC_android_alert_accept( + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Accept the alert or system dialog showing.""" + +def AC_android_alert_dismiss( + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Dismiss the alert or system dialog showing.""" + +def AC_android_app_state( + app_id: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """not_installed, not_running, background or foreground.""" + def AC_android_click_element( text: str | None = ..., resource_id: str | None = ..., @@ -162,9 +184,39 @@ def AC_android_click_element( ) -> Dict[str, int]: """Tap the first widget matching the selectors; return click centre.""" +def AC_android_device_info( + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Backend, versions and capabilities of the device. Sends no input.""" + +def AC_android_drag( + x1: int, + y1: int, + x2: int, + y2: int, + hold_s: float = ..., + duration_s: float = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Press, hold so the item lifts, and move it.""" + def AC_android_dump_hierarchy(serial: str | None = ...) -> str: """Return the device's widget tree as an XML string.""" +def AC_android_find_by_description( + description: str, + model: str | None = ..., + tap: bool = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Ask a vision-language model where something is on the device screen.""" + def AC_android_find_element( text: str | None = ..., resource_id: str | None = ..., @@ -175,18 +227,155 @@ def AC_android_find_element( ) -> Dict[str, int]: """Find an Android widget via uiautomator2; return its bounding rect.""" +def AC_android_find_image( + template_path: str, + detect_threshold: float = ..., + tap: bool = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Template-match in the device screen; answers in device points.""" + +def AC_android_find_text( + text: str, + lang: str = ..., + min_confidence: float = ..., + tap: bool = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """OCR the device screen for text; answers in device points.""" + +def AC_android_get_clipboard( + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Read the device clipboard's text.""" + +def AC_android_install_app( + path: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Install an app package from the host.""" + def AC_android_key(key: str, serial: str | None = ..., adb_path: str | None = ...) -> None: """Send a keycode (``KEYCODE_HOME`` etc.) via ``input keyevent``.""" +def AC_android_launch_app( + app_id: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Launch an app; returns its state.""" + def AC_android_list_devices(adb_path: str | None = ...) -> list: """Return ``{serial, state, model, …}`` for every adb-attached device.""" +def AC_android_long_press( + x: int, + y: int, + duration_s: float = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Press and hold at a point.""" + +def AC_android_pinch( + x: int, + y: int, + scale: float, + duration_s: float = ..., + span: int = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Two-finger pinch: scale above 1 zooms in, below 1 out.""" + +def AC_android_pull_file( + remote_path: str, + local_path: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Copy a device file to the host.""" + +def AC_android_push_file( + local_path: str, + remote_path: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Copy a host file to the device.""" + +def AC_android_screen_info( + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Pixel size, point size, scale and orientation of the current screen.""" + def AC_android_screenshot(file_path: str, serial: str | None = ..., adb_path: str | None = ...) -> str: """Capture the live Android screen and save it as PNG at ``file_path``.""" +def AC_android_self_heal( + template_path: str | None = ..., + description: str | None = ..., + detect_threshold: float = ..., + model: str | None = ..., + tap: bool = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Template first, VLM on a miss, against the device screen.""" + +def AC_android_set_clipboard( + text: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Put text on the device clipboard.""" + def AC_android_shell(command: str, serial: str | None = ..., adb_path: str | None = ...) -> str: """Run an ``adb shell`` command and return its stdout.""" +def AC_android_start_recording( + remote_path: str | None = ..., + time_limit_s: int | None = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Start recording the device screen.""" + +def AC_android_stop_app( + app_id: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Stop an app; returns its state.""" + +def AC_android_stop_recording( + local_path: str, + remote_path: str | None = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Stop recording and save the video on the host.""" + def AC_android_swipe( x1: int, y1: int, @@ -202,7 +391,25 @@ def AC_android_tap(x: int, y: int, serial: str | None = ..., adb_path: str | Non """Send a single ``input tap`` to an Android device.""" def AC_android_text(text: str, serial: str | None = ..., adb_path: str | None = ...) -> None: - """Type a string via ``input text``.""" + """Type a string: ``input text`` for ASCII, a Unicode-capable path otherwise.""" + +def AC_android_type_text( + text: str, + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Type text through a path that can carry it; raises when none can.""" + +def AC_android_wait_for_app( + app_id: str, + timeout_s: float = ..., + state: str = ..., + serial: str | None = ..., + adb_path: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Wait until an app reaches a state; raises on timeout.""" def AC_annotate_screenshot(source: str, annotations: List[Dict[str, Any]] | str, output_path: str) -> Dict[str, Any]: """Executor adapter: draw annotations onto a screenshot and save it.""" @@ -1515,6 +1722,15 @@ def AC_inspector_reset() -> Dict[str, Any]: def AC_inspector_summary() -> Dict[str, Any]: ... +def AC_ios_alert_accept(url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Accept the alert or system dialog showing.""" + +def AC_ios_alert_dismiss(url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Dismiss the alert or system dialog showing.""" + +def AC_ios_app_state(app_id: str, url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """not_installed, not_running, background or foreground.""" + def AC_ios_click_element( name: str | None = ..., class_name: str | None = ..., @@ -1524,9 +1740,33 @@ def AC_ios_click_element( ) -> Dict[str, int]: ... +def AC_ios_device_info(url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Backend, versions and capabilities of the device. Sends no input.""" + +def AC_ios_drag( + x1: int, + y1: int, + x2: int, + y2: int, + hold_s: float = ..., + duration_s: float = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Press, hold so the item lifts, and move it.""" + def AC_ios_dump_source(url: str | None = ...) -> str: ... +def AC_ios_find_by_description( + description: str, + model: str | None = ..., + tap: bool = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Ask a vision-language model where something is on the device screen.""" + def AC_ios_find_element( name: str | None = ..., class_name: str | None = ..., @@ -1536,9 +1776,112 @@ def AC_ios_find_element( ) -> Dict[str, int]: ... +def AC_ios_find_image( + template_path: str, + detect_threshold: float = ..., + tap: bool = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Template-match in the device screen; answers in device points.""" + +def AC_ios_find_text( + text: str, + lang: str = ..., + min_confidence: float = ..., + tap: bool = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """OCR the device screen for text; answers in device points.""" + +def AC_ios_get_clipboard(url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Read the device clipboard's text.""" + +def AC_ios_install_app(path: str, url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Install an app package from the host.""" + +def AC_ios_launch_app(app_id: str, url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Launch an app; returns its state.""" + +def AC_ios_long_press( + x: int, + y: int, + duration_s: float = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Press and hold at a point.""" + +def AC_ios_pinch( + x: int, + y: int, + scale: float, + duration_s: float = ..., + span: int = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Two-finger pinch: scale above 1 zooms in, below 1 out.""" + +def AC_ios_press_key(key: str, url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Press a hardware key (home, volumeUp, volumeDown).""" + +def AC_ios_pull_file( + remote_path: str, + local_path: str, + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Copy a device file to the host.""" + +def AC_ios_push_file( + local_path: str, + remote_path: str, + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Copy a host file to the device.""" + +def AC_ios_screen_info(url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Pixel size, point size, scale and orientation of the current screen.""" + def AC_ios_screenshot(file_path: str, url: str | None = ...) -> str: ... +def AC_ios_self_heal( + template_path: str | None = ..., + description: str | None = ..., + detect_threshold: float = ..., + model: str | None = ..., + tap: bool = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Template first, VLM on a miss, against the device screen.""" + +def AC_ios_set_clipboard(text: str, url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Put text on the device clipboard.""" + +def AC_ios_start_recording( + remote_path: str | None = ..., + time_limit_s: int | None = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Start recording the device screen.""" + +def AC_ios_stop_app(app_id: str, url: str | None = ..., device_timeout_s: float | None = ...) -> Any: + """Stop an app; returns its state.""" + +def AC_ios_stop_recording( + local_path: str, + remote_path: str | None = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Stop recording and save the video on the host.""" + def AC_ios_swipe(x1: int, y1: int, x2: int, y2: int, duration_s: float = ..., url: str | None = ...) -> Dict[str, Any]: ... @@ -1548,6 +1891,15 @@ def AC_ios_tap(x: int, y: int, url: str | None = ...) -> Dict[str, int]: def AC_ios_type(text: str, url: str | None = ...) -> str: ... +def AC_ios_wait_for_app( + app_id: str, + timeout_s: float = ..., + state: str = ..., + url: str | None = ..., + device_timeout_s: float | None = ..., +) -> Any: + """Wait until an app reaches a state; raises on timeout.""" + def AC_is_composing() -> Dict[str, Any]: """Adapter: whether the IME has an uncommitted composition.""" diff --git a/je_auto_control/android/session.py b/je_auto_control/android/session.py index 5a7a6bb8b..f96ac917a 100644 --- a/je_auto_control/android/session.py +++ b/je_auto_control/android/session.py @@ -87,6 +87,20 @@ def _probe(self) -> Dict[str, DeviceCapability]: found[name] = extension.capability(name) return {name: found[name] for name in CAPABILITY_NAMES} + def _versions(self, ready: bool) -> Tuple[str, str, str]: + """adb's build and, when the device answers, its Android release.""" + try: + adb_version = self.adb.version() + except DeviceError: + return "adb", "", "" + if not ready: + return "adb", adb_version, "" + try: + release = self.adb.shell("getprop ro.build.version.release").strip() + except DeviceError: + release = "" + return "adb", adb_version, release + def _blocker(self) -> Optional[Tuple[str, str]]: """Why nothing can be used: ``(state, reason)``, or ``None`` when adb sees the device.""" try: diff --git a/je_auto_control/gui/device_matrix_tab.py b/je_auto_control/gui/device_matrix_tab.py index 51755e5d4..27f8d0590 100644 --- a/je_auto_control/gui/device_matrix_tab.py +++ b/je_auto_control/gui/device_matrix_tab.py @@ -33,12 +33,13 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._tr_init() self._devices = QPlainTextEdit() self._devices.setPlaceholderText( - '[{"platform": "android", "serial": "emulator-5554"}]', + '[{"platform": "android", "serial": "emulator-5554", ' + '"app_id": "com.example.app"}]', ) self._actions = QPlainTextEdit() + # No serial needed: each device's own session is bound while it runs. self._actions.setPlaceholderText( - '[["AC_android_tap", {"x": 1, "y": 2, ' - '"serial": "${device.serial}"}]]', + '[["AC_android_tap", {"x": 1, "y": 2}]]', ) self._parallel = QSpinBox() self._parallel.setRange(1, 64) diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 6bc8d1ebe..df42dbea1 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -226,6 +226,21 @@ # USB sharing panel (AnyDesk-style) "tab_usb_share": "USB Sharing", + "tab_mobile": "Mobile", + "mob_platform": "Platform", + "mob_device": "Device (adb serial / WebDriverAgent URL)", + "mob_adb_path": "adb path (blank = PATH)", + "mob_command": "Command", + "mob_params": "Parameters (JSON object)", + "mob_result": "Result", + "mob_probe": "Probe device", + "mob_run": "Run command", + "mob_fill_params": "Fill parameter template", + "mob_col_capability": "Capability", + "mob_col_state": "State", + "mob_col_reason": "Reason", + "mob_error": "Failed: {error}", + "mob_untested": "Built against fake ADB / WebDriverAgent; not verified on a device.", "usb_share_host_group": "Share USB on this machine", "usb_share_viewer_group": "Use a USB device", "usb_share_enable": "Enable sharing", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 0740bfc51..94d2c7309 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -115,6 +115,21 @@ # USB 共有パネル（AnyDesk 風） "tab_usb_share": "USB 共有", + "tab_mobile": "モバイル", + "mob_platform": "プラットフォーム", + "mob_device": "デバイス（adb シリアル / WebDriverAgent URL）", + "mob_adb_path": "adb のパス（空欄 = PATH）", + "mob_command": "コマンド", + "mob_params": "パラメーター（JSON オブジェクト）", + "mob_result": "結果", + "mob_probe": "デバイスを調べる", + "mob_run": "コマンドを実行", + "mob_fill_params": "パラメーターのひな形を入力", + "mob_col_capability": "機能", + "mob_col_state": "状態", + "mob_col_reason": "理由", + "mob_error": "失敗: {error}", + "mob_untested": "偽の ADB / WebDriverAgent で開発。実機では未検証です。", "usb_share_host_group": "このマシンの USB を共有", "usb_share_viewer_group": "USB デバイスを使う", "usb_share_enable": "共有を有効化", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index d002ab239..c4372461c 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -104,6 +104,21 @@ # USB 共享面板（AnyDesk 风格） "tab_usb_share": "USB 共享", + "tab_mobile": "移动设备", + "mob_platform": "平台", + "mob_device": "设备（adb 序列号 / WebDriverAgent URL）", + "mob_adb_path": "adb 路径（留空 = PATH）", + "mob_command": "命令", + "mob_params": "参数（JSON 对象）", + "mob_result": "结果", + "mob_probe": "探测设备", + "mob_run": "运行命令", + "mob_fill_params": "填入参数模板", + "mob_col_capability": "能力", + "mob_col_state": "状态", + "mob_col_reason": "原因", + "mob_error": "失败：{error}", + "mob_untested": "基于假的 ADB / WebDriverAgent 开发；尚未在真机上验证。", "usb_share_host_group": "共享本机 USB", "usb_share_viewer_group": "使用 USB 设备", "usb_share_enable": "启用共享", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 046af99cd..01ba401d4 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -105,6 +105,21 @@ # USB 分享面板（AnyDesk 風） "tab_usb_share": "USB 分享", + "tab_mobile": "行動裝置", + "mob_platform": "平台", + "mob_device": "裝置（adb serial / WebDriverAgent URL）", + "mob_adb_path": "adb 路徑（留空 = PATH）", + "mob_command": "指令", + "mob_params": "參數（JSON 物件）", + "mob_result": "結果", + "mob_probe": "探測裝置", + "mob_run": "執行指令", + "mob_fill_params": "填入參數範本", + "mob_col_capability": "能力", + "mob_col_state": "狀態", + "mob_col_reason": "原因", + "mob_error": "失敗：{error}", + "mob_untested": "以假的 ADB / WebDriverAgent 開發；尚未在實機上驗證。", "usb_share_host_group": "分享本機 USB", "usb_share_viewer_group": "使用 USB 裝置", "usb_share_enable": "啟用分享", diff --git a/je_auto_control/gui/mobile_tab.py b/je_auto_control/gui/mobile_tab.py new file mode 100644 index 000000000..633fba130 --- /dev/null +++ b/je_auto_control/gui/mobile_tab.py @@ -0,0 +1,186 @@ +"""Mobile tab: probe an Android / iOS device and run one mobile command on it. + +A thin wrapper over the headless API: the command list is +``je_auto_control.MOBILE_COMMANDS``, running one is +``je_auto_control.run_mobile_command`` and probing is +``je_auto_control.device_setup_report``. Nothing here is unreachable from an +action file. +""" +import json +from typing import Any, Dict, List, Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QAbstractItemView, QComboBox, QFormLayout, QLabel, QLineEdit, QPlainTextEdit, + QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, +) + +import je_auto_control as ac +from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) + +_COLS = ("mob_col_capability", "mob_col_state", "mob_col_reason") +_PLATFORMS = (("android", "Android"), ("ios", "iOS")) +#: Parameters the tab fills from its own fields. +_ADDRESS = ("serial", "url", "adb_path") + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class MobileTab(TranslatableMixin, QWidget): + """Pick a device and a command, edit its parameters, see the result.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tr_init() + self._platform = QComboBox() + for key, label in _PLATFORMS: + self._platform.addItem(label, key) + self._device = QLineEdit() + self._device.setPlaceholderText("emulator-5554 / http://192.168.1.20:8100") + self._adb_path = QLineEdit() + self._command = QComboBox() + self._params = QPlainTextEdit() + self._params.setPlaceholderText('{"x": 100, "y": 200}') + self._result = QPlainTextEdit() + self._result.setReadOnly(True) + self._table = QTableWidget(0, len(_COLS)) + self._table.setEditTriggers(QAbstractItemView.NoEditTriggers) + self._table.horizontalHeader().setStretchLastSection(True) + self._note = QLabel() + self._note.setWordWrap(True) + self._apply_headers() + self._build_layout() + self._platform.currentIndexChanged.connect(self._reload_commands) + self._reload_commands() + + def retranslate(self) -> None: + TranslatableMixin.retranslate(self) + self._apply_headers() + + def _apply_headers(self) -> None: + self._table.setHorizontalHeaderLabels([_t(key) for key in _COLS]) + + def _build_layout(self) -> None: + # Probe / run / fill live in the Actions menu; the tab keeps only the + # inputs, the capability table and the result view. + root = QVBoxLayout(self) + root.addWidget(self._tr(self._note, "mob_untested")) + form = QFormLayout() + for key, widget in (("mob_platform", self._platform), ("mob_device", self._device), + ("mob_adb_path", self._adb_path), ("mob_command", self._command)): + form.addRow(self._tr(QLabel(), key), widget) + root.addLayout(form) + root.addWidget(self._tr(QLabel(), "mob_params")) + root.addWidget(self._params) + root.addWidget(self._table, stretch=1) + root.addWidget(self._tr(QLabel(), "mob_result")) + root.addWidget(self._result, stretch=1) + + def menu_actions(self) -> list: + """Expose tab commands to the window-level Actions menu.""" + return [ + ("mob_probe", self._on_probe), + ("mob_run", self._on_run), + ("mob_fill_params", self._on_fill_params), + ] + + # --- state the menu handlers and tests read or set --------------------- + + def command_names(self) -> List[str]: + """Every command the tab offers, across both platforms.""" + return [command.name for command in ac.MOBILE_COMMANDS] + + def set_target(self, platform: str, device_id: str = "", adb_path: str = "") -> None: + """Select the platform and fill the device fields.""" + self._platform.setCurrentIndex(max(0, self._platform.findData(platform))) + self._device.setText(device_id) + self._adb_path.setText(adb_path) + + def set_command(self, name: str, params: Optional[Dict[str, Any]] = None) -> None: + """Select a command and fill its parameters.""" + self._command.setCurrentIndex(max(0, self._command.findData(name))) + self._params.setPlainText(json.dumps(params or {})) + + def result_text(self) -> str: + """What the result view shows.""" + return self._result.toPlainText() + + def capability_rows(self) -> Dict[str, str]: + """Capability name to state, as the table shows it.""" + return {self._table.item(row, 0).text(): self._table.item(row, 1).text() + for row in range(self._table.rowCount())} + + def _current_platform(self) -> str: + return str(self._platform.currentData()) + + def _reload_commands(self) -> None: + platform = self._current_platform() + self._command.clear() + for command in ac.MOBILE_COMMANDS: + if command.platform == platform: + self._command.addItem(f"{command.label} ({command.name})", command.name) + + def _address(self, command: Any) -> Dict[str, Any]: + """The address parameters this command takes, from the tab's fields.""" + names = {param.name for param in command.params} + key = "serial" if command.platform == "android" else "url" + values = {key: self._device.text().strip(), "adb_path": self._adb_path.text().strip()} + return {name: value for name, value in values.items() if value and name in names} + + def _show_error(self, error: Exception) -> None: + self._result.setPlainText(_t("mob_error").replace("{error}", str(error))) + + # --- Actions menu handlers --------------------------------------------- + + def _on_fill_params(self) -> None: + command = self._selected() + if command is None: + return + template = {param.name: param.default for param in command.params + if param.name not in _ADDRESS and (param.required or param.default is not None)} + self._params.setPlainText(json.dumps(template, indent=2)) + + def _selected(self) -> Any: + name = self._command.currentData() + return next((c for c in ac.MOBILE_COMMANDS if c.name == name), None) + + def _on_run(self) -> None: + command = self._selected() + if command is None: + return + try: + params = json.loads(self._params.toPlainText() or "{}") + if not isinstance(params, dict): + raise ValueError("parameters must be a JSON object") + result = ac.run_mobile_command(command.name, {**self._address(command), **params}) + except (ac.AutoControlException, OSError, ValueError, TypeError, LookupError) as error: + self._show_error(error) + return + self._result.setPlainText(json.dumps(result, indent=2, ensure_ascii=False, default=str)) + + def _on_probe(self) -> None: + try: + context = ac.DeviceContext( + self._current_platform(), self._device.text().strip(), + adb_path=self._adb_path.text().strip() or None) + with ac.open_device(context) as session: + report = ac.device_setup_report(session).to_dict() + except (ac.AutoControlException, OSError, ValueError) as error: + self._show_error(error) + return + self._render(report) + + def _render(self, report: Dict[str, Any]) -> None: + capabilities = report.pop("capabilities") + self._table.setRowCount(len(capabilities)) + for row, (name, capability) in enumerate(capabilities.items()): + for col, text in enumerate((name, capability["state"], capability["reason"])): + item = QTableWidgetItem(str(text)) + item.setFlags(item.flags() & ~Qt.ItemIsEditable) + self._table.setItem(row, col, item) + self._result.setPlainText(json.dumps(report, indent=2, ensure_ascii=False)) diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index a9638555d..f0a763b77 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -71,9 +71,27 @@ def _build_specs() -> List[CommandSpec]: _add_window_specs(specs) _add_flow_specs(specs) _add_misc_specs(specs) + _add_mobile_specs(specs) return specs +def _add_mobile_specs(specs: List[CommandSpec]) -> None: + """Android / iOS commands, generated from the table the executor and MCP tools share.""" + from je_auto_control.wrapper.mobile_commands import MOBILE_COMMANDS + kinds = {"int": FieldType.INT, "float": FieldType.FLOAT, "bool": FieldType.BOOL, + "file_path": FieldType.FILE_PATH} + for command in MOBILE_COMMANDS: + specs.append(CommandSpec( + command.name, "Android" if command.platform == "android" else "iOS", + command.label, + fields=tuple( + FieldSpec(param.name, kinds.get(param.kind, FieldType.STRING), + optional=not param.required, default=param.default) + for param in command.params), + description=command.description, + )) + + def _add_mouse_specs(specs: List[CommandSpec]) -> None: specs.append(CommandSpec( "AC_click_mouse", "Mouse", "Click Mouse", diff --git a/je_auto_control/gui/tab_registry.py b/je_auto_control/gui/tab_registry.py index dd9633464..0bc00a888 100644 --- a/je_auto_control/gui/tab_registry.py +++ b/je_auto_control/gui/tab_registry.py @@ -102,6 +102,7 @@ def widget(self) -> Any: TabSpec("flakiness", "tab_flakiness", "system", f"{_GUI}.flakiness_tab", "FlakinessTab"), TabSpec("a11y_audit", "tab_a11y_audit", "core", f"{_GUI}.a11y_audit_tab", "A11yAuditTab"), TabSpec("device_matrix", "tab_device_matrix", "core", f"{_GUI}.device_matrix_tab", "DeviceMatrixTab"), + TabSpec("mobile", "tab_mobile", "core", f"{_GUI}.mobile_tab", "MobileTab"), TabSpec("media_checks", "tab_media_checks", "core", f"{_GUI}.media_checks_tab", "MediaChecksTab"), TabSpec("run_history", "tab_run_history", "automation", f"{_GUI}.run_history_tab", "RunHistoryTab"), TabSpec("profiler", "tab_profiler", "automation", f"{_GUI}.profiler_tab", "ProfilerTab"), diff --git a/je_auto_control/ios/session.py b/je_auto_control/ios/session.py index 2f13825bc..987dbee36 100644 --- a/je_auto_control/ios/session.py +++ b/je_auto_control/ios/session.py @@ -69,6 +69,19 @@ def _probe(self) -> Dict[str, DeviceCapability]: found[name] = extension.capability(name) return {name: found[name] for name in CAPABILITY_NAMES} + def _versions(self, ready: bool) -> Tuple[str, str, str]: + """WebDriverAgent's build and the iOS release, from ``/status``.""" + if not ready: + return "wda", "", "" + try: + status = _wda_status(self.device.handle) + except (DeviceError, OSError): + return "wda", "", "" + build = status.get("build") or {} + system = status.get("os") or {} + version = build.get("version") or build.get("time") or "unknown" + return "wda", str(version), str(system.get("version") or "") + def _blocker(self) -> Optional[Tuple[str, str]]: """Why nothing can be used: ``(state, reason)``, or ``None`` when WDA answers.""" try: diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 45c2c7d52..1168e786b 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -1783,6 +1783,12 @@ def _ac_android_dump_hierarchy(serial: Optional[str] = None) -> str: return dump_hierarchy(device=device) +def _mobile_command_handlers() -> Dict[str, Callable[..., Any]]: + """The generated ``AC_android_*`` / ``AC_ios_*`` handlers (see ``wrapper.mobile_commands``).""" + from je_auto_control.wrapper.mobile_commands import generated_handlers + return generated_handlers() + + # === iOS executor adapters (WebDriverAgent / facebook-wda) ================== def _ios_device(url: Optional[str]) -> Any: @@ -7983,6 +7989,9 @@ def __init__(self): "AC_android_screenshot": _ac_android_screenshot, "AC_android_list_devices": _ac_android_list_devices, "AC_android_shell": _ac_android_shell, + # Android / iOS sessions, gestures, locating, apps and extensions: + # generated from the table the MCP tools and builder schema share. + **_mobile_command_handlers(), # LLM action planner "AC_llm_plan": _llm_plan_for_executor, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fecbc25ae..90bacb133 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1091,6 +1091,20 @@ def ios_tools() -> List[MCPTool]: ] +def mobile_command_tools() -> List[MCPTool]: + """Android / iOS tools generated from the table the executor commands come from.""" + from je_auto_control.wrapper.mobile_commands import mcp_tool_specs + return [ + MCPTool( + name=spec["name"], description=spec["description"], + input_schema=schema(spec["properties"], spec["required"]), + handler=spec["handler"], + annotations=READ_ONLY if spec["read_only"] else DESTRUCTIVE, + ) + for spec in mcp_tool_specs() + ] + + def redaction_tools() -> List[MCPTool]: return [ MCPTool( @@ -9002,7 +9016,8 @@ def media_assert_tools() -> List[MCPTool]: ocr_structure_tools, smart_wait_tools, cost_telemetry_tools, failure_hook_tools, computer_use_tools, dag_tools, presence_tools, chatops_tools, - redaction_tools, android_widget_tools, ios_tools, webrunner_tools, + redaction_tools, android_widget_tools, ios_tools, mobile_command_tools, + webrunner_tools, scheduler_tools, trigger_tools, hotkey_tools, watchdog_tools, unattended_tools, work_queue_tools, synthetic_data_tools, mcp_registry_tools, test_selection_tools, diff --git a/je_auto_control/wrapper/device_context.py b/je_auto_control/wrapper/device_context.py index 52e059096..a84cc986e 100644 --- a/je_auto_control/wrapper/device_context.py +++ b/je_auto_control/wrapper/device_context.py @@ -21,10 +21,10 @@ import contextvars import threading from contextlib import contextmanager -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from enum import Enum from time import monotonic -from typing import Any, Callable, Dict, Iterator, Mapping, Optional, TypeVar, Union +from typing import Any, Callable, Dict, Iterator, Mapping, Optional, Tuple, TypeVar, Union from je_auto_control.utils.exception.exceptions import AutoControlException @@ -115,6 +115,34 @@ def to_dict(self) -> Dict[str, Any]: return asdict(self) +@dataclass(frozen=True) +class DeviceSetupReport: + """What a device's backend is, which version answered, and what it can do. + + ``state`` and ``reason`` are those of the ``input`` capability: whether + the device can be driven at all, and if not, what to fix first. + """ + + platform: str + device_id: str + backend: str + backend_version: str = "" + os_version: str = "" + state: str = STATE_AVAILABLE + reason: str = "" + capabilities: Mapping[str, DeviceCapability] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """JSON-safe form for executor, MCP and GUI consumers.""" + return { + "platform": self.platform, "device_id": self.device_id, + "backend": self.backend, "backend_version": self.backend_version, + "os_version": self.os_version, "state": self.state, "reason": self.reason, + "capabilities": {name: capability.to_dict() + for name, capability in self.capabilities.items()}, + } + + @dataclass(frozen=True) class DeviceContext: """The identity and configuration of one device. @@ -328,6 +356,21 @@ def capabilities(self) -> Dict[str, DeviceCapability]: """What this device can do right now. Sends no input to the device.""" raise NotImplementedError + def setup_report(self) -> DeviceSetupReport: + """Backend, versions and capabilities, for checking a setup. Sends no input.""" + capabilities = self.capabilities() + ready = capabilities["input"] + backend, backend_version, os_version = self.invoke( + "setup_report", lambda: self._versions(ready.available)) + return DeviceSetupReport( + platform=self.platform, device_id=self.device_id, backend=backend, + backend_version=backend_version, os_version=os_version, + state=ready.state, reason=ready.reason, capabilities=capabilities) + + def _versions(self, ready: bool) -> Tuple[str, str, str]: + """``(backend, backend version, OS version)``; empty where it cannot be read.""" + raise NotImplementedError + def capture(self) -> Any: """The current screen as a :class:`~je_auto_control.wrapper.device_frame.DeviceFrame`.""" raise NotImplementedError @@ -474,6 +517,7 @@ def bound_session(platform: str, device_id: Optional[str] = None) -> Optional[De "AlertNotPresentError", "AppState", "CAPABILITY_NAMES", "DEFAULT_TIMEOUT_S", "DeviceCancelledError", "DeviceCapability", "DeviceClosedError", "DeviceContext", "DeviceError", "DevicePermissionError", + "DeviceSetupReport", "DeviceSession", "DeviceTimeoutError", "DeviceUnavailableError", "DeviceUnsupportedError", "Drag", "Gesture", "LongPress", "MOBILE_PLATFORMS", "PLATFORM_ANDROID", "PLATFORM_IOS", "Pinch", "STATE_AVAILABLE", diff --git a/je_auto_control/wrapper/mobile_commands.py b/je_auto_control/wrapper/mobile_commands.py new file mode 100644 index 000000000..53d6ce034 --- /dev/null +++ b/je_auto_control/wrapper/mobile_commands.py @@ -0,0 +1,579 @@ +"""The one description of every ``AC_android_*`` / ``AC_ios_*`` command. + +A mobile capability is only delivered when it can be reached from a JSON +action file, from an MCP client, from the Script Builder and from the GUI. +Writing each of those by hand is how the adb commands ended up with executor +entries but no builder schema and no MCP tool. So the commands are described +once, here, in :data:`MOBILE_COMMANDS`, and the surfaces are generated: + +* the executor takes :func:`generated_handlers`; +* the MCP registry takes :func:`mcp_tool_specs`; +* the Script Builder schema and the Mobile tab read :data:`MOBILE_COMMANDS`. + +Commands that existed before the table are listed too (``legacy``): their +handlers stay where they were, in ``action_executor``, and the table carries +their parameters so the other surfaces cover them. +""" +from __future__ import annotations + +import inspect +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Tuple + +from je_auto_control.wrapper import mobile_extensions as extensions +from je_auto_control.wrapper.device_context import ( + CAPABILITY_NAMES, DEFAULT_TIMEOUT_S, PLATFORM_ANDROID, PLATFORM_IOS, AppState, + DeviceContext, DeviceError, DeviceSession, DeviceSetupReport, Drag, LongPress, + Pinch, Tap, bound_session, open_device, +) + +_KINDS = {"int": int, "float": float, "bool": bool, "string": str, "file_path": str} +_JSON_TYPES = {"int": "integer", "float": "number", "bool": "boolean"} + + +@dataclass(frozen=True) +class Param: + """One command parameter. ``kind`` is int, float, bool, string or file_path.""" + + name: str + kind: str = "string" + required: bool = False + default: Any = None + description: str = "" + + +@dataclass(frozen=True) +class MobileCommand: + """One ``AC_*`` mobile command and everything a surface needs to offer it.""" + + name: str + platform: str + label: str + description: str + params: Tuple[Param, ...] + capability: str + api: str + read_only: bool = False + #: Handled by a function that predates the table, in ``action_executor``. + legacy: bool = False + #: An MCP tool for it was already written by hand in ``_factories``. + handwritten_mcp: bool = False + + @property + def mcp_name(self) -> str: + """The MCP tool name: ``AC_android_tap`` is ``ac_android_tap``.""" + return "ac_" + self.name[3:] + + +#: Commands deliberately not offered as MCP tools, with the reason. +MCP_EXCLUDED: Mapping[str, str] = { + "AC_android_shell": "runs an arbitrary shell command on the device; kept to action " + "files, where the author of the script is the one choosing it", +} + +_ADDRESS = { + PLATFORM_ANDROID: ( + Param("serial", description="adb serial; omit for the bound or only device"), + Param("adb_path", description="adb binary; omit to use PATH"), + Param("device_timeout_s", "float", description="per-call timeout, default 30"), + ), + PLATFORM_IOS: ( + Param("url", description="WebDriverAgent URL; omit for the bound or default device"), + Param("device_timeout_s", "float", description="per-call timeout, default 30"), + ), +} +_ADDRESS_NAMES = ("serial", "url", "adb_path", "device_timeout_s") + + +@contextmanager +def command_session(platform: str, device_id: Optional[str] = None, *, + adb_path: Optional[str] = None, + timeout_s: Optional[float] = None) -> Iterator[DeviceSession]: + """The session a command runs on: the bound one when it is addressed, else a new one. + + A session opened here is closed when the command returns; a bound session + (``use_device``, a device-matrix worker) is left to whoever bound it. + """ + bound = bound_session(platform, device_id or None) + if bound is not None: + yield bound + return + session = open_device(DeviceContext( + platform, device_id or "", adb_path=adb_path or None, + timeout_s=float(timeout_s) if timeout_s else DEFAULT_TIMEOUT_S)) + try: + yield session + finally: + session.close() + + +def device_setup_report(session: DeviceSession) -> DeviceSetupReport: + """Backend, versions and capabilities of ``session``'s device. Sends no input.""" + return session.setup_report() + + +# --- what each operation does, on any session --------------------------- + +def _device_info(session: DeviceSession) -> Dict[str, Any]: + return device_setup_report(session).to_dict() + + +def _screen_info(session: DeviceSession) -> Dict[str, Any]: + return dict(session.capture().to_dict()) + + +def _type_text(session: DeviceSession, text: str) -> None: + session.type_text(text) + + +def _press_key(session: DeviceSession, key: str) -> None: + session.press_key(key) + + +def _long_press(session: DeviceSession, x: int, y: int, duration_s: float = 1.0) -> None: + session.perform(LongPress(x, y, duration_s)) + + +def _drag(session: DeviceSession, x1: int, y1: int, x2: int, y2: int, + hold_s: float = 0.5, duration_s: float = 0.5) -> None: + session.perform(Drag(x1, y1, x2, y2, hold_s, duration_s)) + + +def _pinch(session: DeviceSession, x: int, y: int, scale: float, + duration_s: float = 0.5, span: int = 400) -> None: + session.perform(Pinch(x, y, scale, duration_s, span)) + + +def _located(session: DeviceSession, point: Optional[Tuple[int, int]], + tap: bool) -> Dict[str, Any]: + """The result of a locate, tapping the point first when asked.""" + if point is None: + return {"found": False} + if tap: + session.perform(Tap(int(point[0]), int(point[1]))) + return {"found": True, "x": int(point[0]), "y": int(point[1]), "tapped": bool(tap)} + + +def _find_image(session: DeviceSession, template_path: str, + detect_threshold: float = 0.9, tap: bool = False) -> Dict[str, Any]: + return _located(session, session.capture().locate_image(template_path, detect_threshold), tap) + + +def _find_text(session: DeviceSession, text: str, lang: str = "eng", + min_confidence: float = 60.0, tap: bool = False) -> Dict[str, Any]: + point = session.capture().locate_text(text, lang=lang, min_confidence=min_confidence) + return _located(session, point, tap) + + +def _find_by_description(session: DeviceSession, description: str, + model: Optional[str] = None, tap: bool = False) -> Dict[str, Any]: + return _located(session, session.capture().locate_description(description, model=model), tap) + + +def _self_heal(session: DeviceSession, template_path: Optional[str] = None, + description: Optional[str] = None, detect_threshold: float = 0.9, + model: Optional[str] = None, tap: bool = False) -> Dict[str, Any]: + from je_auto_control.utils.self_healing.locator import self_heal_locate + outcome = self_heal_locate( + template_path=template_path, description=description, + detect_threshold=detect_threshold, model=model, frame=session.capture()) + if tap and outcome.found and outcome.coordinates is not None: + session.perform(Tap(*outcome.coordinates)) + return outcome.to_dict() + + +def _launch_app(session: DeviceSession, app_id: str) -> str: + return str(extensions.launch_app(session, app_id).value) + + +def _stop_app(session: DeviceSession, app_id: str) -> str: + return str(extensions.stop_app(session, app_id).value) + + +def _app_state(session: DeviceSession, app_id: str) -> str: + return str(extensions.app_state(session, app_id).value) + + +def _wait_for_app(session: DeviceSession, app_id: str, timeout_s: float = 10.0, + state: str = "foreground") -> str: + try: + wanted = AppState(state) + except ValueError as error: + raise DeviceError(f"unknown app state {state!r}") from error + return str(extensions.wait_for_app(session, app_id, timeout_s=timeout_s, state=wanted).value) + + +def _alert_accept(session: DeviceSession) -> str: + return extensions.accept_alert(session) + + +def _alert_dismiss(session: DeviceSession) -> str: + return extensions.dismiss_alert(session) + + +def _install_app(session: DeviceSession, path: str) -> str: + return extensions.mobile_extension(session).install_app(path) + + +def _push_file(session: DeviceSession, local_path: str, remote_path: str) -> str: + return extensions.mobile_extension(session).push_file(local_path, remote_path) + + +def _pull_file(session: DeviceSession, remote_path: str, local_path: str) -> str: + return extensions.mobile_extension(session).pull_file(remote_path, local_path) + + +def _get_clipboard(session: DeviceSession) -> str: + return extensions.mobile_extension(session).get_clipboard() + + +def _set_clipboard(session: DeviceSession, text: str) -> None: + extensions.mobile_extension(session).set_clipboard(text) + + +def _start_recording(session: DeviceSession, **options: Any) -> str: + return extensions.mobile_extension(session).start_recording(**options) + + +def _stop_recording(session: DeviceSession, local_path: str, **options: Any) -> str: + return extensions.mobile_extension(session).stop_recording(local_path, **options) + + +@dataclass(frozen=True) +class _Operation: + """One session operation, offered as a command on each platform it lists.""" + + key: str + label: str + description: str + run: Callable[..., Any] + params: Tuple[Param, ...] = () + capability: str = "input" + api: str = "DeviceSession" + read_only: bool = False + platforms: Tuple[str, ...] = (PLATFORM_ANDROID, PLATFORM_IOS) + + +_XY = (Param("x", "int", True), Param("y", "int", True)) +_TAP = Param("tap", "bool", default=False, description="tap the point when found") +_APP = (Param("app_id", required=True, description="Android package or iOS bundle id"),) + +_OPERATIONS: Tuple[_Operation, ...] = ( + _Operation("device_info", "Device Info", + "Backend, versions and capabilities of the device. Sends no input.", + _device_info, api="device_setup_report", read_only=True), + _Operation("screen_info", "Screen Info", + "Pixel size, point size, scale and orientation of the current screen.", + _screen_info, capability="screenshot", api="DeviceFrame", read_only=True), + _Operation("type_text", "Type Text (Unicode)", + "Type text through a path that can carry it; raises when none can.", + _type_text, (Param("text", required=True),), "unicode_text", + platforms=(PLATFORM_ANDROID,)), + _Operation("press_key", "Press Key", "Press a hardware key (home, volumeUp, volumeDown).", + _press_key, (Param("key", required=True),), platforms=(PLATFORM_IOS,)), + _Operation("long_press", "Long Press", "Press and hold at a point.", _long_press, + _XY + (Param("duration_s", "float", default=1.0),), api="LongPress"), + _Operation("drag", "Drag", "Press, hold so the item lifts, and move it.", _drag, + (Param("x1", "int", True), Param("y1", "int", True), + Param("x2", "int", True), Param("y2", "int", True), + Param("hold_s", "float", default=0.5), + Param("duration_s", "float", default=0.5)), api="Drag"), + _Operation("pinch", "Pinch", "Two-finger pinch: scale above 1 zooms in, below 1 out.", + _pinch, _XY + (Param("scale", "float", True), + Param("duration_s", "float", default=0.5), + Param("span", "int", default=400)), + "multi_touch", "Pinch"), + _Operation("find_image", "Find Image", + "Template-match in the device screen; answers in device points.", _find_image, + (Param("template_path", "file_path", True), + Param("detect_threshold", "float", default=0.9), _TAP), + "screenshot", "DeviceFrame"), + _Operation("find_text", "Find Text (OCR)", + "OCR the device screen for text; answers in device points.", _find_text, + (Param("text", required=True), Param("lang", default="eng"), + Param("min_confidence", "float", default=60.0), _TAP), + "screenshot", "DeviceFrame"), + _Operation("find_by_description", "Find By Description (VLM)", + "Ask a vision-language model where something is on the device screen.", + _find_by_description, + (Param("description", required=True), Param("model"), _TAP), + "screenshot", "DeviceFrame"), + _Operation("self_heal", "Self-Heal Locate", + "Template first, VLM on a miss, against the device screen.", _self_heal, + (Param("template_path", "file_path"), Param("description"), + Param("detect_threshold", "float", default=0.9), Param("model"), _TAP), + "screenshot", "self_heal_locate"), + _Operation("launch_app", "Launch App", "Launch an app; returns its state.", _launch_app, + _APP, "app_lifecycle", "launch_app"), + _Operation("stop_app", "Stop App", "Stop an app; returns its state.", _stop_app, + _APP, "app_lifecycle", "stop_app"), + _Operation("app_state", "App State", + "not_installed, not_running, background or foreground.", _app_state, + _APP, "app_lifecycle", "app_state", read_only=True), + _Operation("wait_for_app", "Wait For App", + "Wait until an app reaches a state; raises on timeout.", _wait_for_app, + _APP + (Param("timeout_s", "float", default=10.0), + Param("state", default="foreground")), + "app_lifecycle", "wait_for_app", read_only=True), + _Operation("alert_accept", "Accept Alert", "Accept the alert or system dialog showing.", + _alert_accept, capability="alerts", api="accept_alert"), + _Operation("alert_dismiss", "Dismiss Alert", "Dismiss the alert or system dialog showing.", + _alert_dismiss, capability="alerts", api="dismiss_alert"), + _Operation("install_app", "Install App", "Install an app package from the host.", + _install_app, (Param("path", "file_path", True),), "install", + "mobile_extension"), + _Operation("push_file", "Push File", "Copy a host file to the device.", _push_file, + (Param("local_path", "file_path", True), Param("remote_path", required=True)), + "files", "mobile_extension"), + _Operation("pull_file", "Pull File", "Copy a device file to the host.", _pull_file, + (Param("remote_path", required=True), Param("local_path", "file_path", True)), + "files", "mobile_extension"), + _Operation("get_clipboard", "Get Clipboard", "Read the device clipboard's text.", + _get_clipboard, capability="clipboard", api="mobile_extension", read_only=True), + _Operation("set_clipboard", "Set Clipboard", "Put text on the device clipboard.", + _set_clipboard, (Param("text", required=True),), "clipboard", + "mobile_extension"), + _Operation("start_recording", "Start Screen Recording", + "Start recording the device screen.", _start_recording, + (Param("remote_path"), Param("time_limit_s", "int")), + "recording", "mobile_extension"), + _Operation("stop_recording", "Stop Screen Recording", + "Stop recording and save the video on the host.", _stop_recording, + (Param("local_path", "file_path", True), Param("remote_path")), + "recording", "mobile_extension"), +) + +_PLATFORM_LABELS = {PLATFORM_ANDROID: "Android", PLATFORM_IOS: "iOS"} +_SERIAL = Param("serial") +_ADB = Param("adb_path") +_URL = Param("url") +_TIMEOUT = Param("timeout_s", "float", default=5.0) +_ANDROID_SELECTOR = (Param("text"), Param("resource_id"), Param("description"), + Param("class_name"), _TIMEOUT, _SERIAL) +_IOS_SELECTOR = (Param("name"), Param("class_name"), Param("predicate"), _TIMEOUT, _URL) +_SWIPE = (Param("x1", "int", True), Param("y1", "int", True), + Param("x2", "int", True), Param("y2", "int", True)) + + +def _legacy(name: str, label: str, description: str, params: Tuple[Param, ...], + capability: str = "input", api: str = "DeviceSession", *, + read_only: bool = False, handwritten_mcp: bool = False) -> MobileCommand: + platform = PLATFORM_ANDROID if name.startswith("AC_android_") else PLATFORM_IOS + return MobileCommand( + name, platform, f"{_PLATFORM_LABELS[platform]}: {label}", description, params, + capability, api, read_only=read_only, legacy=True, handwritten_mcp=handwritten_mcp) + + +_LEGACY: Tuple[MobileCommand, ...] = ( + _legacy("AC_android_tap", "Tap", "Tap a point via adb.", _XY + (_SERIAL, _ADB), api="Tap"), + _legacy("AC_android_swipe", "Swipe", "Swipe between two points via adb.", + _SWIPE + (Param("duration_ms", "int", default=250), _SERIAL, _ADB), api="Swipe"), + _legacy("AC_android_key", "Key Event", "Send a keycode (KEYCODE_HOME, BACK, a number).", + (Param("key", required=True), _SERIAL, _ADB)), + _legacy("AC_android_text", "Type Text", + "Type text; ASCII via input text, otherwise a Unicode-capable path.", + (Param("text", required=True), _SERIAL, _ADB), "unicode_text"), + _legacy("AC_android_screenshot", "Screenshot", "Save the device screen as a PNG.", + (Param("file_path", "file_path", True), _SERIAL, _ADB), "screenshot", "DeviceFrame"), + _legacy("AC_android_list_devices", "List Devices", "Every device adb sees, with its state.", + (_ADB,), api="device_setup_report", read_only=True), + _legacy("AC_android_shell", "Shell Command", "Run an adb shell command; returns stdout.", + (Param("command", required=True), _SERIAL, _ADB)), + _legacy("AC_android_find_element", "Find Element", + "Find a widget in the uiautomator2 tree; returns its bounds.", _ANDROID_SELECTOR, + "ui_tree", read_only=True, handwritten_mcp=True), + _legacy("AC_android_click_element", "Click Element", "Tap the first matching widget.", + _ANDROID_SELECTOR, "ui_tree", handwritten_mcp=True), + _legacy("AC_android_dump_hierarchy", "Dump Hierarchy", "The widget tree as XML.", + (_SERIAL,), "ui_tree", read_only=True, handwritten_mcp=True), + _legacy("AC_ios_tap", "Tap", "Tap a point (in points) via WebDriverAgent.", + _XY + (_URL,), api="Tap", handwritten_mcp=True), + _legacy("AC_ios_swipe", "Swipe", "Swipe between two points via WebDriverAgent.", + _SWIPE + (Param("duration_s", "float", default=0.5), _URL), api="Swipe", + handwritten_mcp=True), + _legacy("AC_ios_type", "Type Text", "Type text into the focused field.", + (Param("text", required=True), _URL), "unicode_text", handwritten_mcp=True), + _legacy("AC_ios_screenshot", "Screenshot", "Save the device screen as a PNG.", + (Param("file_path", "file_path", True), _URL), "screenshot", "DeviceFrame", + handwritten_mcp=True), + _legacy("AC_ios_find_element", "Find Element", + "Find an XCUITest element; returns its bounds.", _IOS_SELECTOR, "ui_tree", + read_only=True, handwritten_mcp=True), + _legacy("AC_ios_click_element", "Click Element", "Tap the first matching element.", + _IOS_SELECTOR, "ui_tree", handwritten_mcp=True), + _legacy("AC_ios_dump_source", "Dump Source", "The XCUITest page source as XML.", + (_URL,), "ui_tree", read_only=True, handwritten_mcp=True), +) + + +def _generated() -> Tuple[Tuple[MobileCommand, _Operation], ...]: + pairs = [] + for operation in _OPERATIONS: + for platform in operation.platforms: + pairs.append((MobileCommand( + f"AC_{platform}_{operation.key}", platform, + f"{_PLATFORM_LABELS[platform]}: {operation.label}", operation.description, + operation.params + _ADDRESS[platform], operation.capability, operation.api, + read_only=operation.read_only), operation)) + return tuple(pairs) + + +_GENERATED = _generated() + +#: Every mobile command: the ones that predate the table, then the generated ones. +MOBILE_COMMANDS: Tuple[MobileCommand, ...] = _LEGACY + tuple( + command for command, _operation in _GENERATED) +_BY_NAME: Mapping[str, MobileCommand] = {command.name: command for command in MOBILE_COMMANDS} + + +def _annotation(param: Param) -> Any: + """The parameter's type as the generated stub and help text should show it.""" + kind = _KINDS[param.kind] + return kind if param.required or param.default is not None else Optional[kind] + + +def _signature(params: Tuple[Param, ...]) -> inspect.Signature: + return inspect.Signature([ + inspect.Parameter( + param.name, inspect.Parameter.POSITIONAL_OR_KEYWORD, + default=inspect.Parameter.empty if param.required else param.default, + annotation=_annotation(param)) + for param in params]) + + +def _coerce(command: MobileCommand, param: Param, value: Any) -> Any: + if isinstance(value, bool) and param.kind in ("int", "float"): + raise TypeError(f"{command.name}: {param.name} must be a number, got a boolean") + try: + return _KINDS[param.kind](value) + except (TypeError, ValueError) as error: + raise TypeError( + f"{command.name}: {param.name} must be {param.kind}, got {value!r}") from error + + +def _bind(command: MobileCommand, signature: inspect.Signature, + args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Validate a call against the command's parameters: unknown and missing ones raise.""" + try: + bound = signature.bind(*args, **kwargs) + except TypeError as error: + raise TypeError(f"{command.name}: {error}") from error + bound.apply_defaults() + return {param.name: _coerce(command, param, bound.arguments[param.name]) + for param in command.params if bound.arguments[param.name] is not None} + + +def _handler(command: MobileCommand, operation: _Operation) -> Callable[..., Any]: + signature = _signature(command.params) + + def handler(*args: Any, **kwargs: Any) -> Any: + values = _bind(command, signature, args, kwargs) + address = {name: values.pop(name, None) for name in _ADDRESS_NAMES} + with command_session(command.platform, address["serial"] or address["url"], + adb_path=address["adb_path"], + timeout_s=address["device_timeout_s"]) as session: + return operation.run(session, **values) + + handler.__name__ = "_ac_" + command.name[3:] + handler.__doc__ = command.description + # The executor, the MCP registry and help text introspect handlers. + setattr(handler, "__signature__", signature) + return handler + + +_HANDLERS: Dict[str, Callable[..., Any]] = { + command.name: _handler(command, operation) for command, operation in _GENERATED} + + +def generated_handlers() -> Dict[str, Callable[..., Any]]: + """``AC_*`` name to handler, for every command the table generates.""" + return dict(_HANDLERS) + + +def _executor_handler(name: str) -> Callable[..., Any]: + """A legacy command's handler, looked up in the executor when it is called.""" + def call(**kwargs: Any) -> Any: + from je_auto_control.utils.executor.action_executor import executor + return executor.event_dict[name](**kwargs) + return call + + +def run_mobile_command(name: str, params: Optional[Mapping[str, Any]] = None) -> Any: + """Run one mobile command by name. Only commands in :data:`MOBILE_COMMANDS` are accepted.""" + if name not in _BY_NAME: + raise DeviceError(f"{name!r} is not a mobile command") + handler = _HANDLERS.get(name) or _executor_handler(name) + return handler(**dict(params or {})) + + +def mcp_tool_specs() -> List[Dict[str, Any]]: + """What the MCP registry needs to build a tool for each command that lacks one.""" + specs = [] + for command in MOBILE_COMMANDS: + if command.handwritten_mcp or command.name in MCP_EXCLUDED: + continue + properties: Dict[str, Any] = {} + for param in command.params: + node: Dict[str, Any] = {"type": _JSON_TYPES.get(param.kind, "string")} + if param.kind == "file_path": + node["format"] = "path" + if param.description: + node["description"] = param.description + properties[param.name] = node + specs.append({ + "name": command.mcp_name, "description": command.description, + "properties": properties, + "required": [param.name for param in command.params if param.required], + "read_only": command.read_only, + "handler": _HANDLERS.get(command.name) or _executor_handler(command.name), + }) + return specs + + +#: Desktop features with no mobile counterpart: the limit, and what to use instead. +DESKTOP_ONLY_FEATURES: Tuple[Dict[str, str], ...] = ( + {"feature": "Window management (AC_window_*, focus, move, resize, z-order)", + "limitation": "a phone shows one app at a time; there are no windows to manage", + "alternative": "launch_app / stop_app / app_state to choose what is in front"}, + {"feature": "Mouse buttons, wheel and hover (AC_click_mouse, AC_mouse_scroll)", + "limitation": "touch screens have no pointer, buttons or wheel", + "alternative": "Tap, LongPress, Swipe (to scroll), Drag and Pinch"}, + {"feature": "Keyboard shortcuts and modifier keys (AC_hotkey, AC_press_keyboard_key)", + "limitation": "there is no physical keyboard or modifier state", + "alternative": "type_text for text; AC_android_key / AC_ios_press_key for hardware keys"}, + {"feature": "Desktop accessibility tree (UIA, AX, AT-SPI)", + "limitation": "those APIs describe the host's desktop, not the device", + "alternative": "AC_android_find_element (uiautomator2) / AC_ios_find_element (XCUITest)"}, + {"feature": "COM / Office automation", + "limitation": "COM is a Windows host API", + "alternative": "none on the device; drive the mobile app's UI instead"}, + {"feature": "USB host passthrough and usbip", + "limitation": "these share the host's USB devices; a phone is a USB device, not a host", + "alternative": "none; adb and WebDriverAgent are the device transports"}, + {"feature": "Global hotkeys, triggers on host input, input recording", + "limitation": "they observe the host's keyboard and mouse, which the device does not use", + "alternative": "record executed mobile steps in the action journal"}, + {"feature": "Desktop screen capture and screen recording (AC_screenshot, AC_screen_record)", + "limitation": "they capture the host's monitors", + "alternative": "DeviceSession.capture() and the recording extension"}, +) + + +def mobile_capability_matrix() -> Dict[str, Any]: + """Which commands deliver each capability per platform, and what stays desktop-only.""" + rows = [] + for name in CAPABILITY_NAMES: + row: Dict[str, Any] = {"capability": name} + for platform in (PLATFORM_ANDROID, PLATFORM_IOS): + row[platform] = [command.name for command in MOBILE_COMMANDS + if command.platform == platform and command.capability == name] + rows.append(row) + return {"capabilities": rows, "desktop_only": [dict(row) for row in DESKTOP_ONLY_FEATURES]} + + +__all__ = [ + "DESKTOP_ONLY_FEATURES", "MCP_EXCLUDED", "MOBILE_COMMANDS", "MobileCommand", "Param", + "command_session", "device_setup_report", "generated_handlers", "mcp_tool_specs", + "mobile_capability_matrix", "run_mobile_command", +] diff --git a/test/unit_test/headless/test_mobile_surface_parity.py b/test/unit_test/headless/test_mobile_surface_parity.py new file mode 100644 index 000000000..624e1d02f --- /dev/null +++ b/test/unit_test/headless/test_mobile_surface_parity.py @@ -0,0 +1,295 @@ +"""Every mobile capability is reachable from every surface, from one table. + +``wrapper.mobile_commands.MOBILE_COMMANDS`` is the single description of the +``AC_android_*`` / ``AC_ios_*`` commands. The executor, the MCP registry, the +Script Builder schema and the Mobile tab are all built from it; these tests +fail when one of them drifts. Driven against the fake backends — no device. +""" +import sys +import types + +import pytest + +import je_auto_control as ac +from headless._mobile_doubles import FakeAdbHost, FakeU2, FakeWda +from je_auto_control.android import UIAutomatorDevice +from je_auto_control.ios import IOSDevice +from je_auto_control.utils.executor.action_executor import executor +from je_auto_control.wrapper.device_context import ( + CAPABILITY_NAMES, DeviceContext, DevicePermissionError, DeviceUnsupportedError, + open_device, use_device, +) +from je_auto_control.wrapper.mobile_commands import ( + DESKTOP_ONLY_FEATURES, MCP_EXCLUDED, MOBILE_COMMANDS, device_setup_report, + mobile_capability_matrix, run_mobile_command, +) + +APP = "com.example.shop" + + +@pytest.fixture +def adb_host(monkeypatch): + """A fake ADB host with one attached device, ``phone``.""" + host = FakeAdbHost() + host.add("phone").installed.add(APP) + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", host.run) + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.Popen", host.popen) + monkeypatch.setattr("je_auto_control.android.apps._RECORDINGS", {}) + from je_auto_control.utils.executor import action_executor + monkeypatch.setattr(action_executor, "_android_client_cache", {}) + return host + + +def _ios(handle): + return open_device(DeviceContext("ios", "http://wda.test:8100"), + device=IOSDevice(url="http://wda.test:8100", handle=handle)) + + +# --- one table, every surface ------------------------------------------ + +def _mcp_names(): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + return {tool.name for tool in build_default_tool_registry()} + + +def _gui_commands(): + pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + from PySide6.QtWidgets import QApplication + from je_auto_control.gui.mobile_tab import MobileTab + app = QApplication.instance() or QApplication([]) + tab = MobileTab() + try: + return set(tab.command_names()), [key for key, _handler in tab.menu_actions()] + finally: + tab.deleteLater() + app.processEvents() + + +def test_every_mobile_capability_has_api_command_gui_schema(): + from je_auto_control.gui.script_builder.command_schema import COMMAND_SPECS + mcp_names = _mcp_names() + gui_commands, menu_keys = _gui_commands() + missing_surface_commands = set() + for command in MOBILE_COMMANDS: + surfaces = { + "executor": command.name in executor.event_dict, + "schema": command.name in COMMAND_SPECS, + "mcp": command.name in MCP_EXCLUDED or command.mcp_name in mcp_names, + "gui": command.name in gui_commands, + "api": hasattr(ac, command.api.split(".")[0]) + and command.api.split(".")[0] in ac.__all__, + } + missing_surface_commands |= { + f"{command.name}:{surface}" for surface, present in surfaces.items() if not present} + assert missing_surface_commands == set() + assert menu_keys, "the Mobile tab must expose its commands through the Actions menu" + # The table is the whole mobile surface: no command lives outside it. + in_executor = {name for name in executor.event_dict + if name.startswith(("AC_android_", "AC_ios_"))} + assert in_executor == {command.name for command in MOBILE_COMMANDS} + # An excluded tool says why. + assert all(reason for reason in MCP_EXCLUDED.values()) + + +def test_schema_fields_match_the_command_parameters(): + from je_auto_control.gui.script_builder.command_schema import COMMAND_SPECS + for command in MOBILE_COMMANDS: + spec = COMMAND_SPECS[command.name] + assert [field.name for field in spec.fields] == [p.name for p in command.params] + assert [not field.optional for field in spec.fields] == [ + p.required for p in command.params] + assert spec.category in ("Android", "iOS") + + +def test_mcp_schema_matches_the_command_parameters(): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + tools = {tool.name: tool for tool in build_default_tool_registry()} + for command in MOBILE_COMMANDS: + if command.name in MCP_EXCLUDED: + assert command.mcp_name not in tools + continue + schema = tools[command.mcp_name].input_schema + assert set(schema["properties"]) >= {p.name for p in command.params if p.required} + if not command.handwritten_mcp: + assert set(schema["properties"]) == {p.name for p in command.params} + assert schema.get("required", []) == [p.name for p in command.params if p.required] + assert tools[command.mcp_name].annotations.read_only is command.read_only + + +def test_every_capability_is_covered_on_both_platforms(): + matrix = mobile_capability_matrix() + rows = {row["capability"]: row for row in matrix["capabilities"]} + assert set(rows) == set(CAPABILITY_NAMES) + for name, row in rows.items(): + assert row["android"], f"no Android command uses {name}" + assert row["ios"], f"no iOS command uses {name}" + # Desktop-only features say what the limit is and what to use instead. + assert DESKTOP_ONLY_FEATURES + for row in matrix["desktop_only"]: + assert row["limitation"] and row["alternative"] + + +# --- the generated commands really run --------------------------------- + +def test_generated_commands_drive_the_device(adb_host, tmp_path): + phone = adb_host.devices["phone"] + handle = FakeU2() + session = open_device(DeviceContext("android", "phone", adb_path="fake-adb"), + ui_device=UIAutomatorDevice(handle=handle)) + with use_device(session): + result = executor.execute_action([ + ["AC_android_launch_app", {"app_id": APP}], + ["AC_android_wait_for_app", {"app_id": APP, "timeout_s": 1.0}], + ["AC_android_long_press", {"x": 10, "y": 20, "duration_s": 0.5}], + ["AC_android_drag", [1, 2, 3, 4]], + ["AC_android_pinch", {"x": 500, "y": 800, "scale": 2.0}], + ["AC_android_set_clipboard", {"text": "hi"}], + ["AC_android_stop_app", {"app_id": APP}], + ], raise_on_error=True) + assert list(result.values())[0] == "foreground" + assert list(result.values())[-1] == "not_running" + assert phone.input_calls == [ + f"monkey -p {APP} -c android.intent.category.LAUNCHER 1", + "input swipe 10 20 10 20 500", + "input draganddrop 1 2 3 4 1000", + f"am force-stop {APP}", + ] + assert handle.clipboard == "hi" + # Without a bound session the command opens its own from serial / adb_path. + info = run_mobile_command("AC_android_screen_info", + {"serial": "phone", "adb_path": "fake-adb"}) + assert info["point_size"] == [1080, 1920] + + +def test_unknown_parameters_and_commands_are_rejected(adb_host): + with pytest.raises(TypeError): + run_mobile_command("AC_android_long_press", {"x": 1, "y": 2, "bogus": 3}) + with pytest.raises(TypeError): + run_mobile_command("AC_android_long_press", {"x": 1}) + with pytest.raises(ac.DeviceError): + run_mobile_command("AC_click_mouse", {}) + assert adb_host.devices["phone"].input_calls == [] + + +def test_ios_command_without_a_backend_feature_says_why(tmp_path): + handle = FakeWda() + with use_device(_ios(handle)): + with pytest.raises(DeviceUnsupportedError) as caught: + run_mobile_command("AC_ios_install_app", {"path": str(tmp_path / "a.ipa")}) + assert caught.value.reason and caught.value.alternative + assert run_mobile_command("AC_ios_long_press", {"x": 1, "y": 2}) is None + assert handle.input_calls == [{"op": "tap_hold", "x": 1, "y": 2, "duration": 1.0}] + + +def test_mcp_tool_calls_the_same_handler(adb_host): + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + tools = {tool.name: tool for tool in build_default_tool_registry()} + tools["ac_android_long_press"].invoke( + {"x": 7, "y": 8, "serial": "phone", "adb_path": "fake-adb"}) + tools["ac_android_tap"].invoke({"x": 1, "y": 2, "serial": "phone", "adb_path": "fake-adb"}) + assert adb_host.devices["phone"].input_calls == [ + "input swipe 7 8 7 8 1000", "input tap 1 2"] + + +# --- setup reports ----------------------------------------------------- + +def test_wda_remote_endpoint_setup(monkeypatch): + handle = FakeWda() + session = open_device( + DeviceContext("ios", "http://192.168.1.20:8100"), + device=IOSDevice(url="http://192.168.1.20:8100", handle=handle)) + device_setup_report = ac.device_setup_report(session) + assert device_setup_report.backend_version + assert device_setup_report.backend == "wda" + assert device_setup_report.backend_version == "8.5.2" + assert device_setup_report.os_version == "17.4" + assert device_setup_report.state == "available" + assert handle.input_calls == [] + assert device_setup_report.to_dict()["capabilities"]["install"]["state"] == "needs_dependency" + + # The URL in the context is the endpoint the client is built for. + seen = [] + fake_wda = types.ModuleType("wda") + fake_wda.Client = lambda url: seen.append(url) or handle + monkeypatch.setitem(sys.modules, "wda", fake_wda) + remote = open_device(DeviceContext("ios", "http://10.0.0.7:8100")) + assert ac.device_setup_report(remote).backend_version == "8.5.2" + assert seen == ["http://10.0.0.7:8100"] + + # An endpoint that does not answer is reported, with the reason, not raised. + class _Down(FakeWda): + def status(self): + raise OSError("connection refused") + + down = ac.device_setup_report(_ios(_Down())) + assert down.state == "needs_dependency" + assert "connection refused" in down.reason + assert down.backend_version == "" + + +def test_android_setup_report(adb_host): + report = device_setup_report( + open_device(DeviceContext("android", "phone", adb_path="fake-adb"))) + assert report.backend == "adb" + assert report.backend_version.startswith("1.0.41") + assert report.os_version == "14" + assert report.state == "available" + assert adb_host.devices["phone"].input_calls == [] + + +def test_android_authorization_error(adb_host): + locked = adb_host.add("locked", state="unauthorized") + address = {"serial": "locked", "adb_path": "fake-adb"} + for name, params in (("AC_android_tap", {"x": 1, "y": 2}), + ("AC_android_long_press", {"x": 1, "y": 2}), + ("AC_android_type_text", {"text": "hello"}), + ("AC_android_launch_app", {"app_id": APP})): + with pytest.raises(DevicePermissionError): + run_mobile_command(name, {**params, **address}) + unauthorized_input_calls = locked.input_calls + assert unauthorized_input_calls == [] + report = run_mobile_command("AC_android_device_info", address) + assert report["state"] == "needs_permission" + assert "USB debugging" in report["reason"] + assert report["capabilities"]["input"]["state"] == "needs_permission" + + +# --- the tab ----------------------------------------------------------- + +def test_mobile_tab_is_registered_once_and_translated(): + from je_auto_control.gui.tab_registry import TAB_SPECS + rows = [spec for spec in TAB_SPECS if spec.key == "mobile"] + assert len(rows) == 1 + assert (rows[0].module, rows[0].class_name) == ("je_auto_control.gui.mobile_tab", "MobileTab") + from je_auto_control.gui.language_wrapper import ( + english, japanese, simplified_chinese, traditional_chinese, + ) + catalogues = (english.english_word_dict, japanese.japanese_word_dict, + simplified_chinese.simplified_chinese_word_dict, + traditional_chinese.traditional_chinese_word_dict) + keys = {key for key in catalogues[0] if key.startswith("mob_")} | {"tab_mobile"} + assert len(keys) > 5 + for catalogue in catalogues: + assert keys <= set(catalogue) + + +def test_mobile_tab_runs_a_command_through_the_headless_api(adb_host): + pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + from PySide6.QtWidgets import QApplication + from je_auto_control.gui.mobile_tab import MobileTab + app = QApplication.instance() or QApplication([]) + tab = MobileTab() + try: + tab.set_target("android", "phone", adb_path="fake-adb") + tab.set_command("AC_android_long_press", {"x": 3, "y": 4}) + actions = dict(tab.menu_actions()) + actions["mob_run"]() + assert adb_host.devices["phone"].input_calls == ["input swipe 3 4 3 4 1000"] + actions["mob_probe"]() + assert tab.capability_rows()["input"] == "available" + tab.set_command("AC_android_long_press", {"x": 3}) + actions["mob_run"]() + assert "y" in tab.result_text() + finally: + tab.deleteLater() + app.processEvents() From 9d868e6943ca20765398fce6f33595fdbdeafa8c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:14:52 +0800 Subject: [PATCH 27/44] Name the backtick in words in the keyboard page An inline literal cannot hold a lone backtick: the build raised no warning but rendered the markup, and swallowed the text up to the next literal. --- docs/source/Eng/doc/keyboard/keyboard_doc.rst | 2 +- docs/source/Zh/doc/keyboard/keyboard_doc.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/Eng/doc/keyboard/keyboard_doc.rst b/docs/source/Eng/doc/keyboard/keyboard_doc.rst index 082ce5cee..252fb889f 100644 --- a/docs/source/Eng/doc/keyboard/keyboard_doc.rst +++ b/docs/source/Eng/doc/keyboard/keyboard_doc.rst @@ -39,7 +39,7 @@ alias: Eight Windows names stand for a *character* rather than a key position: ``slash`` (``/``), ``backslash`` (``\``), ``semicolon`` (``;``), ``quote`` -(``'``), ``backquote`` (`` ` ``), ``bracketleft`` (``[``), ``bracketright`` +(``'``), ``backquote`` (the backtick), ``bracketleft`` (``[``), ``bracketright`` (``]``) and ``equal`` (``=``). Each is resolved when it is looked up, by asking the keyboard layout of the foreground window which key types that character (``VkKeyScanExW``), so ``press_keyboard_key("semicolon")`` presses diff --git a/docs/source/Zh/doc/keyboard/keyboard_doc.rst b/docs/source/Zh/doc/keyboard/keyboard_doc.rst index 2e06ea24a..7bffd07fe 100644 --- a/docs/source/Zh/doc/keyboard/keyboard_doc.rst +++ b/docs/source/Zh/doc/keyboard/keyboard_doc.rst @@ -37,7 +37,7 @@ Windows 的按鍵表也收常見的別名（``ctrl``、``alt``、``enter``、``e Windows 上有八個名字代表的是\ *字元*\ 而不是按鍵位置：``slash``\ （``/``\ ）、 ``backslash``\ （``\``\ ）、``semicolon``\ （``;``\ ）、``quote``\ （``'``\ ）、 -``backquote``\ （`` ` ``\ ）、``bracketleft``\ （``[``\ ）、``bracketright``\ （``]``\ ） +``backquote``\ （反引號）、``bracketleft``\ （``[``\ ）、``bracketright``\ （``]``\ ） 與 ``equal``\ （``=``\ ）。它們在查表的當下才解析：向前景視窗的鍵盤配置詢問哪個鍵 打得出該字元（``VkKeyScanExW``\ ），所以在 ``;`` 不在美式位置的配置上， ``press_keyboard_key("semicolon")`` 仍會按到正確的鍵。配置上沒有不加修飾鍵就能 From ff098dcb23fa43884706d70babca3b28fecd6695 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:25:24 +0800 Subject: [PATCH 28/44] Add a cancellable GUI task controller and move the connects, SDP steps, OCR reads and device runs that froze the window onto it --- je_auto_control/gui/_worker_thread.py | 6 + je_auto_control/gui/device_matrix_tab.py | 33 +- .../gui/language_wrapper/english.py | 3 + .../gui/language_wrapper/japanese.py | 3 + .../language_wrapper/simplified_chinese.py | 3 + .../language_wrapper/traditional_chinese.py | 3 + je_auto_control/gui/ocr_tab.py | 73 ++- .../gui/remote_desktop/_connect_task.py | 57 ++ .../gui/remote_desktop/connection_screen.py | 94 +-- .../gui/remote_desktop/viewer_panel.py | 36 +- .../remote_desktop/webrtc_host_connection.py | 48 +- .../gui/remote_desktop/webrtc_host_media.py | 12 +- .../remote_desktop/webrtc_host_sessions.py | 3 +- .../gui/remote_desktop/webrtc_host_trust.py | 3 +- .../gui/remote_desktop/webrtc_host_ui.py | 3 +- .../gui/remote_desktop/webrtc_panel.py | 4 +- .../gui/remote_desktop/webrtc_panel_common.py | 12 + .../webrtc_viewer_address_book.py | 3 +- .../webrtc_viewer_connection.py | 59 +- .../gui/remote_desktop/webrtc_viewer_files.py | 11 +- .../gui/remote_desktop/webrtc_viewer_media.py | 9 +- .../gui/remote_desktop/webrtc_viewer_ui.py | 3 +- je_auto_control/gui/task_controller.py | 446 +++++++++++++ test/unit_test/headless/_qt_settle.py | 22 + .../headless/test_gui_task_lifecycle.py | 607 ++++++++++++++++++ test/unit_test/headless/test_qa_tabs_b_gui.py | 6 + test/unit_test/headless/test_rd_gui_audit.py | 7 +- .../headless/test_remote_desktop_gui.py | 3 + .../test_remote_desktop_quick_connect.py | 4 + .../test_remote_registry_owners_gui.py | 6 + 30 files changed, 1482 insertions(+), 100 deletions(-) create mode 100644 je_auto_control/gui/remote_desktop/_connect_task.py create mode 100644 je_auto_control/gui/task_controller.py create mode 100644 test/unit_test/headless/_qt_settle.py create mode 100644 test/unit_test/headless/test_gui_task_lifecycle.py diff --git a/je_auto_control/gui/_worker_thread.py b/je_auto_control/gui/_worker_thread.py index 1b39ee5e3..1e2d70a31 100644 --- a/je_auto_control/gui/_worker_thread.py +++ b/je_auto_control/gui/_worker_thread.py @@ -23,6 +23,12 @@ the worker thread, are queued to a relay the tab owns; the callbacks always run on the GUI thread and are dropped once the tab is gone. A module-level registry keeps each worker alive until the GUI thread has seen its thread end. + +This module is the thread layer only. Work that waits on a network peer or a +device, and so needs a timeout, cancellation that reaches the backend, typed +errors or progress, goes through :mod:`je_auto_control.gui.task_controller`, +which runs on :func:`start_worker` and shares this registry -- there is one +mechanism, with two levels of use. """ import atexit import threading diff --git a/je_auto_control/gui/device_matrix_tab.py b/je_auto_control/gui/device_matrix_tab.py index 51755e5d4..4cb70a426 100644 --- a/je_auto_control/gui/device_matrix_tab.py +++ b/je_auto_control/gui/device_matrix_tab.py @@ -1,6 +1,9 @@ """Device Matrix tab: run one action list across many devices in parallel. -Thin wrapper over :func:`je_auto_control.run_on_devices`. +Thin wrapper over :func:`je_auto_control.run_on_devices`. The run talks to +every device (adb, WebDriverAgent) and takes as long as the slowest one, so it +goes through the task controller: the window stays responsive, and a tab +destroyed mid-run hears nothing of the result. """ import json from typing import Optional @@ -12,6 +15,7 @@ ) from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.task_controller import TaskHandle, task_controller from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( language_wrapper, ) @@ -47,6 +51,7 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._table.setEditTriggers(QAbstractItemView.NoEditTriggers) self._table.horizontalHeader().setStretchLastSection(True) self._summary = QLabel() + self._task: Optional[TaskHandle] = None # one run at a time self._apply_headers() self._build_layout() @@ -80,16 +85,30 @@ def menu_actions(self) -> list: ] def _on_run(self) -> None: + if self._task is not None: + return try: devices = json.loads(self._devices.toPlainText() or "[]") actions = json.loads(self._actions.toPlainText() or "[]") - report = ac.run_on_devices( - actions, devices, max_parallel=self._parallel.value(), - ) - except (ValueError, RuntimeError) as error: - self._summary.setText(_t("dm_error").replace("{error}", str(error))) + except ValueError as error: + self._show_error(error) return - self._render(report.to_dict()) + max_parallel = self._parallel.value() + self._summary.setText(_t("dm_running")) + # run_on_devices takes neither a timeout nor a cancel signal, so a + # cancelled run finishes on its own and its report is dropped. + self._task = task_controller().submit( + lambda _token: ac.run_on_devices(actions, devices, max_parallel=max_parallel).to_dict(), + owner=self) + self._task.result.connect(self._render) + self._task.error.connect(self._show_error) + self._task.finished.connect(self._on_run_finished) + + def _on_run_finished(self) -> None: + self._task = None + + def _show_error(self, error: object) -> None: + self._summary.setText(_t("dm_error").replace("{error}", str(error))) def _render(self, report: dict) -> None: results = report["results"] diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 6bc8d1ebe..dd9e22693 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -106,6 +106,7 @@ "dm_col_error": "Error", "dm_summary": "{passed} passed, {failed} failed of {total}", "dm_error": "Run failed: {error}", + "dm_running": "Running...", # Media checks tab "media_audio_label": "Audio activity check", @@ -1144,6 +1145,7 @@ "ocr_min_conf_invalid": "Min confidence must be a number", "ocr_regex_required": "Enter a regex pattern first", "ocr_regex_invalid": "Invalid regex", + "ocr_running": "Reading...", # Variables Tab "vars_current_group": "Current scope", @@ -1266,6 +1268,7 @@ "rd_badge_running": "RUNNING · :{port} · {n} viewer(s)", "rd_badge_stopped": "STOPPED", "rd_badge_idle": "NOT CONNECTED", + "rd_viewer_connecting": "CONNECTING...", "rd_badge_live": "LIVE", "rd_host_id_group": "Host ID (share with viewers)", "rd_host_id_label": "Host ID:", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 0740bfc51..dd80a685a 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -1033,6 +1033,8 @@ "ocr_min_conf_invalid": "信頼度は数値で入力", "ocr_regex_required": "先に正規表現を入力", "ocr_regex_invalid": "正規表現が不正", + "ocr_running": "読み取り中...", + "dm_running": "実行中...", # Variables Tab "vars_current_group": "現在のスコープ", @@ -1155,6 +1157,7 @@ "rd_badge_running": "RUNNING · :{port} · ビューア {n} 名", "rd_badge_stopped": "STOPPED", "rd_badge_idle": "未接続", + "rd_viewer_connecting": "接続中...", "rd_badge_live": "接続中", "rd_host_id_group": "ホスト ID（ビューアに伝える）", "rd_host_id_label": "ホスト ID:", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index d002ab239..ecdce2dc4 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -1016,6 +1016,8 @@ "ocr_min_conf_invalid": "置信度需为数字", "ocr_regex_required": "请先输入 regex 表达式", "ocr_regex_invalid": "Regex 不正确", + "ocr_running": "识别中...", + "dm_running": "执行中...", # Variables Tab "vars_current_group": "当前作用域", @@ -1134,6 +1136,7 @@ "rd_badge_running": "RUNNING · :{port} · {n} 个 viewer", "rd_badge_stopped": "STOPPED", "rd_badge_idle": "未连接", + "rd_viewer_connecting": "连接中...", "rd_badge_live": "已连接", "rd_host_id_group": "Host ID（给远程的人）", "rd_host_id_label": "Host ID:", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 046af99cd..f28feeca5 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -1017,6 +1017,8 @@ "ocr_min_conf_invalid": "信心度需為數字", "ocr_regex_required": "請先輸入 regex 樣式", "ocr_regex_invalid": "Regex 不正確", + "ocr_running": "辨識中...", + "dm_running": "執行中...", # Variables Tab "vars_current_group": "目前作用域", @@ -1135,6 +1137,7 @@ "rd_badge_running": "RUNNING · :{port} · {n} 個 viewer", "rd_badge_stopped": "STOPPED", "rd_badge_idle": "尚未連線", + "rd_viewer_connecting": "連線中...", "rd_badge_live": "連線中", "rd_host_id_group": "Host ID（給遠端的人）", "rd_host_id_label": "Host ID：", diff --git a/je_auto_control/gui/ocr_tab.py b/je_auto_control/gui/ocr_tab.py index bc41c9c23..3650e37ff 100644 --- a/je_auto_control/gui/ocr_tab.py +++ b/je_auto_control/gui/ocr_tab.py @@ -1,7 +1,13 @@ -"""OCR Reader tab: dump text in a region, or regex-search for matches.""" +"""OCR Reader tab: dump text in a region, or regex-search for matches. + +Both commands grab the screen and run the OCR backend -- Tesseract as a +subprocess with no timeout, EasyOCR downloading its models on first use -- so +they go through the task controller instead of holding the GUI thread. +""" +import functools import json import re -from typing import Optional +from typing import Any, Callable, Optional from PySide6.QtWidgets import ( QGroupBox, QHBoxLayout, QLabel, QLineEdit, QMessageBox, @@ -13,6 +19,7 @@ language_wrapper, ) from je_auto_control.gui.selector import open_region_selector +from je_auto_control.gui.task_controller import CancellationToken, TaskHandle, task_controller from je_auto_control.utils.ocr.ocr_engine import ( find_text_regex, read_text_in_region, ) @@ -47,6 +54,8 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._result = QTextEdit() self._result.setReadOnly(True) self._status = QLabel() + self._task: Optional[TaskHandle] = None # one read at a time + self._title_key = "ocr_dump_region" self._apply_placeholders() self._build_layout() @@ -122,23 +131,7 @@ def _parse_min_conf(self) -> float: raise ValueError(_t("ocr_min_conf_invalid")) from error def _on_dump(self) -> None: - try: - region = self._parse_region() - min_conf = self._parse_min_conf() - lang = self._lang.text().strip() or "eng" - matches = read_text_in_region( - region=region, lang=lang, min_confidence=min_conf, - ) - except ValueError as error: - self._status.setText(str(error)) - return - except (OSError, RuntimeError) as error: - QMessageBox.warning(self, _t("ocr_dump_region"), str(error)) - return - self._result.setText(_matches_to_json(matches)) - self._status.setText( - _t("ocr_match_count").replace("{n}", str(len(matches))) - ) + self._run_ocr("ocr_dump_region", read_text_in_region, {}) def _on_find_regex(self) -> None: pattern = self._regex.text().strip() @@ -150,20 +143,44 @@ def _on_find_regex(self) -> None: except re.error as error: self._status.setText(f"{_t('ocr_regex_invalid')}: {error}") return + self._run_ocr("ocr_find_regex", find_text_regex, {"pattern": compiled}) + + def _run_ocr(self, title_key: str, backend: Callable[..., list], extra: dict) -> None: + """Read the fields, then run ``backend`` off the GUI thread; one read at a time.""" + if self._task is not None: + return try: - region = self._parse_region() - min_conf = self._parse_min_conf() - lang = self._lang.text().strip() or "eng" - matches = find_text_regex( - compiled, lang=lang, region=region, min_confidence=min_conf, - ) + options = {"region": self._parse_region(), "min_confidence": self._parse_min_conf(), + "lang": self._lang.text().strip() or "eng", **extra} except ValueError as error: self._status.setText(str(error)) return - except (OSError, RuntimeError) as error: - QMessageBox.warning(self, _t("ocr_find_regex"), str(error)) - return + self._title_key = title_key + self._status.setText(_t("ocr_running")) + # The OCR backends take neither a timeout nor a cancel signal: a + # cancelled read runs to its end and its matches are dropped. + self._task = task_controller().submit(functools.partial(_call_backend, backend, options), owner=self) + self._task.result.connect(self._show_matches) + self._task.error.connect(self._show_error) + self._task.finished.connect(self._on_task_finished) + + def _on_task_finished(self) -> None: + self._task = None + + def _show_matches(self, matches: list) -> None: self._result.setText(_matches_to_json(matches)) self._status.setText( _t("ocr_match_count").replace("{n}", str(len(matches))) ) + + def _show_error(self, error: Exception) -> None: + if isinstance(error, ValueError): + self._status.setText(str(error)) + return + self._status.setText("") + QMessageBox.warning(self, _t(self._title_key), str(error)) + + +def _call_backend(backend: Callable[..., list], options: dict, _token: CancellationToken) -> Any: + """Worker thread: one OCR read with the options read from the tab.""" + return backend(**options) diff --git a/je_auto_control/gui/remote_desktop/_connect_task.py b/je_auto_control/gui/remote_desktop/_connect_task.py new file mode 100644 index 000000000..7dadcbddd --- /dev/null +++ b/je_auto_control/gui/remote_desktop/_connect_task.py @@ -0,0 +1,57 @@ +"""Connect a remote-desktop viewer without holding the GUI thread. + +``viewer.connect(timeout=5.0)`` in a button slot froze the window for the TCP +connect, then the TLS / WebSocket upgrade, then the auth exchange -- each +bounded at five seconds, so ten to fifteen against a host that accepts and +says nothing. The legacy viewer panel and the Quick Connect screen both connect +through here instead: the viewer is built on the GUI thread, connected on a +task-controller worker, and handed back on the GUI thread. +""" +import functools +from typing import Any, Callable + +from PySide6.QtCore import QObject + +from je_auto_control.gui.task_controller import CancellationToken, TaskHandle, task_controller +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +#: Seconds the backend gets for each step of the connect. +CONNECT_TIMEOUT_S = 5.0 + + +def _disconnect_quietly(viewer: Any) -> None: + """Close a viewer nobody will use; safe on one that never connected, and twice.""" + try: + viewer.disconnect() + except (OSError, RuntimeError) as error: + autocontrol_logger.debug("dropping an unused viewer: %r", error) + + +def _connect(viewer: Any, timeout_s: float, token: CancellationToken) -> Any: + """Worker thread: connect ``viewer`` and return it.""" + # connect() takes a timeout but no cancel signal: a cancelled attempt + # runs out its timeout, and the viewer it produced is disconnected by + # the task's ``discard``. + token.raise_if_cancelled() + viewer.connect(timeout=token.remaining(timeout_s)) + return viewer + + +def connect_viewer(owner: QObject, viewer: Any, *, + on_connected: Callable[[Any], None], + on_failed: Callable[[Exception], None], + timeout_s: float = CONNECT_TIMEOUT_S) -> TaskHandle: + """Run ``viewer.connect`` off the GUI thread for ``owner``; return the task. + + ``on_connected(viewer)`` or ``on_failed(error)`` runs on the GUI thread, + and only while ``owner`` exists. Cancelling the task, or destroying + ``owner``, disconnects a viewer that connects anyway. + """ + handle = task_controller().submit(functools.partial(_connect, viewer, timeout_s), + owner=owner, discard=_disconnect_quietly) + handle.result.connect(on_connected) + handle.error.connect(on_failed) + return handle + + +__all__ = ["CONNECT_TIMEOUT_S", "connect_viewer"] diff --git a/je_auto_control/gui/remote_desktop/connection_screen.py b/je_auto_control/gui/remote_desktop/connection_screen.py index 9c17e1c28..352045d8e 100644 --- a/je_auto_control/gui/remote_desktop/connection_screen.py +++ b/je_auto_control/gui/remote_desktop/connection_screen.py @@ -26,6 +26,7 @@ ) from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.remote_desktop._connect_task import connect_viewer from je_auto_control.gui.remote_desktop._helpers import ( _StatusBadge, _build_verifying_client_context, _t, displaced_notifier, wire_remote_input, @@ -33,6 +34,7 @@ from je_auto_control.gui.remote_desktop.remote_screen_window import ( RemoteScreenWindow, ) +from je_auto_control.gui.task_controller import TaskHandle from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.remote_desktop import ( PendingViewer, RemoteDesktopHost, RemoteDesktopViewer, @@ -153,6 +155,7 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: # the popup window is open, so we cache the most recent payload # and replay it the moment the window is created. self._pending_frame: Optional[bytes] = None + self._connect_task: Optional[TaskHandle] = None # the connect in progress, off-thread self._book = default_address_book() self._refresh_timer = QTimer(self) self._refresh_timer.setInterval(self._STATUS_INTERVAL_MS) @@ -439,51 +442,65 @@ def _on_displaced(self, slot: str, _by: str) -> None: self._close_screen_window() self._refresh_status() - def _do_tcp_connect(self, host: str, port: int, token: str) -> None: - self._take_slot(SLOT_VIEWER) + def _viewer_callbacks(self) -> dict: + return {"on_frame": self._frame_arrived.emit, + "on_error": lambda exc: self._error_arrived.emit(str(exc)), + "on_cursor": self._cursor_moved.emit} + + def _begin_connect(self, slot: str, build, on_live) -> None: + """Take ``slot``, build the viewer, and connect it off the GUI thread (see ``_connect_task``).""" + self._cancel_pending_connect() + self._take_slot(slot) try: - viewer = RemoteDesktopViewer( - host=host, port=port, token=token, - on_frame=self._frame_arrived.emit, - on_error=lambda exc: self._error_arrived.emit(str(exc)), - on_cursor=self._cursor_moved.emit, - ) - viewer.connect(timeout=5.0) - # ValueError: a host such as "a..b" fails IDNA encoding with - # UnicodeError, which escaped the slot and left the click unanswered. + viewer = build() except (OSError, RuntimeError, ValueError, AutoControlException) as error: - QMessageBox.warning(self, _t("rd_quick_connect_btn"), str(error)) + self._on_connect_failed(error) return - registry.adopt(SLOT_VIEWER, viewer, self._owner, displaced_notifier(self)) - self._remember_tcp(host, port) - self._open_screen_window(f"{host}:{port}") + + def connected(live_viewer) -> None: + registry.adopt(slot, live_viewer, self._owner, displaced_notifier(self)) + on_live() + self._connect_task = connect_viewer( + self, viewer, on_connected=connected, on_failed=self._on_connect_failed) + self._connect_task.finished.connect(self._on_connect_finished) + self._refresh_status() + + def _cancel_pending_connect(self) -> None: + task, self._connect_task = self._connect_task, None + if task is not None: + task.cancel() + + def _on_connect_finished(self) -> None: + if self.sender() is self._connect_task: + self._connect_task = None self._refresh_status() + def _on_connect_failed(self, error: Exception) -> None: + # Any error: a host such as "a..b" fails IDNA encoding with UnicodeError. + QMessageBox.warning(self, _t("rd_quick_connect_btn"), str(error)) + + def _do_tcp_connect(self, host: str, port: int, token: str) -> None: + def live() -> None: + self._remember_tcp(host, port) + self._open_screen_window(f"{host}:{port}") + + self._begin_connect(SLOT_VIEWER, lambda: RemoteDesktopViewer( + host=host, port=port, token=token, **self._viewer_callbacks()), live) + def _do_ws_connect(self, target: ConnectTarget, token: str) -> None: - host = target.host or "" - port = target.port or 0 - path = target.path or "/" - self._take_slot(SLOT_WS_VIEWER) + host, port, path = target.host or "", target.port or 0, target.path or "/" + scheme = "wss" if target.kind == "wss" else "ws" # wss:// was dialled as plain ws://: the session went unencrypted to # a host the operator took for TLS, and a real TLS host was unreachable. - ssl_context = _build_verifying_client_context() if target.kind == "wss" else None - try: - viewer = WebSocketDesktopViewer( - host=host, port=port, token=token, path=path, - on_frame=self._frame_arrived.emit, - on_error=lambda exc: self._error_arrived.emit(str(exc)), - on_cursor=self._cursor_moved.emit, - ssl_context=ssl_context, - ) - viewer.connect(timeout=5.0) - except (OSError, RuntimeError, ValueError, AutoControlException) as error: - QMessageBox.warning(self, _t("rd_quick_connect_btn"), str(error)) - return - registry.adopt(SLOT_WS_VIEWER, viewer, self._owner, displaced_notifier(self)) - scheme = "wss" if target.kind == "wss" else "ws" - self._remember_url(f"{scheme}://{host}:{port}{path}") - self._open_screen_window(f"{scheme}://{host}:{port}") - self._refresh_status() + ssl_context = _build_verifying_client_context() if scheme == "wss" else None + + def live() -> None: + self._remember_url(f"{scheme}://{host}:{port}{path}") + self._open_screen_window(f"{scheme}://{host}:{port}") + + self._begin_connect(SLOT_WS_VIEWER, lambda: WebSocketDesktopViewer( + host=host, port=port, token=token, path=path, ssl_context=ssl_context, + **self._viewer_callbacks()), live) def _on_remote_cursor(self, x: int, y: int) -> None: """Cursor update, delivered on the GUI thread by ``_cursor_moved``.""" @@ -502,6 +519,7 @@ def _handoff_to_webrtc(self, host_id: str, token: str) -> None: def _disconnect(self) -> None: # Either transport, and only a session this screen opened. + self._cancel_pending_connect() registry.release(SLOT_VIEWER, self._owner) registry.release(SLOT_WS_VIEWER, self._owner) self._close_screen_window() @@ -720,6 +738,8 @@ def _refresh_viewer_status(self) -> None: viewer = self._own_viewer() if viewer is not None and viewer.connected: self._viewer_badge.set_state("live", _t("rd_quick_connected")) + elif self._connect_task is not None: + self._viewer_badge.set_state("idle", _t("rd_viewer_connecting")) else: self._viewer_badge.set_state( "idle", _t("rd_quick_disconnected"), diff --git a/je_auto_control/gui/remote_desktop/viewer_panel.py b/je_auto_control/gui/remote_desktop/viewer_panel.py index b0ab654ec..9c7f85259 100644 --- a/je_auto_control/gui/remote_desktop/viewer_panel.py +++ b/je_auto_control/gui/remote_desktop/viewer_panel.py @@ -13,6 +13,7 @@ from je_auto_control.gui._daemon_thread import DaemonThread from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui.remote_desktop._connect_task import connect_viewer from je_auto_control.gui.remote_desktop._helpers import ( _CollapsibleSection, _StatusBadge, _build_insecure_client_context, _build_verifying_client_context, _t, displaced_notifier, @@ -21,6 +22,7 @@ from je_auto_control.gui.remote_desktop.remote_screen_window import ( RemoteScreenWindow, ) +from je_auto_control.gui.task_controller import TaskHandle from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.remote_desktop import ( FileReceiver, RemoteDesktopViewer, WebSocketDesktopViewer, default_download_dir, @@ -90,6 +92,8 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self._disconnect_btn: Optional[QPushButton] = None self._action_row: Optional[QWidget] = None self._connected = False + # The connect in progress, off the GUI thread; None once it answered. + self._connect_task: Optional[TaskHandle] = None self._audio_player: Optional[AudioPlayer] = None self._progress_bar = QProgressBar() self._progress_bar.setVisible(False) @@ -229,6 +233,7 @@ def _connect(self) -> None: viewer_cls = (WebSocketDesktopViewer if transport in ("WebSocket", "WSS") else RemoteDesktopViewer) + self._cancel_pending_connect() registry.evict(SLOT_VIEWER, by=self._owner) try: viewer = viewer_cls( @@ -250,12 +255,34 @@ def _connect(self) -> None: ), base_dir=default_download_dir(), )) - viewer.connect(timeout=5.0) - # ValueError: a host such as "a..b" fails IDNA encoding with - # UnicodeError, which escaped the slot and left the click unanswered. except (OSError, RuntimeError, ValueError, AutoControlException) as error: QMessageBox.warning(self, _t("rd_viewer_connect"), str(error)) return + # The connect itself (TCP, TLS / WebSocket upgrade, auth: up to five + # seconds each) held the GUI thread for all of it. + self._connect_task = connect_viewer( + self, viewer, on_connected=self._on_connected, on_failed=self._on_connect_failed) + self._connect_task.finished.connect(self._on_connect_finished) + self._refresh_status() + + def _cancel_pending_connect(self) -> None: + """Give up a connect that has not answered; a viewer it still produces is dropped.""" + task, self._connect_task = self._connect_task, None + if task is not None: + task.cancel() + + def _on_connect_finished(self) -> None: + if self.sender() is self._connect_task: + self._connect_task = None + self._refresh_status() + + def _on_connect_failed(self, error: Exception) -> None: + # Includes ValueError: a host such as "a..b" fails IDNA encoding with + # UnicodeError, which once escaped the slot and left the click unanswered. + QMessageBox.warning(self, _t("rd_viewer_connect"), str(error)) + + def _on_connected(self, viewer) -> None: + """GUI thread: the viewer this panel asked for is connected.""" registry.adopt(SLOT_VIEWER, viewer, self._owner, displaced_notifier(self)) self._connected = True self._start_audio_player_if_requested() @@ -316,6 +343,7 @@ def _on_displaced(self, _slot: str, _by: str) -> None: self._status.setText(_t("rd_viewer_displaced")) def _disconnect(self) -> None: + self._cancel_pending_connect() registry.release(SLOT_VIEWER, self._owner) self._stop_audio_player() self._connected = False @@ -367,6 +395,8 @@ def _refresh_status(self) -> None: live = self._connected and viewer is not None and viewer.connected if live: self._badge.set_state("live", _t("rd_badge_live")) + elif self._connect_task is not None: + self._badge.set_state("idle", _t("rd_viewer_connecting")) else: self._badge.set_state("idle", _t("rd_badge_idle")) if self._action_row is not None: diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_connection.py b/je_auto_control/gui/remote_desktop/webrtc_host_connection.py index 9f6c6cd3f..6cf216e1c 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_connection.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_connection.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +import functools +from typing import TYPE_CHECKING, Optional, Tuple from PySide6.QtCore import QTimer from PySide6.QtWidgets import ( @@ -17,18 +19,41 @@ from je_auto_control.gui.remote_desktop.webrtc_workers import ( HostPublishLoopWorker, generate_host_id, retire_worker, ) +from je_auto_control.gui.task_controller import CancellationToken, task_controller from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop import ( MultiViewerHost, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _read_webrtc_config, ) +if TYPE_CHECKING: # imported lazily at runtime to keep startup cheap + from je_auto_control.utils.remote_desktop.lan_discovery import HostAdvertiser -class _HostConnectionMixin: + +def _create_offer(host: MultiViewerHost, _token: CancellationToken) -> Tuple[str, str]: + """Worker thread: mint a session and its offer (ICE gathering, up to 12 s).""" + return host.create_session_offer() + + +def _drop_offer(host: MultiViewerHost, outcome: Tuple[str, str]) -> None: + """End the session of an offer nobody will show.""" + try: + host.stop_session(outcome[0]) + except (KeyError, RuntimeError, OSError) as error: + autocontrol_logger.debug("dropping an unused offer: %r", error) + + +class _HostConnectionMixin(_PanelPart): """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" + # State this group owns; the panel's __init__ sets the starting values. + _publish_loop: Optional[HostPublishLoopWorker] + _manual_session_id: Optional[str] + _lan_advertiser: Optional[HostAdvertiser] + def _on_regen_id(self) -> None: self._host_id_edit.setText(generate_host_id()) @@ -122,10 +147,27 @@ def _require_multi_host(self) -> MultiViewerHost: def _produce_offer(self) -> None: try: - session_id, offer = self._require_multi_host().create_session_offer() - except (RuntimeError, OSError) as error: # PermissionError is an OSError + host = self._require_multi_host() + except RuntimeError as error: self._show_error(error) return + # create_session_offer waits for ICE gathering (up to 12 s), which + # used to freeze the window; the backend takes no timeout or cancel. + task = task_controller().submit(functools.partial(_create_offer, host), owner=self, + discard=functools.partial(_drop_offer, host)) + task.result.connect(functools.partial(self._show_offer, host)) + task.error.connect(functools.partial(self._show_offer_error, host)) + + def _show_offer_error(self, host: MultiViewerHost, error: Exception) -> None: + if self._multi_host is host: # a host stopped meanwhile fails by design + self._show_error(error) + + def _show_offer(self, host: MultiViewerHost, outcome: Tuple[str, str]) -> None: + """GUI thread: the offer is ready -- unless the host was stopped or replaced meanwhile.""" + if self._multi_host is not host: + _drop_offer(host, outcome) + return + session_id, offer = outcome self._manual_session_id = session_id self._offer_view.setPlainText(offer) self._status_label.setText(_t("rd_webrtc_offer_ready")) diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_media.py b/je_auto_control/gui/remote_desktop/webrtc_host_media.py index 937fc7576..53a1d7eef 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_media.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_media.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +from typing import Optional + from PySide6.QtGui import QImage from PySide6.QtWidgets import ( @@ -32,13 +34,21 @@ StatsPoller, StatsSnapshot, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _DEFAULT_MONITOR, _QUALITY_DOT_STYLE, _av_frame_to_qimage, ) -class _HostMediaMixin: +class _HostMediaMixin(_PanelPart): """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" + # State this group owns; the panel's __init__ sets the starting values. + _adaptive_controller: Optional[AdaptiveBitrateController] + _adaptive_poller: Optional[StatsPoller] + _blanking: Optional[BlankingOverlay] + _viewer_screen_window: Optional[ViewerScreenWindow] + _annotation_overlay: Optional[HostAnnotationOverlay] + def _on_hw_codec_changed(self) -> None: codec = self._hw_codec_combo.currentData() or "" if not codec: diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py b/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py index ccacd54f8..fadc51a02 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py @@ -22,9 +22,10 @@ from je_auto_control.utils.remote_desktop.webrtc_stats import ( StatsPoller, StatsSnapshot, ) +from je_auto_control.gui.remote_desktop.webrtc_panel_common import _PanelPart -class _HostSessionsMixin: +class _HostSessionsMixin(_PanelPart): """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" def _on_session_count(self, count: int) -> None: diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_trust.py b/je_auto_control/gui/remote_desktop/webrtc_host_trust.py index 29352e1d8..4a22ef7c9 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_trust.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_trust.py @@ -19,11 +19,12 @@ ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _JSON_FILE_FILTER, ) -class _HostTrustMixin: +class _HostTrustMixin(_PanelPart): """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" def _on_view_audit(self) -> None: diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_ui.py b/je_auto_control/gui/remote_desktop/webrtc_host_ui.py index 88e098062..bad1d026c 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_ui.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_ui.py @@ -26,11 +26,12 @@ generate_host_id, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _DEFAULT_FPS, _DEFAULT_SIGNALING_URL, _QUALITY_DOT_STYLE, ) -class _HostUiMixin: +class _HostUiMixin(_PanelPart): """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" def _build_ui(self) -> None: diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel.py b/je_auto_control/gui/remote_desktop/webrtc_panel.py index e96dfaaad..148a77721 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel.py @@ -86,7 +86,7 @@ class _WebRTCHostPanel(_HostUiMixin, _HostTrustMixin, _HostMediaMixin, _HostSessionsMixin, - _HostConnectionMixin, TranslatableMixin, QWidget): + _HostConnectionMixin): """Host: stream this machine's screen and accept viewer input.""" def __init__(self, parent: Optional[QWidget] = None) -> None: @@ -210,7 +210,7 @@ def retranslate(self) -> None: class _WebRTCViewerPanel(_ViewerUiMixin, _ViewerFilesMixin, _ViewerAddressBookMixin, _ViewerMediaMixin, - _ViewerConnectionMixin, TranslatableMixin, QWidget): + _ViewerConnectionMixin): """Viewer: receive screen and send input.""" def __init__(self, parent: Optional[QWidget] = None) -> None: diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel_common.py b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py index b745e9167..890fc7a83 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel_common.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py @@ -5,6 +5,9 @@ from PySide6.QtCore import QObject, Signal from PySide6.QtGui import QImage +from PySide6.QtWidgets import QWidget + +from je_auto_control.gui._i18n_helpers import TranslatableMixin from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop import ( @@ -57,6 +60,15 @@ class _PanelSignals(QObject): annotation = Signal(object) # dict +class _PanelPart(TranslatableMixin, QWidget): + """Base of every WebRTC panel mixin: each one is a slice of a translatable ``QWidget``. + + It adds nothing at run time. It is what lets a mixin use the widget API, + ``_tr`` and the attributes its sibling mixins set, the way the methods did + while they all sat in one class body. + """ + + def _checked_or(panel, attr: str, default: bool = False) -> bool: """Return ``panel..isChecked()`` if the widget exists, else default.""" widget = getattr(panel, attr, None) diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py index c5a1d0de3..0a2f57a19 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py @@ -21,11 +21,12 @@ send_magic_packet, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _JSON_FILE_FILTER, ) -class _ViewerAddressBookMixin: +class _ViewerAddressBookMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" @staticmethod diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py index ba5ef42dc..67d245a93 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +import functools +from typing import TYPE_CHECKING, Optional from PySide6.QtCore import QTimer from PySide6.QtWidgets import ( @@ -18,18 +20,36 @@ ViewerAnswerPushWorker, ViewerSignalingWorker, retire_worker, ) +from je_auto_control.gui.task_controller import CancellationToken, task_controller from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop import ( - WebRTCDesktopViewer, + SessionRecorder, WebRTCDesktopViewer, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _read_webrtc_config, ) +if TYPE_CHECKING: # imported lazily at runtime to keep startup cheap + from je_auto_control.utils.remote_desktop.file_sync import FolderSyncEngine -class _ViewerConnectionMixin: + +def _process_offer(viewer: WebRTCDesktopViewer, offer_sdp: str, expected_dtls: Optional[str], + _token: CancellationToken) -> str: + """Worker thread: answer the host's offer (up to 12 s; the backend takes no timeout or cancel).""" + return viewer.process_offer(offer_sdp, expected_dtls_fingerprint=expected_dtls) + + +class _ViewerConnectionMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + # State this group owns; the panel's __init__ sets the starting values. + _offer_worker: Optional[ViewerSignalingWorker] + _answer_worker: Optional[ViewerAnswerPushWorker] + # Owned by the files and media groups; stopped here with the session. + _sync_engine: Optional[FolderSyncEngine] + _recorder: Optional[SessionRecorder] + def _on_connect_via_server(self) -> None: if not self._validate_required_fields(needs_server=True): return @@ -57,13 +77,32 @@ def _on_offer_received_from_server(self, offer_sdp: str) -> None: def _answer_and_push(self, offer_sdp: str) -> None: host_id = self._host_id_edit.text().strip() expected_dtls = self._known_hosts.dtls_fingerprint_for(host_id) if host_id else None + self._answer_off_thread( + offer_sdp, expected_dtls, + functools.partial(self._push_answer, host_id, expected_dtls, offer_sdp)) + + def _answer_off_thread(self, offer_sdp: str, expected_dtls: Optional[str], on_answer) -> None: + """Run ``process_offer`` on a worker; ``on_answer(answer)`` only if this viewer is still current.""" try: - answer = self._require_viewer().process_offer( - offer_sdp, expected_dtls_fingerprint=expected_dtls, - ) - except (ValueError, RuntimeError, OSError) as error: + viewer = self._require_viewer() + except RuntimeError as error: self._show_error(error) return + + def deliver(answer: str) -> None: + if self._viewer is viewer: # not stopped or replaced while the answer was made + on_answer(answer) + + def fail(error: Exception) -> None: + if self._viewer is viewer: # a viewer stopped meanwhile fails by design + self._show_error(error) + + task = task_controller().submit( + functools.partial(_process_offer, viewer, offer_sdp, expected_dtls), owner=self) + task.result.connect(deliver) + task.error.connect(fail) + + def _push_answer(self, host_id: str, expected_dtls: Optional[str], offer_sdp: str, answer: str) -> None: # First-time TOFU: stash the DTLS fingerprint we just observed if host_id and not expected_dtls: from je_auto_control.utils.remote_desktop.fingerprint import ( @@ -115,11 +154,9 @@ def _require_viewer(self) -> WebRTCDesktopViewer: return viewer def _produce_answer(self, offer: str) -> None: - try: - answer = self._require_viewer().process_offer(offer) - except (ValueError, RuntimeError, OSError) as error: - self._show_error(error) - return + self._answer_off_thread(offer, None, self._show_manual_answer) + + def _show_manual_answer(self, answer: str) -> None: self._answer_view.setPlainText(answer) self._status_label.setText(_t("rd_webrtc_answer_ready")) diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py index 1c5aab251..2fdc84145 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +from typing import TYPE_CHECKING, Optional + from PySide6.QtCore import QTimer from PySide6.QtWidgets import ( @@ -15,11 +17,18 @@ _t, ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.gui.remote_desktop.webrtc_panel_common import _PanelPart + +if TYPE_CHECKING: # imported lazily at runtime to keep startup cheap + from je_auto_control.utils.remote_desktop.file_sync import FolderSyncEngine -class _ViewerFilesMixin: +class _ViewerFilesMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + # State this group owns; the panel's __init__ sets the starting values. + _sync_engine: Optional[FolderSyncEngine] + def _on_sync_browse(self) -> None: path = QFileDialog.getExistingDirectory( self, _t("rd_webrtc_sync_dir"), diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py index 3d05f221b..7aae81a41 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +from typing import Optional + from PySide6.QtGui import QImage from PySide6.QtWidgets import ( @@ -28,13 +30,18 @@ fps_for_preset, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _QUALITY_DOT_STYLE, _av_frame_to_qimage, ) -class _ViewerMediaMixin: +class _ViewerMediaMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" + # State this group owns; the panel's __init__ sets the starting values. + _recorder: Optional[SessionRecorder] + _stats_poller: Optional[StatsPoller] + def _on_send_cad(self) -> None: if self._viewer is None or not self._viewer.authenticated: QMessageBox.information( diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py index 68ff57f44..a2380005e 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_ui.py @@ -27,11 +27,12 @@ BANDWIDTH_PRESETS, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( + _PanelPart, _DEFAULT_SIGNALING_URL, _QUALITY_DOT_STYLE, ) -class _ViewerUiMixin: +class _ViewerUiMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" def _build_ui(self) -> None: diff --git a/je_auto_control/gui/task_controller.py b/je_auto_control/gui/task_controller.py new file mode 100644 index 000000000..0567236df --- /dev/null +++ b/je_auto_control/gui/task_controller.py @@ -0,0 +1,446 @@ +"""Cancellable background work for the GUI: typed outcomes on the GUI thread, nothing after the owner dies. + +:func:`je_auto_control.gui._worker_thread.start_worker` already keeps blocking +work off the GUI thread and outside ``QThread``'s destruction rules. +:class:`TaskController` is the layer above it for work that waits on a network +peer or a device -- the same daemon thread, registry and exit handling, plus +what a bare worker cannot express: + +* **Cancellation reaches the backend.** The work receives a + :class:`CancellationToken`. It polls it (:meth:`~CancellationToken.raise_if_cancelled`, + :meth:`~CancellationToken.wait`), hands its remaining time to a backend that + takes a timeout (:meth:`~CancellationToken.remaining`), and registers how to + let go of what it holds (:meth:`~CancellationToken.on_cancel`): a socket to + close, a session to stop. Cancelling runs those at once, on the cancelling + thread, instead of waiting for the backend to give up. +* **Typed outcomes.** :attr:`TaskHandle.result` carries the return value, + :attr:`TaskHandle.error` the exception object (not its text) and + :attr:`TaskHandle.progress` whatever the work reports. All three are emitted + on the GUI thread, so a lambda connected to them is safe. +* **Owner death.** The handle is a child of ``owner``. Destroying the owner + cancels the work; a result that is ready but not delivered yet, or arrives + later, goes to ``discard`` (close the session nobody will use) and never to + a slot of the dead widget. +* **No worker touches a widget.** Work that is a widget's method, or closes + over one, is refused before it starts: read what the work needs from the + widgets first and pass plain values. +""" +import functools +import threading +import time +from typing import Any, Callable, Dict, List, Optional, Tuple + +from PySide6.QtCore import QObject, QTimer, Signal +from PySide6.QtWidgets import QWidget + +from je_auto_control.gui._worker_thread import WorkerHandle, start_worker +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +Work = Callable[["CancellationToken"], object] + +_RUNNING = "running" +_DONE = "done" +_FAILED = "failed" +_CANCELLED = "cancelled" +_TIMED_OUT = "timed_out" + + +class TaskCancelled(AutoControlException): + """Raised inside the work by :meth:`CancellationToken.raise_if_cancelled`.""" + + +class TaskTimeout(AutoControlException, TimeoutError): + """Delivered through :attr:`TaskHandle.error` when the work outlived ``timeout_s``.""" + + +class TaskUsageError(AutoControlException, TypeError): + """The work handed to :meth:`TaskController.submit` could reach a widget.""" + + +class CancellationToken: + """What the work holds to learn it was cancelled and to release what it took.""" + + def __init__(self, timeout_s: Optional[float] = None) -> None: + self._event = threading.Event() + self._lock = threading.Lock() + self._callbacks: List[Callable[[], object]] = [] + self._deadline = None if timeout_s is None else time.monotonic() + timeout_s + self._progress: Optional[Callable[[object], None]] = None + + @property + def cancelled(self) -> bool: + """Whether the work should stop.""" + return self._event.is_set() + + def cancel(self) -> bool: + """Mark the work cancelled and run its release callbacks, newest first. + + Returns whether this call was the one that cancelled it. The callbacks + run on the calling thread -- normally the GUI thread -- so they must be + quick: close a socket, set an event. + """ + with self._lock: + if self._event.is_set(): + return False + self._event.set() + callbacks = list(reversed(self._callbacks)) + self._callbacks.clear() + for callback in callbacks: + _run_release(callback) + return True + + def on_cancel(self, callback: Callable[[], object]) -> None: + """Register how to release something the work holds; runs at once if already cancelled. + + A resource may be released here and again through the task's + ``discard``, so the callback must tolerate a second call. + """ + with self._lock: + if not self._event.is_set(): + self._callbacks.append(callback) + return + _run_release(callback) + + def raise_if_cancelled(self) -> None: + """Raise :class:`TaskCancelled` once the work has been cancelled.""" + if self._event.is_set(): + raise TaskCancelled("the task was cancelled") + + def wait(self, timeout_s: float) -> bool: + """Sleep up to ``timeout_s``, waking early on cancel; return whether cancelled.""" + return self._event.wait(timeout_s) + + def remaining(self, default: Optional[float] = None) -> Optional[float]: + """Seconds left before the task's timeout, for a backend that takes one. + + ``default`` is returned when the task has no timeout; a task past its + deadline gets ``0.0``. + """ + if self._deadline is None: + return default + return max(0.0, self._deadline - time.monotonic()) + + def report_progress(self, value: object) -> None: + """Send ``value`` to :attr:`TaskHandle.progress` (delivered on the GUI thread).""" + progress = self._progress + if progress is None or self._event.is_set(): + return + try: + progress(value) + except RuntimeError: # reason: the worker object went away with the application + pass + + def _seal(self) -> None: + """Forget the release callbacks: the work is over and owns nothing more.""" + with self._lock: + self._callbacks.clear() + + +def _run_release(callback: Callable[[], object]) -> None: + """Run one release callback; one that fails must not keep the others from running.""" + try: + callback() + except Exception as error: # noqa: BLE001 # reason: logged; the remaining releases must still run + autocontrol_logger.warning(f"GUI task release callback failed: {error!r}") + + +class _TaskState: + """The outcome of one task, shared by its worker thread and the GUI thread.""" + + def __init__(self, token: CancellationToken, discard: Optional[Callable[[Any], object]]) -> None: + self.token = token + self._discard = discard + self._lock = threading.Lock() + self._status = _RUNNING + self._pending: Optional[Tuple[str, Any]] = None + + @property + def status(self) -> str: + """``running``, ``done``, ``failed``, ``cancelled`` or ``timed_out``.""" + with self._lock: + return self._status + + def finish(self, kind: str, value: Any) -> None: + """Worker thread: record a result (``done``) or an error (``failed``).""" + with self._lock: + keep = self._status == _RUNNING and not self.token.cancelled + if keep: + self._status = kind + self._pending = (kind, value) + elif self._status == _RUNNING: + self._status = _CANCELLED + self.token._seal() # noqa: SLF001 # reason: the work returned; nothing left to release + if not keep and kind == _DONE: + self.discard(value) + + def take(self) -> Optional[Tuple[str, Any]]: + """GUI thread: the outcome to deliver, once.""" + with self._lock: + pending, self._pending = self._pending, None + return pending + + def cancel(self, reason: str = _CANCELLED, *, only_running: bool = False) -> bool: + """Stop the work, or drop a result nobody collected; return whether anything changed.""" + with self._lock: + running = self._status == _RUNNING + pending = self._pending + if not running and (only_running or pending is None or pending[0] != _DONE): + return False + self._pending = None + self._status = reason + if pending is not None: + self.discard(pending[1]) + else: + self.token.cancel() + return True + + def discard(self, value: Any) -> None: + """Hand a result that will not be delivered to the caller's ``discard``.""" + if self._discard is None: + return + try: + self._discard(value) + except Exception as error: # noqa: BLE001 # reason: logged; an undelivered result has no caller to tell + autocontrol_logger.warning(f"GUI task discard failed: {error!r}") + + +class _TaskWorker(QObject): + """The ``start_worker`` worker that runs one task's work.""" + + finished = Signal(object) + progressed = Signal(object) + + def __init__(self, work: Work, state: _TaskState) -> None: + super().__init__() + self._work = work + self._state = state + + def run(self) -> None: + """Call the work with its token and record what came of it.""" + state = self._state + try: + value = self._work(state.token) + except TaskCancelled: + state.cancel() + except Exception as error: # noqa: BLE001 # reason: delivered, typed, through TaskHandle.error + state.finish(_FAILED, error) + else: + state.finish(_DONE, value) + # The token reports progress through this object's signal; left in + # place that is a reference cycle through a QObject about to be deleted. + state.token._progress = None # noqa: SLF001 # reason: the work is over + self.finished.emit(None) + + def request_stop(self) -> None: + """Interpreter exit: stop the work at its next checkpoint.""" + self._state.cancel() + + +def _nothing(*_args: object) -> None: + """Stands in for a callback that has been used up.""" + + +class TaskHandle(QObject): + """One submitted task; its signals are emitted on the GUI thread, while the owner lives. + + ``finished`` is emitted once, after ``result`` / ``error`` or as soon as + the task is cancelled or times out -- the moment to re-enable a button. + """ + + result = Signal(object) + error = Signal(object) + progress = Signal(object) + finished = Signal() + + def __init__(self, owner: QObject, state: _TaskState) -> None: + super().__init__(owner) + self._state = state + self._worker_handle: Optional[WorkerHandle] = None + self._announced = False + self._timer: Optional[QTimer] = None + self._on_end: Callable[["TaskHandle"], None] = _nothing + + @property + def state(self) -> str: + """``running``, ``done``, ``failed``, ``cancelled`` or ``timed_out``.""" + return self._state.status + + @property + def token(self) -> CancellationToken: + """The token the work was given.""" + return self._state.token + + def isRunning(self) -> bool: # noqa: N802 # reason: the QThread spelling the tabs already use + """Whether the work's thread has not returned yet (it may, after a cancel).""" + return self._worker_handle is not None and self._worker_handle.isRunning() + + def wait(self, timeout_s: Optional[float] = None) -> bool: + """Block until the work's thread returns or ``timeout_s`` passes; return whether it did.""" + return self._worker_handle is None or self._worker_handle.wait(timeout_s) + + def cancel(self) -> bool: + """Stop the work and release what it holds; no ``result`` or ``error`` follows. + + Returns whether there was anything left to cancel. + """ + changed = self._state.cancel() + if changed: + self._announce_finished() + return changed + + def _start(self, worker: _TaskWorker, timeout_s: Optional[float], + on_end: Callable[["TaskHandle"], None]) -> None: + worker.progressed.connect(self._relay_progress) + if timeout_s is not None: + self._timer = QTimer(self) + self._timer.setSingleShot(True) + self._timer.timeout.connect(self._on_timeout) + self._timer.start(max(0, int(timeout_s * 1000))) + self._on_end = on_end + self._worker_handle = start_worker(self, worker, on_done=self._deliver, + on_thread_done=self._on_thread_done) + + def _on_thread_done(self) -> None: + """The work's thread ended and its outcome was delivered: nothing more will be emitted.""" + on_end, self._on_end = self._on_end, _nothing + on_end(self) + # Python-side state stays readable; only the Qt object goes, so a tab + # that polls for hours does not collect one handle per poll. The + # wrapper must not be left in a reference cycle at this point: PySide + # corrupted the heap when the collector later freed a cycle that ran + # through an already-deleted QObject. + self.deleteLater() + + def _relay_progress(self, value: object) -> None: + # Reported before the work ended but delivered after: still wanted, + # unless the task was cancelled in between. + if self._state.status not in (_CANCELLED, _TIMED_OUT): + self.progress.emit(value) + + def _on_timeout(self) -> None: + if self._state.cancel(_TIMED_OUT, only_running=True): + self.error.emit(TaskTimeout("the task did not finish in time")) + self._announce_finished() + + def _deliver(self, _value: object = None) -> None: + if self._timer is not None: + self._timer.stop() + pending = self._state.take() + if pending is not None: + (self.result if pending[0] == _DONE else self.error).emit(pending[1]) + self._announce_finished() + + def _announce_finished(self) -> None: + if not self._announced: + self._announced = True + self.finished.emit() + + +def _widgets_reachable_from(work: object, depth: int = 0) -> bool: + """Whether ``work`` is a widget's method or carries a widget in its closure or bound arguments.""" + carried = [getattr(work, "__self__", None), *_closure_values(work), + *(getattr(work, "__defaults__", None) or ()), + *(getattr(work, "__kwdefaults__", None) or {}).values()] + inner = [getattr(work, "__wrapped__", None)] + if isinstance(work, functools.partial): + carried.extend(work.args) + carried.extend(work.keywords.values()) + inner.append(work.func) + if any(isinstance(item, QWidget) for item in carried): + return True + return depth < 3 and any(_widgets_reachable_from(item, depth + 1) for item in inner if item is not None) + + +def _closure_values(work: object) -> List[object]: + """What a function's closure cells hold.""" + values: List[object] = [] + for cell in getattr(work, "__closure__", None) or (): + try: + values.append(cell.cell_contents) + except ValueError: # reason: an empty cell holds nothing + continue + return values + + +class TaskController: + """Starts cancellable work for GUI owners and keeps count of what is still running. + + Use from the GUI thread. One shared instance is enough + (:func:`task_controller`); a tab may also keep its own. + """ + + def __init__(self) -> None: + self._active: Dict[TaskHandle, Tuple[QObject, Callable[..., None]]] = {} + + def submit(self, work: Work, *, owner: QObject, timeout_s: Optional[float] = None, + discard: Optional[Callable[[Any], object]] = None) -> TaskHandle: + """Run ``work(token)`` off the GUI thread for ``owner``; return its handle. + + ``timeout_s`` cancels the work when it passes and reports + :class:`TaskTimeout`; the work can read what is left of it from + ``token.remaining()``. ``discard`` receives a result that was produced + but will not be delivered (the task was cancelled, or ``owner`` was + destroyed), so a session opened for a tab that is gone can be closed. + Connect to the handle's signals right after this returns: nothing is + delivered before control goes back to the event loop. + """ + if _widgets_reachable_from(work): + raise TaskUsageError( + "background work must not hold a widget: read what it needs first and pass plain values") + state = _TaskState(CancellationToken(timeout_s), discard) + handle = TaskHandle(owner, state) + worker = _TaskWorker(work, state) + state.token._progress = worker.progressed.emit # noqa: SLF001 # reason: set once, before the thread starts + + def owner_destroyed(*_args: object) -> None: + self._active.pop(handle, None) + state.cancel() + + # A plain function, not a slot of the handle: the handle dies with the + # owner, and this has to run exactly then. + owner.destroyed.connect(owner_destroyed) + self._active[handle] = (owner, owner_destroyed) + handle._start(worker, timeout_s, self._forget) # noqa: SLF001 # reason: the handle's second-phase start + return handle + + def _forget(self, handle: TaskHandle) -> None: + """GUI thread: ``handle``'s thread ended while its owner is still alive.""" + owner, owner_destroyed = self._active.pop(handle, (None, None)) + if owner is None: + return + try: + owner.destroyed.disconnect(owner_destroyed) + except (RuntimeError, TypeError): # reason: the owner is already going away + pass + + def cancel_all(self, owner: Optional[QObject] = None) -> int: + """Cancel every running task, or only ``owner``'s; return how many were cancelled. + + For a tab that is hidden or closed but not destroyed: such a tab + still owns its tasks. + """ + handles = [handle for handle, (task_owner, _hook) in self._active.items() + if owner is None or task_owner is owner] + return sum(1 for handle in handles if handle.cancel()) + + def active_count(self) -> int: + """How many submitted tasks the GUI thread has not yet seen end.""" + return len(self._active) + + +_DEFAULT: Optional[TaskController] = None + + +def task_controller() -> TaskController: + """The controller shared by the tabs, created on first use.""" + global _DEFAULT + if _DEFAULT is None: + _DEFAULT = TaskController() + return _DEFAULT + + +__all__ = [ + "CancellationToken", "TaskCancelled", "TaskController", "TaskHandle", "TaskTimeout", + "TaskUsageError", "task_controller", +] diff --git a/test/unit_test/headless/_qt_settle.py b/test/unit_test/headless/_qt_settle.py new file mode 100644 index 000000000..7d32ae3d8 --- /dev/null +++ b/test/unit_test/headless/_qt_settle.py @@ -0,0 +1,22 @@ +"""Wait for a panel's off-thread work to answer on the GUI thread. + +Connects, device runs and OCR left the GUI thread (plan F3), so a test that +calls the slot has to pump the event loop before it reads the outcome. It must +also not *end* before the outcome is delivered: a warning delivered during +another test's teardown, with ``QMessageBox`` no longer patched, opens a real +modal box and the run hangs. +""" +import time + +from PySide6.QtWidgets import QApplication + + +def settle(owner, attribute: str = "_connect_task", timeout: float = 10.0) -> bool: + """Pump events until ``owner.`` is None again; return whether it is.""" + app = QApplication.instance() + deadline = time.monotonic() + timeout + while getattr(owner, attribute) is not None and time.monotonic() < deadline: + app.processEvents() + time.sleep(0.005) + app.processEvents() + return getattr(owner, attribute) is None diff --git a/test/unit_test/headless/test_gui_task_lifecycle.py b/test/unit_test/headless/test_gui_task_lifecycle.py new file mode 100644 index 000000000..867f00a64 --- /dev/null +++ b/test/unit_test/headless/test_gui_task_lifecycle.py @@ -0,0 +1,607 @@ +"""GUI background work: the window keeps ticking, cancel and owner death release what the work holds. + +Plan F task F3 (offscreen Qt, fakes only -- no network, device or window). A +tab that waited on the network in a slot froze the window for the whole wait; +work handed to a thread had no way to be cancelled, so a session it had opened +stayed open after the tab was closed; and a result that arrived after its tab +was destroyed had nowhere safe to go. +""" +import functools +import os +import threading +import time + +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + +from PySide6.QtCore import QEvent, QObject, QThread, QTimer # noqa: E402 +from PySide6.QtWidgets import QApplication, QLabel, QWidget # noqa: E402 + +from je_auto_control.gui import _worker_thread # noqa: E402 +from je_auto_control.gui.task_controller import ( # noqa: E402 + CancellationToken, TaskCancelled, TaskController, TaskTimeout, TaskUsageError, + task_controller, +) +from je_auto_control.utils.exception.exceptions import AutoControlException # noqa: E402 + + +@pytest.fixture() +def qapp(): + return QApplication.instance() or QApplication([]) + + +def _pump(app, predicate, timeout=5.0): + deadline = time.monotonic() + timeout + while not predicate() and time.monotonic() < deadline: + app.processEvents() + time.sleep(0.005) + app.processEvents() + return predicate() + + +def _destroy(app, widget): + widget.deleteLater() + app.sendPostedEvents(None, QEvent.Type.DeferredDelete.value) + app.processEvents() + + +class _FakeNetwork: + """A backend whose sessions are counted, and whose connect waits like a slow host.""" + + def __init__(self): + self.active = 0 + self.lock = threading.Lock() + self.release = threading.Event() # set: the "host" answers + self.entered = threading.Event() + + def open(self): + with self.lock: + self.active += 1 + return _FakeSession(self) + + def connect(self, token: CancellationToken): + """Open a session, then wait for the host; the session is the result.""" + session = self.open() + token.on_cancel(session.close) + self.entered.set() + while not self.release.wait(0.01): + token.raise_if_cancelled() + return session + + +class _FakeSession: + def __init__(self, network): + self._network = network + self._closed = False + + def close(self): + with self._network.lock: + if not self._closed: + self._closed = True + self._network.active -= 1 + + +def test_ui_tick_continues_during_network_wait(qapp): + network = _FakeNetwork() + owner = QWidget() + ticks, results = [], [] + timer = QTimer(owner) + timer.setInterval(5) + timer.timeout.connect(lambda: ticks.append(time.monotonic())) + timer.start() + handle = TaskController().submit(network.connect, owner=owner) + handle.result.connect(results.append) + assert network.entered.wait(3.0), "the work never started" + # The "network" is still waiting: the event loop must keep delivering ticks. + assert _pump(qapp, lambda: len(ticks) >= 5), "the GUI thread did not tick during the wait" + ui_ticks_during_work = len(ticks) + assert ui_ticks_during_work > 0 and results == [] and handle.isRunning() + network.release.set() + assert _pump(qapp, lambda: bool(results)) + assert handle.state == "done" and network.active == 1 + results[0].close() + timer.stop() + _destroy(qapp, owner) + + +def test_cancel_releases_session(qapp): + network = _FakeNetwork() + owner = QWidget() + controller = TaskController() + outcomes = [] + handle = controller.submit(network.connect, owner=owner) + handle.result.connect(lambda value: outcomes.append(("result", value))) + handle.error.connect(lambda error: outcomes.append(("error", error))) + assert network.entered.wait(3.0) and network.active == 1 + handle.cancel() + # Released at once, on the cancelling thread -- not when the backend gives up. + assert network.active == 0 + assert handle.wait(3.0), "the work ignored the cancellation" + assert _pump(qapp, lambda: not controller.active_count()) + active_sessions_after_close = network.active + assert active_sessions_after_close == 0 + assert outcomes == [] and handle.state == "cancelled" + _destroy(qapp, owner) + + +def test_a_result_finished_before_cancel_is_discarded_not_delivered(qapp): + network = _FakeNetwork() + network.release.set() # the host answers at once + owner = QWidget() + delivered = [] + handle = TaskController().submit(network.connect, owner=owner, + discard=lambda session: session.close()) + handle.result.connect(delivered.append) + assert handle.wait(3.0) and network.active == 1 + handle.cancel() # the result is done but not delivered yet + qapp.processEvents() + assert delivered == [] and network.active == 0 + + +def test_owner_death_drops_result(qapp): + network = _FakeNetwork() + owner = QLabel("owner") + received = [] + + def on_result(session): + received.append(session) + owner.setText("connected") # would raise on a deleted widget + + handle = TaskController().submit(network.connect, owner=owner, + discard=lambda session: session.close()) + handle.result.connect(on_result) + assert network.entered.wait(3.0) + _destroy(qapp, owner) + network.release.set() # the answer arrives after the tab is gone + assert _pump(qapp, lambda: not _worker_thread.running_threads()) + result_after_owner_death = received[0] if received else None + assert result_after_owner_death is None + assert network.active == 0, "the session the dead tab asked for stayed open" + + +def test_owner_death_after_completion_discards_the_undelivered_result(qapp): + network = _FakeNetwork() + network.release.set() + owner = QWidget() + received = [] + handle = TaskController().submit(network.connect, owner=owner, + discard=lambda session: session.close()) + handle.result.connect(received.append) + assert handle.wait(3.0) and network.active == 1 # done; the delivery is still queued + owner.deleteLater() + qapp.sendPostedEvents(None, QEvent.Type.DeferredDelete.value) + qapp.processEvents() + assert received == [] and network.active == 0 + + +def test_no_worker_touches_widget(qapp): + owner = QLabel("idle") + gui_thread = qapp.thread() + seen = {} + + def work(token): + seen["work_on_gui"] = QThread.currentThread() is gui_thread + seen["args"] = type(token) + token.report_progress(40) + return "payload" + + handle = TaskController().submit(work, owner=owner) + # Lambdas: a signal emitted on a worker thread would run these there. + handle.progress.connect(lambda value: seen.update(progress=value, + progress_gui=QThread.currentThread() is gui_thread)) + handle.result.connect(lambda value: (owner.setText(value), + seen.update(result_gui=QThread.currentThread() is gui_thread))) + handle.finished.connect(lambda: seen.update(finished_gui=QThread.currentThread() is gui_thread)) + assert _pump(qapp, lambda: "finished_gui" in seen) + assert seen["work_on_gui"] is False and seen["args"] is CancellationToken + assert seen["progress"] == 40 and seen["progress_gui"] and seen["result_gui"] and seen["finished_gui"] + assert owner.text() == "payload" + + # Work that could reach a widget is refused before it starts. + with pytest.raises(TaskUsageError): + TaskController().submit(owner.setText, owner=owner) # a widget's own method + with pytest.raises(TaskUsageError): + TaskController().submit(lambda _token: owner.text(), owner=owner) # closes over the widget + with pytest.raises(TaskUsageError): + TaskController().submit(functools.partial(lambda label, _token: label, owner), owner=owner) + assert issubclass(TaskUsageError, AutoControlException) + _destroy(qapp, owner) + + +def test_errors_arrive_typed_on_the_gui_thread(qapp): + owner = QWidget() + errors = [] + + class BackendDown(AutoControlException): + pass + + def work(_token): + raise BackendDown("host unreachable") + + handle = TaskController().submit(work, owner=owner) + handle.error.connect(lambda error: errors.append((error, QThread.currentThread() is qapp.thread()))) + assert _pump(qapp, lambda: bool(errors)) + assert isinstance(errors[0][0], BackendDown) and errors[0][1] is True + assert handle.state == "failed" + _destroy(qapp, owner) + + +def test_a_timeout_cancels_the_work_and_reports_it(qapp): + network = _FakeNetwork() + owner = QWidget() + errors, deadlines = [], [] + + def work(token): + deadlines.append(token.remaining(default=99.0)) + return network.connect(token) + + handle = TaskController().submit(work, owner=owner, timeout_s=0.1) + handle.error.connect(errors.append) + assert _pump(qapp, lambda: bool(errors)) + assert isinstance(errors[0], TaskTimeout) and isinstance(errors[0], TimeoutError) + assert handle.state == "timed_out" and handle.wait(3.0) + assert network.active == 0 and 0 < deadlines[0] <= 0.1 + _destroy(qapp, owner) + + +def test_the_token_wakes_a_waiting_backend_and_raises_typed(): + token = CancellationToken() + assert token.remaining(default=7.0) == pytest.approx(7.0) + released = [] + token.on_cancel(lambda: released.append("a")) + threading.Timer(0.05, token.cancel).start() + started = time.monotonic() + assert token.wait(5.0) is True and time.monotonic() - started < 2.0 + assert released == ["a"] and token.cancelled + token.on_cancel(lambda: released.append("late")) # already cancelled: released at once + assert released == ["a", "late"] + with pytest.raises(TaskCancelled): + token.raise_if_cancelled() + assert issubclass(TaskCancelled, AutoControlException) + + +def test_a_failing_release_callback_does_not_stop_the_others(): + token = CancellationToken() + released = [] + + def broken(): + raise OSError("already closed") + + token.on_cancel(lambda: released.append("first")) + token.on_cancel(broken) + token.on_cancel(lambda: released.append("last")) + token.cancel() + assert released == ["last", "first"] + + +def test_cancel_all_for_an_owner_leaves_other_owners_running(qapp): + one, other = _FakeNetwork(), _FakeNetwork() + closing, staying = QWidget(), QWidget() + controller = TaskController() + controller.submit(one.connect, owner=closing) + kept = controller.submit(other.connect, owner=staying) + assert one.entered.wait(3.0) and other.entered.wait(3.0) + controller.cancel_all(closing) + assert one.active == 0 and other.active == 1 and kept.isRunning() + controller.cancel_all() + assert other.active == 0 + assert _pump(qapp, lambda: controller.active_count() == 0) + _destroy(qapp, closing) + _destroy(qapp, staying) + + +def test_the_old_helper_and_the_controller_share_one_registry(qapp): + """``start_worker`` is the thread layer under the controller, not a rival.""" + network = _FakeNetwork() + owner = QObject() + before = _worker_thread.running_threads() + handle = task_controller().submit(network.connect, owner=owner) + assert network.entered.wait(3.0) + assert _worker_thread.running_threads() == before + 1 + handle.cancel() + assert _pump(qapp, lambda: _worker_thread.running_threads() == before) + assert task_controller() is task_controller() + owner.deleteLater() + + +# --- the tabs moved onto the controller --------------------------------------------------------------------- + +class _SlowViewer: + """A remote-desktop viewer whose connect waits like a host that accepts and says nothing.""" + + remote_host_id = None + made = [] + refuse = False + + def __init__(self, **kwargs): + self.kwargs, self.connected, self.disconnects = kwargs, False, 0 + self.answer = threading.Event() + self.timeout = None + type(self).made.append(self) + + def connect(self, timeout=5.0): + self.timeout = timeout + if not self.answer.wait(5.0) or self.refuse: + raise OSError("refused") + self.connected = True + + def disconnect(self, timeout=2.0): + self.disconnects += 1 + self.connected = False + + def set_file_receiver(self, receiver): + self.receiver = receiver + + +@pytest.fixture() +def remote_panels(qapp, monkeypatch, tmp_path): + from PySide6.QtWidgets import QMessageBox + from je_auto_control.gui.remote_desktop import connection_screen, viewer_panel + from je_auto_control.utils.remote_desktop.address_book import AddressBook + from je_auto_control.utils.remote_desktop.registry import registry + warnings = [] + monkeypatch.setattr(QMessageBox, "warning", lambda *args: warnings.append(args[-1])) + for slot in ("_viewer", "_ws_viewer", "_host"): + monkeypatch.setattr(registry, slot, None) + monkeypatch.setattr(_SlowViewer, "made", []) + monkeypatch.setattr(viewer_panel, "RemoteDesktopViewer", _SlowViewer) + monkeypatch.setattr(connection_screen, "RemoteDesktopViewer", _SlowViewer) + monkeypatch.setattr(viewer_panel, "is_audio_backend_available", lambda: False) + book = AddressBook(tmp_path / "book.json") # never the operator's own address book + monkeypatch.setattr(connection_screen, "default_address_book", lambda: book) + return viewer_panel, connection_screen, registry, warnings + + +def _viewer_tab(viewer_panel): + panel = viewer_panel._ViewerPanel() + panel._host_field.setText("desk") + panel._port.setValue(5555) + panel._token.setText("tok") + return panel + + +def test_the_viewer_tab_connects_without_holding_the_gui_thread(qapp, remote_panels): + viewer_panel, _screen, registry, warnings = remote_panels + panel = _viewer_tab(viewer_panel) + started = time.monotonic() + panel._connect() + assert time.monotonic() - started < 1.0, "the slot waited for the host" + viewer = _SlowViewer.made[0] + ticks = [] + timer = QTimer(panel) + timer.timeout.connect(lambda: ticks.append(1)) + timer.start(5) + assert _pump(qapp, lambda: len(ticks) >= 5) and panel._connect_task is not None + assert registry.owned("viewer", panel._owner) is None and panel._screen_window is None + viewer.answer.set() + assert _pump(qapp, lambda: panel._connect_task is None) + assert registry.owned("viewer", panel._owner) is viewer and panel._screen_window is not None + assert viewer.timeout == pytest.approx(5.0) and warnings == [] + timer.stop() + panel._disconnect() + + +def test_disconnect_during_a_connect_drops_the_viewer_that_answers_late(qapp, remote_panels): + viewer_panel, _screen, registry, warnings = remote_panels + panel = _viewer_tab(viewer_panel) + panel._connect() + viewer = _SlowViewer.made[0] + assert _pump(qapp, lambda: viewer.timeout is not None) + panel._disconnect() # the operator gives up + assert panel._connect_task is None + viewer.answer.set() # ... and then the host answers + assert _pump(qapp, lambda: viewer.disconnects >= 1), "the late session stayed open" + assert not viewer.connected and registry.owned("viewer", panel._owner) is None + assert panel._screen_window is None and warnings == [] + + +def test_a_second_connect_supersedes_the_first(qapp, remote_panels): + viewer_panel, _screen, registry, _warnings = remote_panels + panel = _viewer_tab(viewer_panel) + panel._connect() + panel._connect() + first, second = _SlowViewer.made + first.answer.set() + second.answer.set() + assert _pump(qapp, lambda: panel._connect_task is None and first.disconnects >= 1) + assert registry.owned("viewer", panel._owner) is second and not first.connected + panel._disconnect() + + +def test_quick_connect_does_not_hold_the_gui_thread(qapp, remote_panels): + from je_auto_control.utils.remote_desktop.connect_coordinator import parse_target + _panel, connection_screen, registry, warnings = remote_panels + screen = connection_screen.QuickConnectScreen() + started = time.monotonic() + screen._dispatch_target(parse_target("desk:5555"), "tok") + assert time.monotonic() - started < 1.0 + viewer = _SlowViewer.made[0] + assert screen._connect_task is not None and registry.owned("viewer", screen._owner) is None + viewer.answer.set() + assert _pump(qapp, lambda: screen._connect_task is None) + assert registry.owned("viewer", screen._owner) is viewer and screen._screen_window is not None + screen._disconnect() + # A refusal arrives as a warning, on the GUI thread, and adopts nothing. + monkeypatch_refuse = _SlowViewer.refuse + _SlowViewer.refuse = True + try: + screen._dispatch_target(parse_target("desk:5555"), "tok") + _SlowViewer.made[1].answer.set() + assert _pump(qapp, lambda: screen._connect_task is None) + finally: + _SlowViewer.refuse = monkeypatch_refuse + assert warnings == ["refused"] and registry.owned("viewer", screen._owner) is None + screen._refresh_timer.stop() + + +@pytest.fixture() +def webrtc_panel(qapp, monkeypatch): + pytest.importorskip("av") + pytest.importorskip("aiortc") + from PySide6.QtWidgets import QMessageBox + from je_auto_control.gui.remote_desktop import webrtc_panel as module + warnings = [] + monkeypatch.setattr(QMessageBox, "warning", lambda *args: warnings.append(args[-1])) + module.warnings = warnings + yield module + del module.warnings + + +class _SlowSdp: + """Stands in for a WebRTC host or viewer whose SDP step waits on ICE gathering.""" + + def __init__(self): + self.release, self.entered = threading.Event(), threading.Event() + self.stopped, self.on_gui, self.fail = [], [], None + + def _wait(self): + self.on_gui.append(QThread.currentThread() is QApplication.instance().thread()) + self.entered.set() + self.release.wait(5.0) + if self.fail is not None: + raise self.fail + + def create_session_offer(self): + self._wait() + return "session-1", "v=0 offer" + + def process_offer(self, offer_sdp, expected_dtls_fingerprint=None): + self._wait() + return f"answer to {offer_sdp}" + + def stop_session(self, session_id): + self.stopped.append(session_id) + + +def test_the_webrtc_host_makes_its_offer_off_the_gui_thread(qapp, webrtc_panel): + panel = webrtc_panel._WebRTCHostPanel() + host = _SlowSdp() + panel._multi_host = host + started = time.monotonic() + panel._produce_offer() + assert time.monotonic() - started < 1.0 and host.entered.wait(3.0) + assert panel._offer_view.toPlainText() == "" + host.release.set() + assert _pump(qapp, lambda: panel._offer_view.toPlainText() == "v=0 offer") + assert panel._manual_session_id == "session-1" and host.on_gui == [False] + + # Stopped while the offer was being made: the session is ended, nothing is shown or reported. + for failure in (None, RuntimeError("the host is shutting down")): + late = _SlowSdp() + late.fail = failure + panel._multi_host = late + panel._offer_view.setPlainText("") + panel._produce_offer() + assert late.entered.wait(3.0) + panel._multi_host = None + late.release.set() + assert _pump(qapp, lambda: not task_controller().active_count()) + assert panel._offer_view.toPlainText() == "" and webrtc_panel.warnings == [] + assert late.stopped == ([] if failure else ["session-1"]) + panel._annotation_overlay = None + + +def test_the_webrtc_viewer_answers_off_the_gui_thread(qapp, webrtc_panel): + panel = webrtc_panel._WebRTCViewerPanel() + viewer = _SlowSdp() + panel._viewer = viewer + started = time.monotonic() + panel._produce_answer("v=0 offer") + assert time.monotonic() - started < 1.0 and viewer.entered.wait(3.0) + viewer.release.set() + assert _pump(qapp, lambda: panel._answer_view.toPlainText() == "answer to v=0 offer") + assert viewer.on_gui == [False] + + replaced = _SlowSdp() + panel._viewer = replaced + panel._answer_view.setPlainText("") + panel._produce_answer("stale offer") + assert replaced.entered.wait(3.0) + panel._viewer = None # Stop, while the answer is being made + replaced.release.set() + assert _pump(qapp, lambda: not task_controller().active_count()) + assert panel._answer_view.toPlainText() == "" and webrtc_panel.warnings == [] + + failing = _SlowSdp() + failing.fail = ValueError("fingerprint mismatch") + failing.release.set() + panel._viewer = failing + panel._produce_answer("bad offer") + assert _pump(qapp, lambda: webrtc_panel.warnings == ["fingerprint mismatch"]) + panel._viewer = None + + +def test_the_ocr_tab_reads_off_the_gui_thread(qapp, monkeypatch): + from PySide6.QtWidgets import QMessageBox + from je_auto_control.gui import ocr_tab + from je_auto_control.utils.ocr.ocr_engine import TextMatch + warnings = [] + monkeypatch.setattr(QMessageBox, "warning", lambda *args: warnings.append(args[-1])) + release, calls = threading.Event(), [] + + def slow_read(**options): + calls.append((options, QThread.currentThread() is qapp.thread())) + release.wait(5.0) + return [TextMatch("Order#7", 1, 2, 3, 4, 91.0)] + + def failing_find(**options): + raise RuntimeError("tesseract is not installed") + + monkeypatch.setattr(ocr_tab, "read_text_in_region", slow_read) + monkeypatch.setattr(ocr_tab, "find_text_regex", failing_find) + tab = ocr_tab.OCRReaderTab() + tab._region.setText("1, 2, 30, 40") + started = time.monotonic() + tab._on_dump() + assert time.monotonic() - started < 1.0 and tab._task is not None + tab._on_dump() # one read at a time + assert _pump(qapp, lambda: bool(calls)) and tab._result.toPlainText() == "" + release.set() + assert _pump(qapp, lambda: tab._task is None) + assert calls == [({"region": [1, 2, 30, 40], "min_confidence": 60.0, "lang": "eng"}, False)] + assert "Order#7" in tab._result.toPlainText() + tab._regex.setText(r"Order#\d+") + tab._on_find_regex() + assert _pump(qapp, lambda: tab._task is None) + assert warnings == ["tesseract is not installed"] + tab._min_conf.setText("high") + tab._on_dump() # bad input never leaves the GUI thread + assert tab._task is None and len(calls) == 1 + + +def test_the_device_matrix_run_leaves_the_gui_thread(qapp, monkeypatch): + from je_auto_control.gui import device_matrix_tab + release, threads = threading.Event(), [] + + class _Report: + def to_dict(self): + return {"results": [{"device_id": "a", "platform": "android", "success": True, + "duration_s": 0.5, "error": None}], + "passed": 1, "failed": 0, "total": 1} + + def slow_run(actions, devices, max_parallel=4): + threads.append(QThread.currentThread() is qapp.thread()) + release.wait(5.0) + if not devices: + raise ValueError("devices must be a non-empty list of device specs") + return _Report() + + monkeypatch.setattr(device_matrix_tab.ac, "run_on_devices", slow_run) + tab = device_matrix_tab.DeviceMatrixTab() + tab._devices.setPlainText('[{"platform": "android", "serial": "a"}]') + tab._actions.setPlainText("[]") + started = time.monotonic() + tab._on_run() + assert time.monotonic() - started < 1.0 and tab._table.rowCount() == 0 + release.set() + assert _pump(qapp, lambda: tab._task is None) + assert threads == [False] and tab._table.rowCount() == 1 + tab._devices.setPlainText("[]") + tab._on_run() + assert _pump(qapp, lambda: tab._task is None) + assert "non-empty" in tab._summary.text() + tab._devices.setPlainText("{not json") + tab._on_run() + assert tab._task is None and len(threads) == 2 diff --git a/test/unit_test/headless/test_qa_tabs_b_gui.py b/test/unit_test/headless/test_qa_tabs_b_gui.py index cc8873df2..bd9fc837c 100644 --- a/test/unit_test/headless/test_qa_tabs_b_gui.py +++ b/test/unit_test/headless/test_qa_tabs_b_gui.py @@ -1,5 +1,6 @@ """GUI smoke tests for the B-group tabs (a11y audit / matrix / media).""" import os +import time import pytest @@ -34,6 +35,11 @@ def test_device_matrix_tab_runs(app): '[["AC_set_var", {"name":"id","value":"${device.serial}"}]]', ) tab._on_run() + # The run is off the GUI thread now: the table fills once it is delivered. + deadline = time.monotonic() + 10.0 + while tab._task is not None and time.monotonic() < deadline: + app.processEvents() + time.sleep(0.01) assert tab._table.rowCount() == 1 assert "1" in tab._summary.text() diff --git a/test/unit_test/headless/test_rd_gui_audit.py b/test/unit_test/headless/test_rd_gui_audit.py index 3a178fad7..980683b9c 100644 --- a/test/unit_test/headless/test_rd_gui_audit.py +++ b/test/unit_test/headless/test_rd_gui_audit.py @@ -22,6 +22,7 @@ from je_auto_control.utils.remote_desktop.connect_coordinator import parse_target # noqa: E402 from je_auto_control.utils.remote_desktop.registry import registry # noqa: E402 from headless._exit_probe import run_probe # noqa: E402 +from headless._qt_settle import settle # noqa: E402 @pytest.fixture(autouse=True) @@ -56,7 +57,9 @@ def test_quick_connect_dials_wss_with_a_verifying_tls_context(qapp, monkeypatch) made = [] monkeypatch.setattr(connection_screen, "WebSocketDesktopViewer", lambda **kwargs: made.append(_FakeViewer(**kwargs)) or made[-1]) - connection_screen.QuickConnectScreen()._dispatch_target(parse_target("wss://desk:8443/"), "tok") + screen = connection_screen.QuickConnectScreen() + screen._dispatch_target(parse_target("wss://desk:8443/"), "tok") + settle(screen) context = made[0].kwargs.get("ssl_context") assert isinstance(context, ssl.SSLContext) and context.verify_mode == ssl.CERT_REQUIRED assert qapp.warnings == ["refused"] @@ -82,11 +85,13 @@ def test_a_host_that_fails_idna_is_reported_by_both_viewers(qapp): screen._connect_target.setText("a..b:5555") screen._connect_token.setText("tok") screen._connect() + settle(screen) panel = viewer_panel._ViewerPanel() panel._host_field.setText("a..b") panel._port.setValue(5555) panel._token.setText("tok") panel._connect() + settle(panel) assert len(qapp.warnings) == 2 diff --git a/test/unit_test/headless/test_remote_desktop_gui.py b/test/unit_test/headless/test_remote_desktop_gui.py index c7fc536f9..944ed0f53 100644 --- a/test/unit_test/headless/test_remote_desktop_gui.py +++ b/test/unit_test/headless/test_remote_desktop_gui.py @@ -25,6 +25,7 @@ from PySide6.QtWidgets import QApplication # noqa: E402 from je_auto_control.utils.remote_desktop.registry import registry # noqa: E402 +from headless._qt_settle import settle # noqa: E402 @pytest.fixture(scope="module") @@ -83,6 +84,7 @@ def test_viewer_panel_renders_frame_from_host(qapp): panel._port.setValue(host.port) # noqa: SLF001 panel._token.setText("t") # noqa: SLF001 panel._connect() # noqa: SLF001 + settle(panel) assert _process_until( qapp, panel._screen_window.display.has_image, # noqa: SLF001 ) @@ -139,6 +141,7 @@ def test_viewer_input_round_trips_to_dispatcher(qapp): panel._port.setValue(host.port) # noqa: SLF001 panel._token.setText("t") # noqa: SLF001 panel._connect() # noqa: SLF001 + settle(panel) assert _process_until( qapp, panel._screen_window.display.has_image, # noqa: SLF001 ) diff --git a/test/unit_test/headless/test_remote_desktop_quick_connect.py b/test/unit_test/headless/test_remote_desktop_quick_connect.py index ba2ad8b66..b74013ad7 100644 --- a/test/unit_test/headless/test_remote_desktop_quick_connect.py +++ b/test/unit_test/headless/test_remote_desktop_quick_connect.py @@ -13,6 +13,7 @@ from PySide6.QtWidgets import QApplication # noqa: E402 from je_auto_control.utils.remote_desktop.registry import registry # noqa: E402 +from headless._qt_settle import settle # noqa: E402 @pytest.fixture(scope="module") @@ -120,6 +121,7 @@ def test_quick_connect_round_trips_frame_to_popup(qapp): ) screen._connect_token.setText("t") # noqa: SLF001 screen._connect() # noqa: SLF001 + settle(screen) assert registry.viewer is not None # Wait until the pop-out window receives at least one frame. @@ -173,6 +175,7 @@ def test_nine_digit_id_emits_webrtc_handoff(qapp): screen._connect_target.setText("123-456-789") # noqa: SLF001 screen._connect_token.setText("ttt") # noqa: SLF001 screen._connect() # noqa: SLF001 + settle(screen) assert captured == [("123456789", "ttt")] # No TCP viewer should have been created since this is a handoff. assert registry.viewer is None @@ -317,6 +320,7 @@ def test_recent_connections_populated_after_connect(qapp, tmp_path, ) screen._connect_token.setText("t") # noqa: SLF001 screen._connect() # noqa: SLF001 + settle(screen) entries = book.list_entries() expected_url = f"tcp://127.0.0.1:{host.port}" diff --git a/test/unit_test/headless/test_remote_registry_owners_gui.py b/test/unit_test/headless/test_remote_registry_owners_gui.py index d053903bf..7d2c41359 100644 --- a/test/unit_test/headless/test_remote_registry_owners_gui.py +++ b/test/unit_test/headless/test_remote_registry_owners_gui.py @@ -24,6 +24,7 @@ from je_auto_control.utils.remote_desktop.registry import ( # noqa: E402 SCRIPT_OWNER, SLOT_HOST, SLOT_VIEWER, SLOT_WS_VIEWER, registry, ) +from headless._qt_settle import settle # noqa: E402 class _Viewer: @@ -103,6 +104,7 @@ def _quick(target="tcp"): screen = connection_screen.QuickConnectScreen() address = {"tcp": "desk:5555", "ws": "ws://desk:8080/"}[target] screen._dispatch_target(parse_target(address), "tok") + settle(screen) return screen @@ -113,6 +115,7 @@ def _tab(transport="TCP"): panel._token.setText("tok") panel._transport.setCurrentText(transport) panel._connect() + settle(panel) return panel @@ -205,6 +208,7 @@ def test_quick_connect_changing_transport_ends_its_previous_session(): screen = _quick("ws") first = screen._own_viewer() screen._dispatch_target(parse_target("desk:5555"), "tok") + settle(screen) assert not first.connected and registry.owner_of(SLOT_WS_VIEWER) is None assert registry.owner_of(SLOT_VIEWER) == screen._owner and screen._screen_window is not None @@ -213,6 +217,7 @@ def test_a_panel_reconnecting_keeps_its_window(): tab = _tab() first = tab._own_viewer() tab._connect() + settle(tab) assert not first.connected and tab._own_viewer().connected assert tab._screen_window is not None and tab._status.text() != _t("rd_viewer_displaced") @@ -252,6 +257,7 @@ def test_a_panel_takes_over_a_scripts_session(monkeypatch): screen._disconnect() assert scripted.connected screen._dispatch_target(parse_target("desk:5555"), "tok") + settle(screen) assert not scripted.connected and registry.owner_of(SLOT_VIEWER) == screen._owner From 2c0716dcf8d790d1ad415807e9443d39238091e3 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:54:48 +0800 Subject: [PATCH 29/44] Drop qt-material from the gui extra now that nothing imports it The window has drawn its own themes since the token-based style sheet; the extra still pulled qt-material, jinja2 and markupsafe. PyBreeze depends on je_auto_control without the extra and gets qt-material from je-editor, so nothing downstream loses it. --- Third_Party_License.md | 1 - dev.toml | 2 +- dev_requirements.txt | 1 - docs/source/getting_started/installation.rst | 2 - pyproject.toml | 4 +- requirements.txt | 1 - uv.lock | 113 +------------------ 7 files changed, 4 insertions(+), 120 deletions(-) diff --git a/Third_Party_License.md b/Third_Party_License.md index 06c4b5d7a..03d1cfee0 100644 --- a/Third_Party_License.md +++ b/Third_Party_License.md @@ -22,7 +22,6 @@ Full copies of the upstream license texts are archived under `LICENSEs/`. | Package | Version | License | Purpose | |---|---|---|---| | [PySide6](https://pypi.org/project/PySide6/) | 6.11.1 | LGPL-3.0 / Qt Commercial | Qt 6 GUI framework used by `start_autocontrol_gui()` | -| [qt-material](https://pypi.org/project/qt-material/) | 2.17 | BSD-2-Clause | Material Design themes for PySide6 | ### Optional feature dependencies (loaded lazily, not pinned) diff --git a/dev.toml b/dev.toml index df0a3e872..e034a7e4e 100644 --- a/dev.toml +++ b/dev.toml @@ -78,7 +78,7 @@ find = { namespaces = false } ] [project.optional-dependencies] -gui = ["PySide6==6.11.2", "qt-material==2.17"] +gui = ["PySide6==6.11.2"] webrtc = ["aiortc>=1.14.0", "av>=14.0.0"] signaling = ["fastapi>=0.115", "starlette>=1.0.1", "uvicorn>=0.32"] discovery = ["zeroconf>=0.149.16"] diff --git a/dev_requirements.txt b/dev_requirements.txt index bb181fe0a..95d89b516 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,6 @@ twine sphinx sphinx-rtd-theme PySide6==6.11.2 -qt-material==2.17 mss==10.2.0 defusedxml==0.7.1 diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index 0bfb61d74..a853e2b9b 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -70,8 +70,6 @@ Dependencies - Linux X11 backend (auto-installed on Linux) * - ``PySide6`` - GUI application (optional, install with ``[gui]``) - * - ``qt-material`` - - GUI theme (optional, install with ``[gui]``) Platform Support ================ diff --git a/pyproject.toml b/pyproject.toml index 248723735..1d3ad034e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,7 @@ find = { namespaces = false } ] [project.optional-dependencies] -gui = ["PySide6==6.11.2", "qt-material==2.17"] +gui = ["PySide6==6.11.2"] webrtc = ["aiortc>=1.14.0", "av>=14.0.0"] signaling = ["fastapi>=0.115", "starlette>=1.0.1", "uvicorn>=0.32"] discovery = ["zeroconf>=0.149.16"] @@ -275,7 +275,7 @@ ignore_missing_imports = true # unchecked, which is where they already were before the scope widened. module = [ # [gui] - "PySide6", "PySide6.*", "qt_material", "qt_material.*", + "PySide6", "PySide6.*", # [webrtc] / [signaling] / [discovery] "aiortc", "aiortc.*", "av", "av.*", "fastapi", "fastapi.*", "uvicorn", "uvicorn.*", diff --git a/requirements.txt b/requirements.txt index 9e9e11595..9f9a7fae8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,5 @@ je_auto_control>=0.0.217 # when je_auto_control was still unbounded here, and the floor above never # replaced that entry. A direct requirement does. pillow>=12.3.0 -qt-material==2.17 mss==10.2.0 PySide6==6.11.2 diff --git a/uv.lock b/uv.lock index 9b8ba4cef..f03f50518 100644 --- a/uv.lock +++ b/uv.lock @@ -451,7 +451,7 @@ wheels = [ [[package]] name = "je-auto-control" -version = "0.0.227" +version = "0.0.228" source = { editable = "." } dependencies = [ { name = "cryptography", marker = "platform_machine != 'ARM64' or sys_platform != 'win32'" }, @@ -477,7 +477,6 @@ fuzzy = [ ] gui = [ { name = "pyside6" }, - { name = "qt-material" }, ] locale = [ { name = "babel" }, @@ -525,7 +524,6 @@ requires-dist = [ { name = "python-docx", marker = "extra == 'office'", specifier = ">=1.1" }, { name = "python-pptx", marker = "extra == 'office'", specifier = ">=0.6" }, { name = "python-xlib", marker = "platform_system == 'FreeBSD' or platform_system == 'NetBSD' or platform_system == 'OpenBSD' or sys_platform == 'linux'", specifier = "==0.33" }, - { name = "qt-material", marker = "extra == 'gui'", specifier = "==2.17" }, { name = "rapidfuzz", marker = "extra == 'fuzzy'", specifier = ">=3.0" }, { name = "starlette", marker = "extra == 'signaling'", specifier = ">=1.0.1" }, { name = "uvicorn", marker = "extra == 'signaling'", specifier = ">=0.32" }, @@ -547,18 +545,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/00/418352e5080c4c0364609e2a84486f9a4c20dc9625c8b69829d623336915/je_open_cv-0.0.22-py3-none-any.whl", hash = "sha256:847390e3d0d7d9eaf6178faf9cbc297e1b84a51f38281605ba478f24b026b859", size = 7164960, upload-time = "2023-03-27T09:07:39.866Z" }, ] -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - [[package]] name = "jmespath" version = "1.1.0" @@ -686,91 +672,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/a4/441aee36c6f6b249823d20fd91f9be9ab89d7c5a8ae542a4a4ca6d342d56/lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84", size = 3508989, upload-time = "2026-05-18T19:18:38.158Z" }, ] -[[package]] -name = "markupsafe" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" }, - { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" }, - { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" }, - { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" }, - { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" }, - { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" }, - { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" }, - { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" }, - { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, - { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, - { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, - { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, - { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, - { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, - { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, - { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, - { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, - { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, - { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, - { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, - { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, - { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, - { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, - { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, - { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, - { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, - { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, - { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, - { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, - { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, - { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, - { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, - { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, - { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, - { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, - { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, - { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, - { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, - { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, - { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, - { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, - { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, - { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, - { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, - { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, - { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, - { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, - { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, - { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, - { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, - { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, - { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, - { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, - { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, - { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, - { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, - { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, - { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, - { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, - { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, - { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, - { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, - { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, - { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, - { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, - { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, - { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, -] - [[package]] name = "mss" version = "10.2.0" @@ -4479,18 +4380,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/b8/ff33610932e0ee81ae7f1269c890f697d56ff74b9f5b2ee5d9b7fa2c5355/python_xlib-0.33-py2.py3-none-any.whl", hash = "sha256:c3534038d42e0df2f1392a1b30a15a4ff5fdc2b86cfa94f072bf11b10a164398", size = 182185, upload-time = "2022-12-25T18:52:58.662Z" }, ] -[[package]] -name = "qt-material" -version = "2.17" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/57/d6473fc7f9b1a81f6bddea555119270818c54be7717f38d0977bb6161408/qt_material-2.17.tar.gz", hash = "sha256:b500a0c1f5ef8f46a8cf037d1aa5bdf9ea4eb618180b6332539dfbdd0647ad0d", size = 1668475, upload-time = "2025-04-21T17:00:25.783Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/63/e1085fd76bfea103b6c93d133d849f80a5a94b924f31897eb936cabe8459/qt_material-2.17-py3-none-any.whl", hash = "sha256:660523341ae45b79d222bf15e202ec76ba3bf5e7e82170b66759738d7bb4873d", size = 1687312, upload-time = "2025-04-21T17:00:23.761Z" }, -] - [[package]] name = "rapidfuzz" version = "3.14.5" From cca542ede0fe922821506bef945b6a2081f15452 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:54:48 +0800 Subject: [PATCH 30/44] Scroll pages in narrow windows, remember the window's look, build start tabs on first click Pages were squeezed below their minimum size, the theme and panel state lasted one run, Script Builder and Remote Desktop were built before the window showed, and Ctrl+B did nothing because Qt disables a non-closable dock's toggle action. Tab close buttons now show a cross drawn in code, and a tab can be released with an optional dispose() hook. --- je_auto_control/gui/_tab_close_icon.py | 83 ++++ je_auto_control/gui/main_widget.py | 115 +++-- je_auto_control/gui/main_window.py | 80 +++- je_auto_control/gui/tab_registry.py | 31 +- je_auto_control/gui/theme.py | 7 +- je_auto_control/gui/window_settings.py | 130 ++++++ je_auto_control/gui/workspace_tabs.py | 175 ++++++++ test/conftest.py | 2 + .../headless/test_gui_lazy_registry.py | 31 +- .../headless/test_gui_shell_state.py | 417 ++++++++++++++++++ 10 files changed, 1031 insertions(+), 40 deletions(-) create mode 100644 je_auto_control/gui/_tab_close_icon.py create mode 100644 je_auto_control/gui/window_settings.py create mode 100644 je_auto_control/gui/workspace_tabs.py create mode 100644 test/unit_test/headless/test_gui_shell_state.py diff --git a/je_auto_control/gui/_tab_close_icon.py b/je_auto_control/gui/_tab_close_icon.py new file mode 100644 index 000000000..43ab7c209 --- /dev/null +++ b/je_auto_control/gui/_tab_close_icon.py @@ -0,0 +1,83 @@ +"""The cross on a tab's close button, drawn in code and handed to the style sheet. + +Qt's tab bar keeps its own close buttons; only what they show changes. A style +sheet can point ``QTabBar::close-button`` at an image but not at a painter, so +the cross is painted here in the theme's colours and written to a small cache +(``~/.je_auto_control/gui_cache/``) the first time a theme is used. No image +ships with the package. + +Replacing the buttons with widgets of our own was tried first and dropped: a +process that opened a tab in a shown window and then left through ``os._exit`` +(which every GUI probe in the test suite does) died of an access violation in +8 of 70 runs, against 0 of 59 for the stock buttons. +""" +from pathlib import Path +from typing import Any, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +_SIDE = 16 +_SCALES = (1, 2, 3) + + +def cache_directory() -> Path: + """Where the drawn crosses are kept, resolved when asked.""" + return Path.home() / ".je_auto_control" / "gui_cache" + + +def _draw(colour: str, alpha: int, scale: int) -> Any: + from PySide6.QtCore import QPointF, Qt + from PySide6.QtGui import QColor, QImage, QPainter, QPen + side = _SIDE * scale + image = QImage(side, side, QImage.Format.Format_ARGB32_Premultiplied) + image.fill(Qt.GlobalColor.transparent) + stroke = QColor(colour) + stroke.setAlpha(alpha) + pen = QPen(stroke, 1.5 * scale) + pen.setCapStyle(Qt.PenCapStyle.RoundCap) + inset, far = side * 0.3, side * 0.7 + painter = QPainter(image) + painter.setRenderHint(QPainter.RenderHint.Antialiasing, True) + painter.setPen(pen) + painter.drawLine(QPointF(inset, inset), QPointF(far, far)) + painter.drawLine(QPointF(far, inset), QPointF(inset, far)) + painter.end() + return image + + +def cross_file(colour: str, alpha: int, directory: Optional[Path] = None) -> Optional[Path]: + """Path of a cross in ``colour`` at ``alpha`` (0-255), drawing it if it is not cached yet. + + Written at 1x, 2x and 3x (``name@2x.png``), which Qt picks by screen + scale. ``None`` when the cache cannot be written. + """ + folder = directory if directory is not None else cache_directory() + base = folder / f"tab_close_{colour.lstrip('#').lower()}_{int(alpha)}.png" + try: + folder.mkdir(parents=True, exist_ok=True) + for scale in _SCALES: + target = base if scale == 1 else base.with_name(f"{base.stem}@{scale}x.png") + if not target.is_file() and not _draw(colour, alpha, scale).save(str(target), "PNG"): + raise OSError(f"could not write {target}") + except OSError as error: + autocontrol_logger.warning("tab close icon not cached: %r", error) + return None + return base + + +def close_button_rules(tokens: Any, directory: Optional[Path] = None) -> str: + """Style sheet rules that put the drawn cross on every tab's close button. + + An empty string when the cache cannot be written: the style's own icon + stays. + """ + rest = cross_file(tokens.text_muted, 255, directory) + hover = cross_file(tokens.text, 255, directory) + if rest is None or hover is None: + return "" + return ( + f'QTabBar::close-button {{ image: url("{rest.as_posix()}"); subcontrol-position: right;' + f' border-radius: {max(tokens.radius - 2, 2)}px; }}\n' + f'QTabBar::close-button:hover {{ image: url("{hover.as_posix()}"); background-color: {tokens.hover}; }}\n' + f'QTabBar::close-button:pressed {{ background-color: {tokens.selection}; }}' + ) diff --git a/je_auto_control/gui/main_widget.py b/je_auto_control/gui/main_widget.py index 4ee4e4ac0..1cea8517c 100644 --- a/je_auto_control/gui/main_widget.py +++ b/je_auto_control/gui/main_widget.py @@ -4,9 +4,7 @@ from PySide6.QtCore import QTimer, Signal, QObject from PySide6.QtGui import QKeyEvent, Qt -from PySide6.QtWidgets import ( - QWidget, QVBoxLayout, QLabel, QTabWidget, -) +from PySide6.QtWidgets import QWidget, QVBoxLayout, QLabel from je_auto_control.gui._auto_click_tab import AutoClickTabMixin from je_auto_control.gui._i18n_helpers import TranslatableMixin @@ -19,8 +17,10 @@ from je_auto_control.gui.tab_registry import ( TAB_SPECS, MenuActions, TabEntry, WidgetFactory, lazy_factory, ) +from je_auto_control.gui.workspace_tabs import WorkspaceTabWidget from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.json.json_file import read_action_json +from je_auto_control.utils.logging.logging_instance import autocontrol_logger # Kept under its old name: tests and embedders read ``_tab_entries`` rows as this. _TabEntry = TabEntry @@ -52,7 +52,9 @@ def __init__(self, parent=None): self._tab_entries: List[TabEntry] = [] - self.tabs = QTabWidget() + # Pages scroll inside their tab when the window is smaller than they + # are; ``tabs.indexOf(entry.widget)`` and friends still speak in pages. + self.tabs = WorkspaceTabWidget() self.tabs.setTabsClosable(True) self.tabs.setDocumentMode(True) self.tabs.setUsesScrollButtons(True) @@ -61,20 +63,23 @@ def __init__(self, parent=None): self.tabs.setMinimumSize(360, 240) self.tabs.tabCloseRequested.connect(self._on_tab_close_requested) - # The window opens on record / script_builder / remote_desktop. Every - # other tab is registered but not built: its module is imported and - # its widget constructed the first time it is opened, from the - # navigation panel or the View menu. + # The window opens on record / script_builder / remote_desktop, with + # Record in front. Every tab but this widget's own forms is registered + # without being built: its module is imported and its widget + # constructed the first time it is on screen, which for the two other + # start tabs is the first click on them. own_tabs = self._own_tab_builders() for spec in TAB_SPECS: builder, actions = own_tabs.get(spec.key, (None, ())) self._add_tab(spec.key, spec.title_key, builder or lazy_factory(spec.module, spec.class_name), - category=spec.category, default_visible=spec.default_visible, actions=actions) + category=spec.category, default_visible=spec.default_visible, actions=actions, + scrollable=spec.scrollable) layout.addWidget(self.tabs) self.setLayout(layout) + self._build_current_tab() self.tabs.currentChanged.connect(self._on_current_tab_changed) self.timer = QTimer() @@ -170,22 +175,40 @@ def _build_remote_desktop_tab() -> QWidget: def _add_tab( self, key: str, title_key: str, widget: Union[QWidget, WidgetFactory], category: str = "core", default_visible: bool = False, - actions: MenuActions = (), + actions: MenuActions = (), scrollable: bool = True, ) -> None: - """Register a tab from a widget, or from a factory called on first open.""" + """Register a tab from a widget, or from a factory called the first time it is on screen.""" + title = language_wrapper.translate(title_key, title_key) if isinstance(widget, QWidget): built = widget entry = TabEntry(key=key, title_key=title_key, factory=lambda: built, - category=category, default_visible=default_visible, actions=actions) - else: - entry = TabEntry(key=key, title_key=title_key, factory=widget, category=category, default_visible=default_visible, actions=actions, - on_build=lambda page: AutoControlGUIWidget._adopt_hidden_tab(self, page)) + scrollable=scrollable, releasable=False) + self._tab_entries.append(entry) + if default_visible: + self.tabs.addTab(entry.widget, title, scrollable=scrollable) + else: + AutoControlGUIWidget._adopt_hidden_tab(self, entry.widget) + return + entry = TabEntry(key=key, title_key=title_key, factory=widget, + category=category, default_visible=default_visible, actions=actions, + scrollable=scrollable, + on_build=lambda page: AutoControlGUIWidget._place_built_tab(self, key, page)) self._tab_entries.append(entry) if default_visible: - self.tabs.addTab(entry.widget, language_wrapper.translate(title_key, title_key)) - elif isinstance(widget, QWidget): - AutoControlGUIWidget._adopt_hidden_tab(self, entry.widget) + self.tabs.insert_deferred_tab(self.tabs.count(), key, title, scrollable) + + def _place_built_tab(self, key: str, page: QWidget) -> None: + """Put a page built just now where it belongs: its waiting tab, or kept hidden.""" + if not self.tabs.fill_deferred(key, page): + AutoControlGUIWidget._adopt_hidden_tab(self, page) + + def _build_current_tab(self) -> None: + """Build the selected tab's page if it is still waiting for its first show.""" + key = self.tabs.deferred_key(self.tabs.currentIndex()) + entry = self._find_entry(key) if key else None + if entry is not None: + _ = entry.widget def _adopt_hidden_tab(self, widget: QWidget) -> None: # Owned from the start: an unparented hidden tab outlived this @@ -202,10 +225,17 @@ def _is_built(entry: Any) -> bool: def _built_entries(self) -> List[Any]: return [entry for entry in self._tab_entries if self._is_built(entry)] + def _tab_index(self, entry: Any) -> int: + """Where ``entry`` is in the tab bar, or -1; never builds the tab.""" + if self._is_built(entry): + return int(self.tabs.indexOf(entry.widget)) + return int(self.tabs.index_of_deferred(entry.key)) + def _is_open(self, entry: Any) -> bool: - return self._is_built(entry) and self.tabs.indexOf(entry.widget) != -1 + return self._tab_index(entry) != -1 def _on_current_tab_changed(self, _index: int) -> None: + self._build_current_tab() self.current_tab_changed.emit() def current_tab_menu_actions(self) -> list: @@ -282,17 +312,48 @@ def show_tab(self, key: str) -> None: if self._is_open(candidate): target_index += 1 title = language_wrapper.translate(entry.title_key, entry.title_key) - self.tabs.insertTab(target_index, entry.widget, title) + self.tabs.insertTab(target_index, entry.widget, title, scrollable=entry.scrollable) self.tabs.setCurrentWidget(entry.widget) self.tabs_changed.emit() + def open_tab(self, key: str) -> Optional[QWidget]: + """Open and select the tab ``key``; return its widget, or ``None`` for an unknown key.""" + entry = self._find_entry(key) + if entry is None: + return None + self.activate_tab(key) + widget: QWidget = entry.widget + return widget + + def close_tab(self, key: str, release: bool = False) -> bool: + """Close the tab ``key``; with ``release`` also let go of its widget. + + Without ``release`` this is :meth:`hide_tab`: the widget is kept and + the next open shows it as it was left. With it, the widget's optional + ``dispose()`` is called and the widget deleted, so what it held + (timers, threads, listeners) goes with it and the next open builds a + new one. Returns whether a widget was released; the forms this widget + builds for itself are never released. + """ + entry = self._find_entry(key) + if entry is None: + return False + self.hide_tab(key) + if not release: + return False + try: + return bool(entry.release()) + except Exception as error: # noqa: BLE001 # reason: a failing dispose() must not undo the release + autocontrol_logger.error("dispose() of tab %s failed: %r", key, error) + return True + def activate_tab(self, key: str) -> bool: """Bring the tab ``key`` to the front, opening it first if needed.""" entry = self._find_entry(key) if entry is None: return False if self._is_open(entry): - self.tabs.setCurrentWidget(entry.widget) + self.tabs.setCurrentIndex(self._tab_index(entry)) else: self.show_tab(key) return True @@ -300,14 +361,18 @@ def activate_tab(self, key: str) -> bool: def hide_tab(self, key: str) -> None: """Close the tab ``key``; its widget is kept for the next time it is opened.""" entry = self._find_entry(key) - if entry is None or not self._is_built(entry): + if entry is None: return - index = self.tabs.indexOf(entry.widget) + index = self._tab_index(entry) if index != -1: self.tabs.removeTab(index) self.tabs_changed.emit() def _on_tab_close_requested(self, index: int) -> None: + waiting = self.tabs.deferred_key(index) + if waiting: + self.hide_tab(waiting) + return widget = self.tabs.widget(index) for entry in self._built_entries(): if entry.widget is widget: @@ -319,8 +384,8 @@ def _translate(self, key: str) -> str: def retranslate(self) -> None: """Relabel tab titles and propagate into every child tab.""" - for entry in self._built_entries(): - index = self.tabs.indexOf(entry.widget) + for entry in self._tab_entries: + index = self._tab_index(entry) if index != -1: self.tabs.setTabText( index, language_wrapper.translate(entry.title_key, entry.title_key), diff --git a/je_auto_control/gui/main_window.py b/je_auto_control/gui/main_window.py index a8ef06859..ae228333c 100644 --- a/je_auto_control/gui/main_window.py +++ b/je_auto_control/gui/main_window.py @@ -1,8 +1,9 @@ """Top-level window: menu bar, feature navigation, tabbed workspace, themes, live language switching.""" import sys +from typing import Optional -from PySide6.QtCore import Qt -from PySide6.QtGui import QAction, QActionGroup, QKeySequence +from PySide6.QtCore import QByteArray, Qt +from PySide6.QtGui import QAction, QActionGroup, QCloseEvent, QKeySequence from PySide6.QtWidgets import ( QApplication, QDockWidget, QFileDialog, QMainWindow, QMenu, QMessageBox, QWidget, ) @@ -13,8 +14,9 @@ from je_auto_control.gui.main_widget import AutoControlGUIWidget from je_auto_control.gui.navigation import NavigationPanel from je_auto_control.gui.theme import ( - DEFAULT_THEME, THEMES, apply_theme, font_rule, prepare_application, theme_named, + THEMES, apply_theme, font_rule, prepare_application, theme_named, ) +from je_auto_control.gui.window_settings import WindowSettings, WindowState def _t(key: str, default: str = "") -> str: @@ -46,17 +48,24 @@ def _t(key: str, default: str = "") -> str: class AutoControlGUIUI(QMainWindow): - """Main window: menu bar, navigation panel and AutoControlGUIWidget (which owns the tabs).""" + """Main window: menu bar, navigation panel and AutoControlGUIWidget (which owns the tabs). - def __init__(self) -> None: + ``settings`` is where the theme, text size, navigation panel and geometry + are remembered between runs; the default is the per-user store of + :mod:`je_auto_control.gui.window_settings`. + """ + + def __init__(self, settings: Optional[WindowSettings] = None) -> None: super().__init__() + self._settings: WindowSettings = settings if settings is not None else WindowSettings() + self._state: WindowState = self._settings.load() self.app_id = _t("application_name", "AutoControlGUI") if sys.platform in ["win32", "cygwin", "msys"]: from ctypes import windll # type: ignore[attr-defined] # reason: win32-only ctypes windll.shell32.SetCurrentProcessExplicitAppUserModelID(self.app_id) - self._user_font_pt: int = 0 # 0 means auto-detect from screen - self._theme_name: str = DEFAULT_THEME + self._user_font_pt: int = self._state.text_size # 0 means auto-detect from screen + self._theme_name: str = self._state.theme # The theme's style sheet is kept so _apply_font_pt can append the # font rule instead of replacing (and thereby wiping) the theme. self._theme_stylesheet: str = apply_theme(self, theme_named(self._theme_name)) @@ -64,6 +73,8 @@ def __init__(self) -> None: self.setWindowTitle(_t("application_name", "AutoControlGUI")) self.resize(1280, 800) + if self._state.geometry: + self.restoreGeometry(QByteArray(self._state.geometry)) self.auto_control_gui_widget = AutoControlGUIWidget(parent=self) self.setCentralWidget(self.auto_control_gui_widget) @@ -97,7 +108,7 @@ def _build_navigation(self) -> None: self._navigation_dock.setTitleBarWidget(QWidget(self._navigation_dock)) self._navigation_dock.setWidget(self.navigation) self.addDockWidget(Qt.DockWidgetArea.LeftDockWidgetArea, self._navigation_dock) - self.resizeDocks([self._navigation_dock], [260], Qt.Orientation.Horizontal) + self.resizeDocks([self._navigation_dock], [self._state.navigation_width], Qt.Orientation.Horizontal) # Owned by the window, not by a menu: the menu bar is rebuilt on every # language switch, and a shortcut on a per-menu action would be # registered once more each time. @@ -105,9 +116,16 @@ def _build_navigation(self) -> None: self._search_action.setShortcut(QKeySequence("Ctrl+K")) self._search_action.triggered.connect(self._focus_feature_search) self.addAction(self._search_action) - self._sidebar_action = self._navigation_dock.toggleViewAction() + # Not the dock's toggleViewAction(): Qt disables that one for a dock + # without the closable feature, and this dock has no title bar to close. + self._sidebar_action = QAction(self) + self._sidebar_action.setCheckable(True) self._sidebar_action.setShortcut(QKeySequence("Ctrl+B")) + self._sidebar_action.triggered.connect(self.set_navigation_visible) self.addAction(self._sidebar_action) + self._navigation_dock.visibilityChanged.connect(self._sync_sidebar_action) + self._navigation_dock.setVisible(self._state.navigation_visible) + self._sync_sidebar_action() self._refresh_navigation() def _refresh_navigation(self) -> None: @@ -117,9 +135,43 @@ def _refresh_navigation(self) -> None: def _focus_feature_search(self) -> None: """Show the navigation panel and put the cursor in its search box (Ctrl+K).""" - self._navigation_dock.setVisible(True) + self.set_navigation_visible(True) self.navigation.focus_search() + def set_navigation_visible(self, visible: bool) -> None: + """Show or hide the navigation panel (Ctrl+B) and remember the choice.""" + if visible == self._navigation_dock.isHidden(): + # Hiding takes the width with it, so note it while it is still there. + self.window_state() + self._navigation_dock.setVisible(bool(visible)) + self._sync_sidebar_action() + self._remember() + + def _sync_sidebar_action(self, *_args: object) -> None: + self._sidebar_action.setChecked(not self._navigation_dock.isHidden()) + + # --- remembered state ---------------------------------------------------- + + def window_state(self) -> WindowState: + """What would be remembered if the window closed now.""" + state = self._state + state.theme = self._theme_name + state.text_size = self._user_font_pt + state.navigation_visible = not self._navigation_dock.isHidden() + # A dock that is hidden, or in a window never shown, has no width worth keeping. + if self.isVisible() and self._navigation_dock.isVisible(): + state.navigation_width = self._navigation_dock.width() + state.geometry = bytes(self.saveGeometry().data()) + return state + + def _remember(self, *_args: object) -> None: + self._settings.save(self.window_state()) + + def closeEvent(self, event: QCloseEvent) -> None: # noqa: N802 # reason: Qt override + """Save the window's look before it goes.""" + self._remember() + super().closeEvent(event) + # --- menu construction --------------------------------------------------- def _build_menu_bar(self) -> None: @@ -204,6 +256,7 @@ def set_theme(self, name: str) -> None: self._theme_name = tokens.name self._theme_stylesheet = apply_theme(self, tokens) self._apply_font_pt(self._user_font_pt) + self._remember() def _rebuild_tabs_menu(self) -> None: if self._view_menu is None: @@ -273,8 +326,13 @@ def _on_text_size_selected(self) -> None: if not isinstance(action, QAction): return data = action.data() - self._user_font_pt = int(data) if data is not None else 0 + self.set_text_size(int(data) if data is not None else 0) + + def set_text_size(self, point_size: int) -> None: + """Use ``point_size`` for all text; 0 follows the screen's height.""" + self._user_font_pt = max(int(point_size), 0) self._apply_font_pt(self._user_font_pt) + self._remember() def _on_tab_action_toggled(self, checked: bool) -> None: action = self.sender() diff --git a/je_auto_control/gui/tab_registry.py b/je_auto_control/gui/tab_registry.py index dd9633464..15c50e980 100644 --- a/je_auto_control/gui/tab_registry.py +++ b/je_auto_control/gui/tab_registry.py @@ -6,6 +6,10 @@ widget on first access to ``widget``, so the window no longer imports and constructs some fifty panels (with the timers and helper threads several of them start) before it can show the three it opens on. + +Closing a tab keeps its widget for the next open. :meth:`TabEntry.release` +is the other choice: it calls the widget's optional ``dispose()`` and deletes +it, and the next open builds a fresh one. """ from dataclasses import dataclass, field from importlib import import_module @@ -25,6 +29,8 @@ class TabSpec: module: str = "" class_name: str = "" default_visible: bool = False + # False for a panel that lays itself out and must not sit in the page's scroll area. + scrollable: bool = True def lazy_factory(module: str, class_name: str) -> WidgetFactory: @@ -47,6 +53,9 @@ class TabEntry: default_visible: bool = False actions: MenuActions = () on_build: Optional[Callable[[Any], None]] = None + scrollable: bool = True + # False when the factory hands back one fixed widget: releasing it would leave nothing to reopen. + releasable: bool = True _widget: Any = field(default=None, repr=False) @property @@ -63,6 +72,26 @@ def widget(self) -> Any: self.on_build(self._widget) return self._widget + def release(self) -> bool: + """Let go of the widget so the next open builds a new one; ``False`` if nothing was released. + + The tab's side of the contract is optional: a widget with a + ``dispose()`` method has it called first, to stop its timers and + threads and drop the listeners it registered. The widget is then + scheduled for deletion. + """ + widget = self._widget + if widget is None or not self.releasable: + return False + self._widget = None + dispose = getattr(widget, "dispose", None) + try: + if callable(dispose): + dispose() + finally: + widget.deleteLater() + return True + _GUI = "je_auto_control.gui" @@ -111,7 +140,7 @@ def widget(self) -> Any: TabSpec("dag_runner", "tab_dag_runner", "automation", f"{_GUI}.dag_tab", "DagTab"), TabSpec("chatops", "tab_chatops", "automation", f"{_GUI}.chatops_tab", "ChatOpsTab"), TabSpec("trace_replay", "tab_trace_replay", "automation", f"{_GUI}.trace_replay_tab", "TraceReplayTab"), - TabSpec("remote_desktop", "tab_remote_desktop", "system", default_visible=True), + TabSpec("remote_desktop", "tab_remote_desktop", "system", default_visible=True, scrollable=False), TabSpec("presence", "tab_presence", "system", f"{_GUI}.presence_tab", "PresenceTab"), TabSpec("rest_api", "tab_rest_api", "system", f"{_GUI}.rest_api_tab", "RestApiTab"), TabSpec("admin_console", "tab_admin_console", "system", f"{_GUI}.admin_console_tab", "AdminConsoleTab"), diff --git a/je_auto_control/gui/theme.py b/je_auto_control/gui/theme.py index 9a6ad7124..f1277b969 100644 --- a/je_auto_control/gui/theme.py +++ b/je_auto_control/gui/theme.py @@ -4,7 +4,8 @@ window uses; :func:`build_stylesheet` turns it into Qt style sheet text and :func:`apply_theme` puts it on a window together with a matching palette, so the parts Qt draws itself (arrows, check marks, scroll handles) agree with it. -Nothing here loads an image or a font file. +No image or font file ships with it: the one picture, the cross on a tab's close +button, is drawn in code (:mod:`je_auto_control.gui._tab_close_icon`). """ from dataclasses import dataclass from typing import Any, Dict, Optional @@ -167,7 +168,11 @@ def apply_theme(window: Any, tokens: ThemeTokens, point_size: Optional[int] = No ``point_size`` appends :func:`font_rule`, so the size is set on top of the theme instead of replacing it. """ + from je_auto_control.gui._tab_close_icon import close_button_rules sheet = build_stylesheet(tokens) + close_rules = close_button_rules(tokens) + if close_rules: + sheet = f"{sheet}\n{close_rules}" window.setPalette(_palette(tokens)) window.setStyleSheet(sheet if point_size is None else f"{sheet}\n{font_rule(point_size)}") return sheet diff --git a/je_auto_control/gui/window_settings.py b/je_auto_control/gui/window_settings.py new file mode 100644 index 000000000..821472637 --- /dev/null +++ b/je_auto_control/gui/window_settings.py @@ -0,0 +1,130 @@ +"""What the main window remembers between runs, and where it is kept. + +Theme, text size, the navigation panel's visibility and width, and the window +geometry are written to one INI file through ``QSettings``: +``~/.je_auto_control/gui_settings.ini``, resolved when it is used, so a +redirected home directory moves it like the rest of the per-user state. +``JE_AUTOCONTROL_GUI_SETTINGS`` names another file, or switches the store off +(``off``, ``0``, ``none``, ``false`` or empty): nothing is read and nothing is +written, which is what the test suite runs with. +""" +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional, Tuple, Union + +from PySide6.QtCore import QByteArray, QSettings + +from je_auto_control.gui.theme import DEFAULT_THEME, THEMES +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +SETTINGS_ENV = "JE_AUTOCONTROL_GUI_SETTINGS" +DEFAULT_NAVIGATION_WIDTH = 260 +_OFF_VALUES = frozenset({"", "off", "0", "none", "false"}) +_WIDTH_RANGE = (200, 800) +_TEXT_SIZE_RANGE = (6, 48) +_GROUP = "main_window" +_UNSET: Any = object() + + +def settings_path() -> Optional[Path]: + """The settings file in force, or ``None`` when the store is switched off.""" + value = os.environ.get(SETTINGS_ENV) + if value is None: + return Path.home() / ".je_auto_control" / "gui_settings.ini" + if value.strip().lower() in _OFF_VALUES: + return None + return Path(os.path.realpath(os.path.expanduser(value.strip()))) + + +@dataclass +class WindowState: + """The remembered look of the main window; the defaults are a first run.""" + + theme: str = DEFAULT_THEME + text_size: int = 0 + navigation_visible: bool = True + navigation_width: int = DEFAULT_NAVIGATION_WIDTH + geometry: bytes = b"" + + +def _as_bool(value: Any, default: bool) -> bool: + if isinstance(value, bool): + return value + text = str(value).strip().lower() + if text in {"true", "1", "yes"}: + return True + if text in {"false", "0", "no"}: + return False + return default + + +def _as_int(value: Any, default: int, bounds: Tuple[int, int]) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return default + return number if bounds[0] <= number <= bounds[1] else default + + +def _as_bytes(value: Any) -> bytes: + if isinstance(value, QByteArray): + return bytes(value.data()) + return bytes(value) if isinstance(value, (bytes, bytearray)) else b"" + + +class WindowSettings: + """Load and save a :class:`WindowState`; a store without a path does neither.""" + + def __init__(self, path: Union[str, Path, None] = _UNSET) -> None: + resolved = settings_path() if path is _UNSET else path + self._path: Optional[Path] = None if resolved is None else Path(resolved) + + @property + def path(self) -> Optional[Path]: + """The INI file, or ``None`` when nothing is persisted.""" + return self._path + + def _open(self) -> QSettings: + return QSettings(str(self._path), QSettings.Format.IniFormat) + + def load(self) -> WindowState: + """Return what was saved; anything missing or out of range is the default.""" + state = WindowState() + if self._path is None or not self._path.is_file(): + return state + store = self._open() + store.beginGroup(_GROUP) + theme = str(store.value("theme", state.theme)) + state.theme = theme if theme in THEMES else state.theme + # 0 is "follow the screen"; anything else has to be a plausible point size. + state.text_size = _as_int(store.value("text_size", 0), 0, _TEXT_SIZE_RANGE) + state.navigation_visible = _as_bool(store.value("navigation_visible", True), True) + state.navigation_width = _as_int( + store.value("navigation_width", DEFAULT_NAVIGATION_WIDTH), DEFAULT_NAVIGATION_WIDTH, _WIDTH_RANGE) + state.geometry = _as_bytes(store.value("geometry", QByteArray())) + store.endGroup() + return state + + def save(self, state: WindowState) -> bool: + """Write ``state``; return whether it reached the file.""" + if self._path is None: + return False + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + except OSError as error: + autocontrol_logger.warning("GUI settings not saved to %s: %r", self._path, error) + return False + store = self._open() + store.beginGroup(_GROUP) + store.setValue("theme", state.theme) + store.setValue("text_size", int(state.text_size)) + store.setValue("navigation_visible", bool(state.navigation_visible)) + store.setValue("navigation_width", int(state.navigation_width)) + store.setValue("geometry", QByteArray(state.geometry)) + store.endGroup() + store.sync() + if store.status() != QSettings.Status.NoError: + autocontrol_logger.warning("GUI settings not saved to %s: %s", self._path, store.status()) + return False + return True diff --git a/je_auto_control/gui/workspace_tabs.py b/je_auto_control/gui/workspace_tabs.py new file mode 100644 index 000000000..673162d97 --- /dev/null +++ b/je_auto_control/gui/workspace_tabs.py @@ -0,0 +1,175 @@ +"""The workspace's tab widget: pages that scroll, and tabs filled on first use. + +Each page sits in a holder inside the ``QTabWidget``. A scrollable holder is a +``QScrollArea``, so a window narrower than the page shows scroll bars instead +of squeezing the form below its minimum size; a panel that already scrolls +its own content (Remote Desktop) asks for a plain holder and is not +put inside a second scroll area. + +Callers keep speaking in pages: ``addTab`` / ``insertTab`` take the page, +``indexOf`` / ``widget`` / ``currentWidget`` / ``setCurrentWidget`` find and +return it. That is the relation ``tabs.indexOf(entry.widget)`` embedders and +tests rely on, and it holds whether or not the page is wrapped. + +A deferred tab is a holder with a title and no page yet; whoever owns the tab +fills it the first time it is selected. +""" +from typing import Any, Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import QFrame, QScrollArea, QTabWidget, QVBoxLayout, QWidget + + +class PageHolder(QWidget): + """What the tab widget really holds for one tab: the page, scrolled or not.""" + + def __init__(self, scrollable: bool = True, pending_key: str = "", parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self.setObjectName("WorkspacePage") + self.pending_key = pending_key + self._page: Optional[QWidget] = None + self._area: Optional[QScrollArea] = None + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) + if scrollable: + self._area = QScrollArea(self) + self._area.setObjectName("WorkspacePageScroll") + self._area.setWidgetResizable(True) + self._area.setFrameShape(QFrame.Shape.NoFrame) + self._area.setFocusPolicy(Qt.FocusPolicy.NoFocus) + layout.addWidget(self._area) + + @property + def scrollable(self) -> bool: + """Whether the page is inside a scroll area.""" + return self._area is not None + + @property + def page(self) -> Optional[QWidget]: + """The page, or ``None`` for a deferred tab not filled yet.""" + return self._page + + def set_page(self, page: QWidget) -> None: + """Put ``page`` in the holder.""" + self._page = page + if self._area is None: + self.layout().addWidget(page) + page.show() + return + self._area.setWidget(page) + # setWidget() turns this on; the page then paints the palette's window + # colour over the pane instead of sitting on it like an unwrapped page. + page.setAutoFillBackground(False) + + def take_page(self) -> Optional[QWidget]: + """Detach and return the page; the caller gives it a new parent.""" + page, self._page = self._page, None + if page is None: + return None + if self._area is not None: + self._area.takeWidget() + else: + self.layout().removeWidget(page) + return page + + +class WorkspaceTabWidget(QTabWidget): + """``QTabWidget`` whose pages scroll when the window is smaller than they are.""" + + # --- adding -------------------------------------------------------------- + + def insertTab(self, index: int, page: QWidget, *label: Any, # noqa: N802 # reason: Qt name + scrollable: bool = True) -> int: + """Insert ``page`` at ``index``; ``label`` is the title, or an icon and the title.""" + holder = PageHolder(scrollable) + holder.set_page(page) + return int(super().insertTab(index, holder, *label)) + + def addTab(self, page: QWidget, *label: Any, scrollable: bool = True) -> int: # noqa: N802 # reason: Qt name + """Append ``page``; see :meth:`insertTab`.""" + return self.insertTab(self.count(), page, *label, scrollable=scrollable) + + def insert_deferred_tab(self, index: int, key: str, label: str, scrollable: bool = True) -> int: + """Insert a titled tab with no page yet, known by ``key`` until it is filled.""" + return int(super().insertTab(index, PageHolder(scrollable, key), label)) + + def fill_deferred(self, key: str, page: QWidget) -> bool: + """Give the deferred tab ``key`` its page; ``False`` when there is no such tab.""" + holder = self._holder(self.index_of_deferred(key)) + if holder is None: + return False + holder.pending_key = "" + holder.set_page(page) + return True + + # --- finding ------------------------------------------------------------- + + def _holder(self, index: int) -> Optional[PageHolder]: + raw = super().widget(index) if index >= 0 else None + return raw if isinstance(raw, PageHolder) else None + + def index_of_deferred(self, key: str) -> int: + """Index of the unfilled tab ``key``, or -1.""" + if not key: + return -1 + for index in range(self.count()): + holder = self._holder(index) + if holder is not None and holder.pending_key == key: + return index + return -1 + + def deferred_key(self, index: int) -> str: + """Key of the tab at ``index`` if it has no page yet, else ``""``.""" + holder = self._holder(index) + return holder.pending_key if holder is not None and holder.page is None else "" + + def indexOf(self, page: Optional[QWidget]) -> int: # noqa: N802 # reason: Qt name + """Index of the tab showing ``page`` (or holding it), or -1.""" + if page is None: + return -1 + for index in range(self.count()): + raw = super().widget(index) + if raw is page or (isinstance(raw, PageHolder) and raw.page is page): + return index + return -1 + + def widget(self, index: int) -> Optional[QWidget]: + """The page of the tab at ``index``; ``None`` for a deferred tab not filled yet.""" + raw = super().widget(index) + return raw.page if isinstance(raw, PageHolder) else raw + + def currentWidget(self) -> Optional[QWidget]: # noqa: N802 # reason: Qt name + """The page of the selected tab.""" + return self.widget(self.currentIndex()) + + def setCurrentWidget(self, page: QWidget) -> None: # noqa: N802 # reason: Qt name + """Select the tab showing ``page``.""" + index = self.indexOf(page) + if index != -1: + self.setCurrentIndex(index) + + def is_scrollable(self, index: int) -> bool: + """Whether the tab at ``index`` scrolls its page.""" + holder = self._holder(index) + return holder is not None and holder.scrollable + + # --- removing ------------------------------------------------------------ + + def removeTab(self, index: int) -> None: # noqa: N802 # reason: Qt name + """Remove the tab; its page is kept, hidden, as a child of this widget.""" + holder = self._holder(index) + super().removeTab(index) + if holder is None: + return + page = holder.take_page() + if page is not None: + page.setParent(self) + page.hide() + holder.setParent(None) + holder.deleteLater() + + def clear(self) -> None: + """Remove every tab, keeping the pages.""" + while self.count(): + self.removeTab(0) diff --git a/test/conftest.py b/test/conftest.py index 68ae643f8..469cd6180 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -28,3 +28,5 @@ os.environ["HOME"] = str(_HOME) os.environ["USERPROFILE"] = str(_HOME) os.environ["JE_AUTOCONTROL_LOG_FILE"] = str(_SCRATCH / "AutoControlGUI.log") +# The main window remembers its look in a per-user file; one test must not hand the next its theme. +os.environ["JE_AUTOCONTROL_GUI_SETTINGS"] = "off" diff --git a/test/unit_test/headless/test_gui_lazy_registry.py b/test/unit_test/headless/test_gui_lazy_registry.py index f711236c0..0cfc0c04c 100644 --- a/test/unit_test/headless/test_gui_lazy_registry.py +++ b/test/unit_test/headless/test_gui_lazy_registry.py @@ -85,6 +85,9 @@ def test_a_lazy_factory_imports_at_call_time(): "open_at_start": [row["key"] for row in widget.list_registered_tabs() if row["visible"]], "variables_imported_at_start": "je_auto_control.gui.variables_tab" in sys.modules, "presence_imported_at_start": "je_auto_control.gui.presence_tab" in sys.modules, + "script_builder_imported_at_start": "je_auto_control.gui.script_builder" in sys.modules, + "remote_desktop_imported_at_start": "je_auto_control.gui.remote_desktop_tab" in sys.modules, + "current_at_start": widget.current_tab_key(), } widget.list_registered_tabs() widget.retranslate() @@ -107,6 +110,19 @@ def test_a_lazy_factory_imports_at_call_time(): report["current_after_activate"] = widget.current_tab_key() report["changes"] = len(changes) report["unknown"] = widget.activate_tab("no_such_tab") + +# A start tab that has never been in front: closed and reopened without being built, built when selected. +widget.hide_tab("script_builder") +still_open = [r["key"] for r in widget.list_registered_tabs() if r["visible"]] +report["closed_unbuilt"] = not entries["script_builder"].built and "script_builder" not in still_open +widget.show_tab("script_builder") +builder = entries["script_builder"].widget +report["builder_selected"] = [widget.current_tab_key(), widget.tabs.indexOf(builder), + widget.tabs.currentWidget() is builder] +widget.tabs.setCurrentIndex(widget.tabs.count() - 1) +report["remote_built_on_select"] = [entries["remote_desktop"].built, widget.current_tab_key()] +report["scrolls"] = {key: widget.tabs.is_scrollable(widget.tabs.indexOf(entries[key].widget)) + for key in ("record", "script_builder", "remote_desktop")} sys.stdout.write(json.dumps(report)) sys.stdout.flush() os._exit(0) @@ -129,10 +145,21 @@ def test_every_spec_is_registered_in_order(report): def test_only_the_start_tabs_and_the_widgets_own_forms_are_built(report): - assert set(report["built_at_start"]) == OWN_TABS | set(OPEN_AT_START) - assert report["open_at_start"] == OPEN_AT_START + # Record is in front; Script Builder and Remote Desktop have a tab but wait for the first click. + assert set(report["built_at_start"]) == OWN_TABS - {"remote_desktop"} + assert report["open_at_start"] == OPEN_AT_START and report["current_at_start"] == "record" assert not report["variables_imported_at_start"] assert not report["presence_imported_at_start"] + assert not report["script_builder_imported_at_start"] + assert not report["remote_desktop_imported_at_start"] + + +def test_a_start_tab_is_built_when_it_first_comes_to_the_front(report): + assert report["closed_unbuilt"] + assert report["builder_selected"] == ["script_builder", 1, True] + assert report["remote_built_on_select"] == [True, "remote_desktop"] + # Remote Desktop scrolls itself; the other pages scroll in their tab. + assert report["scrolls"] == {"record": True, "script_builder": True, "remote_desktop": False} def test_listing_translating_and_hiding_build_nothing(report): diff --git a/test/unit_test/headless/test_gui_shell_state.py b/test/unit_test/headless/test_gui_shell_state.py new file mode 100644 index 000000000..86bde4664 --- /dev/null +++ b/test/unit_test/headless/test_gui_shell_state.py @@ -0,0 +1,417 @@ +"""The window shell: remembered state, pages that scroll, drawn close crosses, releasing a tab. + +Offscreen Qt throughout; nothing is shown and nothing is clicked on the +desktop. The settings tests write only under ``tmp_path``: the suite itself +runs with the store switched off (``test/conftest.py``), which the first tests +here hold it to. +""" +import json +import os +import pathlib +import subprocess # nosec B404 # reason: runs this file's own probe with a fixed argv +import sys + +import pytest + +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtCore import QSize # noqa: E402 +from PySide6.QtGui import QImage # noqa: E402 +from PySide6.QtWidgets import ( # noqa: E402 + QAbstractButton, QApplication, QLabel, QScrollArea, QTabWidget, QWidget, +) + +from je_auto_control.gui.tab_registry import TAB_SPECS, TabEntry # noqa: E402 +from je_auto_control.gui.window_settings import ( # noqa: E402 + DEFAULT_NAVIGATION_WIDTH, SETTINGS_ENV, WindowSettings, WindowState, settings_path, +) +from je_auto_control.gui._tab_close_icon import cache_directory, close_button_rules, cross_file # noqa: E402 +from je_auto_control.gui.theme import DARK, LIGHT # noqa: E402 +from je_auto_control.gui.workspace_tabs import PageHolder, WorkspaceTabWidget # noqa: E402 + +ROOT = pathlib.Path(__file__).resolve().parents[3] + + +@pytest.fixture(scope="module") +def qapp(): + app = QApplication.instance() or QApplication([]) + yield app + + +def _settle(app) -> None: + for _ in range(3): + app.processEvents() + + +# --- where the state is kept ------------------------------------------------------------------------ + +def test_the_suite_runs_with_the_store_switched_off(): + assert os.environ.get(SETTINGS_ENV) == "off" + store = WindowSettings() + assert settings_path() is None and store.path is None + assert store.save(WindowState(theme="light")) is False + assert store.load() == WindowState() + + +@pytest.mark.parametrize("value", ["", "off", "OFF", "0", "none", " false "]) +def test_the_variable_switches_the_store_off(monkeypatch, value): + monkeypatch.setenv(SETTINGS_ENV, value) + assert settings_path() is None + + +def test_the_default_file_follows_the_home_directory(monkeypatch, tmp_path): + monkeypatch.delenv(SETTINGS_ENV) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + assert settings_path() == tmp_path / ".je_auto_control" / "gui_settings.ini" + + +def test_the_variable_names_another_file(monkeypatch, tmp_path): + monkeypatch.setenv(SETTINGS_ENV, str(tmp_path / "look.ini")) + assert settings_path() == pathlib.Path(os.path.realpath(tmp_path / "look.ini")) + + +def test_state_round_trips_through_the_file(qapp, tmp_path): + path = tmp_path / "deep" / "gui.ini" + saved = WindowState(theme="light", text_size=14, navigation_visible=False, + navigation_width=333, geometry=b"\x01\x02\x00\xff") + assert WindowSettings(path).save(saved) is True + assert path.is_file() + assert WindowSettings(path).load() == saved + + +def test_a_missing_file_is_a_first_run(qapp, tmp_path): + assert WindowSettings(tmp_path / "absent.ini").load() == WindowState() + assert not (tmp_path / "absent.ini").exists() + + +def test_values_out_of_range_fall_back_to_the_defaults(qapp, tmp_path): + path = tmp_path / "gui.ini" + path.write_text("[main_window]\ntheme=neon\ntext_size=999\nnavigation_visible=perhaps\n" + "navigation_width=5\ngeometry=not-bytes\n", encoding="utf-8") + state = WindowSettings(path).load() + assert (state.theme, state.text_size, state.navigation_visible, state.navigation_width) == ( + "dark", 0, True, DEFAULT_NAVIGATION_WIDTH) + + +def test_an_unwritable_place_is_reported_not_raised(qapp, tmp_path): + blocker = tmp_path / "file" + blocker.write_text("x", encoding="utf-8") + assert WindowSettings(blocker / "below" / "gui.ini").save(WindowState()) is False + + +# --- pages scroll; callers still speak in pages ------------------------------------------------------- + +class _Big(QWidget): + """A form that needs more room than a narrow window has.""" + + def minimumSizeHint(self) -> QSize: # noqa: N802 + return QSize(600, 500) + + sizeHint = minimumSizeHint # noqa: N815 + + +def test_the_page_relation_survives_the_wrapper(qapp): + tabs = WorkspaceTabWidget() + first, second = QLabel("one"), QLabel("two") + assert tabs.addTab(first, "One") == 0 + assert tabs.insertTab(0, second, "Two") == 0 + assert (tabs.indexOf(first), tabs.indexOf(second), tabs.indexOf(QLabel())) == (1, 0, -1) + assert tabs.widget(0) is second and tabs.widget(1) is first and tabs.widget(7) is None + tabs.setCurrentWidget(first) + assert tabs.currentIndex() == 1 and tabs.currentWidget() is first + # What Qt itself holds is the holder, with the page inside a scroll area. + holder = QTabWidget.widget(tabs, 1) + assert isinstance(holder, PageHolder) and holder.findChild(QScrollArea) is not None + assert holder.isAncestorOf(first) + tabs.deleteLater() + + +def test_a_removed_page_is_kept_hidden_and_can_come_back(qapp): + tabs = WorkspaceTabWidget() + page = QLabel("kept") + tabs.addTab(page, "Kept") + tabs.removeTab(0) + assert tabs.count() == 0 and tabs.indexOf(page) == -1 + assert page.parent() is tabs and page.isHidden() + tabs.addTab(page, "Kept") + assert tabs.indexOf(page) == 0 and tabs.currentWidget() is page + tabs.clear() + assert tabs.count() == 0 and page.parent() is tabs + tabs.deleteLater() + + +def test_a_narrow_window_scrolls_the_page_instead_of_squeezing_it(qapp): + tabs = WorkspaceTabWidget() + page = _Big() + tabs.addTab(page, "Big") + tabs.resize(320, 260) + tabs.show() + _settle(qapp) + area = QTabWidget.widget(tabs, 0).findChild(QScrollArea) + assert page.width() >= 600 and page.height() >= 500 + assert area.viewport().width() < 600 + assert area.horizontalScrollBar().maximum() > 0 and area.verticalScrollBar().maximum() > 0 + tabs.resize(900, 800) + _settle(qapp) + assert area.horizontalScrollBar().maximum() == 0 and page.width() > 600 + tabs.hide() + tabs.deleteLater() + + +def test_a_panel_that_scrolls_itself_is_not_wrapped_again(qapp): + tabs = WorkspaceTabWidget() + panel = _Big() + tabs.addTab(panel, "Panel", scrollable=False) + holder = QTabWidget.widget(tabs, 0) + assert holder.findChild(QScrollArea) is None and panel.parent() is holder + assert tabs.indexOf(panel) == 0 and tabs.currentWidget() is panel and not tabs.is_scrollable(0) + tabs.resize(320, 260) + tabs.show() + _settle(qapp) + assert panel.width() < 600 + tabs.hide() + tabs.deleteLater() + + +def test_a_deferred_tab_has_a_title_and_no_page_until_filled(qapp): + tabs = WorkspaceTabWidget() + tabs.addTab(QLabel("front"), "Front") + assert tabs.insert_deferred_tab(1, "later", "Later") == 1 + assert tabs.tabText(1) == "Later" and tabs.widget(1) is None + assert tabs.deferred_key(1) == "later" and tabs.index_of_deferred("later") == 1 + assert tabs.deferred_key(0) == "" and tabs.index_of_deferred("") == -1 + page = QLabel("built") + assert tabs.fill_deferred("later", page) is True + assert tabs.widget(1) is page and tabs.indexOf(page) == 1 + assert tabs.deferred_key(1) == "" and tabs.index_of_deferred("later") == -1 + assert tabs.fill_deferred("later", QLabel()) is False + tabs.deleteLater() + + +# --- close buttons -------------------------------------------------------------------------------- + +def test_the_cross_is_drawn_in_code_and_cached(qapp, tmp_path): + path = cross_file("#E7E9EE", 255, tmp_path) + assert path == tmp_path / "tab_close_e7e9ee_255.png" + assert sorted(item.name for item in tmp_path.iterdir()) == [ + "tab_close_e7e9ee_255.png", "tab_close_e7e9ee_255@2x.png", "tab_close_e7e9ee_255@3x.png"] + for name, side in (("tab_close_e7e9ee_255.png", 16), ("tab_close_e7e9ee_255@3x.png", 48)): + image = QImage(str(tmp_path / name)) + assert (image.width(), image.height()) == (side, side) + centre, corner = image.pixelColor(side // 2, side // 2), image.pixelColor(0, 0) + assert corner.alpha() == 0 and centre.alpha() > 200 + assert (centre.red(), centre.green(), centre.blue()) == (0xE7, 0xE9, 0xEE) + stamp = path.stat().st_mtime_ns + assert cross_file("#e7e9ee", 255, tmp_path) == path and path.stat().st_mtime_ns == stamp + + +def test_the_style_sheet_points_the_close_button_at_the_cross(qapp, tmp_path): + dark, light = close_button_rules(DARK, tmp_path), close_button_rules(LIGHT, tmp_path) + assert dark != light + for rules, tokens in ((dark, DARK), (light, LIGHT)): + assert "QTabBar::close-button {" in rules and "QTabBar::close-button:hover {" in rules + rest = tmp_path / f"tab_close_{tokens.text_muted.lstrip('#')}_255.png" + hover = tmp_path / f"tab_close_{tokens.text.lstrip('#')}_255.png" + assert rest.is_file() and hover.is_file() + assert f'url("{rest.as_posix()}")' in rules and f'url("{hover.as_posix()}")' in rules + assert tokens.hover in rules + + +def test_an_unwritable_cache_leaves_the_styles_own_icon(qapp, tmp_path): + blocker = tmp_path / "file" + blocker.write_text("x", encoding="utf-8") + assert cross_file("#ffffff", 255, blocker / "below") is None + assert close_button_rules(DARK, blocker / "below") == "" + + +def test_the_cache_follows_the_home_directory(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + assert cache_directory() == tmp_path / ".je_auto_control" / "gui_cache" + + +def test_the_tab_bar_keeps_qts_own_close_buttons(qapp): + """Buttons of our own in the bar made probes crash on exit; see ``_tab_close_icon``.""" + tabs = WorkspaceTabWidget() + tabs.setTabsClosable(True) + for name in ("a", "b"): + tabs.addTab(QLabel(name), name) + asked = [] + tabs.tabCloseRequested.connect(asked.append) + buttons = [child for child in tabs.tabBar().children() if type(child) is QAbstractButton] + assert len(buttons) == 2 + for button in buttons: + button.click() + assert sorted(asked) == [0, 1] + tabs.deleteLater() + + +# --- releasing a tab (no Qt needed) --------------------------------------------------------------- + +class _Tab: + def __init__(self, log, fail=False): + self.log, self.fail = log, fail + + def dispose(self): + self.log.append("dispose") + if self.fail: + raise RuntimeError("timer already gone") + + def deleteLater(self): # noqa: N802 + self.log.append("deleted") + + +def test_release_disposes_deletes_and_the_next_open_builds_anew(): + log = [] + entry = TabEntry("k", "tab_k", lambda: _Tab(log)) + assert entry.release() is False and log == [] # never built: nothing to let go of + first = entry.widget + assert entry.release() is True + assert log == ["dispose", "deleted"] and not entry.built + assert entry.widget is not first and entry.built + + +def test_release_without_a_dispose_hook_still_deletes(): + log = [] + + class _Plain: + def deleteLater(self): # noqa: N802 + log.append("deleted") + + entry = TabEntry("k", "tab_k", _Plain) + _ = entry.widget + assert entry.release() is True and log == ["deleted"] + + +def test_a_failing_dispose_still_deletes_the_widget(): + log = [] + entry = TabEntry("k", "tab_k", lambda: _Tab(log, fail=True)) + _ = entry.widget + with pytest.raises(RuntimeError): + entry.release() + assert log == ["dispose", "deleted"] and not entry.built + + +def test_a_fixed_widget_is_never_released(): + log = [] + fixed = _Tab(log) + entry = TabEntry("k", "tab_k", lambda: fixed, releasable=False) + _ = entry.widget + assert entry.release() is False and log == [] and entry.widget is fixed + + +def test_only_remote_desktop_opts_out_of_page_scrolling(): + assert [spec.key for spec in TAB_SPECS if not spec.scrollable] == ["remote_desktop"] + + +# --- the real window, twice ------------------------------------------------------------------------ + +_PROBE = r""" +import json, os, sys +os.environ["QT_QPA_PLATFORM"] = "offscreen" +from PySide6.QtCore import QEvent, Qt +from PySide6.QtWidgets import QApplication +from je_auto_control.gui.main_window import AutoControlGUIUI +from je_auto_control.gui.theme import LIGHT, font_rule + +app = QApplication([]) +window = AutoControlGUIUI() +window.show() +for _ in range(3): + app.processEvents() +workspace = window.auto_control_gui_widget +report = { + "theme": window._theme_name, "text": window._user_font_pt, + "sheet": LIGHT.window in window.styleSheet() and font_rule(14) in window.styleSheet(), + "nav_hidden": window._navigation_dock.isHidden(), "nav_width": window._navigation_dock.width(), + "size": [window.width(), window.height()], + "close_rule": "QTabBar::close-button" in window.styleSheet(), + "checked_theme": [a.data() for a in window.findChildren(type(window._search_action)) + if a.isCheckable() and a.isChecked() and a.data() in ("dark", "light")], +} +if sys.argv[1] == "first": + window.set_theme("light") + window.set_text_size(14) + window.resize(700, 500) + window.resizeDocks([window._navigation_dock], [330], Qt.Orientation.Horizontal) + for _ in range(3): + app.processEvents() + report["width_set"] = window._navigation_dock.width() + window.close() +elif sys.argv[1] == "second": + report["toggle_enabled"] = window._sidebar_action.isEnabled() and window._sidebar_action.isChecked() + window._sidebar_action.trigger() # Ctrl+B: saved at once, not only on close + report["hidden_now"] = window._navigation_dock.isHidden() +else: + kept = workspace.open_tab("variables") + report["open_returns_widget"] = kept is workspace._find_entry("variables").widget + report["closed_kept"] = [workspace.close_tab("variables"), workspace._find_entry("variables").widget is kept] + seen = [] + kept.dispose = lambda: seen.append("dispose") + report["released"] = workspace.close_tab("variables", release=True) + app.sendPostedEvents(None, QEvent.Type.DeferredDelete.value) + report["disposed"] = seen + report["built_after_release"] = workspace._find_entry("variables").built + fresh = workspace.open_tab("variables") + report["fresh_widget"] = fresh is not kept and workspace.current_tab_key() == "variables" + report["own_form"] = [workspace.close_tab("record", release=True), workspace._find_entry("record").built] + report["unknown"] = [workspace.open_tab("nope"), workspace.close_tab("nope", release=True)] +sys.stdout.write(json.dumps(report)) +sys.stdout.flush() +os._exit(0) +""" + + +def _run(mode: str, settings: str) -> dict: + env = dict(os.environ, PYTHONPATH=str(ROOT), QT_QPA_PLATFORM="offscreen") + env[SETTINGS_ENV] = settings + done = subprocess.run([sys.executable, "-c", _PROBE, mode], capture_output=True, text=True, # nosec B603 # nosemgrep # reason: fixed argv, no shell + timeout=180, env=env, cwd=str(ROOT), check=False) + assert done.returncode == 0, done.stderr[-2000:] + return json.loads(done.stdout) + + +@pytest.fixture(scope="module") +def runs(tmp_path_factory): + path = tmp_path_factory.mktemp("gui-settings") / "gui.ini" + first = _run("first", str(path)) + second = _run("second", str(path)) + third = _run("third", str(path)) + return first, second, third, path + + +def test_a_first_run_has_the_defaults(runs): + first = runs[0] + assert (first["theme"], first["text"], first["nav_hidden"]) == ("dark", 0, False) + assert first["size"] == [1280, 800] and first["checked_theme"] == ["dark"] + assert first["close_rule"] + + +def test_the_next_run_looks_like_the_last_one(runs): + first, second, _third, path = runs + assert path.is_file() + assert (second["theme"], second["text"], second["sheet"]) == ("light", 14, True) + assert second["checked_theme"] == ["light"] + assert second["size"] == [700, 500] # inside the 800x800 offscreen screen, so not clamped + assert abs(second["nav_width"] - first["width_set"]) <= 2 and first["width_set"] != DEFAULT_NAVIGATION_WIDTH + + +def test_hiding_the_panel_is_remembered_without_closing(runs): + _first, second, third, _path = runs + assert second["toggle_enabled"] and second["nav_hidden"] is False and second["hidden_now"] is True + assert third["nav_hidden"] is True + # A hidden panel keeps the width it had. + assert WindowSettings(runs[3]).load().navigation_width == runs[0]["width_set"] + + +def test_closing_keeps_the_widget_and_releasing_lets_it_go(runs): + third = runs[2] + assert third["open_returns_widget"] + assert third["closed_kept"] == [False, True] + assert third["released"] is True and third["disposed"] == ["dispose"] + assert third["built_after_release"] is False and third["fresh_widget"] + # The widget's own forms hold attributes its slots read; they are closed, never released. + assert third["own_form"] == [False, True] + assert third["unknown"] == [None, False] From 5bd9f33196c609afc535198aee9e67a2da6c46c0 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 02:54:49 +0800 Subject: [PATCH 31/44] Measure GUI start-up and workloads, and hold the rewrite to feature parity There was no number to compare a GUI change against and no test that every tab stayed reachable; the benchmarks run offscreen and the parity test covers the default workflow, the full catalogue, a 640x420 window and mixed-DPI geometry with fake screens. --- benchmarks/_gui_bench.py | 175 ++++++++++ benchmarks/gui_startup.py | 117 +++++++ benchmarks/gui_workloads.py | 151 ++++++++ .../doc/new_features/v223_features_doc.rst | 68 +++- .../Zh/doc/new_features/v223_features_doc.rst | 53 ++- .../headless/test_gui_feature_parity.py | 323 ++++++++++++++++++ 6 files changed, 861 insertions(+), 26 deletions(-) create mode 100644 benchmarks/_gui_bench.py create mode 100644 benchmarks/gui_startup.py create mode 100644 benchmarks/gui_workloads.py create mode 100644 test/unit_test/headless/test_gui_feature_parity.py diff --git a/benchmarks/_gui_bench.py b/benchmarks/_gui_bench.py new file mode 100644 index 000000000..468a7f8c3 --- /dev/null +++ b/benchmarks/_gui_bench.py @@ -0,0 +1,175 @@ +"""Shared pieces of the GUI benchmarks: environment, memory, percentiles, before/after comparison. + +Nothing here imports Qt at module level, and nothing shows a window: the +benchmarks run on the ``offscreen`` platform plugin unless ``QT_QPA_PLATFORM`` +says otherwise, and never read or write the user's GUI settings. +""" +import json +import os +import platform +import statistics +import sys +from typing import Any, Dict, Iterable, List, Optional, Sequence + +SETTINGS_ENV = "JE_AUTOCONTROL_GUI_SETTINGS" +# What has to match for two reports to be comparable. +COMPARABLE_KEYS = ("benchmark", "workload") +COMPARABLE_ENVIRONMENT = ("platform", "machine", "python", "pyside6", "qt_platform", "screens") + + +def prepare_process() -> None: + """Keep this process off the desktop and out of the user's settings.""" + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + os.environ[SETTINGS_ENV] = "off" + + +def rss_mb() -> Optional[float]: + """Resident memory of this process in MiB, or ``None`` when it cannot be read.""" + try: + import psutil + return round(psutil.Process().memory_info().rss / 1048576, 1) + except ImportError: + pass + if sys.platform == "win32": + return _windows_rss_mb() + try: + import resource + except ImportError: + return None + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return round(peak / (1048576 if sys.platform == "darwin" else 1024), 1) + + +def _windows_rss_mb() -> Optional[float]: + import ctypes + from ctypes import wintypes + + class _Counters(ctypes.Structure): + _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), ("PeakPagefileUsage", ctypes.c_size_t)] + + counters = _Counters() + counters.cb = ctypes.sizeof(counters) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.GetCurrentProcess.restype = wintypes.HANDLE + kernel32.K32GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(_Counters), wintypes.DWORD] + if not kernel32.K32GetProcessMemoryInfo(kernel32.GetCurrentProcess(), ctypes.byref(counters), counters.cb): + return None + return round(counters.WorkingSetSize / 1048576, 1) + + +def wait_for_workers(app: Any, timeout: float = 60.0) -> None: + """Pump events until the GUI's worker threads are done, or ``timeout`` seconds pass. + + A tab opened in a shown window starts its first refresh on a worker + thread. Leaving the process with ``os._exit`` while one runs is an access + violation inside ``ExitProcess`` on Windows. + """ + import threading + import time + deadline = time.monotonic() + timeout + while time.monotonic() < deadline and any(t.name.startswith("gui-worker") for t in threading.enumerate()): + app.processEvents() + time.sleep(0.02) + + +def environment(app: Any = None) -> Dict[str, Any]: + """Describe the machine and the Qt it ran on; ``app`` adds what only a live application knows.""" + info: Dict[str, Any] = { + "platform": platform.platform(), + "machine": platform.machine(), + "python": platform.python_version(), + "cpu_count": os.cpu_count(), + "qt_platform": os.environ.get("QT_QPA_PLATFORM", ""), + } + try: + import PySide6 + from PySide6.QtCore import qVersion + info["pyside6"], info["qt"] = PySide6.__version__, qVersion() + except ImportError: + info["pyside6"] = info["qt"] = None + try: + from importlib.metadata import PackageNotFoundError, version + try: + info["je_auto_control"] = version("je_auto_control") + except PackageNotFoundError: + info["je_auto_control"] = None + except ImportError: + info["je_auto_control"] = None + if app is not None: + info["qt_platform"] = app.platformName() + info["screens"] = [[screen.geometry().width(), screen.geometry().height(), screen.devicePixelRatio()] + for screen in app.screens()] + return info + + +def percentile(samples: Sequence[float], percent: float) -> float: + """The ``percent``-th percentile by nearest rank; 0.0 for no samples.""" + if not samples: + return 0.0 + ordered = sorted(samples) + rank = max(1, -(-len(ordered) * percent // 100)) + return float(ordered[int(min(rank, len(ordered))) - 1]) + + +def summary(samples: Iterable[float]) -> Dict[str, Any]: + """Median, extremes and the samples themselves, rounded to a tenth of a millisecond.""" + values = [round(float(value), 1) for value in samples] + if not values: + return {"median": None, "min": None, "max": None, "samples": []} + return {"median": round(statistics.median(values), 1), "min": min(values), "max": max(values), + "samples": values} + + +def _numbers(report: Dict[str, Any], prefix: str = "") -> Dict[str, float]: + found: Dict[str, float] = {} + for key, value in report.items(): + name = f"{prefix}{key}" + if key in ("environment", "samples"): + continue + if isinstance(value, dict): + found.update(_numbers(value, f"{name}.")) + elif isinstance(value, (int, float)) and not isinstance(value, bool): + found[name] = float(value) + return found + + +def compare(before: Dict[str, Any], after: Dict[str, Any]) -> Dict[str, Any]: + """Set two reports of the same workload on the same environment side by side. + + Reports that differ in benchmark, workload or environment are not + compared: the result says which keys differ and carries no numbers. + """ + differing: List[str] = [key for key in COMPARABLE_KEYS if before.get(key) != after.get(key)] + env_before, env_after = before.get("environment", {}), after.get("environment", {}) + differing += [f"environment.{key}" for key in COMPARABLE_ENVIRONMENT if env_before.get(key) != env_after.get(key)] + if differing: + return {"comparable": False, "differs_in": differing, "changes": {}} + old, new = _numbers(before), _numbers(after) + changes = {} + for name in sorted(set(old) & set(new)): + delta = round(new[name] - old[name], 1) + percent = round(delta / old[name] * 100, 1) if old[name] else None + changes[name] = {"before": old[name], "after": new[name], "delta": delta, "percent": percent} + return {"comparable": True, "differs_in": [], "changes": changes} + + +def load(path: str) -> Dict[str, Any]: + """Read a report written by :func:`emit`.""" + with open(path, encoding="utf-8") as handle: + loaded: Dict[str, Any] = json.load(handle) + return loaded + + +def emit(report: Dict[str, Any], output: Optional[str] = None) -> None: + """Write ``report`` as JSON to ``output``, or to stdout.""" + text = json.dumps(report, indent=2, sort_keys=True) + if output: + with open(output, "w", encoding="utf-8") as handle: + handle.write(text + "\n") + return + sys.stdout.write(text + "\n") + sys.stdout.flush() diff --git a/benchmarks/gui_startup.py b/benchmarks/gui_startup.py new file mode 100644 index 000000000..915dc60f6 --- /dev/null +++ b/benchmarks/gui_startup.py @@ -0,0 +1,117 @@ +"""How long the GUI takes to start, and what it costs in memory, from a cold interpreter. + + python benchmarks/gui_startup.py --runs 5 --output after.json + python benchmarks/gui_startup.py --compare before.json after.json + +Each run is a new child interpreter, so imports are paid every time. The +window is built and painted on the ``offscreen`` platform plugin: nothing +appears on the desktop, and the user's GUI settings are neither read nor +written. ``startup_ms`` is from the first line of the child to the first +painted frame; ``process_ms`` adds starting and stopping the interpreter. +""" +import argparse +import json +import os +import subprocess # nosec B404 # reason: starts this script's own child with a fixed argv +import sys +import time +from typing import Any, Dict, List, Optional + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import _gui_bench as bench # noqa: E402 + +WORKLOAD = "main window, default tabs, first frame" + + +def measure_once() -> Dict[str, Any]: + """Start the GUI in this process and return its timings (the child side).""" + start = time.perf_counter() + bench.prepare_process() + from PySide6.QtWidgets import QApplication + qt_imported = time.perf_counter() + from je_auto_control.gui.main_window import AutoControlGUIUI + from je_auto_control.gui.theme import prepare_application + imported = time.perf_counter() + app = QApplication.instance() or QApplication([]) + prepare_application(app) + window = AutoControlGUIUI() + built = time.perf_counter() + window.show() + app.processEvents() + window.grab() + painted = time.perf_counter() + entries = window.auto_control_gui_widget._tab_entries # noqa: SLF001 # reason: counting built tabs + return { + "startup_ms": (painted - start) * 1000, + "phases_ms": {"import_qt": (qt_imported - start) * 1000, "import_gui": (imported - qt_imported) * 1000, + "build_window": (built - imported) * 1000, "first_frame": (painted - built) * 1000}, + "memory": {"rss_mb": bench.rss_mb()}, + "tabs_built": sum(1 for entry in entries if entry.built), + "tabs_registered": len(entries), + "modules_loaded": sum(1 for name in sys.modules if name.startswith("je_auto_control")), + "environment": bench.environment(app), + } + + +def _child() -> None: + sys.stdout.write(json.dumps(measure_once())) + sys.stdout.flush() + # Some tabs start helper threads when built; the numbers are out, so skip teardown. + os._exit(0) + + +def _run_child() -> Dict[str, Any]: + env = dict(os.environ) + env.setdefault("QT_QPA_PLATFORM", "offscreen") + env[bench.SETTINGS_ENV] = "off" + start = time.perf_counter() + done = subprocess.run([sys.executable, os.path.abspath(__file__), "--child"], # nosec B603 # reason: fixed argv + capture_output=True, text=True, env=env, timeout=300, check=False) + elapsed = (time.perf_counter() - start) * 1000 + if done.returncode != 0: + raise SystemExit(f"benchmark child failed ({done.returncode}):\n{done.stderr[-2000:]}") + result: Dict[str, Any] = json.loads(done.stdout) + result["process_ms"] = elapsed + return result + + +def run(runs: int) -> Dict[str, Any]: + """Start the GUI ``runs`` times in child interpreters and summarise.""" + results = [_run_child() for _ in range(max(runs, 1))] + phases = results[0]["phases_ms"] + memory = [r["memory"]["rss_mb"] for r in results if r["memory"]["rss_mb"] is not None] + return { + "benchmark": "gui_startup", + "workload": WORKLOAD, + "runs": len(results), + "environment": results[0]["environment"], + "startup_ms": bench.summary(r["startup_ms"] for r in results), + "process_ms": bench.summary(r["process_ms"] for r in results), + "phases_ms": {name: bench.summary(r["phases_ms"][name] for r in results) for name in phases}, + "memory": {"rss_mb": bench.summary(memory)}, + "tabs_built": results[0]["tabs_built"], + "tabs_registered": results[0]["tabs_registered"], + "modules_loaded": results[0]["modules_loaded"], + } + + +def main(argv: Optional[List[str]] = None) -> int: + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--runs", type=int, default=5, help="cold starts to measure (default 5)") + parser.add_argument("--output", help="write the JSON report here instead of stdout") + parser.add_argument("--compare", nargs=2, metavar=("BEFORE", "AFTER"), help="compare two saved reports") + parser.add_argument("--child", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args(argv) + if args.child: + _child() + if args.compare: + bench.emit(bench.compare(bench.load(args.compare[0]), bench.load(args.compare[1])), args.output) + return 0 + bench.emit(run(args.runs), args.output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/gui_workloads.py b/benchmarks/gui_workloads.py new file mode 100644 index 000000000..362433bf3 --- /dev/null +++ b/benchmarks/gui_workloads.py @@ -0,0 +1,151 @@ +"""What using the GUI costs: first open of each tab, switching, searching, and how long the event loop stalls. + + python benchmarks/gui_workloads.py --output after.json + python benchmarks/gui_workloads.py --tabs record,variables,scheduler + python benchmarks/gui_workloads.py --compare before.json after.json + +One process, the ``offscreen`` platform plugin, the user's GUI settings left +alone. Every step runs from the event loop while a 5 ms timer ticks; how late +each tick arrives is the stall a user would feel, and ``event_loop_p95_ms`` is +its 95th percentile over the whole workload. ``idle_event_loop_p95_ms`` is the +same with every opened tab left running and nothing else happening. +""" +import argparse +import os +import sys +import time +from typing import Any, Callable, Dict, List, Optional + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import _gui_bench as bench # noqa: E402 + +TICK_MS = 5 +IDLE_MS = 500 +SEARCHES = ("rec", "script", "usb", "remote desktop", "zzz", "") + + +class _Ticker: + """Records how late each timer tick arrives.""" + + def __init__(self, timer_type: Any) -> None: + self.late_ms: List[float] = [] + self._last = time.perf_counter() + self.timer = timer_type() + self.timer.setInterval(TICK_MS) + self.timer.timeout.connect(self._tick) + + def _tick(self) -> None: + now = time.perf_counter() + self.late_ms.append(max((now - self._last) * 1000 - TICK_MS, 0.0)) + self._last = now + + def restart(self) -> List[float]: + """Return the samples so far and start a new series.""" + taken, self.late_ms = self.late_ms, [] + self._last = time.perf_counter() + return taken + + +def _timed(step: Callable[[], Any], app: Any) -> float: + start = time.perf_counter() + step() + app.processEvents() + return (time.perf_counter() - start) * 1000 + + +def _steps(window: Any, keys: List[str], app: Any, results: Dict[str, Any]) -> List[Callable[[], None]]: + workspace, navigation = window.auto_control_gui_widget, window.navigation + first_open: Dict[str, float] = results.setdefault("first_open_ms", {}) + switch: List[float] = results.setdefault("_switch", []) + search: List[float] = results.setdefault("_search", []) + theme: List[float] = results.setdefault("_theme", []) + steps: List[Callable[[], None]] = [] + for key in keys: + steps.append(lambda key=key: first_open.__setitem__( + key, round(_timed(lambda: workspace.activate_tab(key), app), 1))) + for key in keys: + steps.append(lambda key=key: switch.append(_timed(lambda: workspace.activate_tab(key), app))) + for text in SEARCHES: + steps.append(lambda text=text: search.append(_timed(lambda: navigation.search.setText(text), app))) + for name in ("light", "dark"): + steps.append(lambda name=name: theme.append(_timed(lambda: window.set_theme(name), app))) + return steps + + +def run(tabs: Optional[List[str]] = None) -> Dict[str, Any]: + """Drive the workload in this process and return the report.""" + bench.prepare_process() + from PySide6.QtCore import QTimer + from PySide6.QtWidgets import QApplication + + from je_auto_control.gui.main_window import AutoControlGUIUI + from je_auto_control.gui.theme import prepare_application + app = QApplication.instance() or QApplication([]) + prepare_application(app) + window = AutoControlGUIUI() + window.show() + app.processEvents() + registered = [row["key"] for row in window.auto_control_gui_widget.list_registered_tabs()] + keys = [key for key in (tabs or registered) if key in registered] + results: Dict[str, Any] = {} + steps = _steps(window, keys, app, results) + ticker = _Ticker(QTimer) + driver = QTimer() + driver.setInterval(0) + + def advance() -> None: + if steps: + steps.pop(0)() + return + driver.stop() + results["_work"] = ticker.restart() + QTimer.singleShot(IDLE_MS, app.quit) + + driver.timeout.connect(advance) + ticker.timer.start() + driver.start() + app.exec() + idle = ticker.restart() + work = results.pop("_work", []) + opened = list(results["first_open_ms"].values()) + return { + "benchmark": "gui_workloads", + "workload": "open, revisit, search, theme: " + ",".join(keys), + "environment": bench.environment(app), + "tabs_opened": len(keys), + "first_open_ms": results["first_open_ms"], + "first_open_summary_ms": {"median": bench.percentile(opened, 50), "p95": bench.percentile(opened, 95), + "max": max(opened, default=0.0), "total": round(sum(opened), 1)}, + "switch_ms": bench.summary(results.pop("_switch")), + "search_ms": bench.summary(results.pop("_search")), + "theme_switch_ms": bench.summary(results.pop("_theme")), + "event_loop_p95_ms": round(bench.percentile(work, 95), 1), + "event_loop_max_ms": round(max(work, default=0.0), 1), + "event_loop_ticks": len(work), + "idle_event_loop_p95_ms": round(bench.percentile(idle, 95), 1), + "memory": {"rss_mb": bench.rss_mb()}, + } + + +def main(argv: Optional[List[str]] = None) -> int: + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--tabs", help="comma-separated tab keys (default: every registered tab)") + parser.add_argument("--output", help="write the JSON report here instead of stdout") + parser.add_argument("--compare", nargs=2, metavar=("BEFORE", "AFTER"), help="compare two saved reports") + args = parser.parse_args(argv) + if args.compare: + bench.emit(bench.compare(bench.load(args.compare[0]), bench.load(args.compare[1])), args.output) + return 0 + tabs = [key.strip() for key in args.tabs.split(",") if key.strip()] if args.tabs else None + bench.emit(run(tabs), args.output) + # Tabs opened here started timers and helper threads; the report is out, so skip teardown + # once the workers have finished what they were doing. + from PySide6.QtWidgets import QApplication + bench.wait_for_workers(QApplication.instance()) + os._exit(0) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/source/Eng/doc/new_features/v223_features_doc.rst b/docs/source/Eng/doc/new_features/v223_features_doc.rst index 8b64c4fc4..7751a50f4 100644 --- a/docs/source/Eng/doc/new_features/v223_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v223_features_doc.rst @@ -36,24 +36,36 @@ the list as you type — by title, by key (``usb_devices``) or by category — a puts the cursor in the search box from anywhere, and ``Ctrl+B`` (**View → Navigation Panel**) hides or shows the panel. -A tab is built the first time it is opened: the window starts with the three -default tabs and the forms the main widget owns, and imports the module of -any other tab only when you open it. +A tab is built the first time it is on screen. The window starts with three +tabs in its bar — Record, Script Builder and Remote Desktop — and Record in +front; the other two are built on the first click, and every other tab when +you open it. Nothing of Script Builder or Remote Desktop (which loads the +WebRTC stack) is imported before that. -The navigation panel --------------------- +Small windows, and what the window remembers +-------------------------------------------- -Every registered tab is listed on the left of the window, grouped by the same -five categories, whether it is open or not; open tabs are shown in bold. -Click a feature to open it (or bring it to the front). The search box filters -the list as you type — by title, by key (``usb_devices``) or by category — and -**Return** opens the first match. ``Ctrl+K`` (**View → Search Features...**) -puts the cursor in the search box from anywhere, and ``Ctrl+B`` -(**View → Navigation Panel**) hides or shows the panel. +A page keeps its minimum size: in a window too small for it, the tab shows +scroll bars instead of squeezing the form. Remote Desktop already scrolls its +own content and is left as it is. For code that embeds +``AutoControlGUIWidget``, ``widget.tabs`` still speaks in pages — +``tabs.indexOf(page)``, ``tabs.widget(index)``, ``tabs.currentWidget()`` and +``tabs.setCurrentWidget(page)`` take and return the tab's own widget, not the +scroll area around it. -A tab is built the first time it is opened: the window starts with the three -default tabs and the forms the main widget owns, and imports the module of -any other tab only when you open it. +The theme, the text size, whether the navigation panel is shown and how wide +it is, and the window's position and size are kept between runs in +``~/.je_auto_control/gui_settings.ini``. Set ``JE_AUTOCONTROL_GUI_SETTINGS`` +to another file to keep them elsewhere, or to ``off`` to remember nothing +(the test suite runs that way). + +Closing a tab keeps its widget, so reopening it shows it as you left it. +``AutoControlGUIWidget.close_tab(key, release=True)`` lets go of it instead: +the widget's optional ``dispose()`` method is called — the place for a tab to +stop its timers and threads and drop its listeners — the widget is deleted, +and the next ``open_tab(key)`` builds a new one. The forms the main widget +builds for itself (Auto Click, Screenshot, Image Detection, Record, Script, +Report) are closed but never released. The View menu ------------- @@ -79,3 +91,29 @@ non-empty ``label_key`` string paired with a callable. A new tab that forgets the hook fails CI instead of silently shipping with no reachable commands. The probe runs the full widget construction in a subprocess so the Qt lifetime cannot destabilise the rest of the headless suite. + +``test/unit_test/headless/test_gui_feature_parity.py`` holds the rest: the +window opens on the three workflow tabs, every registered tab opens from the +navigation panel and fills the Actions menu, a window of 640×420 scrolls +instead of clipping, and the mixed-DPI geometry helpers agree with a +100% + 125% desktop. + +Measuring the GUI +----------------- + +Two scripts print a JSON report and never show a window (they run on Qt's +``offscreen`` platform and leave the GUI settings file alone): + +.. code-block:: bash + + python benchmarks/gui_startup.py --runs 5 --output after.json + python benchmarks/gui_workloads.py --output workloads.json + python benchmarks/gui_startup.py --compare before.json after.json + +``gui_startup.py`` starts the window in fresh interpreters and reports +``startup_ms`` (first line to first painted frame), its phases, and the +process's memory. ``gui_workloads.py`` opens every tab once +(``first_open_ms`` per tab), revisits them, types in the search box and +switches theme while a 5 ms timer ticks; ``event_loop_p95_ms`` is how late +those ticks arrive. ``--compare`` sets two reports side by side and refuses +when the workload or the environment differs. diff --git a/docs/source/Zh/doc/new_features/v223_features_doc.rst b/docs/source/Zh/doc/new_features/v223_features_doc.rst index 6f0e01764..6a20ec045 100644 --- a/docs/source/Zh/doc/new_features/v223_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v223_features_doc.rst @@ -31,20 +31,29 @@ Actions 選單 ``Ctrl+K``(**View → Search Features...**)從任何地方把游標移到搜尋框, ``Ctrl+B``(**View → Navigation Panel**)隱藏或顯示面板。 -分頁在第一次開啟時才建立:視窗啟動時只有三個預設分頁與主元件自己的表單,其他 -分頁的模組要到開啟時才匯入。 +分頁在第一次顯示時才建立。視窗啟動時分頁列上有三個分頁——錄製、Script Builder +與遠端桌面——錄製在最前面;另外兩個在第一次點到時才建立,其他分頁則在開啟時建立。 +在那之前,Script Builder 與遠端桌面(它會載入 WebRTC 相關套件)都不會被匯入。 -導覽面板 --------- +小視窗,以及視窗會記住的事 +-------------------------- -視窗左側依同樣五個分類列出每一個已註冊的分頁,不論是否已開啟;已開啟的以粗體 -顯示。點一下就開啟該功能(已開啟則切到最前面)。搜尋框會隨輸入過濾清單——比對 -標題、鍵名(``usb_devices``)或分類——按 **Return** 開啟第一個符合的項目。 -``Ctrl+K``(**View → Search Features...**)從任何地方把游標移到搜尋框, -``Ctrl+B``(**View → Navigation Panel**)隱藏或顯示面板。 +分頁內容保有自己的最小尺寸:視窗小到放不下時,分頁會出現捲軸,而不是把表單擠扁。 +遠端桌面本來就自己捲動內容,維持原樣。對於內嵌 ``AutoControlGUIWidget`` 的程式, +``widget.tabs`` 仍然以分頁內容為單位——``tabs.indexOf(page)``、 +``tabs.widget(index)``、``tabs.currentWidget()`` 與 ``tabs.setCurrentWidget(page)`` +收與回的都是分頁自己的 widget,不是包在外面的捲動區。 + +主題、字級、導覽面板是否顯示與寬度、視窗位置與大小,會存在 +``~/.je_auto_control/gui_settings.ini``,下次啟動時還原。把 +``JE_AUTOCONTROL_GUI_SETTINGS`` 設成另一個檔案可以換位置,設成 ``off`` 則什麼都 +不記(測試套件就是這樣跑的)。 -分頁在第一次開啟時才建立:視窗啟動時只有三個預設分頁與主元件自己的表單,其他 -分頁的模組要到開啟時才匯入。 +關閉分頁會保留它的 widget,再開啟時維持離開時的樣子。 +``AutoControlGUIWidget.close_tab(key, release=True)`` 則是放掉它:先呼叫 widget +選擇性實作的 ``dispose()``——分頁在這裡停掉計時器與執行緒、移除它註冊的監聽—— +再刪除 widget,下一次 ``open_tab(key)`` 會建立新的。主元件自己建立的表單(自動點擊、 +截圖、影像偵測、錄製、腳本、報告)只會關閉,不會被放掉。 View 選單 --------- @@ -66,3 +75,25 @@ View 選單 新分頁若忘了掛鉤會直接讓 CI 失敗,而不是默默出貨一個沒有任何可觸及指令 的分頁。探測程序在子行程中建構完整 widget,使 Qt 生命週期不會影響無頭 測試套件的其餘部分。 + +``test/unit_test/headless/test_gui_feature_parity.py`` 守其餘的部分:視窗以三個 +工作流程分頁開啟、每個已註冊分頁都能從導覽面板開啟並填入 Actions 選單、640×420 +的視窗會捲動而不是裁切、混合 DPI 的座標換算與 100% + 125% 桌面的實測一致。 + +量測 GUI +-------- + +兩支腳本輸出 JSON 報告,而且不會顯示視窗(使用 Qt 的 ``offscreen`` 平台,也不碰 +GUI 設定檔): + +.. code-block:: bash + + python benchmarks/gui_startup.py --runs 5 --output after.json + python benchmarks/gui_workloads.py --output workloads.json + python benchmarks/gui_startup.py --compare before.json after.json + +``gui_startup.py`` 在全新的直譯器裡啟動視窗,回報 ``startup_ms``(從第一行到畫出 +第一個畫面)、各階段時間與行程的記憶體。``gui_workloads.py`` 把每個分頁各開一次 +(每個分頁的 ``first_open_ms``)、再切回去、在搜尋框輸入、切換主題,同時有一個 5 ms +的計時器在跳;``event_loop_p95_ms`` 就是那些 tick 晚到多久。``--compare`` 把兩份 +報告並排,工作量或環境不同時拒絕比較。 diff --git a/test/unit_test/headless/test_gui_feature_parity.py b/test/unit_test/headless/test_gui_feature_parity.py new file mode 100644 index 000000000..d28797847 --- /dev/null +++ b/test/unit_test/headless/test_gui_feature_parity.py @@ -0,0 +1,323 @@ +"""Nothing was lost in the GUI rewrite: every feature is reachable, and the numbers can be measured. + +The window opens on the three workflow tabs; every registered tab can be +reached from the navigation panel and the View menu and still puts its +commands in the Actions menu; a feature that cannot run says why; the mixed-DPI +geometry helpers agree with what was measured on a 100% + 125% desktop; +``import je_auto_control`` loads no Qt; and both GUI benchmarks produce the +report they document. Offscreen Qt throughout, with fake screens for the +geometry: nothing is shown and no pointer or key is touched. +""" +import importlib.util +import json +import os +import pathlib +import subprocess # nosec B404 # reason: runs this file's own probes with a fixed argv +import sys +import types + +import pytest + +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtCore import QPointF, QRect # noqa: E402 +from PySide6.QtWidgets import QApplication, QLabel # noqa: E402 + +from je_auto_control.gui import _screen_geometry as geo # noqa: E402 +from je_auto_control.gui.tab_registry import TAB_SPECS # noqa: E402 + +ROOT = pathlib.Path(__file__).resolve().parents[3] +BENCHMARKS = ROOT / "benchmarks" +DEFAULT_WORKFLOW = ["record", "script_builder", "remote_desktop"] +# Interactive panels that keep their own controls instead of using the Actions menu. +MENU_EXEMPT_TABS = {"script_builder", "remote_desktop"} + + +def _child(argv, timeout=240, extra_env=None): + env = dict(os.environ, PYTHONPATH=str(ROOT), QT_QPA_PLATFORM="offscreen") + env.update(extra_env or {}) + done = subprocess.run([sys.executable, *argv], capture_output=True, text=True, # nosec B603 # nosemgrep # reason: fixed argv, no shell + timeout=timeout, env=env, cwd=str(ROOT), check=False) + assert done.returncode == 0, done.stderr[-2000:] + return done.stdout + + +# --- the facade stays Qt-free --------------------------------------------------------------------- + +def test_import_facade_is_qt_free(): + probe = ("import sys, je_auto_control; " + "print(sorted(name for name in sys.modules if name.split('.')[0] in ('PySide6', 'shiboken6')))") + assert _child(["-c", probe]).strip() == "[]" + + +# --- the real window: default workflow, full catalogue, small window ------------------------------ + +_PROBE = r""" +import json, os, sys, threading, time +os.environ["QT_QPA_PLATFORM"] = "offscreen" +from PySide6.QtWidgets import QApplication, QScrollArea, QTabWidget +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.gui.main_window import AutoControlGUIUI + +app = QApplication([]) +window = AutoControlGUIUI() +window.show() +app.processEvents() +workspace, navigation = window.auto_control_gui_widget, window.navigation +entries = {entry.key: entry for entry in workspace._tab_entries} +report = { + "registered": list(entries), + "default_keys": [row["key"] for row in workspace.list_registered_tabs() if row["visible"]], + "front": workspace.current_tab_key(), + "navigation_lists": navigation.visible_keys(), + "view_menu_lists": [action.data() for action in window._tab_actions], + "shortcuts_enabled": [window._search_action.isEnabled(), window._sidebar_action.isEnabled()], +} + +unreachable, no_actions, menu_mismatch = [], [], [] +for key in report["navigation_lists"]: + navigation.feature_activated.emit(key) + app.processEvents() + entry = entries[key] + if workspace.current_tab_key() != key or workspace.tabs.indexOf(entry.widget) == -1: + unreachable.append(key) + continue + offered = workspace.current_tab_menu_actions() + shown = [action for action in window._actions_menu.actions() if action.isEnabled()] + if key not in ("script_builder", "remote_desktop") and not offered: + no_actions.append(key) + if len(shown) != len(offered): + menu_mismatch.append([key, len(shown), len(offered)]) +report.update(unreachable=unreachable, no_actions=no_actions, menu_mismatch=menu_mismatch, + all_open=[row["key"] for row in workspace.list_registered_tabs() if row["visible"]]) + +# Language and text size change under a full workspace: the list must still be whole and still open tabs. +window.set_text_size(20) +language_wrapper.reset_language("Traditional_Chinese") +app.processEvents() +report["after_language"] = navigation.visible_keys() +report["translated_title"] = workspace.tabs.tabText(workspace.tabs.indexOf(entries["record"].widget)) +workspace.hide_tab("variables") +navigation.search.setText("variables") +navigation.activate_first_match() +report["opened_after_language"] = workspace.current_tab_key() +navigation.search.setText("") +language_wrapper.reset_language("English") +window.set_text_size(0) + +# A small window: the page keeps its minimum size and its tab scrolls. +window.resize(640, 420) +workspace.activate_tab("auto_click") +for _ in range(3): + app.processEvents() +page = entries["auto_click"].widget +area = QTabWidget.widget(workspace.tabs, workspace.tabs.indexOf(page)).findChild(QScrollArea) +need = page.minimumSizeHint() +report["small"] = { + "window": [window.width(), window.height()], + "page_keeps_minimum": page.width() >= need.width() and page.height() >= need.height(), + "taller_than_view": need.height() > area.viewport().height(), + "scrolls": area.verticalScrollBar().maximum() > 0, +} +sys.stdout.write(json.dumps(report)) +sys.stdout.flush() +# Tabs opened in a shown window start their first refresh on a worker thread (the USB tabs +# enumerate through a child process). Leaving while one runs was an access violation in +# ExitProcess on Windows, two runs in three, so let them finish first. +deadline = time.monotonic() + 60 +while time.monotonic() < deadline and any(t.name.startswith("gui-worker") for t in threading.enumerate()): + app.processEvents() + time.sleep(0.02) +os._exit(0) +""" + + +@pytest.fixture(scope="module") +def window(): + return json.loads(_child(["-c", _PROBE])) + + +def test_default_workflow_and_full_catalog(window): + default_keys = window["default_keys"] + assert default_keys == ['record', 'script_builder', 'remote_desktop'] + assert default_keys == DEFAULT_WORKFLOW and window["front"] == "record" + registered = [spec.key for spec in TAB_SPECS] + assert window["registered"] == registered + assert sorted(window["navigation_lists"]) == sorted(registered) + assert sorted(window["view_menu_lists"]) == sorted(registered) + + +def test_every_registered_tab_opens_from_the_navigation_panel(window): + assert window["unreachable"] == [] + assert sorted(window["all_open"]) == sorted(window["registered"]) + + +def test_every_tab_still_puts_its_commands_in_the_actions_menu(window): + assert window["no_actions"] == [] + assert window["menu_mismatch"] == [] + + +def test_search_and_panel_shortcuts_are_live(window): + # Ctrl+B was the dock's own toggle action, which Qt disables for a dock that cannot be closed. + assert window["shortcuts_enabled"] == [True, True] + + +def test_navigation_survives_a_language_and_text_size_change(window): + assert window["after_language"] == window["navigation_lists"] + assert window["translated_title"] != "Record / Playback" + assert window["opened_after_language"] == "variables" + + +def test_small_window_is_usable(window): + small = window["small"] + assert small["window"] == [640, 420] + horizontal_content_clipped = not small["page_keeps_minimum"] + assert horizontal_content_clipped is False + assert small["taller_than_view"] and small["scrolls"] + + +# --- a feature that cannot run says why ----------------------------------------------------------- + +@pytest.fixture(scope="module") +def qapp(): + app = QApplication.instance() or QApplication([]) + yield app + + +def test_unsupported_feature_has_reason(qapp, monkeypatch): + from je_auto_control.gui.main_widget import AutoControlGUIWidget + + # ``None`` in sys.modules makes the import fail the way a missing extra does. + monkeypatch.setitem(sys.modules, "je_auto_control.gui.remote_desktop_tab", None) + placeholder = AutoControlGUIWidget._build_remote_desktop_tab() # noqa: SLF001 + text = " ".join(label.text() for label in placeholder.findChildren(QLabel)) + unsupported_feature_has_reason = ("pip install je_auto_control[webrtc]" in text + and "Underlying error: ModuleNotFoundError" in text) + assert unsupported_feature_has_reason is True + placeholder.deleteLater() + + +# --- mixed-DPI geometry, with fake screens -------------------------------------------------------- + +class _Screen: + def __init__(self, x, y, width, height, ratio): + self._geometry, self._ratio = QRect(x, y, width, height), ratio + + def geometry(self): + return self._geometry + + def devicePixelRatio(self): # noqa: N802 + return self._ratio + + +# Measured on Windows in a per-monitor-aware process: the 125% screen at (1920, -164) is +# 1536x864 to Qt and 1920x1080 natively; both keep their top-left corner. +PRIMARY = _Screen(0, 0, 1920, 1080, 1.0) +SCALED = _Screen(1920, -164, 1536, 864, 1.25) + + +@pytest.fixture +def desktop(monkeypatch): + monkeypatch.setattr(geo, "sys", types.SimpleNamespace(platform="win32")) + monkeypatch.setattr(geo, "QGuiApplication", types.SimpleNamespace(screens=lambda: [PRIMARY, SCALED])) + + +def test_mixed_dpi_geometry(desktop): + assert geo.capture_ratio(PRIMARY) == 1.0 and geo.capture_ratio(SCALED) == 1.25 + # Scaled within the screen, anchored at its corner. + assert geo.native_region(PRIMARY, QRect(100, 50, 200, 100)) == (100, 50, 200, 100) + assert geo.native_region(SCALED, QRect(0, 0, 1536, 864)) == (1920, -164, 1920, 1080) + assert geo.native_region(SCALED, QRect(100, 40, 200, 100)) == (2045, -114, 250, 125) + + +def test_a_native_point_finds_the_screen_it_is_on(desktop): + assert geo.screen_at_native(0, 0) is PRIMARY and geo.screen_at_native(1919, 1079) is PRIMARY + assert geo.screen_at_native(1920, -164) is SCALED + # Inside the scaled screen's native rectangle, outside its logical one. + assert geo.screen_at_native(1920 + 1900, -164 + 1000) is SCALED + assert geo.screen_at_native(1920 + 1920, 0) is None + assert geo.screen_at_native(1920, -165) is None and geo.screen_at_native(500, 1080) is None + + +def test_native_and_logical_points_round_trip(desktop): + assert geo.logical_point(PRIMARY, 640, 480) == QPointF(640, 480) + assert geo.logical_point(SCALED, 1920 + 1250, -164 + 625) == QPointF(1920 + 1000, -164 + 500) + for x, y in ((0, 0), (300, 200), (1535, 863)): + left, top, _width, _height = geo.native_region(SCALED, QRect(x, y, 1, 1)) + back = geo.logical_point(SCALED, left, top) + assert abs(back.x() - (1920 + x)) <= 0.5 and abs(back.y() - (-164 + y)) <= 0.5 + + +def test_macos_takes_points_whatever_the_ratio(monkeypatch): + monkeypatch.setattr(geo, "sys", types.SimpleNamespace(platform="darwin")) + retina = _Screen(0, 0, 1512, 982, 2.0) + monkeypatch.setattr(geo, "QGuiApplication", types.SimpleNamespace(screens=lambda: [retina])) + assert geo.capture_ratio(retina) == 1.0 + assert geo.native_region(retina, QRect(10, 20, 300, 200)) == (10, 20, 300, 200) + assert geo.screen_at_native(1511, 981) is retina and geo.screen_at_native(1512, 0) is None + assert geo.logical_point(retina, 700, 400) == QPointF(700, 400) + + +# --- the benchmarks ------------------------------------------------------------------------------- + +def _load(name): + spec = importlib.util.spec_from_file_location(name, BENCHMARKS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_percentiles_and_summaries(): + bench = _load("_gui_bench") + samples = list(range(1, 101)) + assert bench.percentile(samples, 95) == 95 and bench.percentile(samples, 50) == 50 + assert bench.percentile([7.0], 95) == 7.0 and bench.percentile([], 95) == 0.0 + assert bench.summary([3, 1, 2]) == {"median": 2, "min": 1, "max": 3, "samples": [3, 1, 2]} + assert bench.summary([])["median"] is None + + +def test_compare_needs_the_same_workload_and_environment(): + bench = _load("_gui_bench") + env = {"platform": "W", "machine": "x", "python": "3", "pyside6": "6", "qt_platform": "offscreen", + "screens": [[800, 800, 1.0]]} + before = {"benchmark": "gui_startup", "workload": "w", "environment": env, + "startup_ms": {"median": 2000.0, "samples": [2000.0]}, "memory": {"rss_mb": {"median": 100.0}}} + after = {"benchmark": "gui_startup", "workload": "w", "environment": dict(env, cpu_count=8), + "startup_ms": {"median": 1500.0, "samples": [1500.0]}, "memory": {"rss_mb": {"median": 110.0}}} + result = bench.compare(before, after) + assert result["comparable"] is True + assert result["changes"]["startup_ms.median"] == {"before": 2000.0, "after": 1500.0, "delta": -500.0, + "percent": -25.0} + assert result["changes"]["memory.rss_mb.median"]["delta"] == 10.0 + other = bench.compare(before, dict(after, environment=dict(env, qt_platform="windows"), workload="x")) + assert other == {"comparable": False, "differs_in": ["workload", "environment.qt_platform"], "changes": {}} + + +def test_the_startup_benchmark_reports_what_it_documents(tmp_path): + report_path = tmp_path / "startup.json" + _child([str(BENCHMARKS / "gui_startup.py"), "--runs", "1", "--output", str(report_path)]) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["benchmark"] == "gui_startup" and report["runs"] == 1 + assert report["environment"]["qt_platform"] == "offscreen" and report["environment"]["python"] + assert report["startup_ms"]["median"] > 0 and report["process_ms"]["median"] >= report["startup_ms"]["median"] + assert set(report["phases_ms"]) == {"import_qt", "import_gui", "build_window", "first_frame"} + assert report["memory"]["rss_mb"]["median"] > 0 + assert report["tabs_registered"] == len(TAB_SPECS) and report["tabs_built"] < 10 + compared = json.loads(_child([str(BENCHMARKS / "gui_startup.py"), "--compare", + str(report_path), str(report_path)], timeout=60)) + assert compared["comparable"] is True and compared["changes"]["startup_ms.median"]["delta"] == 0 + + +def test_the_workload_benchmark_reports_what_it_documents(tmp_path): + report_path = tmp_path / "workloads.json" + _child([str(BENCHMARKS / "gui_workloads.py"), "--tabs", "record,variables,no_such_tab", + "--output", str(report_path)]) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["benchmark"] == "gui_workloads" and report["tabs_opened"] == 2 + assert set(report["first_open_ms"]) == {"record", "variables"} + assert all(value >= 0 for value in report["first_open_ms"].values()) + assert report["event_loop_p95_ms"] >= 0 and report["event_loop_ticks"] > 0 + assert report["idle_event_loop_p95_ms"] >= 0 + assert report["memory"]["rss_mb"] > 0 and report["environment"]["qt_platform"] == "offscreen" + assert len(report["switch_ms"]["samples"]) == 2 and len(report["theme_switch_ms"]["samples"]) == 2 From 4404cd4b17b8f61c3d4fa025fc9295f1f7a1e04b Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:07:09 +0800 Subject: [PATCH 32/44] Name the optional WebRTC classes as types in the split panel modules The package exports them as variables that are None without the extra, which a type checker cannot use in an annotation; the split modules annotated with them and the contract failed on three targets. --- .../gui/remote_desktop/_webrtc_types.py | 26 +++++++++++++++++++ .../remote_desktop/webrtc_host_connection.py | 14 +++++----- .../gui/remote_desktop/webrtc_panel.py | 8 +++--- .../gui/remote_desktop/webrtc_panel_common.py | 3 ++- .../webrtc_viewer_connection.py | 12 +++++---- .../gui/remote_desktop/webrtc_viewer_media.py | 3 ++- 6 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 je_auto_control/gui/remote_desktop/_webrtc_types.py diff --git a/je_auto_control/gui/remote_desktop/_webrtc_types.py b/je_auto_control/gui/remote_desktop/_webrtc_types.py new file mode 100644 index 000000000..4ae91a9e2 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/_webrtc_types.py @@ -0,0 +1,26 @@ +"""The optional WebRTC classes as types, for annotations in the panel modules. + +``je_auto_control.utils.remote_desktop`` exports these names as variables that +are ``None`` without the ``webrtc`` extra, which is right at run time and +useless to a type checker ("variable is not valid as a type"). Annotations use +the names below instead; calls keep using the package's own names. +""" +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from je_auto_control.utils.remote_desktop.multi_viewer import ( + MultiViewerHost as MultiViewerHostT, + ) + from je_auto_control.utils.remote_desktop.session_recorder import ( + SessionRecorder as SessionRecorderT, + ) + from je_auto_control.utils.remote_desktop.webrtc_transport import ( + WebRTCConfig as WebRTCConfigT, + ) + from je_auto_control.utils.remote_desktop.webrtc_viewer import ( + WebRTCDesktopViewer as WebRTCDesktopViewerT, + ) +else: + MultiViewerHostT = SessionRecorderT = WebRTCConfigT = WebRTCDesktopViewerT = Any + +__all__ = ["MultiViewerHostT", "SessionRecorderT", "WebRTCConfigT", "WebRTCDesktopViewerT"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_connection.py b/je_auto_control/gui/remote_desktop/webrtc_host_connection.py index 6cf216e1c..448969f39 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_connection.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_connection.py @@ -13,6 +13,7 @@ QMessageBox, ) +from je_auto_control.gui.remote_desktop._webrtc_types import MultiViewerHostT from je_auto_control.gui.remote_desktop._helpers import ( _t, ) @@ -33,12 +34,12 @@ from je_auto_control.utils.remote_desktop.lan_discovery import HostAdvertiser -def _create_offer(host: MultiViewerHost, _token: CancellationToken) -> Tuple[str, str]: +def _create_offer(host: MultiViewerHostT, _token: CancellationToken) -> Tuple[str, str]: """Worker thread: mint a session and its offer (ICE gathering, up to 12 s).""" return host.create_session_offer() -def _drop_offer(host: MultiViewerHost, outcome: Tuple[str, str]) -> None: +def _drop_offer(host: MultiViewerHostT, outcome: Tuple[str, str]) -> None: """End the session of an offer nobody will show.""" try: host.stop_session(outcome[0]) @@ -50,6 +51,7 @@ class _HostConnectionMixin(_PanelPart): """Methods of ``_WebRTCHostPanel``; the module docstring says which group.""" # State this group owns; the panel's __init__ sets the starting values. + _multi_host: Optional[MultiViewerHostT] _publish_loop: Optional[HostPublishLoopWorker] _manual_session_id: Optional[str] _lan_advertiser: Optional[HostAdvertiser] @@ -138,7 +140,7 @@ def _on_generate_offer(self) -> None: self._offer_view.setPlainText("") QTimer.singleShot(0, self, self._produce_offer) - def _require_multi_host(self) -> MultiViewerHost: + def _require_multi_host(self) -> MultiViewerHostT: """Return the running host, or say the session is not up yet.""" host = self._multi_host if host is None: @@ -158,11 +160,11 @@ def _produce_offer(self) -> None: task.result.connect(functools.partial(self._show_offer, host)) task.error.connect(functools.partial(self._show_offer_error, host)) - def _show_offer_error(self, host: MultiViewerHost, error: Exception) -> None: + def _show_offer_error(self, host: MultiViewerHostT, error: Exception) -> None: if self._multi_host is host: # a host stopped meanwhile fails by design self._show_error(error) - def _show_offer(self, host: MultiViewerHost, outcome: Tuple[str, str]) -> None: + def _show_offer(self, host: MultiViewerHostT, outcome: Tuple[str, str]) -> None: """GUI thread: the offer is ready -- unless the host was stopped or replaced meanwhile.""" if self._multi_host is not host: _drop_offer(host, outcome) @@ -211,7 +213,7 @@ def _validate_required_fields(self, *, needs_server: bool) -> bool: return False return True - def _build_multi_host(self, token: str) -> MultiViewerHost: + def _build_multi_host(self, token: str) -> MultiViewerHostT: whitelist_text = self._ip_whitelist_edit.toPlainText().strip() whitelist = [line.strip() for line in whitelist_text.splitlines() if line.strip() and not line.strip().startswith("#")] diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel.py b/je_auto_control/gui/remote_desktop/webrtc_panel.py index 148a77721..1252102ee 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel.py @@ -25,6 +25,7 @@ QMessageBox, QWidget, ) +from je_auto_control.gui.remote_desktop._webrtc_types import MultiViewerHostT, SessionRecorderT, WebRTCDesktopViewerT from je_auto_control.gui._i18n_helpers import TranslatableMixin from je_auto_control.gui.remote_desktop._helpers import ( _t, @@ -48,7 +49,6 @@ ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop import ( - MultiViewerHost, SessionRecorder, WebRTCDesktopViewer, default_address_book, default_trust_list, is_webrtc_available, load_or_create_viewer_id, @@ -92,7 +92,7 @@ class _WebRTCHostPanel(_HostUiMixin, _HostTrustMixin, _HostMediaMixin, _HostSess def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) self._tr_init() - self._multi_host: Optional[MultiViewerHost] = None + self._multi_host: Optional[MultiViewerHostT] = None self._publish_loop: Optional[HostPublishLoopWorker] = None self._manual_session_id: Optional[str] = None self._adaptive_controller: Optional[AdaptiveBitrateController] = None @@ -216,7 +216,7 @@ class _WebRTCViewerPanel(_ViewerUiMixin, _ViewerFilesMixin, _ViewerAddressBookMi def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) self._tr_init() - self._viewer: Optional[WebRTCDesktopViewer] = None + self._viewer: Optional[WebRTCDesktopViewerT] = None self._offer_worker: Optional[ViewerSignalingWorker] = None self._answer_worker: Optional[ViewerAnswerPushWorker] = None self._address_book = default_address_book() @@ -227,7 +227,7 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: except OSError as error: autocontrol_logger.warning("viewer_id init: %r", error) self._viewer_id = None - self._recorder: Optional[SessionRecorder] = None + self._recorder: Optional[SessionRecorderT] = None self._stats_poller: Optional[StatsPoller] = None self._sync_engine: Optional["FolderSyncEngine"] = None self._auto_reconnect_attempts = 0 diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel_common.py b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py index 890fc7a83..386e029b3 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel_common.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py @@ -7,6 +7,7 @@ from PySide6.QtGui import QImage from PySide6.QtWidgets import QWidget +from je_auto_control.gui.remote_desktop._webrtc_types import WebRTCConfigT from je_auto_control.gui._i18n_helpers import TranslatableMixin from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -89,7 +90,7 @@ def _read_region(panel) -> Optional[tuple]: return tuple(parts) if len(parts) == 4 else None -def _read_webrtc_config(panel) -> WebRTCConfig: +def _read_webrtc_config(panel) -> WebRTCConfigT: """Build a WebRTCConfig from the advanced group + monitor/fps fields.""" from je_auto_control.utils.remote_desktop.webrtc_transport import ( _DEFAULT_STUN_SERVERS, diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py index 67d245a93..08d893bd5 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py @@ -13,6 +13,7 @@ QMessageBox, ) +from je_auto_control.gui.remote_desktop._webrtc_types import SessionRecorderT, WebRTCDesktopViewerT from je_auto_control.gui.remote_desktop._helpers import ( _t, ) @@ -23,7 +24,7 @@ from je_auto_control.gui.task_controller import CancellationToken, task_controller from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop import ( - SessionRecorder, WebRTCDesktopViewer, + WebRTCDesktopViewer, ) from je_auto_control.gui.remote_desktop.webrtc_panel_common import ( _PanelPart, @@ -34,7 +35,7 @@ from je_auto_control.utils.remote_desktop.file_sync import FolderSyncEngine -def _process_offer(viewer: WebRTCDesktopViewer, offer_sdp: str, expected_dtls: Optional[str], +def _process_offer(viewer: WebRTCDesktopViewerT, offer_sdp: str, expected_dtls: Optional[str], _token: CancellationToken) -> str: """Worker thread: answer the host's offer (up to 12 s; the backend takes no timeout or cancel).""" return viewer.process_offer(offer_sdp, expected_dtls_fingerprint=expected_dtls) @@ -44,11 +45,12 @@ class _ViewerConnectionMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" # State this group owns; the panel's __init__ sets the starting values. + _viewer: Optional[WebRTCDesktopViewerT] _offer_worker: Optional[ViewerSignalingWorker] _answer_worker: Optional[ViewerAnswerPushWorker] # Owned by the files and media groups; stopped here with the session. _sync_engine: Optional[FolderSyncEngine] - _recorder: Optional[SessionRecorder] + _recorder: Optional[SessionRecorderT] def _on_connect_via_server(self) -> None: if not self._validate_required_fields(needs_server=True): @@ -146,7 +148,7 @@ def _on_create_answer(self) -> None: self._status_label.setText(_t("rd_webrtc_creating_answer")) QTimer.singleShot(0, self, lambda: self._produce_answer(offer)) - def _require_viewer(self) -> WebRTCDesktopViewer: + def _require_viewer(self) -> WebRTCDesktopViewerT: """Return the live viewer, or say it is not connected yet.""" viewer = self._viewer if viewer is None: @@ -187,7 +189,7 @@ def _validate_required_fields(self, *, needs_server: bool) -> bool: return False return True - def _build_viewer(self, token: str) -> WebRTCDesktopViewer: + def _build_viewer(self, token: str) -> WebRTCDesktopViewerT: viewer = WebRTCDesktopViewer( token=token, config=_read_webrtc_config(self), diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py index 7aae81a41..34f9ac696 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py @@ -13,6 +13,7 @@ QFileDialog, QMessageBox, ) +from je_auto_control.gui.remote_desktop._webrtc_types import SessionRecorderT from je_auto_control.gui.remote_desktop._helpers import ( _t, ) @@ -39,7 +40,7 @@ class _ViewerMediaMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" # State this group owns; the panel's __init__ sets the starting values. - _recorder: Optional[SessionRecorder] + _recorder: Optional[SessionRecorderT] _stats_poller: Optional[StatsPoller] def _on_send_cad(self) -> None: From 5a905d5cea467a4d777f31a95755332ac9e5c654 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:07:46 +0800 Subject: [PATCH 33/44] Re-measure the documented counts for the merged second sweep --- CLAUDE.md | 6 +- README.md | 8 +- README/README_zh-CN.md | 8 +- README/README_zh-TW.md | 8 +- architecture_explore.md | 213 ++++++++++++++++++++-------------------- 5 files changed, 122 insertions(+), 121 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d7a4010c3..1c08ec92c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mouse and keyboard control, image recognition, OCR, accessibility-tree and VLM element location, action scripting, and report generation behind one API. Backends: Windows (Win32 ctypes), macOS (pyobjc/Quartz), Linux X11 (python-Xlib), Linux Wayland (libei / ydotool), Android (adb), iOS (WebDriverAgent). - **Package**: `je_auto_control` · **Python** ≥ 3.10 · **License**: MIT · **Author**: JE-Chen -- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 310 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. +- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 311 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. ## Architecture @@ -18,7 +18,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mo | Template Method | `utils/generate_report/` | HTML / JSON / XML share collect → format → write. | | Backend seam | `backends/` under `accessibility`, `ocr`, `vision`, `llm`, `agent`, `hotkey`, `usb`, `usbip` | Abstract base + concrete impls + null fallback, so dependency-free environments still import. | -Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (310 headless subpackages) → `wrapper/` → per-OS backend. +Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (311 headless subpackages) → `wrapper/` → per-OS backend. ## Development Commands @@ -92,7 +92,7 @@ The map is only useful while it matches the tree, so **update it in the same cha Never adjust one by hand: the counts are `len(text.splitlines())` (what `wc -l` reports), and hand-editing is how the document ended up quoting the same subsystem at two sizes at once — most tables had been counting a phantom trailing line per file while §1 and §8 counted correctly. -- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 310 subpackages; appearing twice or not at all is a defect. +- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 311 subpackages; appearing twice or not at all is a defect. - A new subsystem over ~1,000 lines also needs a file-level table in §5.4.17. - Keep the header's scan date, version, and branch current. - `README.md` and both translations under `README/` cite the same figures (command / subpackage / tab / MCP-tool / example counts) — update all three alongside the map. diff --git a/README.md b/README.md index bb5a725e5..cd6ca070a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. - **One API, seven platforms.** `wrapper/platform_wrapper.py` picks the backend at import time; your script does not change between Windows, macOS, X11, and Wayland. -- **Scriptable without Python.** 779 `AC_*` commands cover the whole feature set, so a +- **Scriptable without Python.** 848 `AC_*` commands cover the whole feature set, so a JSON file can do anything the library can — including loops, branches, try/catch, macros, and variables. - **Headless by default.** `import je_auto_control` never loads Qt. The GUI is an @@ -231,7 +231,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Natural-language planner | `plan_actions`, `run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`, `run_agent` | `AC_run_agent` | Computer Use | | Record & replay | `record`, `stop_record` | `AC_record`, `AC_stop_record` | Record | -| JSON scripting | `execute_action`, `execute_files` | all 779 commands | Script, Script Builder | +| JSON scripting | `execute_action`, `execute_files` | all 848 commands | Script, Script Builder | | Variables & flow control | `execute_action_with_vars` | `AC_set_var`, `AC_loop`, `AC_for_each`, `AC_try`, `AC_retry` | Variables | | Data-driven runs | — | `AC_for_each_row` (CSV / JSON / SQLite / Excel) | Data Sources | | Assertions | `assert_text`, `assert_image` | `AC_assert_text` + 20 more | Assertions | @@ -250,7 +250,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Diagnostics | `run_diagnostics` | `AC_diagnose` | Diagnostics | | Test-code generation | `generate_code` | — | — | -Beyond this table, `utils/` holds 310 headless packages covering assertions, resilience, +Beyond this table, `utils/` holds 311 headless packages covering assertions, resilience, data quality, i18n auditing, redaction, governance, observability, and more. The full per-module map is in **[architecture_explore.md](architecture_explore.md)**. @@ -287,7 +287,7 @@ under a relative name are written below that directory. | Surface | Start it with | Notes | |---|---|---| -| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 680 tools for Claude Desktop / Claude Code / custom tool loops: the full `ac_*` surface plus short model-friendly aliases for common GUI actions. Speaks the stateless MCP 2026-07-28 beside the `initialize`-based revisions. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | +| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 754 tools for Claude Desktop / Claude Code / custom tool loops: the full `ac_*` surface plus short model-friendly aliases for common GUI actions. Speaks the stateless MCP 2026-07-28 beside the `initialize`-based revisions. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | | **REST API** | `je_auto_control start-rest` | Bearer token, per-IP rate limit + lockout, SQLite audit hook, `/metrics`, `/openapi.json`, `/docs` Swagger UI, `/dashboard`. | | **TCP socket server** | `je_auto_control start-server` | Newline-framed JSON action lists. Binds `127.0.0.1` by default. | | **pytest plugin** | installed automatically | Fixtures plus a Gherkin step library for pytest-bdd / behave. | diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 639e4a1bd..7473e2a1e 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -19,7 +19,7 @@ - **一套 API，七个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 Windows、macOS、X11 与 Wayland 上都不需要改写。 -- **不写 Python 也能脚本化。** 779 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 +- **不写 Python 也能脚本化。** 848 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 能做的任何事——包含循环、分支、try/catch、宏与变量。 - **默认无头运行。** `import je_auto_control` 绝不会加载 Qt。GUI 是可选包，包在同一个无头内核之外。 - **四种定位方式。** 模板匹配、OCR、无障碍树、视觉语言模型——可通过锚点定位器与自愈回退串接组合。 @@ -219,7 +219,7 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 自然语言规划 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 录制与回放 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 脚本 | `execute_action`、`execute_files` | 全部 779 个命令 | Script、Script Builder | +| JSON 脚本 | `execute_action`、`execute_files` | 全部 848 个命令 | Script、Script Builder | | 变量与流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 数据驱动执行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 断言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 个 | Assertions | @@ -238,7 +238,7 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 系统诊断 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 测试代码生成 | `generate_code` | — | — | -除了这张表，`utils/` 下还有 310 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 +除了这张表，`utils/` 下还有 311 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 治理、可观测性等等。完整的逐模块地图在 **[architecture_explore.md](../architecture_explore.md)**。 --- @@ -268,7 +268,7 @@ je_auto_control version | 接口 | 启动方式 | 说明 | |---|---|---| -| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 680 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用：完整的 `ac_*` 工具接口，以及常用 GUI 操作的短名称 alias。除了以 `initialize` 握手的各版协议，也支持无状态的 MCP 2026-07-28。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | +| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 754 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用：完整的 `ac_*` 工具接口，以及常用 GUI 操作的短名称 alias。除了以 `initialize` 握手的各版协议，也支持无状态的 MCP 2026-07-28。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、按 IP 限流与锁定、SQLite 审计 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 服务器** | `je_auto_control start-server` | 以换行分隔的 JSON 动作列表。默认绑定 `127.0.0.1`。 | | **pytest 插件** | 安装后自动生效 | 提供 fixture 与供 pytest-bdd／behave 使用的 Gherkin step library。 | diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 63fc74d01..7a5273030 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -18,7 +18,7 @@ - **一套 API，七個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 Windows、macOS、X11 與 Wayland 上都不需要改寫。 -- **不寫 Python 也能腳本化。** 779 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 +- **不寫 Python 也能腳本化。** 848 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 能做的任何事——包含迴圈、分支、try/catch、巨集與變數。 - **預設無頭執行。** `import je_auto_control` 絕不會載入 Qt。GUI 是選用套件，包在同一個無頭核心之外。 - **四種定位方式。** 樣板比對、OCR、無障礙樹、視覺語言模型——可透過錨點定位器與自癒後備串接組合。 @@ -218,7 +218,7 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 自然語言規劃 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 錄製與重播 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 腳本 | `execute_action`、`execute_files` | 全部 779 個指令 | Script、Script Builder | +| JSON 腳本 | `execute_action`、`execute_files` | 全部 848 個指令 | Script、Script Builder | | 變數與流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 資料驅動執行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 斷言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 個 | Assertions | @@ -237,7 +237,7 @@ python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" | 系統診斷 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 測試碼產生 | `generate_code` | — | — | -除了這張表，`utils/` 底下還有 310 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 +除了這張表，`utils/` 底下還有 311 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 治理、可觀測性等等。完整的逐模組地圖在 **[architecture_explore.md](../architecture_explore.md)**。 --- @@ -267,7 +267,7 @@ je_auto_control version | 介面 | 啟動方式 | 說明 | |---|---|---| -| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 680 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用：完整的 `ac_*` 工具介面，以及常用 GUI 操作的短名稱 alias。除了以 `initialize` 握手的各版協定，也支援無狀態的 MCP 2026-07-28。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | +| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 754 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用：完整的 `ac_*` 工具介面，以及常用 GUI 操作的短名稱 alias。除了以 `initialize` 握手的各版協定，也支援無狀態的 MCP 2026-07-28。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、逐 IP 限流與鎖定、SQLite 稽核 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 伺服器** | `je_auto_control start-server` | 以換行分隔的 JSON 動作清單。預設綁 `127.0.0.1`。 | | **pytest 外掛** | 安裝後自動生效 | 提供 fixture 與供 pytest-bdd／behave 使用的 Gherkin step library。 | diff --git a/architecture_explore.md b/architecture_explore.md index e4f1efc4a..2a278beb6 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,13 +19,13 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,077 | -| 程式碼總行數 | 161,551 | -| `je_auto_control/utils/` 子套件數 | 310 | -| `AC_*` 動作指令數（`known_commands()` 實測） | 779 | +| Python 模組總數（含周邊子專案） | 1,152 | +| 程式碼總行數 | 179,640 | +| `je_auto_control/utils/` 子套件數 | 311 | +| `AC_*` 動作指令數（`known_commands()` 實測） | 848 | | 套件門面 `__all__` 公開名稱數 | 1,244 | | GUI 分頁數（`main_widget` 註冊） | 48 | -| MCP 工具數（`build_default_tool_registry()` 實測） | 680 | +| MCP 工具數（`build_default_tool_registry()` 實測） | 754 | | `test_*.py` 測試檔／測試函式 | 478 / 4,654 | | 範例腳本 | 27 | @@ -55,7 +55,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 └───────────────────────────────┬──────────────────────────────────────────┘ │ ┌───────────────────────────────▼──────────────────────────────────────────┐ -│ 能力層 utils/（310 個子套件，全部無 Qt 相依） │ +│ 能力層 utils/（311 個子套件，全部無 Qt 相依） │ │ 影像辨識 │ OCR │ 無障礙樹 │ 定位自癒 │ AI/Agent │ 遠端桌面 │ USB │ │ 報表觀測 │ 資料 │ 安全 │ 韌性 │ 系統整合 │ 排程觸發 │ 網路協定 │ └───────────────────────────────┬──────────────────────────────────────────┘ @@ -92,7 +92,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 | 模式 | 落點 | 說明 | | --- | --- | --- | | **Strategy** | `wrapper/platform_wrapper.py` | 依 `sys.platform` 只匯入當前 OS 的 `keyboard`／`mouse`／`screen`／`recorder` 實作；Linux 再細分 Wayland／X11，Wayland 後端不可用時自動退回 XWayland 並記警告。新增平台不需要改 wrapper。 | -| **Facade** | `je_auto_control/__init__.py` | 把 1,264 個公開名稱集中再匯出，使用者只 `import je_auto_control`。`api/core.py` 另提供一個小而穩定的版本化門面（7 個名稱）給新整合使用。 | +| **Facade** | `je_auto_control/__init__.py` | 把 1,377 個公開名稱集中再匯出，使用者只 `import je_auto_control`。`api/core.py` 另提供一個小而穩定的版本化門面（7 個名稱）給新整合使用。 | | **Command** | `utils/executor/action_executor.py` | `Executor.event_dict` 是字串 → callable 的分派表；JSON 動作檔即指令序列，因此可錄製、序列化、重播、簽章。 | | **Observer** | `utils/callback/`、`utils/observer/`、`utils/triggers/` | 動作完成後觸發回呼；畫面出現／消失／變化與外部事件（webhook／IMAP／檔案）驅動腳本。 | | **Template Method** | `utils/generate_report/` | HTML／JSON／XML 三個產生器共用「收集紀錄 → 格式化 → 寫檔」骨架，各自實作渲染。 | @@ -154,9 +154,9 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `je_auto_control/__init__.py` | 1,994 | **套件門面**。集中匯入並再匯出 1,264 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | +| `je_auto_control/__init__.py` | 2,100 | **套件門面**。集中匯入並再匯出 1,377 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | | `je_auto_control/__main__.py` | 98 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | -| `je_auto_control/cli.py` | 372 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | +| `je_auto_control/cli.py` | 422 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | | `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | | `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約以此為起點，現已擴到整包（見「設定基線」）。 | | `je_auto_control/utils/cli_output.py` | 44 | 本套件命令列工具與 stdio 伺服器的標準串流：不論碼頁一律 UTF-8。 | @@ -173,7 +173,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | --- | ---: | --- | | `wrapper/platform_wrapper.py` | 119 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；八個名稱都帶著 `backend_contract` 的型別出去，其中 `keyboard`／`mouse` 因為四個分支綁的是三種互不相容的形狀，先落在私有的 `_keyboard`／`_mouse`（`Any`）上再標合約；載入失敗直接拋 `AutoControlException`（fail fast）。 | | `wrapper/backend_contract.py` | 249 | 平台縫的型別合約：`ScreenBackend`／`KeyboardCheckBackend`／`RecorderBackend` 三個跨平台 Protocol，加上 `keyboard`／`mouse` 各自的三份——`Win32*`（SendInput 與 Interception）、`Darwin*`（Quartz）、`X11Unix*`（XTest／uinput／Wayland／BSD），因為這兩個名稱的呼叫形狀真的因平台而異；`KeyboardBackend`／`MouseBackend` 依 `sys.platform` 別名到其中一組，所以呼叫端被檢查的是它真的會走到的簽章。四個 `_platform_*` 組裝模組各自標注自己綁的是什麼，少一個成員就在該後端自己的檔案裡紅掉，而不是在三層之上的呼叫點。 | -| `wrapper/_platform_windows.py` | 366 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表（含 OEM 鍵與 `keyboard_key_aliases` 別名表）+ 選用 Interception 驅動。 | +| `wrapper/_platform_windows.py` | 376 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表（含 OEM 鍵與 `keyboard_key_aliases` 別名表）+ 選用 Interception 驅動。 | | `wrapper/_platform_osx.py` | 162 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | | `wrapper/_platform_linux.py` | 278 | X11 後端組裝（python-Xlib + 選用 uinput）。 | | `wrapper/_platform_wayland.py` | 63 | Wayland 後端組裝（libei／ydotool／grim）。 | @@ -187,7 +187,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.3 平台後端 -#### Windows（`windows/`，23 檔／2,154 行） +#### Windows（`windows/`，24 檔／2,266 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -233,7 +233,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 44 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 130 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，17 檔／2,921 行） +#### Linux Wayland（`linux_wayland/`，23 檔／5,153 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -241,49 +241,50 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `_ydotool_cli.py` | 134 | 判定安裝的是哪一代 ydotool 命令列，擋掉會靜默失效的 0.1.x（對本專案送的 argv 回傳 0 卻不送任何事件）。 | | `_ctypes_bind.py` | 75 | libei／liboeffis 共用的 ctypes 載入與 prototype 綁定。 | | `_dbus_client.py` | 24 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | -| `_select_input.py` | 85 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | +| `_select_input.py` | 224 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | | `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | -| `oeffis.py` | 197 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | -| `libei.py` | 655 | libei 綁定與完整握手（seat 綁定能力 → 由事件取得 device → start_emulating → 每次發送後 frame）。另負責絕對指標的座標空間:讀回裝置的 region,把版面座標映射進去,沒有任何 region 涵蓋就拒絕（libei 對這種移動是靜靜丟掉的）。 | +| `oeffis.py` | 230 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | +| `libei.py` | 712 | libei 綁定與完整握手（seat 綁定能力 → 由事件取得 device → start_emulating → 每次發送後 frame）。另負責絕對指標的座標空間:讀回裝置的 region,把版面座標映射進去,沒有任何 region 涵蓋就拒絕（libei 對這種移動是靜靜丟掉的）。 | | `mouse.py` | 384 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | | `keyboard.py` | 173 | 鍵盤後端：libei 優先，退回 ydotool／wtype。 | | `keymap.py` | 155 | 友善鍵名 → evdev key code。 | | `capture.py` | 246 | 擷取分層：操作者自訂指令 → grim → gnome-screenshot → spectacle → portal。 | -| `portal.py` | 207 | `org.freedesktop.portal.Screenshot` 最後備援,經 `_dbus_client` 直接講 D-Bus（不再需要安裝 `gdbus`,只要有 session bus）。 | +| `portal.py` | 224 | `org.freedesktop.portal.Screenshot` 最後備援,經 `_dbus_client` 直接講 D-Bus（不再需要安裝 `gdbus`,只要有 session bus）。 | | `screen.py` | 296 | 螢幕後端；發布 `grab_image` 與 `layout_origin`（擷取畫面左上角的版面座標，有螢幕在主螢幕左側／上方時為負），全框架的擷取都經由它。 | -| `listener.py` / `record.py` | 48 / 34 | 監聽與錄製 stub（Wayland 限制）。 | +| `listener.py` / `record.py` | 58 / 45 | 監聽與錄製 stub（Wayland 限制）。 | #### 行動裝置 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `android/adb_client.py` | 213 | `adb` CLI 的薄封裝。 | +| `android/adb_client.py` | 342 | `adb` CLI 的薄封裝。 | | `android/client.py` | 129 | `uiautomator2.Device` 的延遲封裝。 | | `android/find.py` | 108 | uiautomator2 widget 樹的元素查詢。 | | `ios/client.py` | 124 | `facebook-wda`（WebDriverAgent）封裝。 | | `ios/find.py` | 94 | XCUITest 無障礙查詢。 | -| `ios/input.py` | 51 | iOS 觸控與按鍵原語。 | -| `ios/screen.py` | 34 | iOS 裝置螢幕擷取與尺寸。 | +| `ios/input.py` | 83 | iOS 觸控與按鍵原語。 | +| `ios/screen.py` | 85 | iOS 裝置螢幕擷取與尺寸。 | -### 5.4 能力層 `utils/`（310 個子套件） +### 5.4 能力層 `utils/`（311 個子套件） 以下依主題分組。每個子套件都是獨立可匯入的無頭模組，不含任何 Qt 相依。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 15,224 行。 +> 25 個套件、約 16,748 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/action_lint/` | 429 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | -| `utils/action_signing/` | 716 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | +| `utils/action_signing/` | 771 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | | `utils/checkpoint/` | 129 | 流程檢查點與續跑，讓長 action list 具持久性 | -| `utils/codegen/` | 294 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | +| `utils/codegen/` | 695 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | | `utils/dag/` | 560 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 112 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 116 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 9,666 | **核心**。`Executor` 指令分派表（779 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/executor/` | 9,972 | **核心**。`Executor` 指令分派表（848 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/action_journal/` | 718 | 選用的動作日誌：執行邊界記錄 typed 輸入／結果與 parent id，先遮蔽再寫入 JSONL；`codegen.journal_import` 由它產生候選腳本 | | `utils/flow_debugger/` | 166 | action list 的單步除錯器與追蹤器 | | `utils/input_macro/` | 462 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | | `utils/json/` | 99 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | @@ -294,16 +295,16 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/project/` | 183 | 專案腳手架：建立目錄結構與範本 action 檔 | | `utils/recording_edit/` | 165 | 不重錄的前提下裁切／過濾／縮放已錄製的 action list | | `utils/saga/` | 103 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | -| `utils/script_vars/` | 287 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | +| `utils/script_vars/` | 310 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | | `utils/skill_library/` | 145 | 具名可重用 action 序列（skill）的持久化倉庫 | -| `utils/state_machine/` | 268 | 宣告式有限狀態機驅動 action JSON | +| `utils/state_machine/` | 289 | 宣告式有限狀態機驅動 action JSON | | `utils/stubs/` | 311 | 為 `AC_*` 指令面產生型別 stub | | `utils/test_record/` | 70 | 全域測試紀錄單例，記錄每個動作的參數與例外 | | `utils/work_queue/` | 280 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | ### 5.4.2 框架基礎設施 -> 14 個套件、約 3,256 行。 +> 14 個套件、約 3,277 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -312,11 +313,11 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/critical_exit/` | 132 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | | `utils/diagnostics/` | 330 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | | `utils/dbus_client/` | 714 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | -| `utils/exception/` | 223 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/exception/` | 234 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | | `utils/failure_bundle/` | 229 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | | `utils/file_process/` | 40 | 目錄檔案列舉（`execute_dir` 的後端） | | `utils/logging/` | 168 | `autocontrol_logger` 單例 + 家目錄共用記錄檔 handler（`JE_AUTOCONTROL_LOG_FILE` 可改） | -| `utils/package_manager/` | 179 | 動態載入套件並把 executor 注入其中；載入前先過套件閘門（允許清單加上只能從 Python 呼叫的開關，工作區 X-12） | +| `utils/package_manager/` | 189 | 動態載入套件並把 executor 注入其中；載入前先過套件閘門（允許清單加上只能從 Python 呼叫的開關，工作區 X-12） | | `utils/path_guard/` | 231 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | | `utils/platform_id/` | 62 | 作業系統家族的單一判定點。`sys.platform` 原本在一百多處跟字面清單比對，而那些清單都沒有 BSD；`is_x11_unix()` 問的是「這是不是 X11 unix」，這才是守衛一直想問的問題 | | `utils/shell_process/` | 279 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | @@ -324,20 +325,20 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.3 排程、觸發與背景監看 -> 11 個套件、約 4,302 行。 +> 11 個套件、約 4,336 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/hotkey/` | 852 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | +| `utils/hotkey/` | 861 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | | `utils/idle_keepawake/` | 245 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | | `utils/lock_session/` | 166 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | | `utils/observer/` | 234 | 反應式畫面觀察者，在出現／消失／變化時觸發 | | `utils/recurrence/` | 470 | RFC 5545 重複規則解析與發生時間展開 | -| `utils/scheduler/` | 506 | 間隔式與 cron 式的 action JSON 排程器 | +| `utils/scheduler/` | 510 | 間隔式與 cron 式的 action JSON 排程器 | | `utils/session_guard/` | 62 | 驅動輸入前先偵測工作階段是否已鎖定／非互動 | -| `utils/triggers/` | 1,383 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | +| `utils/triggers/` | 1,396 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | | `utils/voice/` | 99 | 語音指令路由：把辨識到的語句對應到 `AC_*` action list | -| `utils/watchdog/` | 195 | 背景彈窗／中斷看門狗，供無人值守自動化 | +| `utils/watchdog/` | 203 | 背景彈窗／中斷看門狗，供無人值守自動化 | | `utils/watcher/` | 90 | 無頭輪詢原語：滑鼠位置、像素顏色、log tail | ### 5.4.4 輸入模擬與動作品質 @@ -464,7 +465,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.8 元素定位、自我修復與智慧等待 -> 23 個套件、約 4,426 行。 +> 23 個套件、約 5,754 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -480,54 +481,54 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/element_scoring/` | 105 | 加權候選評分（角色 + 名稱相似度 + 鄰近度 + 啟用狀態） | | `utils/expect_poll/` | 149 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | | `utils/grounding_consensus/` | 173 | 對同一目標的多個接地提案做自我一致性投票 | -| `utils/heal_analytics/` | 77 | 自癒事件記錄的分析（治癒率、脆弱定位器） | +| `utils/heal_analytics/` | 95 | 自癒事件記錄的分析（治癒率、脆弱定位器） | | `utils/locator_chain/` | 112 | 可組合／可過濾的候選定位器（chained-locator 慣用法） | | `utils/locator_repair/` | 147 | 自癒回寫：把修正後的定位器持久化 | | `utils/observation/` | 89 | 供 VLM／agent 接地用的 token 預算內、帶索引的 a11y 文字觀察 | | `utils/observation_delta/` | 122 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | | `utils/screen_state/` | 191 | 語義畫面狀態：快照／差異與結構化畫面描述 | | `utils/scroll_find/` | 103 | 捲動直到目標影像／文字可見 | -| `utils/self_healing/` | 359 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | +| `utils/self_healing/` | 1,669 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | | `utils/semantic_recording/` | 498 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | | `utils/settle_detector/` | 79 | 以純函式介面判定 UI 是否已靜止 | | `utils/smart_waits/` | 672 | 智慧等待：以影格差異取代 `time.sleep` | ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 23,676 行。 +> 13 個套件、約 25,361 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/a2a/` | 92 | A2A（agent-to-agent）agent card 產生 | -| `utils/agent/` | 2,142 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | +| `utils/agent/` | 2,251 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | | `utils/agent_memory/` | 166 | agent 的持久化情節記憶（goal → trajectory → outcome） | | `utils/agent_replay/` | 67 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | | `utils/agent_trace/` | 172 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | | `utils/cost_telemetry/` | 345 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | | `utils/cua_action/` | 204 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | -| `utils/llm/` | 365 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | +| `utils/llm/` | 376 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 97 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 19,161 | **無頭 MCP 伺服器**（16K LOC，預設註冊 680 個工具＝661 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 20,726 | **無頭 MCP 伺服器**（16K LOC，預設註冊 754 個工具＝735 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 195 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 132 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 538 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 20,018 行。 +> 6 個套件、約 22,996 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/admin/` | 418 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | -| `utils/config_sync/` | 332 | 透過訊令伺服器做跨機器設定同步 | -| `utils/device_matrix/` | 143 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 13,301 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,816 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/config_sync/` | 2,910 | 透過訊令伺服器做跨機器設定同步 | +| `utils/device_matrix/` | 201 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | +| `utils/remote_desktop/` | 13,513 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/usb/` | 4,946 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 1,008 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 -> 24 個套件、約 7,101 行。 +> 24 個套件、約 7,140 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -549,7 +550,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/otp/` | 37 | TOTP 一次性密碼產生（自動化 2FA 登入） | | `utils/outbox/` | 107 | 交易式 outbox，保證至少一次的事件投遞 | | `utils/pytest_plugin/` | 283 | pytest 外掛 + BDD step library（`pytest11` entry point） | -| `utils/rest_api/` | 2,039 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | +| `utils/rest_api/` | 2,078 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | | `utils/socket_server/` | 163 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | | `utils/sse_client/` | 142 | Server-Sent Events 用戶端解析 | | `utils/tls_acme/` | 473 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | @@ -558,7 +559,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 7,721 行。 +> 34 個套件、約 7,734 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -583,7 +584,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/profiler/` | 451 | 逐動作效能剖析器 + 資源剖析器 | | `utils/quarantine/` | 204 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | | `utils/run_diff/` | 143 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | -| `utils/run_history/` | 454 | 執行歷史儲存與產出物管理 | +| `utils/run_history/` | 467 | 執行歷史儲存與產出物管理 | | `utils/sarif/` | 187 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | | `utils/slo/` | 115 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | | `utils/smoothing/` | 67 | 數列移動平均平滑 | @@ -630,7 +631,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.14 安全、機密與合規 -> 13 個套件、約 3,403 行。 +> 13 個套件、約 3,861 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -639,7 +640,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/governance/` | 242 | 治理：maker-checker 核准閘門與即時憑證租約 | | `utils/license_policy/` | 240 | 以 SBOM 元件評估 SPDX 授權允許／拒絕政策 | | `utils/provenance/` | 126 | SLSA 建置來源證明（in-toto v1） | -| `utils/rbac/` | 670 | 角色型存取控制：使用者、角色與權杖驗證（尚未接到 REST／MCP） | +| `utils/rbac/` | 1,128 | 角色型存取控制：使用者、角色與權杖驗證（尚未接到 REST／MCP） | | `utils/redaction/` | 508 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | | `utils/sbom/` | 170 | SBOM（CycloneDX）產生 | | `utils/secret_ref/` | 238 | URI scheme 形式的值參照解析 | @@ -696,35 +697,35 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 -#### `utils/executor/`（9,666 行）— 執行核心 +#### `utils/executor/`（9,972 行）— 執行核心 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `action_executor.py` | 8,435 | `Executor` 類別與 `event_dict` 分派表（779 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | -| `flow_control.py` | 656 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | +| `action_executor.py` | 8,675 | `Executor` 類別與 `event_dict` 分派表（848 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | +| `flow_control.py` | 659 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | | `flow_data_commands.py` | 272 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | -| `action_schema.py` | 159 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | +| `action_schema.py` | 222 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `action_redaction.py` | 86 | 記錄與紀錄鍵用的遮蔽：`AC_secret_*` 的參數（金庫通行碼、機密值）在寫進 log、當成結果紀錄的鍵之前換成 `***`，巢狀在區塊指令裡的也一樣。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | | `flags.py` | 19 | 旗標讀取 `as_bool`：依拼法（`"true"`／`"yes"`／`"on"`／`"1"`）而非 Python 真值判斷，所以 JSON、CLI、MCP 傳來的 `"false"` 是關；所有 adapter 與流程指令共用。 | -#### `utils/mcp_server/`（19,161 行，680 個工具）— 最大子系統 +#### `utils/mcp_server/`（20,726 行，754 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `tools/_factories.py` | 9,063 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | +| `tools/_factories.py` | 9,370 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | | `tools/_handlers.py` | 547 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter；主題模組拆完之後這裡留的是資料／文字／HTTP 那一類與 WebRunner 橋接。 | -| `tools/_handlers_qa.py` | 419 | 同一種 adapter，QA 主題：斷言 DSL、資料驅動、SQL／PDF／郵件／HTTP 步驟、codegen、視覺回歸、狀態機、flaky 偵測與隔離、suite runner、無障礙稽核、裝置矩陣、媒體斷言。從 `_handlers.py` 依主題拆出的第一塊（750 行上限）；兩者互不引用。 | +| `tools/_handlers_qa.py` | 433 | 同一種 adapter，QA 主題：斷言 DSL、資料驅動、SQL／PDF／郵件／HTTP 步驟、codegen、視覺回歸、狀態機、flaky 偵測與隔離、suite runner、無障礙稽核、裝置矩陣、媒體斷言。從 `_handlers.py` 依主題拆出的第一塊（750 行上限）；兩者互不引用。 | | `tools/_handlers_input.py` | 221 | 同一種 adapter，輸入主題：滑鼠、鍵盤、虛擬手把（ViGEm）。 | | `tools/_handlers_screen.py` | 344 | 同一種 adapter，螢幕主題：擷取、像素、影像與文字搜尋、螢幕錄影。 | -| `tools/_handlers_system.py` | 575 | 同一種 adapter，桌面工作階段：視窗、行程與 shell、開檔、閒置與睡眠、音量、鎖定、輸入法狀態、欄位驗證與重試、色彩對比、變更排序、元件分類、剪貼簿。 | -| `tools/_handlers_runs.py` | 110 | 同一種 adapter，執行主題：executor、執行歷史、錄製、動作檔。 | +| `tools/_handlers_system.py` | 599 | 同一種 adapter，桌面工作階段：視窗、行程與 shell、開檔、閒置與睡眠、音量、鎖定、輸入法狀態、欄位驗證與重試、色彩對比、變更排序、元件分類、剪貼簿。 | +| `tools/_handlers_runs.py` | 139 | 同一種 adapter，執行主題：executor、執行歷史、錄製、動作檔。 | | `tools/_handlers_scheduling.py` | 200 | 同一種 adapter，排程主題：排程器、觸發器、熱鍵常駐。 | | `tools/_handlers_remote.py` | 66 | 同一種 adapter，遠端桌面的 host 與 viewer。 | | `tools/_handlers_executor_bridge.py` | 1,429 | 252 個純委派（中位數 3 行，最長的 16 行全是參數簽章）：每個都是 `from action_executor import _x` 再 `return _x(...)`，沒有分支邏輯。超過 750 行,理由記在 `Progress.md` 的豁免表（再切只能照 MCP 工廠領域分,會把同一種委派散進十幾個沒有語意邊界的檔）。 | -| `tools/_handlers_locators.py` | 436 | 同一種 adapter，定位主題：無障礙樹、智慧等待、自我修復、螢幕觀察、座標空間、視覺與 OCR、影像去重、元件倉庫、A/B 定位。 | -| `tools/_handlers_operations.py` | 629 | 同一種 adapter，營運主題：agent 與其記憶／追蹤、治理與合規、成本與遙測、失敗掛鉤、看門狗、速率限制、檢查點、核可、產物與資產、測試選擇與分片、佇列與 saga。 | -| `server.py` | 734 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器；握手時代的方法表（`_run_method`），兩個協定時代的逐請求分派在 `_stateless.py`。 | +| `tools/_handlers_locators.py` | 482 | 同一種 adapter，定位主題：無障礙樹、智慧等待、自我修復、螢幕觀察、座標空間、視覺與 OCR、影像去重、元件倉庫、A/B 定位。 | +| `tools/_handlers_operations.py` | 654 | 同一種 adapter，營運主題：agent 與其記憶／追蹤、治理與合規、成本與遙測、失敗掛鉤、看門狗、速率限制、檢查點、核可、產物與資產、測試選擇與分片、佇列與 saga。 | +| `server.py` | 643 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器；握手時代的方法表（`_run_method`），兩個協定時代的逐請求分派在 `_stateless.py`。 | | `http_transport.py` | 727 | MCP 的 HTTP 傳輸；宣告 2026-07-28 的請求走 `_http_stateless.py` 的標頭規則，不發 session。 | | `_http_stateless.py` | 185 | MCP 2026-07-28 在 Streamable HTTP 上的規則：`MCP-Protocol-Version`／`Mcp-Method`／`Mcp-Name` 必須與 body 相符（`=?base64?…?=` 先解碼），不符是 400＋`HeaderMismatch`；版本與中繼資料錯誤 400、未知方法 404。純函式，由 `http_transport.py` 回覆。 | | `http_sessions.py` | 247 | MCP 的 HTTP 傳輸用的 session 身分:`Mcp-Session-Id` 註冊表,以及每個 session 那條常駐的 server→client SSE 串流。 | @@ -737,16 +738,16 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `prompts.py` | 220 | MCP prompt 目錄。 | | `fake_backend.py` | 187 | CI／無頭測試用的記憶體內假後端。 | | `plugin_watcher.py` | 168 | 檔案變更時熱重載外掛工具的背景 watcher。 | -| `tools/_base.py` | 153 | 工具註冊表的共用型別與輔助。 | +| `tools/_base.py` | 190 | 工具註冊表的共用型別與輔助。 | | `tools/_validation.py` | 122 | MCP 工具用到的 JSON Schema 子集驗證器。 | | `tools/plugin_tools.py` | 89 | 把外掛載入的 `AC_*` callable 包成 `MCPTool`。 | | `log_bridge.py` | 118 | 把 Python logging 記錄橋接成 MCP `notifications/message`；2026-07-28 的請求只收到自己設了 `logLevel` 時產生的記錄。 | | `audit.py` | 112 | MCP 工具呼叫稽核記錄。 | | `context.py` | 71 | 傳給 opt-in 工具處理器的每次呼叫上下文。 | | `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | -| `__main__.py` | 92 | `je_auto_control_mcp` console script 進入點。 | +| `__main__.py` | 107 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（13,301 行／56 檔） +#### `utils/remote_desktop/`（13,513 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 @@ -761,7 +762,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `registry.py` | 564 | `AC_remote_*` 指令使用的行程級單例。 | | `webrtc_transport.py` | 421 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | | `multi_viewer.py` | 357 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | -| `signaling_server.py` | 427 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | +| `signaling_server.py` | 516 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | | `audit_log.py` | 355 | SQLite 雜湊鏈稽核記錄。 | | `host_capture.py` | 323 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | | `ws_protocol.py` | 318 | 最小 RFC 6455 WebSocket 框架與握手。 | @@ -788,26 +789,26 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `input_dispatch.py` | 161 | 在主機端套用輸入訊息。 | | `session_recorder.py` | 139 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | | `totp.py` | 160 | RFC 6238 TOTP（零外部相依）。 | -| `file_sync.py` | 141 | 輪詢式資料夾鏡像。 | +| `file_sync.py` | 211 | 輪詢式資料夾鏡像。 | | `transport.py` | 126 | 可插拔的型別化訊息傳輸。 | | `host_access.py` | 112 | TCP 主機的檢視端核准與存取控制：`PendingViewer`、權限字串、分享碼的 TOTP 候選值、IP 白名單。`host` 與 `host_client` 共用,所以獨立成模組。 | | `protocol.py` | 98 | 長度前綴的 TCP 框架。 | | `resume_tokens.py` / `session_quality_cache.py` / `rate_limit.py` | 94 / 85 / 84 | 快速重連 token、每 session 品質快取、檢視端限流。 | | `host_id.py` / `viewer_id.py` | 83 / 79 | 主機與檢視端的持久身分。 | -| `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 64 / 74 / 56 / 40 / 28 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | +| `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 64 / 127 / 56 / 40 / 28 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | | `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 40 / 29 / 146 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,816 行）與 `utils/usbip/`（1,008 行） +#### `utils/usb/`（4,946 行）與 `utils/usbip/`（1,008 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `usb/passthrough/session.py` | 690 | 逐 peer 的 USB 直通 session。 | -| `usb/passthrough/viewer_client.py` | 748 | 檢視端的直通協定用戶端。 | +| `usb/passthrough/viewer_client.py` | 552 | 檢視端的直通協定用戶端。 | | `usb/passthrough/backend.py` | 488 | 後端 ABC + libusb 實作。 | | `usb/passthrough/winusb_backend.py` | 488 | Windows WinUSB 後端（ctypes）。 | -| `usb/passthrough/acl.py` | 495 | 逐裝置 ACL。 | +| `usb/passthrough/acl.py` | 505 | 逐裝置 ACL。 | | `usb/passthrough/iokit_backend.py` | 221 | macOS IOKit 後端。 | -| `usb/passthrough/webrtc_channel.py` | 180 | 把直通協定橋到 WebRTC `usb` DataChannel。 | +| `usb/passthrough/webrtc_channel.py` | 203 | 把直通協定橋到 WebRTC `usb` DataChannel。 | | `usb/passthrough/loopback.py` | 159 | 行程內 loopback 傳輸（測試用）。 | | `usb/passthrough/protocol.py` | 177 | 線路框格式。 | | `usb/passthrough/descriptor.py` | 134 | USB 標準裝置描述元解析。 | @@ -820,16 +821,16 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `usbip/libusb_backend.py` | 224 | 以 PyUSB／libusb 執行 URB 的正式後端。 | | `usbip/backend.py` | 87 | 可插拔 URB 執行後端。 | -#### `utils/rest_api/`（2,039 行） +#### `utils/rest_api/`（2,078 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `rest_server.py` | 634 | HTTP 前端主體。 | -| `rest_handlers.py` | 528 | 端點實作。 | +| `rest_server.py` | 636 | HTTP 前端主體。 | +| `rest_handlers.py` | 534 | 端點實作。 | | `rest_openapi.py` | 443 | 走訪路由表產生 OpenAPI 3.1 規格。 | -| `rest_auth.py` | 202 | Bearer token 驗證 + 逐 client 限流閘門。 | +| `rest_auth.py` | 220 | Bearer token 驗證 + 逐 client 限流閘門。 | | `rest_metrics.py` | 75 | Prometheus 曝露端點。 | -| `rest_registry.py` | 77 | 保存執行中 REST 伺服器的行程級單例。 | +| `rest_registry.py` | 90 | 保存執行中 REST 伺服器的行程級單例。 | | `__main__.py` | 66 | `python -m je_auto_control.utils.rest_api` 進入點。 | #### 其他多檔子套件 @@ -881,23 +882,23 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 模組 | 行數 | 職責 | | --- | ---: | --- | | `gui/__init__.py` | 25 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | -| `main_window.py` | 379 | `QMainWindow`：選單列（File／Actions／View／…）、左側導覽面板 dock（`Ctrl+K` 搜尋、`Ctrl+B` 收合）、即時語言切換、字級預設、深色／淺色主題（`theme.py` 的 token，不再用 qt-material）。分頁分為 core／editing／detection／automation／system 五類。 | -| `main_widget.py` | 368 | 工作區：擁有 `QTabWidget`，依 `tab_registry.TAB_SPECS` 註冊 48 個分頁，並暴露 show/hide/activate/list API 給選單列與導覽面板。只有預設開啟的三個分頁與自己的 mixin 表單在啟動時建立，其餘第一次開啟才匯入模組、建立 widget。核心分頁在 `_own_tab_builders` 宣告 `(label_key, handler)` 動作對。 | -| `tab_registry.py` | 125 | 分頁表：每個分頁一筆 `TabSpec`（鍵、標題鍵、分類、模組與類別名），`TabEntry` 在第一次存取 `widget` 時才呼叫 factory。不匯入 Qt。 | +| `main_window.py` | 437 | `QMainWindow`：選單列（File／Actions／View／…）、左側導覽面板 dock（`Ctrl+K` 搜尋、`Ctrl+B` 收合）、即時語言切換、字級預設、深色／淺色主題（`theme.py` 的 token，不再用 qt-material）。分頁分為 core／editing／detection／automation／system 五類。 | +| `main_widget.py` | 433 | 工作區：擁有 `QTabWidget`，依 `tab_registry.TAB_SPECS` 註冊 48 個分頁，並暴露 show/hide/activate/list API 給選單列與導覽面板。只有預設開啟的三個分頁與自己的 mixin 表單在啟動時建立，其餘第一次開啟才匯入模組、建立 widget。核心分頁在 `_own_tab_builders` 宣告 `(label_key, handler)` 動作對。 | +| `tab_registry.py` | 156 | 分頁表：每個分頁一筆 `TabSpec`（鍵、標題鍵、分類、模組與類別名），`TabEntry` 在第一次存取 `widget` 時才呼叫 factory。不匯入 Qt。 | | `navigation.py` | 200 | `NavigationPanel`：搜尋框 + 依分類的功能樹，列出每個已註冊分頁（開啟中的以粗體標示），只回報被選的鍵，開啟分頁仍由視窗負責。 | -| `theme.py` | 178 | 設計 token（`ThemeTokens`：顏色、圓角、間距、字族）、深色與淺色兩組值、由 token 產生的樣式表與對應的 `QPalette`；不載入圖檔或字型檔。 | +| `theme.py` | 183 | 設計 token（`ThemeTokens`：顏色、圓角、間距、字族）、深色與淺色兩組值、由 token 產生的樣式表與對應的 `QPalette`；不載入圖檔或字型檔。 | | `_auto_click_tab.py` | 291 | 自動點擊分頁的 mixin 建構器。 | | `_screenshot_tab.py` | 137 | 截圖／取像素分頁 mixin。 | | `_image_detect_tab.py` | 115 | 影像偵測分頁 mixin。 | | `_script_tab.py` | 115 | 腳本執行分頁 mixin。 | | `_record_tab.py` | 114 | 錄製／回放分頁 mixin。 | | `_report_tab.py` | 88 | 報表分頁 mixin。 | -| `_i18n_helpers.py` | 66 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | +| `_i18n_helpers.py` | 75 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | | `_validators.py` | 29 | `int_validator()`／`double_validator()`：以 C locale 驗證的數字輸入框 validator，接受的正是 `int()`／`float()` 讀得懂的寫法（預設 locale 在法文、德文下只收小數逗號）。所有數字 `QLineEdit` 都用它。 | | `_screen_geometry.py` | 55 | Qt 邏輯座標與截圖用的原生像素互轉：`native_region()`、`screen_at_native()`、`logical_point()`（每個螢幕的左上角在兩者相同，螢幕內依 device pixel ratio 縮放）。區域選取與主機端標註覆蓋層都用它。 | | `_daemon_thread.py` | 79 | `DaemonThread`：`QThread` 的替代品，保留遠端桌面 worker 用到的介面（`start`／`run`／`isRunning`／`wait`／`requestInterruption`／`started`／`finished`），但 `run()` 跑在 daemon `threading.Thread` 上，刪除物件或程式結束都不會銷毀執行中的執行緒。 | -| `_worker_thread.py` | 216 | `start_worker()`：在 daemon `threading.Thread` 上執行 `QObject` worker 的 `run()`（沒有 `QThread` 可被銷毀），並經由分頁擁有的中繼物件回報結果（回呼一律在 GUI 執行緒；worker 沒處理的例外也送到 `on_fail`）；worker 留在模組登錄表直到 GUI 執行緒看到它結束，回傳 `WorkerHandle`（`isRunning()`）；程式結束時先呼叫 worker 的 `request_stop()`，最多等 10 秒，仍在跑的隨行程結束。 | -| `language_wrapper/` | 5,067 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | +| `_worker_thread.py` | 222 | `start_worker()`：在 daemon `threading.Thread` 上執行 `QObject` worker 的 `run()`（沒有 `QThread` 可被銷毀），並經由分頁擁有的中繼物件回報結果（回呼一律在 GUI 執行緒；worker 沒處理的例外也送到 `on_fail`）；worker 留在模組登錄表直到 GUI 執行緒看到它結束，回傳 `WorkerHandle`（`isRunning()`）；程式結束時先呼叫 worker 的 `request_stop()`，最多等 10 秒，仍在跑的隨行程結束。 | +| `language_wrapper/` | 5,765 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | | `selector/` | 216 | 拖曳選取螢幕區域的半透明全螢幕覆蓋層與樣板裁切工具（互動式，但都有對應的程式化 API）。 | > **分頁指令一律走 Actions 選單**：分頁本身只放輸入、表格與結果檢視，指令由視窗層選單暴露。 @@ -957,16 +958,16 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | -#### 遠端桌面 GUI（`gui/remote_desktop/`，19 檔／6,735 行） +#### 遠端桌面 GUI（`gui/remote_desktop/`，32 檔／7,340 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `webrtc_panel.py` | 2,527 | WebRTC 子分頁主體。 | +| `webrtc_panel.py` | 372 | WebRTC 子分頁主體。 | | `webrtc_dialogs.py` | 519 | WebRTC GUI 用的自訂對話框與清單元件（待審檢視者、信任清單、通訊錄、遠端檔案表、稽核記錄、LAN 瀏覽）。 | | `advanced_group.py` | 92 | 兩個 WebRTC 面板共用的 Advanced STUN／TURN（含選用硬體編碼器）群組，含它寫回面板的 Protocol。 | | `trusted_group.py` | 70 | WebRTC host 面板的信任 viewer 清單群組（移除／清空／匯入／匯出），含它寫回面板的 Protocol。 | -| `connection_screen.py` | 729 | Quick Connect —— AnyDesk 風格單畫面入口。 | -| `viewer_panel.py` | 543 | 「控制另一台機器」子分頁。 | +| `connection_screen.py` | 749 | Quick Connect —— AnyDesk 風格單畫面入口。 | +| `viewer_panel.py` | 573 | 「控制另一台機器」子分頁。 | | `webrtc_known_hosts.py` | 346 | TOFU 釘選庫瀏覽器：`KnownHostsDialog` 與帶外釘選用的小表單。由 `webrtc_dialogs` 再匯出。 | | `host_panel.py` | 371 | 「分享這台機器」子分頁。 | | `frame_display.py` | 228 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | @@ -1079,26 +1080,26 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 98 | 28,507 | -| `utils/mcp_server/` | 37 | 19,161 | -| `utils/remote_desktop/` | 56 | 13,301 | -| `utils/executor/` | 8 | 9,666 | -| `utils/usb/` | 17 | 4,816 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,464 | +| `gui/` | 119 | 32,099 | +| `utils/mcp_server/` | 41 | 20,726 | +| `utils/remote_desktop/` | 56 | 13,513 | +| `utils/executor/` | 8 | 9,972 | +| `utils/usb/` | 20 | 4,946 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,620 | | `utils/accessibility/` | 14 | 3,143 | -| `wrapper/` | 19 | 4,099 | -| `windows/` | 23 | 2,154 | -| `utils/rest_api/` | 8 | 2,039 | -| `utils/agent/` | 9 | 2,142 | +| `wrapper/` | 24 | 6,071 | +| `windows/` | 24 | 2,266 | +| `utils/rest_api/` | 8 | 2,078 | +| `utils/agent/` | 13 | 2,251 | | `linux_with_x11/` | 19 | 1,281 | -| `linux_wayland/` | 17 | 2,921 | -| `utils/triggers/` | 4 | 1,383 | +| `linux_wayland/` | 23 | 5,153 | +| `utils/triggers/` | 4 | 1,396 | | `utils/ocr/` | 10 | 1,381 | | `utils/usbip/` | 5 | 1,008 | | `utils/assertion/` | 3 | 887 | | `osx/` | 17 | 981 | | `autocontrol-lsp/` | 8 | 744 | -| `utils/hotkey/` | 7 | 852 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 689 | 58,556 | -| **總計** | **1,071** | **161,486** | +| `utils/hotkey/` | 7 | 861 | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 714 | 65,448 | +| **總計** | **1,140** | **178,825** | From b9437ad923830415802bf95608d8ab34afe21060 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:12:05 +0800 Subject: [PATCH 34/44] Record the second sweep: what plans B to G and the follow-ups delivered and what is still unverified --- CHANGELOG.md | 96 +++++++++++++++++++- Progress.md | 197 +++++++++++++++++++++++++++------------- README.md | 2 + README/README_zh-CN.md | 2 + README/README_zh-TW.md | 2 + architecture_explore.md | 13 ++- docs/updates/2026-10.md | 66 ++++++++++++++ docs/updates/README.md | 13 ++- 8 files changed, 320 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d32a82885..e6b386226 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,51 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's ### Added +- Action journal: `start_action_journal` / `stop_action_journal` / + `read_events` / `list_journal_runs` (`AC_journal_*`, `ac_journal_*`), an + opt-in JSON-lines record of executed actions with secrets masked before + they are written. `generate_candidate_from_log` (`AC_generate_code_from_journal`, + `ac_generate_code_from_log`, `je_auto_control codegen --from-log`) builds a + reviewable candidate script from one run without executing anything from it. +- Self-healing measurement: `evaluate_locators`, `evaluate_healing_dataset`, + `EvaluationSample`, `HealingComparison`, template revisions + (`propose_` / `preview_` / `accept_` / `revert_` / `list_template_revision(s)`), + `heal_context`, six `AC_self_heal_*` commands and MCP tools, and + `benchmarks/self_healing`. +- Config sync: `ConfigStore` (SQLite, revision-checked commits), version + vectors and `merge_entries`, `SyncOutbox`, `SyncAdapter` / `sync_assets`, + `config_sync_run` / `_status` / `_resolve` / `_full_resync` (`AC_config_sync_*`, + `ac_config_sync_*`) and a Config Sync tab. Signaling server flags + `--config-db` and `--allow-blind-config-writes`, env `AC_SIGNALING_CONFIG_DB`. +- Mobile: `DeviceContext`, `open_device`, `DeviceSession`, `DeviceFrame`, + gestures (`Tap`, `LongPress`, `Swipe`, `Drag`, `Pinch`), app lifecycle, + `MobileExtension`, 46 new `AC_android_*` / `AC_ios_*` commands with MCP tools + and Script Builder entries, and a Mobile tab. Built against fake ADB / WDA + transports only; not run on a device. +- Wayland: `probe_capabilities` / `AC_probe_capabilities` / + `ac_probe_capabilities`, `CapabilityStatus`, `WaylandAuthorisationError`, + `reset_input_authorisation`, `PhysicalRecorder`, `InputStepLog`, + `StopShortcutSession`, an opt-in libei helper process + (`JE_AUTOCONTROL_WAYLAND_EI_WORKER`), `JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES`. +- MCP tool modes: `JE_AUTOCONTROL_MCP_TOOL_MODE` / `--tool-mode` + (`full`, `progressive`, `static`), `JE_AUTOCONTROL_MCP_TOOL_PROFILE`, the + session tools `ac_tools_search` / `_schema` / `_enable` / `_disable` / + `_state`, `ToolIndex`, `ToolView`, `ToolMode`; `tools/list` is paged in the + two new modes. +- RBAC: user management (`AC_user_*`, `ac_user_*`, `je_auto_control users`, + `python -m je_auto_control.utils.rbac`, a Users group in the REST API tab), + `Capability.READ_DATA`, `DeferredOwner` / `capture_owner` / `owner_scope`. +- Signing: optional passphrase for the Ed25519 private key + (`passphrase=`, `JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE`); + `AutoControlSignatureException`. +- GUI: `gui/task_controller.py` (cancellable background work), + `AutoControlGUIWidget.open_tab` / `close_tab(key, release=False)` and an + optional `dispose()` hook on tabs, remembered window state + (`JE_AUTOCONTROL_GUI_SETTINGS`), `benchmarks/gui_startup.py` and + `gui_workloads.py`. +- Windows key names that follow the keyboard layout: `slash`, `backslash`, + `semicolon`, `quote`, `backquote`, `bracketleft`, `bracketright`, `equal`. +- `examples/28`–`33` (see `examples/README.md`). - Action files can be signed with an Ed25519 key pair, so an endpoint that verifies does not hold what signs: `create_signing_keypair(private_path, public_path)` / `AC_create_signing_keypair`, `private_key_path=` on @@ -193,6 +238,55 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's ### Changed +- **Breaking wire change: config sync PUT `/config/{user_id}` is version 2.** + The body is `{"version": 2, "base_revision", "operation_id", "bucket"}`; a + stale base answers 409 and writes nothing. An older client's push is refused + with HTTP 428 unless the server runs with `--allow-blind-config-writes`; a + newer client refuses an older server, so upgrade the server first. + `ConfigSyncClient.push` returns the revision and raises `ConfigSyncConflict` + instead of overwriting. Buckets are stored in SQLite. +- **Under RBAC, a viewer no longer gets 42 read-only MCP tools that return + data rather than screen state** (clipboard, files, databases, references and + tokens, processes); they need the new `read_data` capability, which operator + and admin hold. A scheduler job, trigger, hotkey, webhook, e-mail trigger or + watchdog rule registered by an authenticated user now runs as that user, + with the role re-read when it fires. Without RBAC nothing changes. +- REST `POST /execute_file` answers 403 with the reason for a refused + signature (was 500). `rest_api_registry.status()` reports `rbac` and + `users_path`, and `token` is `None` when a user store is in use. +- **Wayland: a refused portal consent no longer falls back to ydotool.** + Input raises `WaylandAuthorisationError` until `reset_input_authorisation()`; + a session the compositor disconnected refuses further input the same way. + Other portal failures fall back as before. +- `AdbClient.text()` raises `AdbUnsupportedError` for text `adb input text` + cannot carry (non-ASCII, a literal `%s`) instead of reporting success; + `AC_android_text` routes such text through ADBKeyBoard or uiautomator2. adb + failures are typed (`AdbTimeoutError`, `AdbUnauthorizedError`, + `AdbDeviceMissingError`). A device-matrix step without an address targets + that worker's own device. +- `self_heal_locate` / `self_heal_click`: `screen_region` now confines the + template match as well as the VLM; `self_heal_click` logs one event after + the click; results and `HealEvent` gain optional fields. +- `je_auto_control codegen`: the positional script is optional (use + `--from-log`), and `--style` defaults by source. +- Variable scope: `run --dry-run --var`, observer callbacks, `AC_llm_run` and + `AC_run_state_machine` each get a deliberate scope instead of the process + one. A state machine's `on_enter` given as a single action now runs; it was + always rejected before. +- Package gate: a list that loads an allowed package and then uses its + commands runs; it used to fail validation with "unknown command". +- USB passthrough: a transfer waits for its own `timeout_ms` plus + `reply_timeout_s`; channel-level errors reach the caller at once. + Concurrent changes to one `UsbAcl` no longer lose rules. +- MCP `ac_window_minimize` / `_maximize` / `_restore` return a tool error + when the window refused. +- GUI: pages scroll in a narrow window instead of being squeezed; Script + Builder and Remote Desktop are built when first selected; connecting a + remote desktop, WebRTC offer / answer, OCR reads and device-matrix runs no + longer freeze the window; `Ctrl+B` works; tab close buttons are drawn in + the theme's colours. `gui/remote_desktop/webrtc_panel.py` is split into + twelve modules. +- The `[gui]` extra no longer installs `qt-material`. - **Breaking: the package gate refuses by default.** `AC_add_package_to_executor` / `AC_add_package_to_callback_executor` (and the `package_manager` methods) no longer import a package that has not been @@ -278,7 +372,7 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's An embedder that relied on every tab existing after construction (a tab's timer or listener started at start-up) has to open the tab first. - The main window no longer imports `qt_material`; it styles itself from - `gui/theme.py`. The `[gui]` extra still lists `qt-material` for now. + `gui/theme.py`. (The `[gui]` extra listed `qt-material` until 2026-10-09.) - GUI text size "Auto" is 10 / 11 / 13 pt by screen height (was 12 / 14 / 16), and the default window is 1280×800 (was 1000×760). The font family is the platform's UI font instead of Lato. diff --git a/Progress.md b/Progress.md index 268e6e4ea..6888d2a2e 100644 --- a/Progress.md +++ b/Progress.md @@ -2,29 +2,12 @@ ## 跨平台與 GUI 全面改版 -`TODO` — 重設 UI、重寫並優化 GUI、修正 Wayland 與函式庫問題、跨機器同步、 -深化全套 mypy、補齊 iOS／Android、MCP 逐步揭露、自愈定位器量測、動作日誌 codegen、 -完整範例與文件。涉及 `gui/`、`linux_wayland/`、`android/`、`ios/`、 -`utils/{config_sync,remote_desktop,mcp_server,self_healing,codegen,executor}/` 與型別／文件驗證。 +`WIP` — 計畫 A–H 的程式都已交付（A 在 U-20261009-01…14，B–H 在 U-20261009-15…26）；還沒做到的是**實機驗證**與下面各節列的缺口 + 核准設計：[跨平台自動化與 GUI 改版](docs/superpowers/specs/2026-10-02-platform-gui-modernization-design.md)。 -實作計畫：[分階段交付計畫](docs/superpowers/plans/2026-10-02-modernization-index.md)，待審閱。 -現有 `[Answer]` 決策沿用。 - -`WIP` — 計畫 A（既有決策與執行契約）已交付，A2、A7 在 2026-10-08、其餘在 2026-10-09（U-20261009-01…14）； -各項沒驗到或沒做完的部分在本檔下面三節（輸入與視窗、macOS、這一輪修正留下的後續）。計畫 F（GUI）：F1 的延遲分頁註冊與 F2 的導覽／搜尋／主題已交付（U-20261008-02）。 -計畫 B（動作日誌、自愈量測、codegen）、C（持久化同步與遠端 session；其中 C4 的擁有者模型已做）、D（Wayland）、E（Android／iOS）、G（MCP 逐步揭露）、H（型別深化與總驗收）尚未開始。F 還缺： - -- **窄視窗的內容是被擠壓而不是可捲動**：`gui/main_widget.py` 給 `QTabWidget` 明確的最小尺寸讓視窗能縮到 640×420， - 但分頁內容沒有包進 `QScrollArea`；包進去會改變 `tabs.indexOf(entry.widget)` 這個 PyBreeze 與測試都在用的關係，要一起設計。 -- **主題與面板狀態不會記住**：`AutoControlGUIUI.set_theme`、導覽面板的顯示與寬度、字級都只活在當次執行； - 要用 `QSettings` 存，並讓測試不寫到使用者的設定。 -- **`qt-material` 還在 `[gui]` extra**：`gui/main_window.py` 已不匯入它。移除要同時改 `pyproject.toml`、`dev.toml`、 - `requirements.txt`、`uv.lock` 與 mypy 的 override，並先確認 PyBreeze 沒有靠這個 extra 取得它。 -- **分頁的關閉鈕是 Fusion 內建圖示**：計畫規定主題不新增點陣圖相依，換圖示要用 Qt 內建向量或既有資產。 -- **Remote Desktop 與 Script Builder 仍在啟動時建立**（預設開啟），啟動時間 2.6–2.7 秒裡大半是它們與門面匯入。 -- **F1 的 `TabRegistry.open/close` 介面與 `close` 釋放訂閱**：現在關閉分頁只是從分頁列移除，widget 留著。 -- **F3**（共用 worker、取消、關閉時不碰已銷毀物件、`webrtc_panel.py` 拆分）與 **F4**（啟動／記憶體基準、mixed-DPI、 - 功能對等測試）尚未開始。 +實作計畫：[分階段交付計畫](docs/superpowers/plans/2026-10-02-modernization-index.md)。 +這一批幾乎全部只對假後端測試：沒有在真的桌面打字或點擊、沒有 Mac、沒有 Linux 桌面、沒有 Android／iOS 裝置、沒有真的 MCP client、 +沒有對付費 API 送過請求。每一節寫的是該領域還缺什麼證據或功能。 **只記未完成的事。** 完成的工作記在 [docs/updates/](docs/updates/README.md)（每月一個批次檔， 索引與查詢指令在它的 README），相容性變更寫進 [CHANGELOG.md](CHANGELOG.md)；完成的項目 @@ -119,7 +102,7 @@ sys_platform != 'win32' or platform_machine != 'ARM64' | `cryptography>=50.0.0` | **沒有** | wheel 只出到 **46.0.3**，46.0.4 起上游就不再發 win_arm64。`>=50.0.0` 是**安全下限**（GHSA-537c-gmf6-5ccf 與 GHSA-g6cj-pr64-35w5），不能為了 arm64 降回去。 | | `pillow==12.3.0` | 有 | `pillow-12.3.0-cp3xx-win_arm64.whl` 一直都在。**曾經被寫成卡點，那是猜的，它從來不是。** | | `mss`／`defusedxml` | 有 | 純 Python。這三個加上 Pillow 就是 arm64 實際裝到的全部。 | -| `PySide6==6.11.1`／`qt-material==2.17` | 有 | `[gui]` extra 在 arm64 上裝得起來。 | +| `PySide6==6.11.1` | 有 | `[gui]` extra 在 arm64 上裝得起來（量的時候 extra 還含 `qt-material==2.17`，2026-10-09 已移除）。 | | `aiortc` | **沒有** | 卡在傳遞相依 `google-crc32c`，與本專案的選擇無關；`av` 自己有 wheel。 | 重驗指令（不需要 arm64 機器，也不需要 runner）： @@ -174,6 +157,20 @@ pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version **緩解**:驗不到的擷取部分有逃生門——`JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` 讓操作者 直接指定自己的擷取指令（`{output}` 會被換成暫存 PNG 路徑）,優先於所有偵測。 +### 2026-10-09 加進來、還沒在 Linux 上跑過的（U-20261009-21） + +能力狀態機（`wrapper/capabilities.py`）、授權記錄（`linux_wayland/authorisation.py`）、實體事件讀取與停止快捷鍵（`input_events.py`、`global_shortcuts.py`）、 +選用的 libei helper 行程（`JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`，預設關閉）都是在 Windows 上對假物件寫的。要看 CI 的 `portal-verification`、 +`eis-verification`、`wayland-verification` 三個 job 的輸出（新增的都是 `info` 行，不改變 job 的結果）： + +- **「同意被拒絕就不退回 ydotool」靠 liboeffis 的錯誤文字含 "denied" 來判斷**——沒對真的 liboeffis 驗過；判斷不出來時行為與以前相同（退回 ydotool）。 + 同意對話框放著不回答（逾時）仍會退回 ydotool。 +- 沒有任何腳本從 server 端切斷一個連線中的 client，所以 `EI_EVENT_DISCONNECT` 對應到 `revoked` 沒有實證。 +- helper：SIGTERM 時放開按鍵、經 portal（fd 路徑）而非 socket 路徑，沒有 job 會跑到。 +- `compositor_identity`、uinput／藍牙裝置在 sysfs 的位置、`struct input_event` 的配置、GlobalShortcuts 的線上格式與哪些桌面有實作、XWayland 的可及範圍： + 都只有 `test/manual_test/wayland_authorisation_checklist.md` 的人工步驟，**沒有執行過**。 +- `record.py`／`listener.py` 仍然丟 `NotImplementedError`（既有測試要求），`PhysicalRecorder` 沒有接到 `record()`。 + --- ## 輸入、視窗與擷取修正：還沒在真的桌面與 Jeffrey_RPA 上驗過 @@ -196,12 +193,6 @@ pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version `MoveWindow` 對最大化視窗與另一個 DPI 的螢幕是否能原樣還原，都只用假的 `user32` 驗過。 - **X11／Wayland**：大寫與 Shift 標點的判斷假設 `keysym_to_keycode` 把大寫字母對到小寫的 keycode，沒有對著 X server 跑過。 -**還沒決定的小事**：Windows 鍵表沒有 `slash` 這個名字。`plus`／`minus`／`comma`／`period` 與 `oem_1`…`oem_8` 都有， -但鍵表的註解刻意不替 `oem_*` 取好讀的別名，因為它們隨鍵盤配置而變（`oem_2` 只有在美式配置上是 `/`）。 -要加的話是 `wrapper/_platform_windows.py` 的 `keyboard_key_aliases` 一行。 - -**MCP 的 `_show_command`**（`utils/mcp_server/tools/_handlers_system.py`，`window_minimize` 等）仍不看 `show_window` 的新回傳值。 - --- ## macOS 的修正：CI 驗到一半，其餘要一台真的 Mac @@ -224,41 +215,108 @@ U-20261009-06 與 -04 的 macOS 部分是在 Windows 上對假的 pyobjc／Quart --- -## 這一輪修正留下的後續 - -`TODO` — 各項都是 2026-10-09 那批（U-20261009-01…14）交付時明確沒做的部分 - -- **Intel Mac 安裝要編譯 `cryptography`**：下限拉到 50 之後 `macosx_10_9_x86_64` 沒有 wheel（探測：最新只到 48.0.1）， - `pip install je_auto_control` 在 Intel Mac 需要 Rust 工具鏈。沒有在 Intel Mac 上實際編過。 -- **簽章分離擋不住能寫檔的人**：能執行寫檔指令（shell、檔案類指令）的人仍能換掉公鑰檔本身，保護那個檔是作業系統權限的事； - 私鑰是未加密的 PEM（0600，Windows 上沒驗權限位元），沒有通行碼選項；遷移模式開著時 HMAC 簽章會被接受。 - 強制簽章經 socket／REST／MCP 的端到端沒有跑過，覆蓋的是共用的 `read_executable_action_json` 與指令本身。 -- **RBAC**（`JE_AUTOCONTROL_RBAC_USERS`，預設關閉）： - - 延後執行的工作不帶角色：operator 註冊的排程、觸發器、熱鍵、watchdog 之後執行時沒有身分，裡面的特權指令不會被擋。 - - 使用者只能用 Python 管（`UserStore.add_user/set_role/rotate_token/remove_user`），沒有 `AC_*`、CLI 或 GUI；`Capability.MANAGE_USERS` 因此沒有任何路由或工具對應。 - - `gui/rest_api_tab.py` 仍顯示共用 token，RBAC 開啟後那個 token 會被拒絕。 - - viewer 拿得到所有標成唯讀的工具，包括 `ac_sql_query`、`ac_load_dotenv`、`ac_get_clipboard`、`ac_jwt_encode`，沒有重新分類。 - - 只實作 `check()` 的自製 REST gate 現在會 `AttributeError`（要有 `authenticate()`）。 - - 沒對真的 client（Claude Desktop、VS Code、內建 dashboard）試過；SSE、無狀態與 `subscriptions/listen` 路徑、TLS 加 RBAC 沒有測試。 -- **MCP 路徑根目錄**（`JE_AUTOCONTROL_MCP_PATH_ROOTS`，預設關閉）管不到的參數：執行動作清單的工具、 - 有時才是路徑的參數（`ac_open_path`／`ac_plan_open`／`ac_file_association`／`ac_act_in_view` 的 `target`，`ac_launch_process`／`ac_shell` 的 `argv`／`command`）、 - `ac_handle_file_dialog` 的 `path`、自由格式物件裡的路徑（`ac_run_suite` 的 `spec`、`ac_run_dag` 的 `definition`、`ac_assert_all` 的 `specs`）、沒標註的外掛工具。 +## 安全與伺服器：留下的缺口 + +`TODO` — 各項都是 U-20261009-08、-09、-13、-22、-23 交付時明確沒做的部分 + +- **Intel Mac 安裝要編譯 `cryptography`**：下限 50 沒有 `macosx_10_9_x86_64` wheel（探測：最新只到 48.0.1），沒有在 Intel Mac 上實際編過。 +- **簽章分離擋不住能寫檔的人**：能執行寫檔指令的人仍能換掉公鑰檔本身；Windows 上沒驗私鑰檔的權限位元；遷移模式開著時 HMAC 簽章會被接受； + 強制簽章經 socket server 的端到端沒有測試（REST 與 MCP 已有）。 +- **RBAC**： + - MCP 的 HTTP session 沒有綁定建立它的使用者：每個請求仍以自己的 token 授權，但知道 session id 的任何已驗證使用者都能接上它的 GET 串流或 DELETE 它 + （在 `utils/mcp_server/http_transport.py`，727 行，要先拆）。漸進模式下第二個使用者拿同一個 session id 會看到第一個人啟用的工具名稱（再經自己的角色過濾）。 + - observer 回呼、LLM planner、狀態機這幾條延後路徑不帶擁有者。 + - `AC_user_*` 沒有 Script Builder schema。 + - `utils/rbac/policy.py` 的 `DATA_TOOLS`（42 個需要 `read_data` 的工具）是實作時分的類，**請維護者過目**；刻意留在 `read_screen` 的有 + `ac_list_run_history`、`ac_costs_*`、`ac_trace_export`、`ac_self_heal_log_list`、`ac_usb_acl_list`、`ac_vlm_locate`／`ac_self_heal_locate`（會花 VLM 費用）。 + - 被拒絕的擁有者的 watchdog 規則每次輪詢都記一筆 info，沒有節流。 + - 沒對真的 client（Claude Desktop、VS Code、內建 dashboard）試過。 +- **MCP 路徑根目錄**管不到的參數：執行動作清單的工具、有時才是路徑的參數（`ac_open_path`／`ac_plan_open`／`ac_file_association`／`ac_act_in_view` 的 `target`， + `ac_launch_process`／`ac_shell` 的 `argv`／`command`）、`ac_handle_file_dialog` 的 `path`、自由格式物件裡的路徑、沒標註的外掛工具。 檔案 symlink 的跳脫在這台機器上建不出來（只跑了目錄 junction）；POSIX 的 `:` 分隔與 `~` 沒在 Linux／macOS 跑。 - `roots/list` 來的根目錄要另外開 `JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT` 才算數（維護者 2026-10-09 確認）。 -- **USB passthrough**：client 仍在 `reply_timeout_s`（10 秒）放棄，即使呼叫端要的 `timeout_ms` 更長（host 接受到 60 秒）； - 對新 host 逾時已經無害，對舊 host 正常使用就會進入「需重連」。通道層的 ERROR（passthrough 關閉、bad frame、host 沒有 session）在 session 之前送出、不帶請求編號， - 呼叫仍是等到逾時。沒對真的舊版 host、真的 WebRTC 通道或真的 USB 裝置跑過。 -- **變數範圍**：`je_auto_control run --dry-run --var`、observer 回呼（`AC_observe_add` 與 MCP 的 observe bridge）、 - `utils/llm/planner.py` 與狀態機預設 runner 仍用行程層級的範圍。排程／觸發／熱鍵／webhook／e-mail 的隔離是在 `run_counting_failures` 測的，沒有把每個 daemon 真的觸發一次。 - free-threaded 版（執行緒繼承 context）上，一次執行裡另開的執行緒會看到該次執行的範圍，沒測。 -- **遠端桌面的擁有者模型**：Quick Connect 沒有狀態列，它的連線被取代時視窗只是關掉；主機的 Stop 會停任何擁有者開的 host（刻意的）； - 只用假的 viewer／host 測過，真的 viewer 被擠掉時會不會觸發 `on_error` 而多跳一個警告框，沒看過。 -- **套件閘門**：`execute_action` 會先驗完所有指令名稱才執行，所以「載入套件」與「用它的指令」寫在同一份清單裡會以 unknown command 失敗，與閘門無關（既有限制）。 - repo 根目錄的舊範例 `AutoControl/keyword/keyword1.json` 載入 `time`，現在預設會被拒絕。 -- **`pytest --cov` 沒有重新量過**：進入點搬到 `je_auto_control_pytest` 之後，它是否還少算，未知；規則仍是 `coverage run -m pytest`。 -- **快到 750 行上限的檔案**：`utils/usb/passthrough/viewer_client.py`（748）、`gui/remote_desktop/connection_screen.py`（729）、 - `utils/mcp_server/server.py`（734）、`utils/mcp_server/http_transport.py`（727）、`utils/agent/backends/anthropic_computer_use.py`（723）——下一次要加東西就得先拆。 -- **Sphinx 沒有建置過**：這批改了 35 個 `.rst`，都沒有渲染檢查。 +- **MCP 完整模式的唯讀漏洞**：唯讀只在建立 registry 時過濾，所以完整模式下外掛啟動後才註冊的變更類工具仍會被列出與執行。 + 漸進與靜態模式已經擋住；完整模式要擋會改變預設行為，待決定。 +- **MCP 漸進揭露**（`JE_AUTOCONTROL_MCP_TOOL_MODE`，預設關閉）：沒對任何真的 client 跑過，client 會不會跟 `nextCursor`、收到 `list_changed` 會不會重列，未知； + `tools/list` 的分頁游標與 `-32602` 是憑對規格的記憶寫的，repo 裡沒有可對照的既有實作；外掛由 watcher 執行緒註冊時，通知到不了 HTTP session。 + `start_mcp_http_server` 沒有 `tool_mode` 參數（要用環境變數或自己建 `MCPServer`）。沒有 GUI 設定入口（GUI 裡本來就沒有 MCP 設定面）。 +- **USB passthrough**：沒對真的舊版 host、真的 WebRTC 通道或真的 USB 裝置跑過。`UsbAcl` 的鎖是每個實例一把， + 兩個實例或兩個行程仍可能在資料檔與 `.sig` 之間互相踩到，要檔案鎖或單一原子檔。 +- **設定同步**（U-20261009-17）： + - CI 沒裝 `[signaling]` extra，所以 signaling server 端的測試在 CI 全部 `importorskip`；只有 `ConfigStore` 的測試在 CI 跑。 + - 用舊的 `upsert`／`remove`（不帶 `origin=`）寫的項目仍是 last-write-wins；只用舊 `sync()` 的裝置不會被記成 peer，墓碑回收因此等不到它。 + - 同一個 `operation_id` 帶不同內容不會被發現。複合觸發器（all-of／any-of／sequence）不同步。衝突項目要呼叫 `config_sync_resolve` 才會解。 + - `note_received` 與 `ClipboardEchoGuard` 有測試但沒有呼叫端；資料夾鏡像對寫到一半的檔案沒有處理。 + - 資產傳輸只有 `DirectoryAssetTransport`（兩台機器都看得到的資料夾），signaling server 沒有 blob 端點。分頁不記表單欄位。 + - `config_sync/client.py` 717 行。沒有做過真的兩台機器同步。 + +--- + +## 動作日誌、候選腳本與自愈量測：留下的缺口 + +`TODO` — U-20261009-15、-16 交付時明確沒做的部分 + +- 日誌事件沒有 `artifacts` 欄位；`run_history` 的列沒有連到日誌的 run。 +- 只有 `AC_parallel` 把父步驟交給工作執行緒；DAG runner、device matrix、bulkhead 跑的步驟 `parent_id` 是 None， + 產生候選腳本時靠時間包含關係猜，猜出來的列為 `detached`、不進腳本。 +- 錯誤文字經 `redact_secret_text` 遮蔽，那是比對樣式：例外訊息裡以樣式沒涵蓋的形式出現的秘密仍會被寫進去。 +- 候選腳本沒有 GUI 的 diff 檢視；Robot 輸出沒有語法檢查。沒量過日誌在真的執行上的負擔，也沒試過多個行程寫同一個日誌檔。 +- 自愈量測：資料集 JSON 只能指定 `template` 策略，VLM 版本要從 Python 傳 callable，沒有模型成本欄； + `AC_self_heal_click` 帶不了驗證（JSON 步驟沒有 callable），`action_verified` 永遠是 `None`；GUI 以 JSON 文字顯示結果，沒有比較表或影像 diff； + 區域限定的樣板比對沒對真的螢幕跑過。 + +--- + +## 行動裝置（Android／iOS）：完全沒有在裝置上跑過 + +`BLOCKED` — 需要 Android 模擬器或實機、iPhone 與 WebDriverAgent；這台機器連 `adb` 都沒有 + +U-20261009-18 的每一個裝置面呼叫都只對假的 ADB／uiautomator2／WDA 測過。`uiautomator2` 與 `facebook-wda` 沒有安裝在開發環境裡， +方法名稱與參數是憑記憶寫的，第一次接上真的 SDK 很可能要修： + +- uiautomator2：`send_keys`、`drag`、空 selector 的 `gesture(...)`（捏合）、`clipboard`／`set_clipboard`、對話框按鈕的 resource id。 +- facebook-wda：`app_launch`／`app_terminate`／`app_state`、`alert.*`、`set_clipboard`、`status()`、`orientation` 的值、元素的 `pinch`、`screenshot()` 回傳 PIL 影像。 +- 旋轉方向：螢幕截圖以面板自然方向回來時，`landscape_left` 轉 270°、`landscape_right` 轉 90°，是從文件推的，沒看過。 +- Android：`input draganddrop` 是否存在、`monkey` 啟動輸出、`pidof`／`pm path` 的結束碼、`dumpsys activity activities` 的格式、`pkill -INT screenrecord` 能否收尾 MP4。 + ADBKeyBoard 的送達無法由裝置確認，是從 IME 被選取推定的。 +- 已知限制：iOS 捏合忽略 `x`／`y`／`span`；iOS `app_state` 不會回 `not_installed`；iOS 的安裝／檔案／錄影沒有 adapter（只有 `MobileExtension` 協定與註冊點）； + Mobile 分頁的 probe／run 會卡住 GUI 執行緒到裝置回應或逾時。`AC_android_shell` 刻意不做成 MCP 工具，**請維護者確認**。 +- 沒有 `examples/` 的實機腳本以外的 smoke；Xcode／WDA 的建置與簽章只寫成前置條件。 + +--- + +## GUI：留下的缺口 + +`TODO` — U-20261009-19、-20 交付時明確沒做的部分 + +- **43 處會卡住 GUI 執行緒的呼叫還沒搬**（稽核找到 52 處，搬了 9 處：遠端桌面連線、WebRTC offer／answer、OCR、Device Matrix）。最需要單獨設計的是 + 在 GUI 執行緒上執行腳本的分頁（`_script_tab`、Script Builder、LLM planner、錄製回放、test suite、ChatOps）：executor 沒有逾時或停止事件， + 搬到別的執行緒會改變重入與 Stop 的語意。其餘：檔案傳輸（`connection_screen._on_files_dropped`、WebRTC 推檔）、`webrunner_tab`、`assertions_tab`、 + `self_healing_tab`、`a11y_audit_tab`、`accessibility_tab`、`media_checks_tab`、`webrtc_host_connection._on_apply_answer`、 + 各種會 join 執行緒的 Stop（2–6 秒）、每個輸入事件的 socket 寫入（要在後端加送出佇列）。清單在 U-20261009-19。 +- **一個沒查明的間歇失敗**：`test_remote_desktop_gui.py` + `test_remote_registry_owners_gui.py` + `test_gui_task_lifecycle.py` 連跑約 38 輪出現過一次失敗， + 沒抓到是哪個測試，之後 30 輪乾淨。 +- **把 Python 建的 widget 放進分頁列會在結束時崩潰**（70 次裡 8 次存取違規，原因沒找到），所以關閉鈕改成樣式表圖片， + 圖是執行時用 `QPainter` 畫好寫到 `~/.je_auto_control/gui_cache/`。另外發現：已 `deleteLater()` 又還在 Python 參照循環裡的 `QObject`，被垃圾回收時會弄壞堆積（0xc0000374）。 +- 還沒有任何分頁實作 `dispose()`；全部 50 個分頁開著時切換主題要 1.0–1.4 秒；基準（`benchmarks/gui_*.py`）只回報、CI 不設門檻。 +- `gui/remote_desktop/connection_screen.py` 749 行，離上限一行。 +- `test_region_selector_screens.py:34` 仍掃 `QApplication.topLevelWidgets()`，在 Python 3.10／3.11 有與下面 segfault 相同的風險；某個 GUI 測試會漏一個 `QComboBox` 的彈出容器，沒找出是哪個。 +- 沒在 macOS／Linux 上看過新的關閉鈕與捲動；沒在真的多螢幕桌面試過視窗位置還原；PyBreeze 沒有對這一版跑過（只讀了它的相依宣告）。 +- 各語系新增的字串（日文、簡體中文）沒有母語者看過。 + +--- + +## 雜項 + +`TODO` — 小項目，各自獨立 + +- **`DECIDE`：`pytest --cov` 可以放行了嗎**。進入點搬到 `je_auto_control_pytest` 之後重新量過（2026-10-09，同一棵樹、各跑一次完整套件）： + `coverage run -m pytest` 87.35%，`pytest --cov` 87.32%，差 21 個敘述（19 個在 `je_auto_control_pytest.py` 自己）。原本少算 24 個百分點的問題已經不存在。 + `CLAUDE.md` 與 `test_coverage_measurement.py` 仍規定只能用 `coverage run`；要不要放寬由維護者決定。 +- **本機的共用 `.venv` 裝著 `je_auto_control_dev 0.0.136`**，它的 `pytest11` 進入點還是舊的重量級路徑，所以從那個環境跑的測試仍會載入舊外掛；重裝即可，不影響 CI。 +- **Sphinx 還有 184 個既有警告**（較舊的 `vN_features_doc.rst`、`getting_started/run_in_ci.rst`、`conf.py` 的 `_static` 不存在）；這個月改過的頁面是 0 個。 +- **變數範圍在 free-threaded 版**（執行緒繼承 context）沒有測試。 +- **Windows 的版面鍵名**（`slash` 等 8 個，依前景視窗的鍵盤配置解析）只在美式配置上實際呼叫過，非美式配置用的是假的 `user32`。 +- **狀態機 `on_enter` 的單一動作寫法**以前一直被拒絕、從沒執行過（已修，U-20261009-24）：依賴「它不會跑」的既有狀態機現在會跑它。 --- @@ -292,6 +350,9 @@ U-20261009-06 與 -04 的 macOS 部分是在 Windows 上對假的 pyobjc／Quart 重驗方式就是跑那兩支腳本（`eis-verification` job 已經在跑）；哪天 banner 不再 出現，就把 `_teardown` 的迴避拿掉。形狀與 arm64 那條一樣：卡上游、有一行重驗。 +2026-10-09：新增選用的 helper 行程（`linux_wayland/ei_worker.py`、`ei_client.py`、`ei_transport.py`，`JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`）把 libei 隔離在子行程裡， +**預設關閉，原本的行程內路徑與刻意的洩漏沒有動**。兩支 verify 腳本現在會分類 teardown 並量 helper 的延遲與洩漏；要等 CI 的輸出才知道 helper 是否真的可用。 + --- ## Agent 的歷史壓縮還沒對真的 API 跑過 @@ -309,9 +370,15 @@ U-20261009-11 之後，Anthropic 的兩條路徑（`anthropic.py`、`anthropic_c --- -## `test_usb_acl_prompt.py` 讓 Python 3.10 的 headless 測試間歇 segfault +## Python 3.10 的 headless segfault：已修，等 CI 確認 + +`TODO` — 修正在 U-20261009-25；要看 ubuntu-22.04 與 macos-14 的 3.10 `pytest-headless` 連續幾次都綠才能結案 -`TODO` — `test/unit_test/headless/test_usb_acl_prompt.py::test_bridge_remember_persists_acl_rule` 在 `coverage run -m pytest` 下讓行程 SIGSEGV（exit 139），整個 `pytest-headless` job 因此失敗：2026-09-26 連續三次 AutoControl Code Quality（ubuntu-22.04／3.10），2026-09-30 一次（macos-14／3.10），2026-10-08 一次（ubuntu-22.04／3.10，PR #501，重跑該 job 後通過）；同一次其他版本都過，之後的 run 又過，所以是間歇的。原因還沒查：先在 3.10 開 `faulthandler` 重跑這一支，看崩在哪個原生呼叫。 +`test_usb_acl_prompt.py::test_bridge_remember_persists_acl_rule` 在 3.10 間歇 SIGSEGV（2026-09-26 三次、09-30 一次、2026-10-08 四次）。 +查到的機制：每個 `UsbPassthroughPromptDialog` 都因 `TranslatableMixin._tr(self, …)` 把自己存進自己的登錄表而成為參照循環，被丟掉後要等循環回收； +3.10／3.11 的回收會在配置物件時發生，測試掃 `QApplication.topLevelWidgets()` 的當下若剛好回收，清單裡的指標就指向已銷毀的 widget。 +在 Windows 的 3.10.22 上以探針確定性重現（存取違規），修掉循環後不再發生；CI 那個崩潰本身沒在這台機器重現。 +被銷毀的是不是那兩個被丟掉的對話框、為什麼只有 Linux／macOS，是推論。 --- diff --git a/README.md b/README.md index cd6ca070a..e35d6bd71 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,8 @@ All servers bind to `127.0.0.1` unless you opt in explicitly. **Opt-in hardening.** Each of these is off until configured, and a server without them behaves as before. `JE_AUTOCONTROL_RBAC_USERS=` makes the REST API and the MCP HTTP transport resolve the bearer token to a user and authorise each route, tool and privileged `AC_*` command by role (viewer / operator / admin); the shared token is then refused. `JE_AUTOCONTROL_MCP_PATH_ROOTS` (directories separated by the OS path separator) confines every MCP tool argument that is a file path to those directories, `JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT=1` adds the client's `roots/list`, and `JE_AUTOCONTROL_MCP_ENV_REF_ALLOW` limits which `env://` names `ac_resolve_ref` may read. `JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY` makes an endpoint verify Ed25519-signed action files without being able to sign them (`create_signing_keypair`; the private key stays on the signing machine). A remote-desktop viewer writes files pushed by a host only below `~/Downloads/AutoControl` (`JE_AUTOCONTROL_REMOTE_DOWNLOAD_DIR`); that one is on by default. +**Also configurable.** `JE_AUTOCONTROL_MCP_TOOL_MODE=progressive` (or `je_auto_control_mcp --tool-mode`) starts an MCP session with five core tools that search, describe and enable the rest; `static` serves a fixed profile (`JE_AUTOCONTROL_MCP_TOOL_PROFILE`); the default `full` mode is unchanged. Users for RBAC are managed with `je_auto_control users add|remove|set-role|rotate-token|list`, the `AC_user_*` commands or the REST API tab. `JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE` unlocks a passphrase-protected signing key. The action journal (`start_action_journal`, `AC_journal_*`) records executed actions with secrets masked, and `je_auto_control codegen --from-log` turns one run into a candidate script. Config sync keeps buckets in SQLite with revision-checked writes (`--config-db`, `AC_SIGNALING_CONFIG_DB`; older clients need `--allow-blind-config-writes`). Android and iOS devices get their own sessions (`open_device`, `AC_android_*`, `AC_ios_*`); none of the mobile code has been run on a device yet. On Wayland, `probe_capabilities` / `AC_probe_capabilities` report what input and capture can do and why, and `JE_AUTOCONTROL_WAYLAND_EI_WORKER=1` moves libei into a helper process. The GUI remembers theme, text size, panel and window geometry in `~/.je_auto_control/gui_settings.ini` (`JE_AUTOCONTROL_GUI_SETTINGS`). + ### How the remote-desktop wire protocol works Worth knowing before you expose a host, and described nowhere else in the diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 7473e2a1e..f414e26ed 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -281,6 +281,8 @@ je_auto_control version **可选的加固设置。** 下列各项在设置之前都是关闭的，没设置的服务器行为与先前相同。`JE_AUTOCONTROL_RBAC_USERS=<用户文件>` 让 REST API 与 MCP HTTP 传输把 bearer token 对应到用户，并按角色（viewer／operator／admin）授权每一个路由、工具与特权 `AC_*` 命令；此时共用 token 会被拒绝。`JE_AUTOCONTROL_MCP_PATH_ROOTS`（以操作系统的路径分隔符分开的目录）把每一个属于文件路径的 MCP 工具参数限制在这些目录里，`JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT=1` 再加上 client 的 `roots/list`，`JE_AUTOCONTROL_MCP_ENV_REF_ALLOW` 限制 `ac_resolve_ref` 可以读哪些 `env://` 名称。`JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY` 让端点只能验证 Ed25519 签名的动作文件而不能签名（`create_signing_keypair`；私钥留在签名的机器上）。远程桌面的 viewer 只把 host 推来的文件写在 `~/Downloads/AutoControl` 下面（`JE_AUTOCONTROL_REMOTE_DOWNLOAD_DIR`）；这一项默认就是开的。 +**其他可设置的项目。** `JE_AUTOCONTROL_MCP_TOOL_MODE=progressive`（或 `je_auto_control_mcp --tool-mode`）让 MCP session 从五个核心工具开始，再搜索、查询与启用其他工具；`static` 提供固定的工具组（`JE_AUTOCONTROL_MCP_TOOL_PROFILE`）；默认的 `full` 模式不变。RBAC 的用户以 `je_auto_control users add|remove|set-role|rotate-token|list`、`AC_user_*` 命令或 REST API 标签页管理。`JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE` 用来解开有口令保护的签名私钥。动作日志（`start_action_journal`、`AC_journal_*`）记录执行过的动作并先遮蔽秘密，`je_auto_control codegen --from-log` 把一次执行转成候选脚本。配置同步把数据存在 SQLite 并检查版本（`--config-db`、`AC_SIGNALING_CONFIG_DB`；旧版 client 需要 `--allow-blind-config-writes`）。Android 与 iOS 设备各有自己的 session（`open_device`、`AC_android_*`、`AC_ios_*`）；移动设备的代码还没有在任何设备上跑过。Wayland 上 `probe_capabilities`／`AC_probe_capabilities` 报告输入与捕获能做什么、为什么，`JE_AUTOCONTROL_WAYLAND_EI_WORKER=1` 把 libei 移到辅助进程。GUI 会把主题、字号、面板与窗口位置记在 `~/.je_auto_control/gui_settings.ini`（`JE_AUTOCONTROL_GUI_SETTINGS`）。 + ### 远程桌面的线路协议 把主机开放出去之前值得先了解，而且这一段在其他文档里都没有写。默认传输是**裸 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 7a5273030..399b33a78 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -280,6 +280,8 @@ je_auto_control version **選用的強化設定。** 下列各項在設定之前都是關閉的，沒設定的伺服器行為與先前相同。`JE_AUTOCONTROL_RBAC_USERS=<使用者檔>` 讓 REST API 與 MCP HTTP 傳輸把 bearer token 對到使用者，並依角色（viewer／operator／admin）授權每一個路由、工具與特權 `AC_*` 命令；此時共用 token 會被拒絕。`JE_AUTOCONTROL_MCP_PATH_ROOTS`（以作業系統的路徑分隔字元分開的目錄）把每一個屬於檔案路徑的 MCP 工具參數限制在這些目錄裡，`JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT=1` 再加上 client 的 `roots/list`，`JE_AUTOCONTROL_MCP_ENV_REF_ALLOW` 限制 `ac_resolve_ref` 可以讀哪些 `env://` 名稱。`JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY` 讓端點只能驗證 Ed25519 簽章的動作檔而不能簽章（`create_signing_keypair`；私鑰留在簽章的機器上）。遠端桌面的 viewer 只把 host 推來的檔案寫在 `~/Downloads/AutoControl` 底下（`JE_AUTOCONTROL_REMOTE_DOWNLOAD_DIR`）；這一項預設就是開的。 +**其他可設定的項目。** `JE_AUTOCONTROL_MCP_TOOL_MODE=progressive`（或 `je_auto_control_mcp --tool-mode`）讓 MCP session 從五個核心工具開始，再搜尋、查詢與啟用其他工具；`static` 提供固定的工具組（`JE_AUTOCONTROL_MCP_TOOL_PROFILE`）；預設的 `full` 模式不變。RBAC 的使用者以 `je_auto_control users add|remove|set-role|rotate-token|list`、`AC_user_*` 命令或 REST API 分頁管理。`JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE` 用來解開有通行碼保護的簽章私鑰。動作日誌（`start_action_journal`、`AC_journal_*`）記錄執行過的動作並先遮蔽秘密，`je_auto_control codegen --from-log` 把一次執行轉成候選腳本。設定同步把資料存在 SQLite 並檢查版本（`--config-db`、`AC_SIGNALING_CONFIG_DB`；舊版 client 需要 `--allow-blind-config-writes`）。Android 與 iOS 裝置各有自己的 session（`open_device`、`AC_android_*`、`AC_ios_*`）；行動裝置的程式還沒有在任何裝置上跑過。Wayland 上 `probe_capabilities`／`AC_probe_capabilities` 回報輸入與擷取能做什麼、為什麼，`JE_AUTOCONTROL_WAYLAND_EI_WORKER=1` 把 libei 移到輔助行程。GUI 會把主題、字級、面板與視窗位置記在 `~/.je_auto_control/gui_settings.ini`（`JE_AUTOCONTROL_GUI_SETTINGS`）。 + ### 遠端桌面的線路協定 把主機開出去之前值得先知道，而且這段在其他文件裡都沒有寫。預設傳輸是**裸 TCP diff --git a/architecture_explore.md b/architecture_explore.md index 2a278beb6..52e47d18e 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -6,7 +6,7 @@ > 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 > 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 > -> **掃描時間**：2026-10-08　**版本**：`pyproject.toml` version `0.0.225`　**分支**：`feat/coverage-to-80` +> **掃描時間**：2026-10-09　**版本**：`pyproject.toml` version `0.0.228`　**分支**：`feat/progress-sweep-2` --- @@ -874,7 +874,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.5 GUI 層（`gui/`，84 檔／26,367 行） -GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-material），且刻意保持「薄」： +GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6），且刻意保持「薄」： 每個分頁只把使用者輸入翻譯成對 `utils/` 無頭核心的呼叫。 #### 骨架 @@ -883,7 +883,10 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | --- | ---: | --- | | `gui/__init__.py` | 25 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | | `main_window.py` | 437 | `QMainWindow`：選單列（File／Actions／View／…）、左側導覽面板 dock（`Ctrl+K` 搜尋、`Ctrl+B` 收合）、即時語言切換、字級預設、深色／淺色主題（`theme.py` 的 token，不再用 qt-material）。分頁分為 core／editing／detection／automation／system 五類。 | -| `main_widget.py` | 433 | 工作區：擁有 `QTabWidget`，依 `tab_registry.TAB_SPECS` 註冊 48 個分頁，並暴露 show/hide/activate/list API 給選單列與導覽面板。只有預設開啟的三個分頁與自己的 mixin 表單在啟動時建立，其餘第一次開啟才匯入模組、建立 widget。核心分頁在 `_own_tab_builders` 宣告 `(label_key, handler)` 動作對。 | +| `main_widget.py` | 433 | 工作區：擁有 `QTabWidget`，依 `tab_registry.TAB_SPECS` 註冊 50 個分頁，並暴露 show/hide/activate/list API 給選單列與導覽面板。只有預設開啟的三個分頁與自己的 mixin 表單在啟動時建立，其餘第一次開啟才匯入模組、建立 widget。核心分頁在 `_own_tab_builders` 宣告 `(label_key, handler)` 動作對。 | +| `workspace_tabs.py` | 175 | `WorkspaceTabWidget`：每頁包在可捲動的 holder 裡，對 Python 呼叫端仍以頁面本身為單位（`indexOf(page)`、`setCurrentWidget(page)`）。 | +| `window_settings.py` | 130 | 以 `QSettings` INI 記住主題、字級、導覽面板與視窗位置；`JE_AUTOCONTROL_GUI_SETTINGS` 可改路徑或關閉。 | +| `task_controller.py` | 446 | 可取消的背景工作：結果／錯誤／進度以 typed signal 回到 GUI 執行緒，擁有者銷毀後的結果直接丟棄。建在 `_worker_thread.start_worker` 之上。 | | `tab_registry.py` | 156 | 分頁表：每個分頁一筆 `TabSpec`（鍵、標題鍵、分類、模組與類別名），`TabEntry` 在第一次存取 `widget` 時才呼叫 factory。不匯入 Qt。 | | `navigation.py` | 200 | `NavigationPanel`：搜尋框 + 依分類的功能樹，列出每個已註冊分頁（開啟中的以粗體標示），只回報被選的鍵，開啟分頁仍由視窗負責。 | | `theme.py` | 183 | 設計 token（`ThemeTokens`：顏色、圓角、間距、字族）、深色與淺色兩組值、由 token 產生的樣式表與對應的 `QPalette`；不載入圖檔或字型檔。 | @@ -905,7 +908,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate > 核心分頁在 `main_widget.py` 的 `_own_tab_builders` 宣告動作；功能分頁實作 `menu_actions()`（目前 40 個檔案有此 hook）。 > `test/unit_test/headless/test_actions_menu_gui.py` 會守住這個契約——沒有動作宣告的新分頁會讓 CI 失敗。 -#### 48 個分頁 +#### 50 個分頁 | 分頁 | 模組 | 行數 | 職責 | | --- | --- | ---: | --- | @@ -937,6 +940,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | flakiness | `flakiness_tab.py` | 117 | 依執行歷史排序間歇失敗的腳本。 | | a11y_audit | `a11y_audit_tab.py` | 114 | 從即時樹找出無障礙／i18n 缺陷。 | | device_matrix | `device_matrix_tab.py` | 111 | 一份 action list 跨多裝置平行執行。 | +| mobile | `mobile_tab.py` | 0 | Android／iOS 裝置：能力探測、手勢、App 生命週期，指令來自 `wrapper/mobile_commands.py` 的同一張表。 | | media_checks | `media_checks_tab.py` | 116 | 音訊活動與影片動態斷言。 | | run_history | `run_history_tab.py` + `run_history_timeline.py` | 462 | 瀏覽過去的排程／觸發／熱鍵執行，含自訂時間軸元件。 | | profiler | `profiler_tab.py` | 131 | 視覺化逐動作耗時熱點。 | @@ -948,6 +952,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | trace_replay | `trace_replay_tab.py` | 187 | 拖曳捲動時光回溯錄製內容。 | | remote_desktop | `remote_desktop/`（17 檔） | 6,240 | 見下。 | | presence | `presence_tab.py` | 152 | 多檢視者在場名單。 | +| config_sync | `config_sync_tab.py` | 0 | 設定同步：revision、待送、衝突與離線狀態；可取消的 worker。 | | rest_api | `rest_api_tab.py` | 198 | 啟停 HTTP 前端並顯示 URL 與 token。 | | admin_console | `admin_console_tab.py` | 313 | 管理多個遠端 AutoControl REST 端點。 | | audit_log | `audit_log_tab.py` | 192 | 瀏覽並驗證防竄改雜湊鏈。 | diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index e2f1d8e36..bd0e70c05 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -302,3 +302,69 @@ Jeffrey_RPA kept its own `KEY_ALIASES` (40 friendly names) and `_EXTRA_KEY_CODES - **What**: `PackageManager.allow_arbitrary_packages` defaults to `False`; `JE_AUTOCONTROL_ALLOWED_PACKAGES` is read where the package manager is created, so every entry point honours it; `je_auto_control run --allow-package`. - **Tests**: `test_package_gate.py` (12). - **Open items**: none beyond the pre-existing same-list limitation (`Progress.md`). + +## U-20261009-15 · 2026-10-09 · Opt-in action journal, and candidate scripts built from one run · #done #journal #codegen #B1 #B3 + +- **What**: `utils/action_journal/` (events, store, sanitize, recorder) records a start and an end line per action, masked first, parent ids across nested blocks and `AC_parallel`; off by default with no per-action allocation. `utils/codegen/journal_import.py` rebuilds the steps, labels a block it cannot recover as observed, keeps secret references, and validates the product with `ast.parse`, known command names and a dry run on a private executor. Six commands, six MCP tools, `codegen --from-log`, Run History / Recording Editor / Script Builder entries. +- **Tests**: `test_action_journal.py` (14), `test_codegen_from_journal.py` (14), `test_action_journal_gui.py` (5). +- **Open items**: `Progress.md` › 動作日誌、候選腳本與自愈量測. + +## U-20261009-16 · 2026-10-09 · Self-healing locator versions are measured on the same labelled frames · #done #self-healing #B2 + +- **What**: `evaluation.py`, `eval_strategies.py`, `template_revision.py`; `HealEvent` schema 2 with optional version / context fields (old lines load); a located point is no longer read as a verified action; `screen_region` confines the template match too. `benchmarks/self_healing`: v1-exact 6/9 accurate, v2-multiscale 8/9, v3-loose 7/9 with one false positive, on 10 drawn frames. +- **Tests**: `test_self_healing_evaluation.py` (26), `_template_revision.py` (22), `_eval_surfaces.py` (11). +- **Open items**: `Progress.md` › 動作日誌、候選腳本與自愈量測. + +## U-20261009-17 · 2026-10-09 · Config sync survives restarts, refuses stale writes and merges by causality · #done #sync #protocol #C1 #C2 #C3 + +- **What**: `config_sync/store.py` (SQLite, one transaction per commit, idempotent by operation id), `versions.py` (version vectors, concurrent edits kept as conflicts), `outbox.py`, `adapters.py`, `assets.py`, `session.py`, a Config Sync tab. Wire version 2 on the signaling server; the old blind write is behind `--allow-blind-config-writes`. +- **Tests**: `test_config_sync_persistence.py` (28), `_convergence.py` (23), `test_sync_adapters.py` (22), `test_config_sync_tab_gui.py` (9). The server tests need the `[signaling]` extra, which CI does not install. +- **Open items**: `Progress.md` › 安全與伺服器. + +## U-20261009-18 · 2026-10-09 · Each mobile device has its own session; gestures, frames, app lifecycle and every surface from one command table · #mobile #E1 #E2 #E3 #E4 + +- **What**: `wrapper/device_context.py`, `device_frame.py`, `mobile_extensions.py`, `mobile_commands.py`; `android/{session,input,screen,apps}.py`, `ios/{session,apps}.py`; a Mobile tab. Text adb cannot carry is refused or routed instead of reported as sent. 46 commands, 52 MCP tools. +- **Tests**: `test_mobile_context_isolation.py` (13), `_input_capture.py` (19), `_app_lifecycle.py` (13), `_surface_parity.py` (13), all against fake ADB / uiautomator2 / WDA. +- **Open items**: not `#done`: nothing ran on a device, and the SDK calls are unverified (`Progress.md` › 行動裝置). + +## U-20261009-19 · 2026-10-09 · A cancellable GUI task controller; the WebRTC panel split into twelve modules · #gui #F3 + +- **What**: `gui/task_controller.py` on top of `start_worker`: typed result / error / progress on the GUI thread, results dropped when the owner dies, cancel runs release callbacks. Nine blocking sites moved (TCP / WS connects, WebRTC offer and answer, OCR dump and find, device-matrix run). `webrtc_panel.py` 2,527 → 372 lines; the first commit is a pure move (162 methods compared, 0 differences). Blocking sites found and left, by group: script execution on the GUI thread (8 tabs), file transfer (4), other unbounded network / VLM / device calls (8), `_on_apply_answer`, stops that join a thread (about 18), per-event socket writes (3), short local probes. +- **Tests**: `test_gui_task_lifecycle.py` (20); five existing GUI tests now pump events after a connect or run. +- **Open items**: not `#done`: 43 of 52 sites remain (`Progress.md` › GUI). + +## U-20261009-20 · 2026-10-09 · GUI: scrolling pages, remembered window state, drawn close buttons, start tabs built on first click, benchmarks · #done #gui #F1 #F4 + +- **What**: `gui/workspace_tabs.py` keeps the page-level API (`indexOf(page)`, `setCurrentWidget(page)`) while each page sits in a scroll area; `gui/window_settings.py` (`QSettings` INI, `JE_AUTOCONTROL_GUI_SETTINGS`, off in tests); `gui/_tab_close_icon.py`; `close_tab(key, release=True)` and `dispose()`; `qt-material` removed from the extra, the lock and the mypy override. `Ctrl+B` never worked on a non-closable dock and is now the window's own action. Measured here: start-up 2.1–2.3 s → 1.7–1.8 s, window build 251–275 ms → 51–55 ms, 8 → 6 widgets built. +- **Tests**: `test_gui_shell_state.py` (32), `test_gui_feature_parity.py` (16). +- **Open items**: `Progress.md` › GUI. + +## U-20261009-21 · 2026-10-09 · Wayland capability states, explicit authorisation, opt-in physical recording and an opt-in libei helper · #wayland #D1 #D2 #D3 + +- **What**: `wrapper/capabilities.py`, `linux_wayland/{authorisation,input_events,global_shortcuts,ei_transport,ei_worker,ei_client}.py`; a capability table in the Diagnostics tab. A refused consent or a revoked session stops input instead of falling back. The helper is off by default and the in-process teardown is unchanged. The step log class is `InputStepLog` (renamed at integration: the action journal already exports `ActionJournal`). +- **Tests**: `test_wayland_capability_states.py` (20), `_worker_cleanup.py` (17), `_recording_paths.py` (19), `test_diagnostics_capabilities_gui.py` (4). Written on Windows against fakes. +- **Open items**: not `#done`: no Linux run yet (`Progress.md` › Wayland, libei). + +## U-20261009-22 · 2026-10-09 · MCP sessions can start with five tools and enable what they need · #done #mcp #G1 #G2 #G3 + +- **What**: `mcp_server/discovery.py`, `disclosure.py`, `_core_tools.py`, `_connection_state.py` (`server.py` 734 → 643 lines). Full mode is byte-identical to before (ten wire replies compared by SHA-256). Measured: `tools/list` 327,378 bytes and 43.7 ms in full mode, 2,411 bytes and 0.95 ms in progressive mode. +- **Tests**: `test_mcp_discovery_index.py` (20), `_progressive_sessions.py` (16), `_disclosure_policy.py` (16). +- **Open items**: no real client was used (`Progress.md` › 安全與伺服器). + +## U-20261009-23 · 2026-10-09 · RBAC: deferred work runs as its owner, users have a management surface, data tools need read_data · #done #security #rbac + +- **What**: `utils/rbac/{deferred,admin,cli}.py`, `gui/rbac_users_panel.py`; `Capability.READ_DATA`; passphrase for the signing key; REST answers 403 for a refused signature; a `check()`-only gate works again. MCP SSE, stateless, `subscriptions/listen` and TLS paths tested with RBAC on (18 tests, no code change needed). +- **Tests**: `test_rbac_deferred_owner.py` (34), `_user_admin.py` (22), `_mcp_transports.py` (18), `test_signing_passphrase_e2e.py` (12), `test_rbac_users_gui.py` (10). An audit-row test from the earlier round raced the server's write and now waits for the row. +- **Open items**: `Progress.md` › 安全與伺服器. + +## U-20261009-24 · 2026-10-09 · Follow-ups: USB waits, run-level scopes, load-and-use in one list, layout key names, file splits, Sphinx · #done #followups + +- **What**: USB transfers wait for their own `timeout_ms`; channel errors echo the request id; `viewer_client.py` 748 → 552 lines. `run_level_scope()` for the planner and the state machine, snapshots for observer callbacks; five daemons fired end to end with fakes. `anthropic_computer_use.py` 723 → 234 lines. Sphinx: 211 → 184 warnings, 0 in pages changed since v0.0.225. Coverage measured both ways: 87.35% (`coverage run`) and 87.32% (`pytest --cov`). +- **Tests**: seven new test files, 125 tests. +- **Open items**: `Progress.md` › 雜項. + +## U-20261009-25 · 2026-10-09 · The Python 3.10 headless segfault: a prompt dialog that was its own reference cycle · #bugfix #gui #ci + +- **What**: `TranslatableMixin._tr(self, …)` stored the widget in its own registry, so a dropped `UsbPassthroughPromptDialog` waited for the cycle collector; on 3.10 a collection inside `QApplication.topLevelWidgets()` destroys widgets still in the returned list. Reproduced on Windows 3.10.22 with a probe (access violation every time) and gone after the fix. `UsbAcl` changes are serialised per instance (4 threads adding 60 rules kept 43 before). +- **Tests**: `test_usb_prompt_lifetime.py` (6), one new ACL test; `PYTHONFAULTHANDLER=1` on the CI test step. +- **Open items**: not `#done`: the CI crash itself did not reproduce here; repeated green 3.10 runs on Linux / macOS are the confirmation. diff --git a/docs/updates/README.md b/docs/updates/README.md index e6dbf4fcf..7e2ed2de4 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,17 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261009-25 | 2026-10-09 | The Python 3.10 headless segfault: a prompt dialog that was its own reference cycle | #bugfix #gui #ci | [2026-10](2026-10.md) | +| U-20261009-24 | 2026-10-09 | Follow-ups: USB waits, run-level scopes, load-and-use in one list, layout key names, file splits, Sphinx | #done #followups | [2026-10](2026-10.md) | +| U-20261009-23 | 2026-10-09 | RBAC: deferred work runs as its owner, users have a management surface, data tools need read_data | #done #security #rbac | [2026-10](2026-10.md) | +| U-20261009-22 | 2026-10-09 | MCP sessions can start with five tools and enable what they need | #done #mcp #G1 #G2 #G3 | [2026-10](2026-10.md) | +| U-20261009-21 | 2026-10-09 | Wayland capability states, explicit authorisation, opt-in physical recording and an opt-in libei helper | #wayland #D1 #D2 #D3 | [2026-10](2026-10.md) | +| U-20261009-20 | 2026-10-09 | GUI: scrolling pages, remembered window state, drawn close buttons, start tabs built on first click, benchmarks | #done #gui #F1 #F4 | [2026-10](2026-10.md) | +| U-20261009-19 | 2026-10-09 | A cancellable GUI task controller; the WebRTC panel split into twelve modules | #gui #F3 | [2026-10](2026-10.md) | +| U-20261009-18 | 2026-10-09 | Each mobile device has its own session; gestures, frames, app lifecycle and every surface from one command table | #mobile #E1 #E2 #E3 #E4 | [2026-10](2026-10.md) | +| U-20261009-17 | 2026-10-09 | Config sync survives restarts, refuses stale writes and merges by causality | #done #sync #protocol #C1 #C2 #C3 | [2026-10](2026-10.md) | +| U-20261009-16 | 2026-10-09 | Self-healing locator versions are measured on the same labelled frames | #done #self-healing #B2 | [2026-10](2026-10.md) | +| U-20261009-15 | 2026-10-09 | Opt-in action journal, and candidate scripts built from one run | #done #journal #codegen #B1 #B3 | [2026-10](2026-10.md) | | U-20261009-14 | 2026-10-09 | The package gate refuses unlisted packages by default | #done #security #breaking | [2026-10](2026-10.md) | | U-20261009-13 | 2026-10-09 | REST routes and MCP tools are authorised by role when a user store is configured | #done #security #rbac #A10 | [2026-10](2026-10.md) | | U-20261009-12 | 2026-10-09 | Remote-desktop hosts and viewers have owners | #done #remote-desktop #gui #C4 | [2026-10](2026-10.md) | @@ -363,7 +374,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-10.md](2026-10.md) | 2026-10 | 27 | +| [2026-10.md](2026-10.md) | 2026-10 | 38 | | [2026-09-e.md](2026-09-e.md) | 2026-09 | 7 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 55 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | From fac6006d1dd8cd2cfda9be8f302e1b0580867373 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:12:28 +0800 Subject: [PATCH 35/44] Give the two new tab rows their measured sizes --- architecture_explore.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/architecture_explore.md b/architecture_explore.md index 52e47d18e..a95fa29b1 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -940,7 +940,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6），且 | flakiness | `flakiness_tab.py` | 117 | 依執行歷史排序間歇失敗的腳本。 | | a11y_audit | `a11y_audit_tab.py` | 114 | 從即時樹找出無障礙／i18n 缺陷。 | | device_matrix | `device_matrix_tab.py` | 111 | 一份 action list 跨多裝置平行執行。 | -| mobile | `mobile_tab.py` | 0 | Android／iOS 裝置：能力探測、手勢、App 生命週期，指令來自 `wrapper/mobile_commands.py` 的同一張表。 | +| mobile | `mobile_tab.py` | 186 | Android／iOS 裝置：能力探測、手勢、App 生命週期，指令來自 `wrapper/mobile_commands.py` 的同一張表。 | | media_checks | `media_checks_tab.py` | 116 | 音訊活動與影片動態斷言。 | | run_history | `run_history_tab.py` + `run_history_timeline.py` | 462 | 瀏覽過去的排程／觸發／熱鍵執行，含自訂時間軸元件。 | | profiler | `profiler_tab.py` | 131 | 視覺化逐動作耗時熱點。 | @@ -952,7 +952,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6），且 | trace_replay | `trace_replay_tab.py` | 187 | 拖曳捲動時光回溯錄製內容。 | | remote_desktop | `remote_desktop/`（17 檔） | 6,240 | 見下。 | | presence | `presence_tab.py` | 152 | 多檢視者在場名單。 | -| config_sync | `config_sync_tab.py` | 0 | 設定同步：revision、待送、衝突與離線狀態；可取消的 worker。 | +| config_sync | `config_sync_tab.py` | 236 | 設定同步：revision、待送、衝突與離線狀態；可取消的 worker。 | | rest_api | `rest_api_tab.py` | 198 | 啟停 HTTP 前端並顯示 URL 與 token。 | | admin_console | `admin_console_tab.py` | 313 | 管理多個遠端 AutoControl REST 端點。 | | audit_log | `audit_log_tab.py` | 192 | 瀏覽並驗證防竄改雜湊鏈。 | From 8179adbd504504ff2414558cf5dac7d87d16db6c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:18:23 +0800 Subject: [PATCH 36/44] Hold every module written since v0.0.225 to full annotations and typed containers The package-wide mypy floor checks bodies but accepts an unannotated def and a bare dict, so new code could land at the same depth as the oldest. A strict override now names the 83 modules added since v0.0.225 and the pytest plugin, and a test compares that list with the tree against a shrink-only baseline, so a new module cannot be left out by forgetting it. The 57 findings were annotations only. typing_contract_verify.py also checks the py-modules the distribution ships. --- je_auto_control/gui/config_sync_tab.py | 4 +- je_auto_control/gui/mobile_tab.py | 4 +- je_auto_control/gui/rbac_users_panel.py | 4 +- .../gui/remote_desktop/_webrtc_types.py | 8 +- .../gui/remote_desktop/webrtc_host_media.py | 5 +- .../remote_desktop/webrtc_host_sessions.py | 10 +- .../gui/remote_desktop/webrtc_panel_common.py | 12 +- .../webrtc_viewer_address_book.py | 16 +- .../webrtc_viewer_connection.py | 5 +- .../gui/remote_desktop/webrtc_viewer_files.py | 18 +- .../gui/remote_desktop/webrtc_viewer_media.py | 4 +- je_auto_control/linux_wayland/ei_transport.py | 2 +- .../utils/agent/backends/_computer_actions.py | 12 +- .../utils/codegen/journal_import.py | 15 +- .../utils/mcp_server/_connection_state.py | 17 +- je_auto_control/utils/path_guard/policy.py | 6 +- .../windows/core/utils/win32_layout_vk.py | 2 +- je_auto_control/wrapper/device_frame.py | 4 +- je_auto_control_pytest.py | 15 +- pyproject.toml | 103 ++ .../test_typing_modernization_contract.py | 256 ++++ test/verify/typing_contract_verify.py | 13 +- test/verify/typing_strict_baseline.txt | 1064 +++++++++++++++++ 23 files changed, 1525 insertions(+), 74 deletions(-) create mode 100644 test/unit_test/headless/test_typing_modernization_contract.py create mode 100644 test/verify/typing_strict_baseline.txt diff --git a/je_auto_control/gui/config_sync_tab.py b/je_auto_control/gui/config_sync_tab.py index 4af5e6366..3c02b1630 100644 --- a/je_auto_control/gui/config_sync_tab.py +++ b/je_auto_control/gui/config_sync_tab.py @@ -12,7 +12,7 @@ """ import threading import time -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple from PySide6.QtCore import Qt from PySide6.QtWidgets import ( @@ -89,7 +89,7 @@ def retranslate(self) -> None: self._apply_translations() self._render() - def menu_actions(self) -> list: + def menu_actions(self) -> List[Tuple[str, Callable[[], None]]]: """Expose tab commands to the window-level Actions menu.""" return [ ("config_sync_run_btn", self.sync_now), diff --git a/je_auto_control/gui/mobile_tab.py b/je_auto_control/gui/mobile_tab.py index 633fba130..d111d113d 100644 --- a/je_auto_control/gui/mobile_tab.py +++ b/je_auto_control/gui/mobile_tab.py @@ -7,7 +7,7 @@ action file. """ import json -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple from PySide6.QtCore import Qt from PySide6.QtWidgets import ( @@ -81,7 +81,7 @@ def _build_layout(self) -> None: root.addWidget(self._tr(QLabel(), "mob_result")) root.addWidget(self._result, stretch=1) - def menu_actions(self) -> list: + def menu_actions(self) -> List[Tuple[str, Callable[[], None]]]: """Expose tab commands to the window-level Actions menu.""" return [ ("mob_probe", self._on_probe), diff --git a/je_auto_control/gui/rbac_users_panel.py b/je_auto_control/gui/rbac_users_panel.py index 4976f6268..f3b6b2e6c 100644 --- a/je_auto_control/gui/rbac_users_panel.py +++ b/je_auto_control/gui/rbac_users_panel.py @@ -9,7 +9,7 @@ ``rotate``; the next operation clears it. It is never written to a log. """ import os -from typing import List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from PySide6.QtCore import Qt from PySide6.QtGui import QGuiApplication @@ -161,7 +161,7 @@ def _target(self) -> Optional[Tuple[str, str]]: return None return path, user_id - def _show_token(self, issued: dict) -> None: + def _show_token(self, issued: Dict[str, Any]) -> None: self._reload() self._token_value.setText(str(issued["token"])) self._status_label.setText( diff --git a/je_auto_control/gui/remote_desktop/_webrtc_types.py b/je_auto_control/gui/remote_desktop/_webrtc_types.py index 4ae91a9e2..87644cca2 100644 --- a/je_auto_control/gui/remote_desktop/_webrtc_types.py +++ b/je_auto_control/gui/remote_desktop/_webrtc_types.py @@ -7,6 +7,12 @@ """ from typing import TYPE_CHECKING, Any +#: An ``av.VideoFrame``. PyAV belongs to the optional ``webrtc`` extra, which the +#: typing contract reads as ``Any`` on purpose (its result must not depend on +#: which extras are installed), so the frame cannot be named more precisely. +#: This alias is the one place the panel modules let that ``Any`` in. +AvFrameT = Any + if TYPE_CHECKING: from je_auto_control.utils.remote_desktop.multi_viewer import ( MultiViewerHost as MultiViewerHostT, @@ -23,4 +29,4 @@ else: MultiViewerHostT = SessionRecorderT = WebRTCConfigT = WebRTCDesktopViewerT = Any -__all__ = ["MultiViewerHostT", "SessionRecorderT", "WebRTCConfigT", "WebRTCDesktopViewerT"] +__all__ = ["AvFrameT", "MultiViewerHostT", "SessionRecorderT", "WebRTCConfigT", "WebRTCDesktopViewerT"] diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_media.py b/je_auto_control/gui/remote_desktop/webrtc_host_media.py index 53a1d7eef..32e189f69 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_media.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_media.py @@ -16,6 +16,7 @@ from je_auto_control.gui.remote_desktop._helpers import ( _t, ) +from je_auto_control.gui.remote_desktop._webrtc_types import AvFrameT from je_auto_control.gui.remote_desktop.blanking_overlay import BlankingOverlay from je_auto_control.gui.remote_desktop.annotation_overlay import ( HostAnnotationOverlay, @@ -244,7 +245,7 @@ def _stop_adaptive(self) -> None: self._adaptive_poller = None self._adaptive_controller = None - def _on_annotation_event(self, data) -> None: + def _on_annotation_event(self, data: object) -> None: if not isinstance(data, dict): return if self._annotation_overlay is None: @@ -263,7 +264,7 @@ def _on_session_authed(self, session_id: str) -> None: return host.set_viewer_video_callback(self._on_viewer_video_av_frame) - def _on_viewer_video_av_frame(self, frame) -> None: + def _on_viewer_video_av_frame(self, frame: AvFrameT) -> None: image = _av_frame_to_qimage(frame) if image is not None: self._signals.viewer_video_frame.emit(image) diff --git a/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py b/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py index fadc51a02..b1dcfd66e 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py +++ b/je_auto_control/gui/remote_desktop/webrtc_host_sessions.py @@ -5,9 +5,9 @@ """ from __future__ import annotations -from typing import Optional +from typing import Callable, Dict, Optional -from PySide6.QtCore import Qt +from PySide6.QtCore import QPoint, Qt from PySide6.QtWidgets import ( QTableWidgetItem, ) @@ -86,7 +86,7 @@ def _sync_session_pollers(self) -> None: poller.start() self._session_pollers[sid] = poller - def _make_session_stats_handler(self, session_id: str): + def _make_session_stats_handler(self, session_id: str) -> Callable[[StatsSnapshot], None]: """Closure capturing session_id for the per-session poller.""" def _handle(snapshot: StatsSnapshot) -> None: default_webrtc_inspector().record(snapshot) @@ -163,7 +163,7 @@ def _refresh_sessions_table(self) -> None: self._sessions_table.setItem(row, 3, QTableWidgetItem(state)) self._sessions_table.setItem(row, 4, QTableWidgetItem(connected)) - def _on_sessions_context_menu(self, position) -> None: + def _on_sessions_context_menu(self, position: QPoint) -> None: from PySide6.QtWidgets import QMenu if self._multi_host is None: return @@ -189,7 +189,7 @@ def _on_sessions_context_menu(self, position) -> None: ) self._dispatch_session_menu(chosen, actions, sid, viewer_id) - def _dispatch_session_menu(self, chosen, actions: dict, + def _dispatch_session_menu(self, chosen: object, actions: Dict[str, object], sid: str, viewer_id: str) -> None: """Run the action chosen from the sessions context menu.""" if chosen is actions["disconnect"]: diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel_common.py b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py index 386e029b3..be92aaa66 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel_common.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel_common.py @@ -1,13 +1,13 @@ """Signals, defaults and config readers shared by the WebRTC host and viewer panels.""" from __future__ import annotations -from typing import Optional +from typing import Optional, Tuple from PySide6.QtCore import QObject, Signal from PySide6.QtGui import QImage from PySide6.QtWidgets import QWidget -from je_auto_control.gui.remote_desktop._webrtc_types import WebRTCConfigT +from je_auto_control.gui.remote_desktop._webrtc_types import AvFrameT, WebRTCConfigT from je_auto_control.gui._i18n_helpers import TranslatableMixin from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -27,7 +27,7 @@ _JSON_FILE_FILTER = "JSON (*.json);;All (*)" -def _av_frame_to_qimage(frame) -> Optional[QImage]: +def _av_frame_to_qimage(frame: AvFrameT) -> Optional[QImage]: """Convert an aiortc/av video frame to a Qt-owned QImage.""" try: arr = frame.to_ndarray(format="rgb24") @@ -70,13 +70,13 @@ class _PanelPart(TranslatableMixin, QWidget): """ -def _checked_or(panel, attr: str, default: bool = False) -> bool: +def _checked_or(panel: QWidget, attr: str, default: bool = False) -> bool: """Return ``panel..isChecked()`` if the widget exists, else default.""" widget = getattr(panel, attr, None) return widget.isChecked() if widget is not None else default -def _read_region(panel) -> Optional[tuple]: +def _read_region(panel: QWidget) -> Optional[Tuple[int, ...]]: edit = getattr(panel, "_region_edit", None) if edit is None: return None @@ -90,7 +90,7 @@ def _read_region(panel) -> Optional[tuple]: return tuple(parts) if len(parts) == 4 else None -def _read_webrtc_config(panel) -> WebRTCConfigT: +def _read_webrtc_config(panel: QWidget) -> WebRTCConfigT: """Build a WebRTCConfig from the advanced group + monitor/fps fields.""" from je_auto_control.utils.remote_desktop.webrtc_transport import ( _DEFAULT_STUN_SERVERS, diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py index 0a2f57a19..a6d3c04ab 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_address_book.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +from typing import Any, Dict, Optional from PySide6.QtWidgets import ( QFileDialog, QInputDialog, QMessageBox, @@ -30,14 +31,15 @@ class _ViewerAddressBookMixin(_PanelPart): """Methods of ``_WebRTCViewerPanel``; the module docstring says which group.""" @staticmethod - def _wol_defaults(entry) -> tuple[str, str]: + def _wol_defaults(entry: Optional[Dict[str, Any]]) -> tuple[str, str]: """Return (mac, broadcast) pre-fill values from a book entry.""" if entry is None: return "", "" return (entry.get("mac_address", "") or "", entry.get("broadcast_address", "") or "") - def _persist_wol_entry(self, entry, mac: str, broadcast: str) -> None: + def _persist_wol_entry(self, entry: Optional[Dict[str, Any]], mac: str, + broadcast: str) -> None: """Save the MAC / broadcast just used back onto the book entry.""" if entry is None: return @@ -156,7 +158,7 @@ def _refresh_address_book(self) -> None: self._address_book.list_entries(), tag_filter=active_tag, ) - def _on_address_tags(self, entry: dict) -> None: + def _on_address_tags(self, entry: Dict[str, Any]) -> None: existing = entry.get("tags", []) or [] text, ok = QInputDialog.getText( self, _t("rd_webrtc_edit_tags"), @@ -176,19 +178,19 @@ def _on_address_tags(self, entry: dict) -> None: autocontrol_logger.debug("set_tags: %r", error) self._refresh_address_book() - def _on_address_chosen(self, entry: dict) -> None: + def _on_address_chosen(self, entry: Dict[str, Any]) -> None: self._server_edit.setText(entry.get("server_url", "")) self._host_id_edit.setText(entry.get("host_id", "")) self._on_connect_via_server() - def _on_address_removed(self, entry: dict) -> None: + def _on_address_removed(self, entry: Dict[str, Any]) -> None: self._address_book.remove( host_id=entry.get("host_id", ""), server_url=entry.get("server_url", ""), ) self._refresh_address_book() - def _on_address_favorite(self, entry: dict) -> None: + def _on_address_favorite(self, entry: Dict[str, Any]) -> None: try: self._address_book.toggle_favorite( host_id=entry.get("host_id", ""), @@ -228,7 +230,7 @@ def _on_lan_browse(self) -> None: dialog.chosen.connect(self._on_lan_chosen) dialog.exec() - def _on_lan_chosen(self, svc: dict) -> None: + def _on_lan_chosen(self, svc: Dict[str, Any]) -> None: host_id = svc.get("host_id", "") signaling = svc.get("signaling_url", "") if host_id: diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py index 08d893bd5..42faf7ed9 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_connection.py @@ -6,7 +6,7 @@ from __future__ import annotations import functools -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Callable, Optional from PySide6.QtCore import QTimer from PySide6.QtWidgets import ( @@ -83,7 +83,8 @@ def _answer_and_push(self, offer_sdp: str) -> None: offer_sdp, expected_dtls, functools.partial(self._push_answer, host_id, expected_dtls, offer_sdp)) - def _answer_off_thread(self, offer_sdp: str, expected_dtls: Optional[str], on_answer) -> None: + def _answer_off_thread(self, offer_sdp: str, expected_dtls: Optional[str], + on_answer: Callable[[str], None]) -> None: """Run ``process_offer`` on a worker; ``on_answer(answer)`` only if this viewer is still current.""" try: viewer = self._require_viewer() diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py index 2fdc84145..28674472b 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_files.py @@ -5,7 +5,7 @@ """ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Optional, Sequence from PySide6.QtCore import QTimer @@ -99,7 +99,7 @@ def _on_browse_delete_button(self) -> None: return self._on_delete_names(names) - def _on_pull_names(self, names) -> None: + def _on_pull_names(self, names: Sequence[str]) -> None: if self._viewer is None or not self._viewer.authenticated: return try: @@ -108,7 +108,7 @@ def _on_pull_names(self, names) -> None: except (RuntimeError, OSError, ValueError) as error: QMessageBox.warning(self, "WebRTC", str(error)) - def _on_delete_names(self, names) -> None: + def _on_delete_names(self, names: Sequence[str]) -> None: if not names or self._viewer is None or not self._viewer.authenticated: return confirm_text = ( @@ -128,7 +128,7 @@ def _on_delete_names(self, names) -> None: except (RuntimeError, OSError, ValueError) as error: QMessageBox.warning(self, "WebRTC", str(error)) - def _on_upload_paths(self, paths) -> None: + def _on_upload_paths(self, paths: Sequence[str]) -> None: if self._viewer is None or not self._viewer.authenticated: QMessageBox.information( self, "WebRTC", _t("rd_webrtc_cad_not_connected"), @@ -157,11 +157,11 @@ def _on_copy_name(self, name: str) -> None: if clipboard is not None: clipboard.setText(name) - def _on_inbox_listing(self, files) -> None: + def _on_inbox_listing(self, files: object) -> None: from datetime import datetime if not isinstance(files, list): return - def _format_mtime(value): + def _format_mtime(value: Any) -> str: # a field of the remote host's JSON listing try: return datetime.fromtimestamp(float(value)).strftime( "%Y-%m-%d %H:%M:%S", @@ -170,7 +170,7 @@ def _format_mtime(value): return str(value) self._remote_files_table.populate(files, _format_mtime) - def _on_inbox_op_result(self, name: str, ok: bool, error) -> None: + def _on_inbox_op_result(self, name: str, ok: bool, error: object) -> None: if ok: self._status_label.setText( _t("rd_webrtc_browse_op_ok").format(name=name), @@ -208,13 +208,13 @@ def _on_send_file(self) -> None: except (RuntimeError, OSError, ValueError) as error: QMessageBox.warning(self, "WebRTC", str(error)) - def _on_received_file(self, path) -> None: + def _on_received_file(self, path: object) -> None: # Called from the asyncio thread, which has no Qt event loop: # QTimer.singleShot would never fire. Emit a signal — Qt queues the # status update onto the GUI thread. self._signals.file_received.emit(path) - def _on_file_received_ui(self, path) -> None: + def _on_file_received_ui(self, path: object) -> None: self._status_label.setText( _t("rd_webrtc_file_received").format(name=str(path)), ) diff --git a/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py index 34f9ac696..c2ea76029 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py +++ b/je_auto_control/gui/remote_desktop/webrtc_viewer_media.py @@ -13,7 +13,7 @@ QFileDialog, QMessageBox, ) -from je_auto_control.gui.remote_desktop._webrtc_types import SessionRecorderT +from je_auto_control.gui.remote_desktop._webrtc_types import AvFrameT, SessionRecorderT from je_auto_control.gui.remote_desktop._helpers import ( _t, ) @@ -194,7 +194,7 @@ def _on_pen_clear(self) -> None: pass # called from asyncio thread - def _on_av_frame(self, frame) -> None: + def _on_av_frame(self, frame: AvFrameT) -> None: if self._recorder is not None: try: self._recorder.write_frame(frame) diff --git a/je_auto_control/linux_wayland/ei_transport.py b/je_auto_control/linux_wayland/ei_transport.py index 620f25e17..665375a35 100644 --- a/je_auto_control/linux_wayland/ei_transport.py +++ b/je_auto_control/linux_wayland/ei_transport.py @@ -223,7 +223,7 @@ def decode_batch(raw: Any) -> List[InputEvent]: return events -def _checked(kind: Any, code: Any, value: Any) -> tuple: +def _checked(kind: Any, code: Any, value: Any) -> Tuple[int, int, int]: """One event's three fields, or a refusal naming what is wrong.""" for part in (kind, code, value): if isinstance(part, bool) or not isinstance(part, int): diff --git a/je_auto_control/utils/agent/backends/_computer_actions.py b/je_auto_control/utils/agent/backends/_computer_actions.py index 025c9de33..1d369a233 100644 --- a/je_auto_control/utils/agent/backends/_computer_actions.py +++ b/je_auto_control/utils/agent/backends/_computer_actions.py @@ -80,27 +80,27 @@ def _click_repeats(action: str) -> int: return {"double_click": 2, "triple_click": 3}.get(action, 1) -def _action_screenshot(_payload): +def _action_screenshot(_payload: Dict[str, Any]) -> Dict[str, Any]: return {"tool": "AC_screenshot", "input": {}} -def _action_cursor_position(_payload): +def _action_cursor_position(_payload: Dict[str, Any]) -> Dict[str, Any]: return {"tool": "AC_get_mouse_position", "input": {}} -def _action_mouse_move(payload): +def _action_mouse_move(payload: Dict[str, Any]) -> Dict[str, Any]: x, y = _xy(payload.get("coordinate")) return {"tool": "AC_set_mouse_position", "input": {"x": x, "y": y}} -def _action_type(payload): +def _action_type(payload: Dict[str, Any]) -> Dict[str, Any]: return { "tool": "AC_write", "input": {"write_string": str(payload.get("text") or "")}, } -def _action_wait(payload): +def _action_wait(payload: Dict[str, Any]) -> Dict[str, Any]: duration = payload.get("duration") # An explicit 0 is a valid "don't wait" — only fall back when unset. seconds = _number(duration, "duration") if duration is not None else 1.0 @@ -156,7 +156,7 @@ def _with_modifiers(decision: Dict[str, Any], combo: str) -> Dict[str, Any]: "input": {"modifiers": keys, "actions": actions}} -def _click_decision(action: str, coordinate) -> Dict[str, Any]: +def _click_decision(action: str, coordinate: Any) -> Dict[str, Any]: button = _click_button(action) repeats = _click_repeats(action) inputs: Dict[str, Any] = {"mouse_keycode": button} diff --git a/je_auto_control/utils/codegen/journal_import.py b/je_auto_control/utils/codegen/journal_import.py index c122be6a5..320c6da1e 100644 --- a/je_auto_control/utils/codegen/journal_import.py +++ b/je_auto_control/utils/codegen/journal_import.py @@ -34,7 +34,9 @@ import re from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union +from typing import ( + Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Set, Tuple, Union, +) from je_auto_control.utils.action_journal.events import ( SCHEMA_VERSION, STATUS_ERROR, STATUS_INCOMPLETE, STATUS_OK, ActionEvent, @@ -91,7 +93,7 @@ def to_dict(self) -> Dict[str, Any]: "observed_path_only": self.observed_path_only} -def _nesting_commands() -> frozenset: +def _nesting_commands() -> FrozenSet[str]: """Commands whose arguments hold action lists the executor runs.""" from je_auto_control.utils.executor.action_executor import Executor return frozenset(FLOW_BODY_KEYS) | frozenset(FLOW_BRANCH_LIST_KEYS) | frozenset( @@ -106,8 +108,8 @@ def __init__(self, contents: JournalContents) -> None: self.lines = contents.lines self.children: Dict[str, List[ActionEvent]] = {} self.roots: List[ActionEvent] = [] - self.orphans: set = set() - seen: set = set() + self.orphans: Set[str] = set() + seen: Set[str] = set() for event in self.events: if event.parent_id is None: self.roots.append(event) @@ -119,7 +121,7 @@ def __init__(self, contents: JournalContents) -> None: self.orphans.add(event.step_id) seen.add(event.step_id) - def detached(self, nesting: frozenset) -> set: + def detached(self, nesting: FrozenSet[str]) -> Set[str]: """Roots that ran on another thread *inside* a nesting root step. A runner that hands work to a thread pool (the DAG runner, a device @@ -166,7 +168,8 @@ def __init__(self, run: _Run) -> None: self.steps: List[Dict[str, Any]] = [] self.warnings: List[str] = [] self.observed = False - self._macros: set = set() + # Names as the journal recorded them: read from a file, so not known to be strings. + self._macros: Set[object] = set() def build(self) -> None: """Walk every root step of the run.""" diff --git a/je_auto_control/utils/mcp_server/_connection_state.py b/je_auto_control/utils/mcp_server/_connection_state.py index 8c3f398bc..174b46d08 100644 --- a/je_auto_control/utils/mcp_server/_connection_state.py +++ b/je_auto_control/utils/mcp_server/_connection_state.py @@ -10,7 +10,10 @@ """ import contextlib import threading -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional + +_Notifier = Callable[[str, Dict[str, Any]], None] +_Writer = Callable[[str], None] class ConnectionStateMixin: @@ -54,7 +57,7 @@ def _notifier(self) -> Optional[Callable[[str, Dict[str, Any]], None]]: return getattr(self._local, "notifier", None) or self._default_notifier @_notifier.setter - def _notifier(self, value) -> None: + def _notifier(self, value: Optional[_Notifier]) -> None: self._default_notifier = value @property @@ -62,7 +65,7 @@ def _writer(self) -> Optional[Callable[[str], None]]: return getattr(self._local, "writer", None) or self._default_writer @_writer.setter - def _writer(self, value) -> None: + def _writer(self, value: Optional[_Writer]) -> None: self._default_writer = value @property @@ -73,7 +76,7 @@ def _concurrent_tools(self) -> bool: return scoped @_concurrent_tools.setter - def _concurrent_tools(self, value) -> None: + def _concurrent_tools(self, value: object) -> None: self._default_concurrent_tools = bool(value) @property @@ -102,8 +105,10 @@ def _client_capabilities(self, value: Dict[str, Any]) -> None: self._client_caps_by_conn[conn] = value @contextlib.contextmanager - def connection_scope(self, *, notifier=None, writer=None, - concurrent_tools=None, connection_id=None): + def connection_scope(self, *, notifier: Optional[_Notifier] = None, + writer: Optional[_Writer] = None, + concurrent_tools: Optional[bool] = None, + connection_id: Any = None) -> Iterator[Any]: """Bind notifier/writer/concurrency/identity to the calling thread only. Transports that serve more than one peer must wrap each request in diff --git a/je_auto_control/utils/path_guard/policy.py b/je_auto_control/utils/path_guard/policy.py index 1a3ba5f81..ada558ef0 100644 --- a/je_auto_control/utils/path_guard/policy.py +++ b/je_auto_control/utils/path_guard/policy.py @@ -41,7 +41,7 @@ class PathPolicy: refused rather than let through while the roots are still unknown. """ - def __init__(self, roots: Iterable[os.PathLike | str] = (), *, + def __init__(self, roots: Iterable[os.PathLike[str] | str] = (), *, use_client_roots: bool = False) -> None: self._static: Tuple[Path, ...] = tuple(_canonical(root) for root in roots) self._use_client_roots = bool(use_client_roots) @@ -71,7 +71,7 @@ def use_client_roots(self) -> bool: """Whether roots reported by the client are honoured.""" return self._use_client_roots - def set_client_roots(self, roots: Iterable[os.PathLike | str]) -> None: + def set_client_roots(self, roots: Iterable[os.PathLike[str] | str]) -> None: """Replace the client-reported roots; ignored unless they are honoured.""" if not self._use_client_roots: return @@ -84,7 +84,7 @@ def roots(self) -> Tuple[Path, ...]: with self._lock: return self._static + self._client - def validate(self, path: os.PathLike | str, *, operation: str) -> Path: + def validate(self, path: os.PathLike[str] | str, *, operation: str) -> Path: """Return ``path`` canonicalised, or raise :class:`PathNotAllowedError`. ``operation`` says what wanted the path and is quoted in the refusal. diff --git a/je_auto_control/windows/core/utils/win32_layout_vk.py b/je_auto_control/windows/core/utils/win32_layout_vk.py index 7d08f388b..0417087c4 100644 --- a/je_auto_control/windows/core/utils/win32_layout_vk.py +++ b/je_auto_control/windows/core/utils/win32_layout_vk.py @@ -73,7 +73,7 @@ def layout_virtual_key(character: str) -> Optional[int]: return virtual_key if shift_state == 0 and virtual_key else None -class LayoutKeyTable(dict): +class LayoutKeyTable(Dict[str, int]): """A key table whose layout-dependent names are resolved at lookup time. A plain ``dict`` in every other respect: the layout names are stored with diff --git a/je_auto_control/wrapper/device_frame.py b/je_auto_control/wrapper/device_frame.py index 597ba5321..79bd6682e 100644 --- a/je_auto_control/wrapper/device_frame.py +++ b/je_auto_control/wrapper/device_frame.py @@ -17,7 +17,7 @@ import os from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union from je_auto_control.utils.exception.exceptions import ImageNotFoundException from je_auto_control.wrapper.device_context import DeviceError @@ -128,7 +128,7 @@ def save(self, file_path: Union[str, "os.PathLike[str]"]) -> str: self.image.save(str(target), format="PNG") return str(target) - def to_dict(self) -> dict: + def to_dict(self) -> Dict[str, Any]: """Geometry only (no pixels), for executor / MCP results.""" return {"platform": self.platform, "device_id": self.device_id, "orientation": self.orientation, diff --git a/je_auto_control_pytest.py b/je_auto_control_pytest.py index cefdffc29..e3e74c0eb 100644 --- a/je_auto_control_pytest.py +++ b/je_auto_control_pytest.py @@ -24,11 +24,15 @@ import os from pathlib import Path -from typing import Optional +from types import ModuleType +from typing import TYPE_CHECKING, Any, Generator, Optional import pytest +if TYPE_CHECKING: # the executor pulls in the whole facade; fixtures import it on use + from je_auto_control.utils.executor.action_executor import Executor + _MARKER_NAME = "autocontrol" _DEFAULT_SUBDIR = "autocontrol_screenshots" @@ -42,14 +46,14 @@ def pytest_configure(config: "pytest.Config") -> None: @pytest.fixture -def autocontrol(): +def autocontrol() -> ModuleType: """Return the ``je_auto_control`` module without forcing a global import.""" import je_auto_control return je_auto_control @pytest.fixture -def autocontrol_screenshot_dir(tmp_path) -> Path: +def autocontrol_screenshot_dir(tmp_path: Path) -> Path: """Per-test directory under ``tmp_path`` for screenshots / artefacts.""" target = tmp_path / _DEFAULT_SUBDIR target.mkdir(parents=True, exist_ok=True) @@ -57,7 +61,7 @@ def autocontrol_screenshot_dir(tmp_path) -> Path: @pytest.fixture -def autocontrol_executor(): +def autocontrol_executor() -> "Executor": """Yield the executor singleton (callable via ``executor.event_dict``).""" from je_auto_control.utils.executor.action_executor import executor return executor @@ -84,7 +88,8 @@ def _capture_failure_screenshot(item: "pytest.Item", @pytest.hookimpl(hookwrapper=True, tryfirst=True) -def pytest_runtest_makereport(item, call): # noqa: D401 +def pytest_runtest_makereport(item: "pytest.Item", + call: "pytest.CallInfo[None]") -> Generator[None, Any, None]: """Attach a screenshot path to the failure report for ``autocontrol`` tests.""" outcome = yield report = outcome.get_result() diff --git a/pyproject.toml b/pyproject.toml index 1d3ad034e..8bb06582b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -326,3 +326,106 @@ module = ["numpy", "numpy.*"] follow_imports = "skip" follow_imports_for_stubs = true ignore_missing_imports = true + +[[tool.mypy.overrides]] +# The strict tier: every function fully annotated, every generic container +# parameterised. It holds each module written since v0.0.225 (plus the pytest +# plugin that ships beside the package), so new code starts at this depth +# instead of at the package-wide floor above, and an older module joins by +# being fixed and named here. +# +# The list is checked against the tree, not remembered: +# `test/unit_test/headless/test_typing_modernization_contract.py` fails when a +# module exists that is neither in `test/verify/typing_strict_baseline.txt` +# (the modules that predate this tier) nor matched below. A package that is new +# as a whole is named with `.*` as well, so a file added to it is covered the +# day it is written. Nothing here loosens a setting from `[tool.mypy]`. +module = [ + "je_auto_control.android.apps", + "je_auto_control.android.input", + "je_auto_control.android.screen", + "je_auto_control.android.session", + "je_auto_control.gui._journal_import", + "je_auto_control.gui._tab_close_icon", + "je_auto_control.gui.config_sync_tab", + "je_auto_control.gui.mobile_tab", + "je_auto_control.gui.navigation", + "je_auto_control.gui.rbac_users_panel", + "je_auto_control.gui.remote_desktop._connect_task", + "je_auto_control.gui.remote_desktop._webrtc_types", + "je_auto_control.gui.remote_desktop.webrtc_host_connection", + "je_auto_control.gui.remote_desktop.webrtc_host_media", + "je_auto_control.gui.remote_desktop.webrtc_host_sessions", + "je_auto_control.gui.remote_desktop.webrtc_host_trust", + "je_auto_control.gui.remote_desktop.webrtc_host_ui", + "je_auto_control.gui.remote_desktop.webrtc_panel_common", + "je_auto_control.gui.remote_desktop.webrtc_viewer_address_book", + "je_auto_control.gui.remote_desktop.webrtc_viewer_connection", + "je_auto_control.gui.remote_desktop.webrtc_viewer_files", + "je_auto_control.gui.remote_desktop.webrtc_viewer_media", + "je_auto_control.gui.remote_desktop.webrtc_viewer_ui", + "je_auto_control.gui.tab_registry", + "je_auto_control.gui.task_controller", + "je_auto_control.gui.theme", + "je_auto_control.gui.window_settings", + "je_auto_control.gui.workspace_tabs", + "je_auto_control.ios.apps", + "je_auto_control.ios.session", + "je_auto_control.linux_wayland.authorisation", + "je_auto_control.linux_wayland.ei_client", + "je_auto_control.linux_wayland.ei_transport", + "je_auto_control.linux_wayland.ei_worker", + "je_auto_control.linux_wayland.global_shortcuts", + "je_auto_control.linux_wayland.input_events", + "je_auto_control.utils.action_journal", + "je_auto_control.utils.action_journal.*", + "je_auto_control.utils.action_journal.events", + "je_auto_control.utils.action_journal.recorder", + "je_auto_control.utils.action_journal.sanitize", + "je_auto_control.utils.action_journal.store", + "je_auto_control.utils.action_signing.asymmetric", + "je_auto_control.utils.action_signing.config", + "je_auto_control.utils.agent.backends._computer_actions", + "je_auto_control.utils.agent.backends._computer_beta_path", + "je_auto_control.utils.agent.backends._computer_messages", + "je_auto_control.utils.agent.backends._computer_toolset_path", + "je_auto_control.utils.codegen.journal_import", + "je_auto_control.utils.config_sync.adapters", + "je_auto_control.utils.config_sync.assets", + "je_auto_control.utils.config_sync.outbox", + "je_auto_control.utils.config_sync.session", + "je_auto_control.utils.config_sync.store", + "je_auto_control.utils.config_sync.versions", + "je_auto_control.utils.generate_report.report_path", + "je_auto_control.utils.mcp_server._argument_policy", + "je_auto_control.utils.mcp_server._authz", + "je_auto_control.utils.mcp_server._connection_state", + "je_auto_control.utils.mcp_server._core_tools", + "je_auto_control.utils.mcp_server.disclosure", + "je_auto_control.utils.mcp_server.discovery", + "je_auto_control.utils.monitor_layout.macos_frame", + "je_auto_control.utils.ocr.tesseract_setup", + "je_auto_control.utils.path_guard.policy", + "je_auto_control.utils.rbac.__main__", + "je_auto_control.utils.rbac.admin", + "je_auto_control.utils.rbac.authorization", + "je_auto_control.utils.rbac.cli", + "je_auto_control.utils.rbac.deferred", + "je_auto_control.utils.rbac.policy", + "je_auto_control.utils.script_vars.execution", + "je_auto_control.utils.self_healing.eval_strategies", + "je_auto_control.utils.self_healing.evaluation", + "je_auto_control.utils.self_healing.template_revision", + "je_auto_control.utils.usb.passthrough._client_requests", + "je_auto_control.utils.usb.passthrough.client_errors", + "je_auto_control.utils.usb.passthrough.client_handle", + "je_auto_control.windows.core.utils.win32_layout_vk", + "je_auto_control.wrapper.capabilities", + "je_auto_control.wrapper.device_context", + "je_auto_control.wrapper.device_frame", + "je_auto_control.wrapper.mobile_commands", + "je_auto_control.wrapper.mobile_extensions", + "je_auto_control_pytest", +] +disallow_untyped_defs = true +disallow_any_generics = true diff --git a/test/unit_test/headless/test_typing_modernization_contract.py b/test/unit_test/headless/test_typing_modernization_contract.py new file mode 100644 index 000000000..3df40a04c --- /dev/null +++ b/test/unit_test/headless/test_typing_modernization_contract.py @@ -0,0 +1,256 @@ +"""The strict typing tier covers every module written since it was introduced. + +``pyproject.toml`` turns ``disallow_untyped_defs`` and ``disallow_any_generics`` +on for a named list of modules. A list a person has to remember to extend is a +list that stops growing, so this compares it against the tree: a module is +either one that predates the tier (``test/verify/typing_strict_baseline.txt``, +which may only shrink) or it is in the tier. There is no third state to forget +a module into. + +mypy itself is run by ``test/verify/typing_contract_verify.py`` (three target +platforms, a job of its own in CI). What is checked here needs no mypy: the +lists, and -- read straight from the source -- that the modules in the tier +really do annotate every function, so the suite notices on any interpreter. +""" +import ast +import re +import shutil +import subprocess # nosec B404 # reason: fixed `git` argv, no shell +from pathlib import Path +from typing import Dict, Iterator, List, Set, Tuple + +import pytest + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10, where pytest itself depends on tomli + tomllib = pytest.importorskip("tomli") + +REPO_ROOT = Path(__file__).resolve().parents[3] +PACKAGE = "je_auto_control" +VERIFY_DIR = REPO_ROOT / "test" / "verify" +EXEMPT_FILE = VERIFY_DIR / "typing_contract_exempt.txt" +BASELINE_FILE = VERIFY_DIR / "typing_strict_baseline.txt" +#: The tag the baseline was measured at. +BASELINE_TAG = "v0.0.225" +STRICT_SETTINGS = ("disallow_untyped_defs", "disallow_any_generics") + +#: Modules written after the baseline that are NOT in the strict tier, each +#: with the reason. Empty: every one of them was made clean. An entry here is +#: a debt someone agreed to, not a way to turn a red build green. +NOT_YET_STRICT: Dict[str, str] = {} + +#: The only modules that may bind a name to ``Any`` at module level. Each is a +#: typed adapter for an optional SDK the contract deliberately reads as ``Any`` +#: (see the ``follow_imports = "skip"`` block in ``pyproject.toml``); the +#: modules around it import the alias instead of spelling ``Any`` themselves. +SDK_ANY_ADAPTERS = { + # av.VideoFrame and the aiortc-backed host / viewer / recorder classes, + # which are ``None`` at run time without the ``webrtc`` extra. + "je_auto_control.gui.remote_desktop._webrtc_types", +} + +_BARE_IGNORE = re.compile(r"#\s*type:\s*ignore(?!\[)") + + +def _module_name(path: Path) -> str: + parts = path.relative_to(REPO_ROOT).with_suffix("").parts + if parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(parts) + + +def _tree_modules() -> Dict[str, Path]: + """Every module the distribution ships: the package plus its ``py-modules``.""" + found = {_module_name(path): path + for path in (REPO_ROOT / PACKAGE).rglob("*.py") + if "__pycache__" not in path.parts} + for name in _pyproject()["tool"]["setuptools"].get("py-modules", []): + found[name] = REPO_ROOT / f"{name}.py" + return found + + +def _pyproject() -> dict: + with (REPO_ROOT / "pyproject.toml").open("rb") as handle: + return tomllib.load(handle) + + +def _listed(path: Path) -> Set[str]: + lines = path.read_text(encoding="utf-8").splitlines() + return {line.strip() for line in lines if line.strip() and not line.startswith("#")} + + +def _strict_patterns() -> List[str]: + """The module patterns of the override block that carries both strict settings.""" + blocks = [block for block in _pyproject()["tool"]["mypy"]["overrides"] + if all(block.get(setting) is True for setting in STRICT_SETTINGS)] + assert len(blocks) == 1, "expected exactly one strict [[tool.mypy.overrides]] block" + return list(blocks[0]["module"]) + + +def _matches(module: str, patterns: List[str]) -> bool: + for pattern in patterns: + if pattern.endswith(".*"): + if module.startswith(pattern[:-1]): + return True + elif module == pattern: + return True + return False + + +def _strict_modules() -> Dict[str, Path]: + patterns = _strict_patterns() + return {name: path for name, path in _tree_modules().items() if _matches(name, patterns)} + + +_SCOPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda) + + +def _functions(tree: ast.AST, in_class: bool = False) -> Iterator[Tuple[ast.AST, bool]]: + """Each function with whether it is a method (its nearest scope is a class). + + The nearest *scope*, not the direct parent: a method declared under + ``if TYPE_CHECKING:`` in a class body is still a method. + """ + for child in ast.iter_child_nodes(tree): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + yield child, in_class + inner = isinstance(child, ast.ClassDef) if isinstance(child, _SCOPES) else in_class + yield from _functions(child, inner) + + +def _missing_annotations(node: ast.AST, in_class: bool) -> List[str]: + """What mypy's ``disallow_untyped_defs`` would name on one function.""" + arguments = node.args + positional = [*arguments.posonlyargs, *arguments.args] + decorators = {getattr(item, "id", getattr(item, "attr", "")) for item in node.decorator_list} + if in_class and positional and "staticmethod" not in decorators: + positional = positional[1:] # self / cls + every = [*positional, *arguments.kwonlyargs, arguments.vararg, arguments.kwarg] + missing = [arg.arg for arg in every if arg is not None and arg.annotation is None] + if node.returns is None: + missing.append("return") + return missing + + +def _untyped(tree: ast.AST) -> List[Tuple[int, str, List[str]]]: + return [(node.lineno, node.name, missing) for node, is_method in _functions(tree) + for missing in [_missing_annotations(node, is_method)] if missing] + + +def _any_aliases(tree: ast.Module) -> List[str]: + """Module-level names bound to ``Any`` (``X = Any``, at any nesting of ``if``).""" + names: List[str] = [] + pending = list(tree.body) + while pending: + node = pending.pop() + if isinstance(node, ast.If): + pending.extend([*node.body, *node.orelse]) + elif isinstance(node, ast.Assign) and getattr(node.value, "id", "") == "Any": + names.extend(target.id for target in node.targets if isinstance(target, ast.Name)) + return sorted(names) + + +def test_exemptions_remain_empty(): + exemptions = _listed(EXEMPT_FILE) + assert exemptions == set() + + +def test_the_strict_tier_never_relaxes_a_package_wide_setting(): + base = _pyproject()["tool"]["mypy"] + assert base["check_untyped_defs"] is True and base["no_implicit_optional"] is True + assert "ignore_errors" not in base + for block in base["overrides"]: + assert "ignore_errors" not in block, f"ignore_errors in the override for {block['module']}" + assert block.get("check_untyped_defs", True) is True + for setting in STRICT_SETTINGS: + assert block.get(setting, True) is True, f"{setting} switched off for {block['module']}" + + +def test_every_module_is_strict_or_predates_the_tier(): + """A module in neither list is new code that nobody asked mypy to hold to the tier.""" + patterns = _strict_patterns() + baseline = _listed(BASELINE_FILE) + modules = _tree_modules() + unassigned = sorted(name for name in modules + if name not in baseline and name not in NOT_YET_STRICT + and not _matches(name, patterns)) + assert unassigned == [], ( + "add these to the strict [[tool.mypy.overrides]] block in pyproject.toml " + "(the baseline list may not grow)") + + +def test_the_lists_name_only_modules_that_exist(): + modules = set(_tree_modules()) + stale_baseline = sorted(_listed(BASELINE_FILE) - modules) + assert stale_baseline == [], "delete these lines from typing_strict_baseline.txt" + stale_strict = sorted(pattern for pattern in _strict_patterns() + if not pattern.endswith(".*") and pattern not in modules) + assert stale_strict == [], "these strict overrides match no module" + assert sorted(set(NOT_YET_STRICT) - modules) == [] + assert all(reason.strip() for reason in NOT_YET_STRICT.values()) + + +def test_the_baseline_is_the_package_at_its_tag(): + """Where the tag is available, the baseline is what it says it is -- or smaller.""" + git = shutil.which("git") + if git is None: + pytest.skip("git is not installed") + listing = subprocess.run( # nosec B603 # nosemgrep # reason: fixed argv, no shell, no input + [git, "ls-tree", "-r", "--name-only", BASELINE_TAG, "--", PACKAGE], + cwd=REPO_ROOT, capture_output=True, text=True, check=False) + if listing.returncode != 0: + pytest.skip(f"{BASELINE_TAG} is not in this clone (shallow checkout)") + at_tag = {_module_name(REPO_ROOT / line) for line in listing.stdout.split() + if line.endswith(".py")} + grown = sorted(_listed(BASELINE_FILE) - at_tag) + assert grown == [], f"modules newer than {BASELINE_TAG} were added to the baseline" + + +def test_the_pytest_plugin_module_is_type_checked(): + """``py-modules`` ship beside the package and are held to the same contract.""" + shipped = _pyproject()["tool"]["setuptools"]["py-modules"] + verify = (VERIFY_DIR / "typing_contract_verify.py").read_text(encoding="utf-8") + for name in shipped: + assert f'"{name}.py"' in verify, f"typing_contract_verify.py does not check {name}.py" + assert _matches(name, _strict_patterns()) + + +def test_new_modules_have_complete_annotations(): + untyped_public_signatures = [] + for name, path in sorted(_strict_modules().items()): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + untyped_public_signatures += [ + f"{name}:{line} {function}({', '.join(missing)})" + for line, function, missing in _untyped(tree)] + assert untyped_public_signatures == [] + + +def test_adapter_is_only_sdk_any_boundary(): + sdk_any_leaks = [] + for name, path in sorted(_strict_modules().items()): + source = path.read_text(encoding="utf-8") + if name not in SDK_ANY_ADAPTERS: + sdk_any_leaks += [f"{name}: {alias} = Any" for alias in _any_aliases(ast.parse(source))] + sdk_any_leaks += [f"{name}:{number} unscoped type: ignore" + for number, line in enumerate(source.splitlines(), 1) + if _BARE_IGNORE.search(line)] + assert sdk_any_leaks == [] + assert SDK_ANY_ADAPTERS <= set(_strict_modules()), "an adapter must itself be strict" + + +@pytest.mark.parametrize("source, expected", [ + ("def f(a, b: int): ...", [(1, "f", ["a", "return"])]), + ("class C:\n def m(self, x) -> None: ...", [(2, "m", ["x"])]), + ("class C:\n @staticmethod\n def s(x) -> None: ...", [(3, "s", ["x"])]), + ("def f(*args, **kwargs) -> None: ...", [(1, "f", ["args", "kwargs"])]), + ("def f(a: int, *, b: str = '') -> None:\n def g(c: int) -> int: return c", []), +]) +def test_the_annotation_scan_sees_what_mypy_would(source, expected): + """The scan above is only evidence if it fails on an untyped function.""" + assert _untyped(ast.parse(source)) == expected + + +def test_the_alias_scan_sees_an_any_alias(): + tree = ast.parse("from typing import Any\nif True:\n A = B = Any\nelse:\n C = Any\nD = int") + assert _any_aliases(tree) == ["A", "B", "C"] diff --git a/test/verify/typing_contract_verify.py b/test/verify/typing_contract_verify.py index 6d157d770..caaa3c88b 100644 --- a/test/verify/typing_contract_verify.py +++ b/test/verify/typing_contract_verify.py @@ -39,6 +39,11 @@ REPO_ROOT = Path(__file__).resolve().parents[2] PACKAGE = "je_auto_control" +# Top-level modules the distribution ships beside the package (`py-modules` in +# `pyproject.toml`). They are installed and imported like the package, so they +# are checked like it; a path list that only named the package left the pytest +# plugin outside the contract without anyone having decided that. +EXTRA_MODULES = ("je_auto_control_pytest.py",) EXEMPT_FILE = Path(__file__).with_name("typing_contract_exempt.txt") # mypy resolves `sys.platform` tests against a single target. The supported @@ -80,8 +85,8 @@ def _failing_modules(platform: str) -> set[str]: # `nosemgrep` only on the exact line it reports, and the audit rule reports # the call, not the argument. See `je_auto_control/android/adb_client.py` # for the same shape. - completed = subprocess.run( # nosec B603 # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit # reason: argv is `sys.executable` plus literals and one value from the module-level PLATFORMS tuple; no shell, no environment, no caller input - [sys.executable, "-m", "mypy", "--platform", platform, "-O", "json", PACKAGE], + completed = subprocess.run( # nosec B603 # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit # reason: argv is `sys.executable` plus literals and values from the module-level PLATFORMS / EXTRA_MODULES tuples; no shell, no environment, no caller input + [sys.executable, "-m", "mypy", "--platform", platform, "-O", "json", PACKAGE, *EXTRA_MODULES], cwd=REPO_ROOT, capture_output=True, text=True, @@ -100,7 +105,7 @@ def _failing_modules(platform: str) -> set[str]: if record.get("severity") != "error": continue path = str(record.get("file", "")).replace("\\", "/") - if path.startswith(f"{PACKAGE}/") or path == f"{PACKAGE}.py": + if path.startswith(f"{PACKAGE}/") or path in (f"{PACKAGE}.py", *EXTRA_MODULES): modules.add(_module_name(path)) if not modules and completed.returncode not in (0, 1): raise SystemExit( @@ -153,7 +158,7 @@ def main() -> int: ) args = parser.parse_args() - print(f"Type-checking {PACKAGE} for {len(PLATFORMS)} target platforms...") + print(f"Type-checking {PACKAGE} (+ {', '.join(EXTRA_MODULES)}) for {len(PLATFORMS)} target platforms...") failing = _measure() if args.fix: diff --git a/test/verify/typing_strict_baseline.txt b/test/verify/typing_strict_baseline.txt new file mode 100644 index 000000000..1a3b74e3c --- /dev/null +++ b/test/verify/typing_strict_baseline.txt @@ -0,0 +1,1064 @@ +# Modules that predate the strict typing tier. +# +# `pyproject.toml` holds a `[[tool.mypy.overrides]]` block that turns on +# `disallow_untyped_defs` and `disallow_any_generics`. Every module of the +# package must be either matched by that block or named here -- so a module +# written from now on is strict by default, because it cannot be here. +# +# This is the package as it stood at v0.0.225, when the tier was introduced. +# Rules (enforced by `test_typing_modernization_contract.py`): +# * The list may only shrink. Annotate a module, name it in the override +# block, delete its line. +# * A new module may not join it. Fix the types instead. +# * A deleted or renamed module loses its line in the same change; a module +# written later under the same name is new and therefore strict. +# +je_auto_control +je_auto_control.__main__ +je_auto_control.android +je_auto_control.android.adb_client +je_auto_control.android.client +je_auto_control.android.find +je_auto_control.api +je_auto_control.api.core +je_auto_control.cli +je_auto_control.gui +je_auto_control.gui._auto_click_tab +je_auto_control.gui._daemon_thread +je_auto_control.gui._i18n_helpers +je_auto_control.gui._image_detect_tab +je_auto_control.gui._record_tab +je_auto_control.gui._report_tab +je_auto_control.gui._screen_geometry +je_auto_control.gui._screenshot_tab +je_auto_control.gui._script_tab +je_auto_control.gui._validators +je_auto_control.gui._worker_thread +je_auto_control.gui.a11y_audit_tab +je_auto_control.gui.accessibility_tab +je_auto_control.gui.admin_console_tab +je_auto_control.gui.assertions_tab +je_auto_control.gui.audit_log_tab +je_auto_control.gui.chatops_tab +je_auto_control.gui.computer_use_tab +je_auto_control.gui.dag_tab +je_auto_control.gui.data_source_tab +je_auto_control.gui.device_matrix_tab +je_auto_control.gui.diagnostics_tab +je_auto_control.gui.email_triggers_tab +je_auto_control.gui.flakiness_tab +je_auto_control.gui.flow_editor +je_auto_control.gui.flow_editor.layout +je_auto_control.gui.flow_editor.scene +je_auto_control.gui.flow_editor.tab +je_auto_control.gui.hotkeys_tab +je_auto_control.gui.inspector_tab +je_auto_control.gui.language_wrapper +je_auto_control.gui.language_wrapper.english +je_auto_control.gui.language_wrapper.japanese +je_auto_control.gui.language_wrapper.multi_language_wrapper +je_auto_control.gui.language_wrapper.simplified_chinese +je_auto_control.gui.language_wrapper.traditional_chinese +je_auto_control.gui.live_hud_tab +je_auto_control.gui.llm_planner_tab +je_auto_control.gui.main_widget +je_auto_control.gui.main_window +je_auto_control.gui.media_checks_tab +je_auto_control.gui.ocr_tab +je_auto_control.gui.plugins_tab +je_auto_control.gui.presence_tab +je_auto_control.gui.profiler_tab +je_auto_control.gui.recording_editor_tab +je_auto_control.gui.remote_desktop +je_auto_control.gui.remote_desktop._helpers +je_auto_control.gui.remote_desktop.advanced_group +je_auto_control.gui.remote_desktop.annotation_overlay +je_auto_control.gui.remote_desktop.blanking_overlay +je_auto_control.gui.remote_desktop.connection_screen +je_auto_control.gui.remote_desktop.frame_display +je_auto_control.gui.remote_desktop.host_panel +je_auto_control.gui.remote_desktop.remote_screen_window +je_auto_control.gui.remote_desktop.sparkline +je_auto_control.gui.remote_desktop.tab +je_auto_control.gui.remote_desktop.tray_icon +je_auto_control.gui.remote_desktop.trusted_group +je_auto_control.gui.remote_desktop.viewer_panel +je_auto_control.gui.remote_desktop.viewer_screen_window +je_auto_control.gui.remote_desktop.webrtc_dialogs +je_auto_control.gui.remote_desktop.webrtc_known_hosts +je_auto_control.gui.remote_desktop.webrtc_panel +je_auto_control.gui.remote_desktop.webrtc_workers +je_auto_control.gui.remote_desktop_tab +je_auto_control.gui.rest_api_tab +je_auto_control.gui.run_history_tab +je_auto_control.gui.run_history_timeline +je_auto_control.gui.scheduler_tab +je_auto_control.gui.script_builder +je_auto_control.gui.script_builder.builder_tab +je_auto_control.gui.script_builder.command_schema +je_auto_control.gui.script_builder.step_form_view +je_auto_control.gui.script_builder.step_list_view +je_auto_control.gui.script_builder.step_model +je_auto_control.gui.secrets_tab +je_auto_control.gui.selector +je_auto_control.gui.selector.region_overlay +je_auto_control.gui.selector.region_selector +je_auto_control.gui.selector.template_cropper +je_auto_control.gui.self_healing_tab +je_auto_control.gui.test_suite_tab +je_auto_control.gui.trace_replay_tab +je_auto_control.gui.triggers_tab +je_auto_control.gui.usb_browser_tab +je_auto_control.gui.usb_devices_tab +je_auto_control.gui.usb_passthrough_panel +je_auto_control.gui.usb_passthrough_prompt +je_auto_control.gui.variables_tab +je_auto_control.gui.vlm_tab +je_auto_control.gui.webhooks_tab +je_auto_control.gui.webrunner_tab +je_auto_control.gui.window_tab +je_auto_control.ios +je_auto_control.ios.client +je_auto_control.ios.find +je_auto_control.ios.input +je_auto_control.ios.screen +je_auto_control.linux_wayland +je_auto_control.linux_wayland._ctypes_bind +je_auto_control.linux_wayland._dbus_client +je_auto_control.linux_wayland._detect +je_auto_control.linux_wayland._layout +je_auto_control.linux_wayland._select_input +je_auto_control.linux_wayland._ydotool_cli +je_auto_control.linux_wayland.capture +je_auto_control.linux_wayland.keyboard +je_auto_control.linux_wayland.keymap +je_auto_control.linux_wayland.libei +je_auto_control.linux_wayland.listener +je_auto_control.linux_wayland.mouse +je_auto_control.linux_wayland.oeffis +je_auto_control.linux_wayland.portal +je_auto_control.linux_wayland.record +je_auto_control.linux_wayland.screen +je_auto_control.linux_with_x11 +je_auto_control.linux_with_x11.core +je_auto_control.linux_with_x11.core.utils +je_auto_control.linux_with_x11.core.utils.x11_linux_display +je_auto_control.linux_with_x11.core.utils.x11_linux_vk +je_auto_control.linux_with_x11.keyboard +je_auto_control.linux_with_x11.keyboard.x11_linux_keyboard_control +je_auto_control.linux_with_x11.listener +je_auto_control.linux_with_x11.listener.x11_linux_listener +je_auto_control.linux_with_x11.mouse +je_auto_control.linux_with_x11.mouse.x11_linux_mouse_control +je_auto_control.linux_with_x11.record +je_auto_control.linux_with_x11.record.x11_linux_record +je_auto_control.linux_with_x11.screen +je_auto_control.linux_with_x11.screen.x11_linux_screen +je_auto_control.linux_with_x11.uinput +je_auto_control.linux_with_x11.uinput._device +je_auto_control.linux_with_x11.uinput.keyboard +je_auto_control.linux_with_x11.uinput.mouse +je_auto_control.osx +je_auto_control.osx.core +je_auto_control.osx.core.utils +je_auto_control.osx.core.utils.osx_vk +je_auto_control.osx.keyboard +je_auto_control.osx.keyboard.osx_keyboard +je_auto_control.osx.keyboard.osx_keyboard_check +je_auto_control.osx.listener +je_auto_control.osx.listener.osx_listener +je_auto_control.osx.mouse +je_auto_control.osx.mouse.osx_mouse +je_auto_control.osx.pid +je_auto_control.osx.pid.pid_control +je_auto_control.osx.record +je_auto_control.osx.record.osx_record +je_auto_control.osx.screen +je_auto_control.osx.screen.osx_screen +je_auto_control.utils +je_auto_control.utils.a11y_audit +je_auto_control.utils.a11y_audit.audit +je_auto_control.utils.a11y_audit.wcag +je_auto_control.utils.a2a +je_auto_control.utils.a2a.agent_card +je_auto_control.utils.ab_locator +je_auto_control.utils.ab_locator.runner +je_auto_control.utils.ab_locator.store +je_auto_control.utils.accessibility +je_auto_control.utils.accessibility.accessibility_api +je_auto_control.utils.accessibility.backends +je_auto_control.utils.accessibility.backends.base +je_auto_control.utils.accessibility.backends.linux_backend +je_auto_control.utils.accessibility.backends.macos_backend +je_auto_control.utils.accessibility.backends.null_backend +je_auto_control.utils.accessibility.backends.windows_backend +je_auto_control.utils.accessibility.backends.windows_query +je_auto_control.utils.accessibility.backends.windows_reads +je_auto_control.utils.accessibility.backends.windows_state +je_auto_control.utils.accessibility.element +je_auto_control.utils.accessibility.recorder +je_auto_control.utils.accessibility.tree +je_auto_control.utils.acme_v2 +je_auto_control.utils.acme_v2.client +je_auto_control.utils.acme_v2.jws +je_auto_control.utils.act_in_view +je_auto_control.utils.act_in_view.act_in_view +je_auto_control.utils.act_modes +je_auto_control.utils.act_modes.act_modes +je_auto_control.utils.action_effect +je_auto_control.utils.action_effect.action_effect +je_auto_control.utils.action_grounding +je_auto_control.utils.action_grounding.action_grounding +je_auto_control.utils.action_lint +je_auto_control.utils.action_lint.__main__ +je_auto_control.utils.action_lint.linter +je_auto_control.utils.action_lint.schema +je_auto_control.utils.action_signing +je_auto_control.utils.action_signing._key_file +je_auto_control.utils.action_signing.cipher +je_auto_control.utils.action_signing.signer +je_auto_control.utils.actionability +je_auto_control.utils.actionability.actionability +je_auto_control.utils.adaptive_timeout +je_auto_control.utils.adaptive_timeout.adaptive_timeout +je_auto_control.utils.admin +je_auto_control.utils.admin.admin_client +je_auto_control.utils.agent +je_auto_control.utils.agent.agent_loop +je_auto_control.utils.agent.backends +je_auto_control.utils.agent.backends._computer_toolset +je_auto_control.utils.agent.backends.anthropic +je_auto_control.utils.agent.backends.anthropic_computer_use +je_auto_control.utils.agent.backends.base +je_auto_control.utils.agent.backends.openai +je_auto_control.utils.agent.computer_use +je_auto_control.utils.agent_memory +je_auto_control.utils.agent_memory.agent_memory +je_auto_control.utils.agent_replay +je_auto_control.utils.agent_replay.agent_replay +je_auto_control.utils.agent_trace +je_auto_control.utils.agent_trace.agent_trace +je_auto_control.utils.anchor_locator +je_auto_control.utils.anchor_locator.locator +je_auto_control.utils.annotate +je_auto_control.utils.annotate.annotate +je_auto_control.utils.anomaly +je_auto_control.utils.anomaly.anomaly +je_auto_control.utils.app_idle +je_auto_control.utils.app_idle.app_idle +je_auto_control.utils.approval +je_auto_control.utils.approval.approval_test +je_auto_control.utils.artifact_store +je_auto_control.utils.artifact_store.s3_store +je_auto_control.utils.assertion +je_auto_control.utils.assertion.assertions +je_auto_control.utils.assertion.combinators +je_auto_control.utils.assets +je_auto_control.utils.assets.assets +je_auto_control.utils.ax_events +je_auto_control.utils.ax_events.ax_events +je_auto_control.utils.ax_props +je_auto_control.utils.ax_props.ax_props +je_auto_control.utils.ax_text +je_auto_control.utils.ax_text.ax_text +je_auto_control.utils.ax_tree_walk +je_auto_control.utils.ax_tree_walk.ax_tree_walk +je_auto_control.utils.baggage +je_auto_control.utils.baggage.baggage +je_auto_control.utils.barcode +je_auto_control.utils.barcode.barcode +je_auto_control.utils.bidi_check +je_auto_control.utils.bidi_check.bidi_check +je_auto_control.utils.bulkhead +je_auto_control.utils.bulkhead.bulkhead +je_auto_control.utils.callback +je_auto_control.utils.callback.callback_function_executor +je_auto_control.utils.canonical_log +je_auto_control.utils.canonical_log.canonical_log +je_auto_control.utils.change_localize +je_auto_control.utils.change_localize.change_localize +je_auto_control.utils.chaos +je_auto_control.utils.chaos.chaos +je_auto_control.utils.chatops +je_auto_control.utils.chatops.handlers +je_auto_control.utils.chatops.router +je_auto_control.utils.chatops.slack_bot +je_auto_control.utils.checkpoint +je_auto_control.utils.checkpoint.checkpoint +je_auto_control.utils.checksum +je_auto_control.utils.checksum.checksum +je_auto_control.utils.ci_annotations +je_auto_control.utils.ci_annotations.ci_annotations +je_auto_control.utils.cli_output +je_auto_control.utils.clipboard +je_auto_control.utils.clipboard.clipboard +je_auto_control.utils.clipboard.win32_clipboard_api +je_auto_control.utils.clipboard_files +je_auto_control.utils.clipboard_files.clipboard_files +je_auto_control.utils.clipboard_formats +je_auto_control.utils.clipboard_formats.clipboard_formats +je_auto_control.utils.clipboard_history +je_auto_control.utils.clipboard_history.clipboard_history +je_auto_control.utils.clipboard_rich_formats +je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats +je_auto_control.utils.codegen +je_auto_control.utils.codegen.codegen +je_auto_control.utils.color_match +je_auto_control.utils.color_match.color_match +je_auto_control.utils.color_region +je_auto_control.utils.color_region.color_region +je_auto_control.utils.color_stats +je_auto_control.utils.color_stats.color_stats +je_auto_control.utils.column_layout +je_auto_control.utils.column_layout.column_layout +je_auto_control.utils.compliance +je_auto_control.utils.compliance.compliance_report +je_auto_control.utils.config_bundle +je_auto_control.utils.config_bundle.__main__ +je_auto_control.utils.config_bundle.config_bundle +je_auto_control.utils.config_redaction +je_auto_control.utils.config_redaction.config_redaction +je_auto_control.utils.config_schema +je_auto_control.utils.config_schema.config_schema +je_auto_control.utils.config_sync +je_auto_control.utils.config_sync.client +je_auto_control.utils.confusables +je_auto_control.utils.confusables.confusables +je_auto_control.utils.contrast_map +je_auto_control.utils.contrast_map.contrast_map +je_auto_control.utils.control_patterns +je_auto_control.utils.control_patterns.control_patterns +je_auto_control.utils.cookie_jar +je_auto_control.utils.cookie_jar.cookie_jar +je_auto_control.utils.coordinate_space +je_auto_control.utils.coordinate_space.coordinate_space +je_auto_control.utils.cost_telemetry +je_auto_control.utils.cost_telemetry.pricing +je_auto_control.utils.cost_telemetry.store +je_auto_control.utils.critic_features +je_auto_control.utils.critic_features.critic_features +je_auto_control.utils.critical_exit +je_auto_control.utils.critical_exit.critical_exit +je_auto_control.utils.cua_action +je_auto_control.utils.cua_action.cua_action +je_auto_control.utils.cv2_utils +je_auto_control.utils.cv2_utils.blobs +je_auto_control.utils.cv2_utils.frame_clock +je_auto_control.utils.cv2_utils.image_file +je_auto_control.utils.cv2_utils.optional +je_auto_control.utils.cv2_utils.region_capture +je_auto_control.utils.cv2_utils.screen_grabber +je_auto_control.utils.cv2_utils.screen_record +je_auto_control.utils.cv2_utils.screenshot +je_auto_control.utils.cv2_utils.template_detection +je_auto_control.utils.cv2_utils.video_recording +je_auto_control.utils.cvd_simulate +je_auto_control.utils.cvd_simulate.cvd_simulate +je_auto_control.utils.dag +je_auto_control.utils.dag.graph +je_auto_control.utils.dag.runner +je_auto_control.utils.data_drift +je_auto_control.utils.data_drift.data_drift +je_auto_control.utils.data_profile +je_auto_control.utils.data_profile.data_profile +je_auto_control.utils.data_quality +je_auto_control.utils.data_quality.data_quality +je_auto_control.utils.data_source +je_auto_control.utils.data_source.data_source +je_auto_control.utils.dataset_diff +je_auto_control.utils.dataset_diff.dataset_diff +je_auto_control.utils.dbus_client +je_auto_control.utils.dbus_client.session_bus +je_auto_control.utils.decision_table +je_auto_control.utils.decision_table.decision_table +je_auto_control.utils.dedup_window +je_auto_control.utils.dedup_window.dedup_window +je_auto_control.utils.deprecation +je_auto_control.utils.deterministic +je_auto_control.utils.deterministic.deterministic +je_auto_control.utils.device_matrix +je_auto_control.utils.device_matrix.matrix +je_auto_control.utils.diagnostics +je_auto_control.utils.diagnostics.__main__ +je_auto_control.utils.diagnostics.diagnostics +je_auto_control.utils.dotenv +je_auto_control.utils.dotenv.dotenv +je_auto_control.utils.edge_lines +je_auto_control.utils.edge_lines.edge_lines +je_auto_control.utils.edge_match +je_auto_control.utils.edge_match.edge_match +je_auto_control.utils.egress +je_auto_control.utils.egress.egress_policy +je_auto_control.utils.element_diff +je_auto_control.utils.element_diff.element_diff +je_auto_control.utils.element_parse +je_auto_control.utils.element_parse.element_parse +je_auto_control.utils.element_proposal +je_auto_control.utils.element_proposal.element_proposal +je_auto_control.utils.element_repository +je_auto_control.utils.element_repository.element_repository +je_auto_control.utils.element_scoring +je_auto_control.utils.element_scoring.element_scoring +je_auto_control.utils.email_send +je_auto_control.utils.email_send.email_sender +je_auto_control.utils.ensure_state +je_auto_control.utils.ensure_state.ensure_state +je_auto_control.utils.events +je_auto_control.utils.events.cloud_events +je_auto_control.utils.exception +je_auto_control.utils.exception.exception_tags +je_auto_control.utils.exception.exceptions +je_auto_control.utils.executor +je_auto_control.utils.executor.action_executor +je_auto_control.utils.executor.action_redaction +je_auto_control.utils.executor.action_schema +je_auto_control.utils.executor.flags +je_auto_control.utils.executor.flow_control +je_auto_control.utils.executor.flow_data_commands +je_auto_control.utils.executor.mouse_aliases +je_auto_control.utils.expect_poll +je_auto_control.utils.expect_poll.expect_poll +je_auto_control.utils.failure_bundle +je_auto_control.utils.failure_bundle.bundle +je_auto_control.utils.failure_hooks +je_auto_control.utils.failure_hooks.backends +je_auto_control.utils.failure_hooks.manager +je_auto_control.utils.failure_hooks.report +je_auto_control.utils.failure_signature +je_auto_control.utils.failure_signature.failure_signature +je_auto_control.utils.feature_flags +je_auto_control.utils.feature_flags.feature_flags +je_auto_control.utils.feature_match +je_auto_control.utils.feature_match.feature_match +je_auto_control.utils.field_entry +je_auto_control.utils.field_entry.field_entry +je_auto_control.utils.file_assoc +je_auto_control.utils.file_assoc.file_assoc +je_auto_control.utils.file_dialog +je_auto_control.utils.file_dialog.file_dialog +je_auto_control.utils.file_drop +je_auto_control.utils.file_drop.file_drop +je_auto_control.utils.file_process +je_auto_control.utils.file_process.get_dir_file_list +je_auto_control.utils.flake_cluster +je_auto_control.utils.flake_cluster.flake_cluster +je_auto_control.utils.flakiness +je_auto_control.utils.flakiness.flakiness +je_auto_control.utils.flow_debugger +je_auto_control.utils.flow_debugger.flow_debugger +je_auto_control.utils.focus_order +je_auto_control.utils.focus_order.focus_order +je_auto_control.utils.form_fields +je_auto_control.utils.form_fields.form_fields +je_auto_control.utils.fuzzy +je_auto_control.utils.fuzzy.fuzzy_match +je_auto_control.utils.gamepad +je_auto_control.utils.gamepad._facade +je_auto_control.utils.generate_report +je_auto_control.utils.generate_report.generate_html_report +je_auto_control.utils.generate_report.generate_json_report +je_auto_control.utils.generate_report.generate_xml_report +je_auto_control.utils.gettext_catalog +je_auto_control.utils.gettext_catalog.gettext_catalog +je_auto_control.utils.governance +je_auto_control.utils.governance.credential_broker +je_auto_control.utils.governance.governance +je_auto_control.utils.grid_locator +je_auto_control.utils.grid_locator.grid_locator +je_auto_control.utils.grounding_consensus +je_auto_control.utils.grounding_consensus.grounding_consensus +je_auto_control.utils.guardrail +je_auto_control.utils.guardrail.guardrail +je_auto_control.utils.heading_segment +je_auto_control.utils.heading_segment.heading_segment +je_auto_control.utils.heal_analytics +je_auto_control.utils.heal_analytics.heal_analytics +je_auto_control.utils.hotkey +je_auto_control.utils.hotkey.backends +je_auto_control.utils.hotkey.backends.base +je_auto_control.utils.hotkey.backends.linux_backend +je_auto_control.utils.hotkey.backends.macos_backend +je_auto_control.utils.hotkey.backends.windows_backend +je_auto_control.utils.hotkey.hotkey_daemon +je_auto_control.utils.hsv_segment +je_auto_control.utils.hsv_segment.hsv_segment +je_auto_control.utils.http_cassette +je_auto_control.utils.http_cassette.http_cassette +je_auto_control.utils.http_client +je_auto_control.utils.http_client.http_client +je_auto_control.utils.http_conditional +je_auto_control.utils.http_conditional.http_conditional +je_auto_control.utils.http_content +je_auto_control.utils.http_content.http_content +je_auto_control.utils.http_headers +je_auto_control.utils.http_problem +je_auto_control.utils.http_problem.http_problem +je_auto_control.utils.humanize +je_auto_control.utils.humanize.motion +je_auto_control.utils.humanize.typing +je_auto_control.utils.i18n_test +je_auto_control.utils.i18n_test.i18n_test +je_auto_control.utils.icon_classify +je_auto_control.utils.icon_classify.icon_classify +je_auto_control.utils.idempotency +je_auto_control.utils.idempotency.idempotency +je_auto_control.utils.idle_keepawake +je_auto_control.utils.idle_keepawake.idle_keepawake +je_auto_control.utils.image_dedup +je_auto_control.utils.image_dedup.perceptual_hash +je_auto_control.utils.image_quality +je_auto_control.utils.image_quality.image_quality +je_auto_control.utils.ime_state +je_auto_control.utils.ime_state.ime_state +je_auto_control.utils.img_histogram +je_auto_control.utils.img_histogram.img_histogram +je_auto_control.utils.input_macro +je_auto_control.utils.input_macro.input_macro +je_auto_control.utils.input_macro.recorder_base +je_auto_control.utils.input_reach +je_auto_control.utils.input_reach.input_reach +je_auto_control.utils.json +je_auto_control.utils.json.json_file +je_auto_control.utils.json_contract +je_auto_control.utils.json_contract.json_contract +je_auto_control.utils.json_patch +je_auto_control.utils.json_patch.json_patch +je_auto_control.utils.json_schema +je_auto_control.utils.json_schema.ecma_regex +je_auto_control.utils.json_schema.json_schema +je_auto_control.utils.json_store +je_auto_control.utils.json_store.json_store +je_auto_control.utils.jsonpath +je_auto_control.utils.jsonpath.jsonpath +je_auto_control.utils.jwt +je_auto_control.utils.jwt.jwt_codec +je_auto_control.utils.key_hold +je_auto_control.utils.key_hold.key_hold +je_auto_control.utils.keyboard_layout +je_auto_control.utils.keyboard_layout.keyboard_layout +je_auto_control.utils.layered_config +je_auto_control.utils.layered_config.layered_config +je_auto_control.utils.legacy_accessible +je_auto_control.utils.legacy_accessible.legacy_accessible +je_auto_control.utils.license_policy +je_auto_control.utils.license_policy.license_policy +je_auto_control.utils.link_header +je_auto_control.utils.link_header.link_header +je_auto_control.utils.list_format +je_auto_control.utils.list_format.list_format +je_auto_control.utils.llm +je_auto_control.utils.llm.backends +je_auto_control.utils.llm.backends.anthropic_backend +je_auto_control.utils.llm.backends.base +je_auto_control.utils.llm.backends.null_backend +je_auto_control.utils.llm.planner +je_auto_control.utils.locale_collation +je_auto_control.utils.locale_collation.locale_collation +je_auto_control.utils.locale_parse +je_auto_control.utils.locale_parse.locale_parse +je_auto_control.utils.locator_chain +je_auto_control.utils.locator_chain.locator_chain +je_auto_control.utils.locator_repair +je_auto_control.utils.locator_repair.locator_repair +je_auto_control.utils.lock_session +je_auto_control.utils.lock_session.lock_session +je_auto_control.utils.logging +je_auto_control.utils.logging.logging_instance +je_auto_control.utils.loop_guard +je_auto_control.utils.loop_guard.loop_guard +je_auto_control.utils.marks_layout +je_auto_control.utils.marks_layout.marks_layout +je_auto_control.utils.match_autothresh +je_auto_control.utils.match_autothresh.match_autothresh +je_auto_control.utils.match_ensemble +je_auto_control.utils.match_ensemble.match_ensemble +je_auto_control.utils.match_stability +je_auto_control.utils.match_stability.match_stability +je_auto_control.utils.match_trust +je_auto_control.utils.match_trust.match_trust +je_auto_control.utils.mcp_registry +je_auto_control.utils.mcp_registry.registry +je_auto_control.utils.mcp_server +je_auto_control.utils.mcp_server.__main__ +je_auto_control.utils.mcp_server._client_requests +je_auto_control.utils.mcp_server._http_stateless +je_auto_control.utils.mcp_server._input_required +je_auto_control.utils.mcp_server._protocol +je_auto_control.utils.mcp_server._stateless +je_auto_control.utils.mcp_server._subscriptions +je_auto_control.utils.mcp_server.audit +je_auto_control.utils.mcp_server.context +je_auto_control.utils.mcp_server.fake_backend +je_auto_control.utils.mcp_server.http_sessions +je_auto_control.utils.mcp_server.http_transport +je_auto_control.utils.mcp_server.log_bridge +je_auto_control.utils.mcp_server.plugin_watcher +je_auto_control.utils.mcp_server.prompts +je_auto_control.utils.mcp_server.rate_limit +je_auto_control.utils.mcp_server.resources +je_auto_control.utils.mcp_server.server +je_auto_control.utils.mcp_server.tools +je_auto_control.utils.mcp_server.tools._base +je_auto_control.utils.mcp_server.tools._factories +je_auto_control.utils.mcp_server.tools._handlers +je_auto_control.utils.mcp_server.tools._handlers_executor_bridge +je_auto_control.utils.mcp_server.tools._handlers_input +je_auto_control.utils.mcp_server.tools._handlers_locators +je_auto_control.utils.mcp_server.tools._handlers_operations +je_auto_control.utils.mcp_server.tools._handlers_qa +je_auto_control.utils.mcp_server.tools._handlers_remote +je_auto_control.utils.mcp_server.tools._handlers_runs +je_auto_control.utils.mcp_server.tools._handlers_scheduling +je_auto_control.utils.mcp_server.tools._handlers_screen +je_auto_control.utils.mcp_server.tools._handlers_system +je_auto_control.utils.mcp_server.tools._validation +je_auto_control.utils.mcp_server.tools.plugin_tools +je_auto_control.utils.media_assert +je_auto_control.utils.media_assert.media +je_auto_control.utils.message_format +je_auto_control.utils.message_format.message_format +je_auto_control.utils.modifier_state +je_auto_control.utils.modifier_state.modifier_state +je_auto_control.utils.monitor_layout +je_auto_control.utils.monitor_layout.logical_frame +je_auto_control.utils.monitor_layout.monitor_layout +je_auto_control.utils.motion_regions +je_auto_control.utils.motion_regions.motion_regions +je_auto_control.utils.mouse_path +je_auto_control.utils.mouse_path.mouse_path +je_auto_control.utils.mouse_relative +je_auto_control.utils.mouse_relative.mouse_relative +je_auto_control.utils.multipart +je_auto_control.utils.multipart.multipart +je_auto_control.utils.near_dup +je_auto_control.utils.near_dup.near_dup +je_auto_control.utils.notify +je_auto_control.utils.notify.notifier +je_auto_control.utils.notify_channels +je_auto_control.utils.notify_channels.notify_channels +je_auto_control.utils.observability +je_auto_control.utils.observability.exporter +je_auto_control.utils.observability.metrics +je_auto_control.utils.observability.tracing +je_auto_control.utils.observation +je_auto_control.utils.observation.observation +je_auto_control.utils.observation_delta +je_auto_control.utils.observation_delta.observation_delta +je_auto_control.utils.observer +je_auto_control.utils.observer.observer +je_auto_control.utils.ocr +je_auto_control.utils.ocr.backends +je_auto_control.utils.ocr.backends.base +je_auto_control.utils.ocr.backends.easyocr_backend +je_auto_control.utils.ocr.backends.paddleocr_backend +je_auto_control.utils.ocr.backends.tesseract_backend +je_auto_control.utils.ocr.ocr_engine +je_auto_control.utils.ocr.structure +je_auto_control.utils.ocr.text_span +je_auto_control.utils.office +je_auto_control.utils.office.office +je_auto_control.utils.optimistic +je_auto_control.utils.optimistic.optimistic +je_auto_control.utils.otlp_export +je_auto_control.utils.otlp_export.otlp_export +je_auto_control.utils.otp +je_auto_control.utils.otp.otp +je_auto_control.utils.outbox +je_auto_control.utils.outbox.outbox +je_auto_control.utils.package_manager +je_auto_control.utils.package_manager.package_manager_class +je_auto_control.utils.path_guard +je_auto_control.utils.path_guard.path_guard +je_auto_control.utils.pdf +je_auto_control.utils.pdf.pdf_reader +je_auto_control.utils.percentiles +je_auto_control.utils.percentiles.percentiles +je_auto_control.utils.perceptual_diff +je_auto_control.utils.perceptual_diff.perceptual_diff +je_auto_control.utils.pii_text +je_auto_control.utils.pii_text.pii_text +je_auto_control.utils.platform_id +je_auto_control.utils.plugin_loader +je_auto_control.utils.plugin_loader.plugin_loader +je_auto_control.utils.plugin_sdk +je_auto_control.utils.plugin_sdk.plugin_sdk +je_auto_control.utils.postcondition +je_auto_control.utils.postcondition.postcondition +je_auto_control.utils.preprocess +je_auto_control.utils.preprocess.preprocess +je_auto_control.utils.process_doc +je_auto_control.utils.process_doc.process_doc +je_auto_control.utils.process_mining +je_auto_control.utils.process_mining.process_mining +je_auto_control.utils.profiler +je_auto_control.utils.profiler.profiler +je_auto_control.utils.profiler.resource_profiler +je_auto_control.utils.project +je_auto_control.utils.project.create_project_structure +je_auto_control.utils.project.template +je_auto_control.utils.project.template.template_executor +je_auto_control.utils.project.template.template_keyword +je_auto_control.utils.provenance +je_auto_control.utils.provenance.provenance +je_auto_control.utils.pytest_plugin +je_auto_control.utils.pytest_plugin.bdd_steps +je_auto_control.utils.pytest_plugin.keywords +je_auto_control.utils.pytest_plugin.plugin +je_auto_control.utils.qr +je_auto_control.utils.qr.qr +je_auto_control.utils.quarantine +je_auto_control.utils.quarantine.store +je_auto_control.utils.rate_limit +je_auto_control.utils.rate_limit.rate_limit +je_auto_control.utils.rbac +je_auto_control.utils.rbac.users +je_auto_control.utils.readability +je_auto_control.utils.readability.readability +je_auto_control.utils.reading_flow +je_auto_control.utils.reading_flow.reading_flow +je_auto_control.utils.recording_edit +je_auto_control.utils.recording_edit.editor +je_auto_control.utils.recurrence +je_auto_control.utils.recurrence.recurrence +je_auto_control.utils.redaction +je_auto_control.utils.redaction.engine +je_auto_control.utils.redaction.policies +je_auto_control.utils.redaction.rules +je_auto_control.utils.referential +je_auto_control.utils.referential.referential +je_auto_control.utils.remote_desktop +je_auto_control.utils.remote_desktop.adaptive_bitrate +je_auto_control.utils.remote_desktop.address_book +je_auto_control.utils.remote_desktop.audio +je_auto_control.utils.remote_desktop.audit_log +je_auto_control.utils.remote_desktop.auth +je_auto_control.utils.remote_desktop.clipboard_sync +je_auto_control.utils.remote_desktop.connect_coordinator +je_auto_control.utils.remote_desktop.file_sync +je_auto_control.utils.remote_desktop.file_transfer +je_auto_control.utils.remote_desktop.fingerprint +je_auto_control.utils.remote_desktop.host +je_auto_control.utils.remote_desktop.host_access +je_auto_control.utils.remote_desktop.host_capture +je_auto_control.utils.remote_desktop.host_client +je_auto_control.utils.remote_desktop.host_id +je_auto_control.utils.remote_desktop.host_service +je_auto_control.utils.remote_desktop.hw_codec +je_auto_control.utils.remote_desktop.input_dispatch +je_auto_control.utils.remote_desktop.jpeg_recorder +je_auto_control.utils.remote_desktop.jpeg_recorder_encrypted +je_auto_control.utils.remote_desktop.lan_discovery +je_auto_control.utils.remote_desktop.multi_viewer +je_auto_control.utils.remote_desktop.permissions +je_auto_control.utils.remote_desktop.presence +je_auto_control.utils.remote_desktop.protocol +je_auto_control.utils.remote_desktop.rate_limit +je_auto_control.utils.remote_desktop.registry +je_auto_control.utils.remote_desktop.relay +je_auto_control.utils.remote_desktop.resume_tokens +je_auto_control.utils.remote_desktop.session_actions +je_auto_control.utils.remote_desktop.session_quality_cache +je_auto_control.utils.remote_desktop.session_recorder +je_auto_control.utils.remote_desktop.signaling_client +je_auto_control.utils.remote_desktop.signaling_server +je_auto_control.utils.remote_desktop.totp +je_auto_control.utils.remote_desktop.transport +je_auto_control.utils.remote_desktop.trust_list +je_auto_control.utils.remote_desktop.turn_config +je_auto_control.utils.remote_desktop.video_codec +je_auto_control.utils.remote_desktop.viewer +je_auto_control.utils.remote_desktop.viewer_id +je_auto_control.utils.remote_desktop.wake_on_lan +je_auto_control.utils.remote_desktop.webrtc_audio +je_auto_control.utils.remote_desktop.webrtc_files +je_auto_control.utils.remote_desktop.webrtc_host +je_auto_control.utils.remote_desktop.webrtc_host_auth +je_auto_control.utils.remote_desktop.webrtc_host_media +je_auto_control.utils.remote_desktop.webrtc_inspector +je_auto_control.utils.remote_desktop.webrtc_mic +je_auto_control.utils.remote_desktop.webrtc_stats +je_auto_control.utils.remote_desktop.webrtc_transport +je_auto_control.utils.remote_desktop.webrtc_viewer +je_auto_control.utils.remote_desktop.ws_host +je_auto_control.utils.remote_desktop.ws_protocol +je_auto_control.utils.remote_desktop.ws_viewer +je_auto_control.utils.resilience +je_auto_control.utils.resilience.resilience +je_auto_control.utils.rest_api +je_auto_control.utils.rest_api.__main__ +je_auto_control.utils.rest_api.rest_auth +je_auto_control.utils.rest_api.rest_handlers +je_auto_control.utils.rest_api.rest_metrics +je_auto_control.utils.rest_api.rest_openapi +je_auto_control.utils.rest_api.rest_registry +je_auto_control.utils.rest_api.rest_server +je_auto_control.utils.retry_budget +je_auto_control.utils.retry_budget.retry_budget +je_auto_control.utils.rich_clipboard +je_auto_control.utils.rich_clipboard.rich_clipboard +je_auto_control.utils.rotated_match +je_auto_control.utils.rotated_match.rotated_match +je_auto_control.utils.run_diff +je_auto_control.utils.run_diff.run_diff +je_auto_control.utils.run_history +je_auto_control.utils.run_history.artifact_manager +je_auto_control.utils.run_history.history_store +je_auto_control.utils.run_history.run_outcome +je_auto_control.utils.saga +je_auto_control.utils.saga.saga +je_auto_control.utils.saliency +je_auto_control.utils.saliency.saliency +je_auto_control.utils.sarif +je_auto_control.utils.sarif.sarif +je_auto_control.utils.sbom +je_auto_control.utils.sbom.sbom +je_auto_control.utils.scale_detect +je_auto_control.utils.scale_detect.scale_detect +je_auto_control.utils.scheduler +je_auto_control.utils.scheduler.cron +je_auto_control.utils.scheduler.scheduler +je_auto_control.utils.schema_compat +je_auto_control.utils.schema_compat.schema_compat +je_auto_control.utils.screen_grid +je_auto_control.utils.screen_grid.screen_grid +je_auto_control.utils.screen_state +je_auto_control.utils.screen_state.screen_state +je_auto_control.utils.script_vars +je_auto_control.utils.script_vars.interpolate +je_auto_control.utils.script_vars.scope +je_auto_control.utils.scroll_find +je_auto_control.utils.scroll_find.scroll_find +je_auto_control.utils.search_index +je_auto_control.utils.search_index.search_index +je_auto_control.utils.secret_ref +je_auto_control.utils.secret_ref.secret_ref +je_auto_control.utils.secrets +je_auto_control.utils.secrets.secret_store +je_auto_control.utils.secrets_scan +je_auto_control.utils.secrets_scan.secrets_scan +je_auto_control.utils.selection_view +je_auto_control.utils.selection_view.selection_view +je_auto_control.utils.self_healing +je_auto_control.utils.self_healing.heal_log +je_auto_control.utils.self_healing.locator +je_auto_control.utils.semantic_recording +je_auto_control.utils.semantic_recording.enrich +je_auto_control.utils.semantic_recording.replay +je_auto_control.utils.semantic_recording.self_healing +je_auto_control.utils.sequence_gap +je_auto_control.utils.sequence_gap.sequence_gap +je_auto_control.utils.session_guard +je_auto_control.utils.session_guard.session_guard +je_auto_control.utils.set_of_marks +je_auto_control.utils.set_of_marks.set_of_marks +je_auto_control.utils.settle_detector +je_auto_control.utils.settle_detector.settle_detector +je_auto_control.utils.shape_locator +je_auto_control.utils.shape_locator.shape_locator +je_auto_control.utils.shell_open +je_auto_control.utils.shell_open.shell_open +je_auto_control.utils.shell_process +je_auto_control.utils.shell_process.shell_exec +je_auto_control.utils.skill_library +je_auto_control.utils.skill_library.skill_library +je_auto_control.utils.slo +je_auto_control.utils.slo.slo +je_auto_control.utils.smart_waits +je_auto_control.utils.smart_waits.waits +je_auto_control.utils.smoothing +je_auto_control.utils.smoothing.smoothing +je_auto_control.utils.socket_server +je_auto_control.utils.socket_server.auto_control_socket_server +je_auto_control.utils.soft_assert +je_auto_control.utils.soft_assert.soft_assert +je_auto_control.utils.sql +je_auto_control.utils.sql.sql_query +je_auto_control.utils.sqlite_support +je_auto_control.utils.sse_client +je_auto_control.utils.sse_client.sse_client +je_auto_control.utils.ssim +je_auto_control.utils.ssim.ssim +je_auto_control.utils.start_exe +je_auto_control.utils.start_exe.start_another_process +je_auto_control.utils.state_machine +je_auto_control.utils.state_machine.engine +je_auto_control.utils.stats +je_auto_control.utils.stats.stats +je_auto_control.utils.step_repair +je_auto_control.utils.step_repair.step_repair +je_auto_control.utils.step_timeline +je_auto_control.utils.step_timeline.step_timeline +je_auto_control.utils.stubs +je_auto_control.utils.stubs.__main__ +je_auto_control.utils.stubs.generator +je_auto_control.utils.subpixel_match +je_auto_control.utils.subpixel_match.subpixel_match +je_auto_control.utils.system_volume +je_auto_control.utils.system_volume.system_volume +je_auto_control.utils.table_grid_fill +je_auto_control.utils.table_grid_fill.table_grid_fill +je_auto_control.utils.table_pattern +je_auto_control.utils.table_pattern.table_pattern +je_auto_control.utils.test_data +je_auto_control.utils.test_data.test_data +je_auto_control.utils.test_record +je_auto_control.utils.test_record.record_test_class +je_auto_control.utils.test_select +je_auto_control.utils.test_select.test_select +je_auto_control.utils.test_shard +je_auto_control.utils.test_shard.test_shard +je_auto_control.utils.test_suite +je_auto_control.utils.test_suite.reports +je_auto_control.utils.test_suite.result +je_auto_control.utils.test_suite.runner +je_auto_control.utils.text_blocks +je_auto_control.utils.text_blocks.text_blocks +je_auto_control.utils.text_diff +je_auto_control.utils.text_diff.text_diff +je_auto_control.utils.text_normalize +je_auto_control.utils.text_normalize.text_normalize +je_auto_control.utils.text_regions +je_auto_control.utils.text_regions.text_regions +je_auto_control.utils.text_similarity +je_auto_control.utils.text_similarity.text_similarity +je_auto_control.utils.text_unicode +je_auto_control.utils.text_unicode.text_unicode +je_auto_control.utils.theme_normalize +je_auto_control.utils.theme_normalize.theme_normalize +je_auto_control.utils.time_travel +je_auto_control.utils.time_travel.controller +je_auto_control.utils.time_travel.player +je_auto_control.utils.timeouts +je_auto_control.utils.timeseries +je_auto_control.utils.timeseries.timeseries +je_auto_control.utils.tls_acme +je_auto_control.utils.tls_acme.challenge +je_auto_control.utils.tls_acme.keys +je_auto_control.utils.tls_acme.renewal +je_auto_control.utils.tool_use_schema +je_auto_control.utils.tool_use_schema.schema +je_auto_control.utils.trace_context +je_auto_control.utils.trace_context.trace_context +je_auto_control.utils.trajectory_eval +je_auto_control.utils.trajectory_eval.trajectory_eval +je_auto_control.utils.transform_window +je_auto_control.utils.transform_window.transform_window +je_auto_control.utils.trash +je_auto_control.utils.trash.trash +je_auto_control.utils.triggers +je_auto_control.utils.triggers.email_trigger +je_auto_control.utils.triggers.trigger_engine +je_auto_control.utils.triggers.webhook_server +je_auto_control.utils.tween_drag +je_auto_control.utils.tween_drag.tween_drag +je_auto_control.utils.url_canon +je_auto_control.utils.url_canon.url_canon +je_auto_control.utils.usb +je_auto_control.utils.usb.passthrough +je_auto_control.utils.usb.passthrough.acl +je_auto_control.utils.usb.passthrough.backend +je_auto_control.utils.usb.passthrough.commands +je_auto_control.utils.usb.passthrough.descriptor +je_auto_control.utils.usb.passthrough.flags +je_auto_control.utils.usb.passthrough.iokit_backend +je_auto_control.utils.usb.passthrough.key_provider +je_auto_control.utils.usb.passthrough.loopback +je_auto_control.utils.usb.passthrough.protocol +je_auto_control.utils.usb.passthrough.session +je_auto_control.utils.usb.passthrough.viewer_client +je_auto_control.utils.usb.passthrough.webrtc_channel +je_auto_control.utils.usb.passthrough.winusb_backend +je_auto_control.utils.usb.usb_devices +je_auto_control.utils.usb.usb_watcher +je_auto_control.utils.usbip +je_auto_control.utils.usbip.backend +je_auto_control.utils.usbip.libusb_backend +je_auto_control.utils.usbip.protocol +je_auto_control.utils.usbip.server +je_auto_control.utils.verify_field +je_auto_control.utils.verify_field.verify_field +je_auto_control.utils.vex +je_auto_control.utils.vex.vex +je_auto_control.utils.video_report +je_auto_control.utils.video_report.video_report +je_auto_control.utils.virtualized +je_auto_control.utils.virtualized.virtualized +je_auto_control.utils.vision +je_auto_control.utils.vision.backends +je_auto_control.utils.vision.backends._parse +je_auto_control.utils.vision.backends.anthropic_backend +je_auto_control.utils.vision.backends.base +je_auto_control.utils.vision.backends.null_backend +je_auto_control.utils.vision.backends.openai_backend +je_auto_control.utils.vision.vlm_api +je_auto_control.utils.visual_match +je_auto_control.utils.visual_match.visual_match +je_auto_control.utils.visual_regression +je_auto_control.utils.visual_regression.compare +je_auto_control.utils.voice +je_auto_control.utils.voice.voice_router +je_auto_control.utils.vuln_scan +je_auto_control.utils.vuln_scan.vuln_scan +je_auto_control.utils.watchdog +je_auto_control.utils.watchdog.popup_watchdog +je_auto_control.utils.watcher +je_auto_control.utils.watcher.watcher +je_auto_control.utils.webrunner_bridge +je_auto_control.utils.webrunner_bridge.bridge +je_auto_control.utils.window_capture +je_auto_control.utils.window_capture.window_capture +je_auto_control.utils.window_geometry +je_auto_control.utils.window_geometry.window_geometry +je_auto_control.utils.window_layout +je_auto_control.utils.window_layout.window_layout +je_auto_control.utils.window_zorder +je_auto_control.utils.window_zorder.window_zorder +je_auto_control.utils.work_queue +je_auto_control.utils.work_queue.work_queue +je_auto_control.utils.xml +je_auto_control.utils.xml.change_xml_structure +je_auto_control.utils.xml.change_xml_structure.change_xml_structure +je_auto_control.utils.xml.xml_file +je_auto_control.utils.xml.xml_file.xml_file +je_auto_control.windows +je_auto_control.windows.core +je_auto_control.windows.core.utils +je_auto_control.windows.core.utils.win32_ctype_input +je_auto_control.windows.core.utils.win32_keypress_check +je_auto_control.windows.core.utils.win32_vk +je_auto_control.windows.interception +je_auto_control.windows.interception._dll +je_auto_control.windows.interception.keyboard +je_auto_control.windows.interception.mouse +je_auto_control.windows.keyboard +je_auto_control.windows.keyboard.win32_ctype_keyboard_control +je_auto_control.windows.message +je_auto_control.windows.message.window_message +je_auto_control.windows.mouse +je_auto_control.windows.mouse.win32_ctype_mouse_control +je_auto_control.windows.record +je_auto_control.windows.record.win32_input_hook +je_auto_control.windows.record.win32_record +je_auto_control.windows.screen +je_auto_control.windows.screen.win32_screen +je_auto_control.windows.window +je_auto_control.windows.window.windows_window_manage +je_auto_control.wrapper +je_auto_control.wrapper._platform_linux +je_auto_control.wrapper._platform_osx +je_auto_control.wrapper._platform_wayland +je_auto_control.wrapper._platform_windows +je_auto_control.wrapper.auto_control_image +je_auto_control.wrapper.auto_control_keyboard +je_auto_control.wrapper.auto_control_mouse +je_auto_control.wrapper.auto_control_record +je_auto_control.wrapper.auto_control_screen +je_auto_control.wrapper.auto_control_window +je_auto_control.wrapper.backend_contract +je_auto_control.wrapper.platform_wrapper +je_auto_control.wrapper.window_backends +je_auto_control.wrapper.window_backends.base +je_auto_control.wrapper.window_backends.macos_backend +je_auto_control.wrapper.window_backends.null_backend +je_auto_control.wrapper.window_backends.windows_backend +je_auto_control.wrapper.window_backends.x11_backend From 8577eba047a44c76a3028a1c72f9bafaa782abde Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:51:10 +0800 Subject: [PATCH 37/44] Add offline-validated examples, a configuration reference and a capability matrix checked against the code The six newest features had prose and no script a reader could run without a desktop, a device or a server. Each example now has a --validate path that exercises the real API against temp-directory, in-memory or loopback stand-ins, and a test runs all six as child processes with every input, capture, process and network seam replaced by a tripwire. Four environment variables the package reads were in no document; a configuration page lists all 41 in both languages and a test compares it with the code. CAPABILITY_MATRIX.md gains the probe and mobile tables generated from probe_capabilities() and mobile_capability_matrix(); API_LIFECYCLE.md records the journal, config-sync wire, signature sidecar and USB request_id formats. Ten Sphinx warnings in Zh pages changed this round are fixed. --- docs/API_LIFECYCLE.md | 56 ++++ docs/CAPABILITY_MATRIX.md | 91 ++++++ .../doc/configuration/configuration_doc.rst | 263 ++++++++++++++++ docs/source/Eng/doc/examples/examples_doc.rst | 170 ++++++++++ .../doc/new_features/v223_features_doc.rst | 40 +++ docs/source/Eng/eng_index.rst | 2 + .../Zh/doc/config_sync/config_sync_doc.rst | 8 +- .../doc/configuration/configuration_doc.rst | 247 +++++++++++++++ docs/source/Zh/doc/examples/examples_doc.rst | 164 ++++++++++ .../Zh/doc/new_features/v223_features_doc.rst | 39 +++ .../Zh/doc/new_features/v2_features_doc.rst | 2 +- .../Zh/doc/new_features/v5_features_doc.rst | 4 +- .../Zh/doc/new_features/v81_features_doc.rst | 2 +- .../doc/wayland/wayland_capabilities_doc.rst | 4 +- docs/source/Zh/zh_index.rst | 2 + examples/28_wayland_diagnostics.py | 155 +++++++++ examples/29_config_sync.py | 290 +++++++++++++++++ examples/30_mobile_devices.py | 193 ++++++++++++ examples/31_healing_comparison.py | 149 +++++++++ examples/32_codegen_from_log.py | 143 +++++++++ examples/33_mcp_progressive.py | 148 +++++++++ examples/README.md | 43 ++- test/unit_test/headless/_example_guard.py | 142 +++++++++ .../headless/test_modernization_examples.py | 294 ++++++++++++++++++ 24 files changed, 2639 insertions(+), 12 deletions(-) create mode 100644 docs/source/Eng/doc/configuration/configuration_doc.rst create mode 100644 docs/source/Eng/doc/examples/examples_doc.rst create mode 100644 docs/source/Zh/doc/configuration/configuration_doc.rst create mode 100644 docs/source/Zh/doc/examples/examples_doc.rst create mode 100644 examples/28_wayland_diagnostics.py create mode 100644 examples/29_config_sync.py create mode 100644 examples/30_mobile_devices.py create mode 100644 examples/31_healing_comparison.py create mode 100644 examples/32_codegen_from_log.py create mode 100644 examples/33_mcp_progressive.py create mode 100644 test/unit_test/headless/_example_guard.py create mode 100644 test/unit_test/headless/test_modernization_examples.py diff --git a/docs/API_LIFECYCLE.md b/docs/API_LIFECYCLE.md index 11058c613..008ff8d76 100644 --- a/docs/API_LIFECYCLE.md +++ b/docs/API_LIFECYCLE.md @@ -15,3 +15,59 @@ available for compatibility but is not expanded with new integrations. The project remains pre-1.0. A 1.0 release requires passing stable capability tests on every claimed platform, documented recovery/diagnostic behavior, and no unresolved critical security advisories. + +## Formats and wire versions + +A file or a message that outlives the process that wrote it is an interface +too. Each of the formats below carries its version in the data, and a reader +that meets a version it does not know refuses it instead of guessing. + +| Format | Version | Status | What changed, and what an older peer sees | +|---|---|---|---| +| Action journal (`*.jsonl`) | `schema_version` 1 | new, beta | One `start` and one `end` record per executed action, grouped by `run_id`. A line with another `schema_version` raises `JournalFormatError`; a line cut off mid-write is reported as torn, not as an error. Secrets are masked before the line is written. | +| Config-sync wire (`GET` / `PUT /config/{user_id}`) | `version` 2 | **changed** | A `PUT` is now an envelope `{"version": 2, "base_revision", "operation_id", "bucket"}` and is committed only while the stored bucket is at `base_revision`; otherwise `409` with the current revision. A bare bucket (what clients sent before) is answered `428` unless the server runs with `--allow-blind-config-writes`. A `GET` reply gained `revision` and `version`, which older clients ignore. Buckets moved from memory to SQLite (`AC_SIGNALING_CONFIG_DB`). | +| Config-sync entries | version vector per entry | **changed** | An entry written with a device id carries a version vector and merges causally; two changes made apart are both kept as a conflict. Entries without one keep the older rule (the later `last_modified` wins). | +| Action-file signature sidecar (`.sig`) | envelope `version` 2 | new format beside the old one | Version 2 is a JSON envelope `{"version": 2, "algorithm": "ed25519", "key_id", "signature"}`; the private key signs, the public key only verifies. The earlier sidecar (hex HMAC-SHA256) is still written when no key pair is configured. Once a public key is configured an HMAC sidecar is **refused** unless `JE_AUTOCONTROL_ACCEPT_LEGACY_ACTION_SIGNATURES` is set. | +| USB passthrough payloads | optional `request_id` | **changed**, backward compatible | Every viewer request carries a `request_id` (1–64 characters) and the host echoes it in the reply, so a late reply can no longer be handed to the next request. `LIST` and `CLOSE` used to have an empty payload and now send `{"request_id": "..."}`. A host that does not echo ids still works; the viewer then treats a timeout the cautious way. | +| Self-healing evaluation dataset (`*.json`) | `schema_version` 1 | new, beta | Samples, version configs and thresholds for `evaluate_healing_dataset`. Another `schema_version` raises `HealingEvaluationError`. | +| GUI settings (`gui_settings.ini`) | unversioned | new, internal | Theme, text size, navigation panel and window geometry. Not an interface: an unreadable value falls back to its default. | + +## Surfaces added in the 2026-10 round + +Everything below is reachable from `import je_auto_control`. None of it is in +`je_auto_control.api` yet, so by the first paragraph of this document it is +**beta** unless marked otherwise: usable and tested, and free to change with +one release note. + +| Area | Names | Status | Verified by | +|---|---|---|---| +| Capability states | `probe_capabilities`, `BackendContext`, `Capability`, `CapabilitySnapshot`, `CapabilityStatus`, `reset_input_authorisation`, `close_input_session`, `WaylandAuthorisationError` | beta | headless tests on desktops described through `BackendContext`; the probe itself is never a test of the desktop | +| Recording without a hook | `InputStepLog`, `PhysicalRecorder`, `InputDevice`, `InputEvent`, `list_input_devices`, `StopShortcutSession` | beta; `PhysicalRecorder` and `StopShortcutSession` experimental | fakes; no CI job reads a real `/dev/input` device or a real GlobalShortcuts portal | +| Config sync | `config_sync_run` / `_status` / `_resolve` / `_full_resync`, `ConfigStore`, `SyncOutbox`, `SyncAdapter`, `SyncEntry`, `SyncOperation`, `merge_entries`, `sync_assets`, `AssetManifest` | beta | headless tests against a real SQLite store and the FastAPI app in-process | +| Mobile device sessions | `DeviceContext`, `DeviceSession`, `open_device`, `use_device`, `Tap` / `LongPress` / `Swipe` / `Drag` / `Pinch`, `DeviceFrame`, `device_setup_report`, `run_mobile_command`, `mobile_capability_matrix`, `MobileExtension` | experimental | fake `adb` host and fake WebDriverAgent only; **no real device** | +| Healing evaluation | `evaluate_locators`, `evaluate_healing_dataset`, `EvaluationSample`, `HealingComparison`, `template_match_strategy`, `propose_` / `preview_` / `accept_` / `revert_template_revision` | beta | `benchmarks/self_healing/run.py` on drawn frames; no VLM is called | +| Action journal and candidates | `start_action_journal`, `stop_action_journal`, `action_journal_status`, `read_events`, `list_journal_runs`, `generate_candidate_from_log`, `CandidateScript` | beta | headless tests; a candidate is validated by parsing and dry run, never executed | +| MCP tool modes | `ToolMode`, `ToolIndex`, `ToolSummary`, `ToolView`, the `ac_tools_*` tools, `--tool-mode` | beta | in-process server tests and `benchmarks/mcp_discovery.py` | +| Roles | `UserStore`, `AuthorizationContext`, `authorization_scope`, `rbac_add_user` / `_remove_user` / `_set_user_role` / `_rotate_user_token` / `_list_users`, `DeferredOwner`, `capture_owner`, `owner_scope` | beta | headless tests of the REST and MCP HTTP transports | +| Signing | `create_signing_keypair`, `SigningConfig`, `action_signing_config`, `AutoControlSignatureException`, `CryptographyUnavailableError` | beta | headless tests | + +"Experimental" here means exactly what it means at the top: it may change in +any release. The mobile row is experimental because nothing in this +repository has driven a real phone, not because the code is unfinished. + +## Compatibility notes for callers + +- `je_auto_control.execute_action(actions)` takes the action list only. + `dry_run`, `raise_on_error` and `step_callback` are parameters of + `je_auto_control.executor.execute_action`. +- A refused or revoked Wayland input consent raises + `WaylandAuthorisationError` and no longer falls back to `ydotool`. It is + deliberately not a `RuntimeError`. +- Every framework error still derives from `AutoControlException`; the new + ones keep a builtin base where callers may already catch one + (`DeviceError` is a `RuntimeError`, `DeviceTimeoutError` a `TimeoutError`, + `JournalFormatError` and `JournalImportError` a `ValueError`, + `AuthorizationError` a `PermissionError`). +- `python -m mypy` holds every module written since v0.0.225 to + `disallow_untyped_defs` and `disallow_any_generics`; see + `test/verify/typing_strict_baseline.txt` for the modules that predate that. diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index be434f06a..5bf8f3847 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -333,3 +333,94 @@ in that release. Debian trixie ships no `ydotool` package; bookworm and every current Ubuntu ship 0.1.8, which answers this argv with exit code 0 and no events. AutoControl refuses that version up front rather than reporting success for input it never sent. + +## Capability states the code reports + +The table at the top says what CI exercises. This section says what the +running program reports about itself, which is a different question and has a +machine-readable answer. + +`probe_capabilities()` diagnoses four desktop capabilities separately — +`input`, `capture`, `recording` and `stop_shortcut` — and gives each one of +these states: + +| State | Meaning | Usable now | +|---|---|---| +| `available` | Nothing stands in the way. | yes | +| `not_requested` | Usable; the desktop asks for consent on first use. | yes | +| `session_closed` | This process closed its session; the next use asks again. | yes | +| `requesting` | A consent request is on screen right now. | no | +| `needs_permission` | Someone has to say yes: a refused or unanswered consent, a device node this user may not read. | no | +| `needs_setup` | Something has to be installed or configured. | no | +| `revoked` | The compositor took a granted session away. | no | +| `compositor_restarted` | The grant came from a compositor that is no longer running. | no | +| `unsupported` | The platform cannot do this. | no | +| `unknown` | Not determinable without side effects. | no | + +A state is a diagnosis, not a test result. On Linux every state comes from +what the probe could read without side effects — the environment, `PATH`, the +loader and the authorisation ledger — so `available` there means "nothing was +found in the way", not "an event was delivered". On the other two desktops the +probe reads nothing at all, and says so: + + +| Platform | Capability | State | Backend | Evidence | +|---|---|---|---|---| +| `win32` | `input` | `available` | `win32` | reported, not probed: nothing to authorise | +| `win32` | `capture` | `available` | `win32` | reported, not probed: nothing to authorise | +| `win32` | `recording` | `available` | `win32` | reported, not probed: nothing to authorise | +| `win32` | `stop_shortcut` | `available` | `win32` | reported, not probed: nothing to authorise | +| `darwin` | `input` | `unknown` | `quartz` | not probed: permission for this platform is granted outside the process and is not probed here | +| `darwin` | `capture` | `unknown` | `quartz` | not probed: permission for this platform is granted outside the process and is not probed here | +| `darwin` | `recording` | `unknown` | `quartz` | not probed: permission for this platform is granted outside the process and is not probed here | +| `darwin` | `stop_shortcut` | `unknown` | `quartz` | not probed: permission for this platform is granted outside the process and is not probed here | + + +So a Windows or macOS row in a capability report is the backend's name and a +constant, never evidence that input works: on macOS the Accessibility and +Screen Recording permissions are granted in System Settings and are not +readable here. The `macos-capabilities` and Windows jobs in the table above +are what exercise those backends. + +## Mobile devices + +`mobile_capability_matrix()` is the source of the tables below: which +`AC_android_*` / `AC_ios_*` commands deliver each capability, and which +desktop features have no mobile counterpart. A session reports each capability +per device with `device_setup_report()`, in one of four states — `available`, +`needs_permission` (USB debugging not authorised), `needs_dependency` (for +example `uiautomator2` for multi-touch, or the ADB Keyboard IME for non-ASCII +text) and `unsupported` — together with the backend and the versions that +answered. + +Every command below is exercised in CI against a fake `adb` host and a fake +WebDriverAgent client. That proves which commands are sent. **No job drives a +real Android device or a real iPhone**, so nothing here is hardware-verified, +and the row in the table at the top stays `mocked CI`. + + +| Capability | Android commands | iOS commands | +|---|---|---| +| `input` | `AC_android_tap`, `AC_android_swipe`, `AC_android_key`, `AC_android_list_devices`, `AC_android_shell`, `AC_android_device_info`, `AC_android_long_press`, `AC_android_drag` | `AC_ios_tap`, `AC_ios_swipe`, `AC_ios_device_info`, `AC_ios_press_key`, `AC_ios_long_press`, `AC_ios_drag` | +| `unicode_text` | `AC_android_text`, `AC_android_type_text` | `AC_ios_type` | +| `multi_touch` | `AC_android_pinch` | `AC_ios_pinch` | +| `screenshot` | `AC_android_screenshot`, `AC_android_screen_info`, `AC_android_find_image`, `AC_android_find_text`, `AC_android_find_by_description`, `AC_android_self_heal` | `AC_ios_screenshot`, `AC_ios_screen_info`, `AC_ios_find_image`, `AC_ios_find_text`, `AC_ios_find_by_description`, `AC_ios_self_heal` | +| `ui_tree` | `AC_android_find_element`, `AC_android_click_element`, `AC_android_dump_hierarchy` | `AC_ios_find_element`, `AC_ios_click_element`, `AC_ios_dump_source` | +| `app_lifecycle` | `AC_android_launch_app`, `AC_android_stop_app`, `AC_android_app_state`, `AC_android_wait_for_app` | `AC_ios_launch_app`, `AC_ios_stop_app`, `AC_ios_app_state`, `AC_ios_wait_for_app` | +| `alerts` | `AC_android_alert_accept`, `AC_android_alert_dismiss` | `AC_ios_alert_accept`, `AC_ios_alert_dismiss` | +| `install` | `AC_android_install_app` | `AC_ios_install_app` | +| `files` | `AC_android_push_file`, `AC_android_pull_file` | `AC_ios_push_file`, `AC_ios_pull_file` | +| `clipboard` | `AC_android_get_clipboard`, `AC_android_set_clipboard` | `AC_ios_get_clipboard`, `AC_ios_set_clipboard` | +| `recording` | `AC_android_start_recording`, `AC_android_stop_recording` | `AC_ios_start_recording`, `AC_ios_stop_recording` | + +| Desktop-only feature | Why | Use instead | +|---|---|---| +| Window management (AC_window_*, focus, move, resize, z-order) | a phone shows one app at a time; there are no windows to manage | launch_app / stop_app / app_state to choose what is in front | +| Mouse buttons, wheel and hover (AC_click_mouse, AC_mouse_scroll) | touch screens have no pointer, buttons or wheel | Tap, LongPress, Swipe (to scroll), Drag and Pinch | +| Keyboard shortcuts and modifier keys (AC_hotkey, AC_press_keyboard_key) | there is no physical keyboard or modifier state | type_text for text; AC_android_key / AC_ios_press_key for hardware keys | +| Desktop accessibility tree (UIA, AX, AT-SPI) | those APIs describe the host's desktop, not the device | AC_android_find_element (uiautomator2) / AC_ios_find_element (XCUITest) | +| COM / Office automation | COM is a Windows host API | none on the device; drive the mobile app's UI instead | +| USB host passthrough and usbip | these share the host's USB devices; a phone is a USB device, not a host | none; adb and WebDriverAgent are the device transports | +| Global hotkeys, triggers on host input, input recording | they observe the host's keyboard and mouse, which the device does not use | record executed mobile steps in the action journal | +| Desktop screen capture and screen recording (AC_screenshot, AC_screen_record) | they capture the host's monitors | DeviceSession.capture() and the recording extension | + diff --git a/docs/source/Eng/doc/configuration/configuration_doc.rst b/docs/source/Eng/doc/configuration/configuration_doc.rst new file mode 100644 index 000000000..322fe5284 --- /dev/null +++ b/docs/source/Eng/doc/configuration/configuration_doc.rst @@ -0,0 +1,263 @@ +======================= +Configuration Reference +======================= + +Every environment variable AutoControl reads, in one place. Each row names the +page that explains the feature; this page only says what the variable is, what +it accepts and what happens when it is unset. + +Nothing here is required. With no variable set, AutoControl picks the platform +backend by itself, servers bind ``127.0.0.1``, and every opt-in feature +(signature enforcement, RBAC, USB passthrough, tool-path roots) is off. + +``test/unit_test/headless/test_modernization_examples.py`` compares this page +with the code: a variable the package reads and this page does not name fails +CI, in both languages. + +.. contents:: + :local: + :depth: 1 + +Platform backends +================= + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - Variable + - Values (default first) + - Effect + * - ``JE_AUTOCONTROL_WIN32_BACKEND`` + - ``sendinput`` / ``interception`` + - Windows keyboard and mouse backend. ``interception`` uses the + Interception driver and falls back to ``SendInput`` with a warning when + the driver or DLL is missing. + * - ``JE_AUTOCONTROL_INTERCEPTION_DLL`` + - unset / a path + - Full path of ``interception.dll``. Unset: looked up on ``PATH``, then + next to the package. + * - ``JE_AUTOCONTROL_INTERCEPTION_KEYBOARD`` + - ``1`` / ``1``–``10`` + - Interception device id keyboard events are sent to. + * - ``JE_AUTOCONTROL_INTERCEPTION_MOUSE`` + - ``11`` / ``11``–``20`` + - Interception device id mouse events are sent to. + * - ``JE_AUTOCONTROL_LINUX_BACKEND`` + - ``x11`` / ``uinput`` + - X11 input backend. ``uinput`` writes kernel events and falls back to + XTest with a warning when ``/dev/uinput`` is not writable. + * - ``JE_AUTOCONTROL_LINUX_DISPLAY_SERVER`` + - ``auto`` / ``wayland`` / ``x11`` + - Which Linux backend loads. ``auto`` reads ``XDG_SESSION_TYPE`` and + ``WAYLAND_DISPLAY``; ``x11`` on a Wayland session drives XWayland + windows only. + +Wayland +======= + +See :doc:`../wayland/wayland_capabilities_doc`. ``probe_capabilities()`` shows +the effect of each of these without side effects. + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - Variable + - Values (default first) + - Effect + * - ``JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND`` + - ``auto`` / ``cli`` + - ``cli`` sends input through ``ydotool`` and never asks the desktop + portal. It is a choice made before starting: after a refused consent + AutoControl does not switch to it by itself. + * - ``JE_AUTOCONTROL_WAYLAND_EI_WORKER`` + - unset / ``1`` + - Runs the libei session in a helper process instead of in-process. + * - ``JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL`` + - ``warn`` / ``flat`` / ``strict`` + - Applies to absolute moves on the ``ydotool`` path only, which are + relative motion the compositor accelerates. ``warn`` (also when unset) + warns once and moves; ``flat`` declares acceleration off and moves + silently; ``strict`` refuses the move. + * - ``JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND`` + - unset / a command line + - Your own screenshot command, with ``{output}`` where the PNG path goes. + Takes precedence over ``grim``, ``gnome-screenshot``, ``spectacle`` and + the portal. + * - ``JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES`` + - unset / comma-separated paths + - The ``/dev/input/event*`` devices ``PhysicalRecorder`` may read. Unset + means none: physical recording is opt-in per device. + +Logging and local state +======================= + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - Variable + - Values (default first) + - Effect + * - ``JE_AUTOCONTROL_LOG_FILE`` + - unset / a path + - Where the log file is written. Unset: + ``~/.je_auto_control/logs/AutoControlGUI.log``. The null device + (``/dev/null``, ``NUL``) switches the file off. Read when the first + record is written, not at import. + * - ``JE_AUTOCONTROL_GUI_SETTINGS`` + - unset / a path / ``off`` + - File the main window keeps its theme, text size, navigation panel and + geometry in. Unset: ``~/.je_auto_control/gui_settings.ini``. ``off``, + ``0``, ``none``, ``false`` or empty: nothing is read or written. + * - ``JE_AUTOCONTROL_ENV`` + - ``default`` / a name + - The active environment of the asset store (``active_environment()``), + so one script reads different values in ``dev`` and ``prod``. + * - ``JE_AUTOCONTROL_REDACTION`` + - ``off`` / ``moderate`` / ``strict`` + - Default screenshot redaction policy. An unknown name is an error, not + ``off``. + * - ``JE_AUTOCONTROL_REMOTE_DOWNLOAD_DIR`` + - unset / a directory + - Where a remote-desktop viewer stores files its host sends. Unset: + ``~/Downloads/AutoControl``. A received path is confined to it. + * - ``JE_AUTOCONTROL_PYTEST_ARTIFACTS`` + - unset / a directory + - Where the pytest plugin writes failure screenshots when a test did not + request the ``autocontrol_screenshot_dir`` fixture. Unset: + ``./autocontrol_screenshots``. + +Executing and signing action files +================================== + +See :doc:`../keyword_and_executor/keyword_and_executor_doc` and +:doc:`../new_features/v4_features_doc`. + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - Variable + - Values (default first) + - Effect + * - ``JE_AUTOCONTROL_ALLOWED_PACKAGES`` + - unset / comma-separated names + - Packages ``AC_add_package_to_executor`` may load (submodules included), + for every entry point. Read once, when the process starts. + * - ``JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS`` + - unset / ``1`` + - Every path that runs an action file refuses one without a valid + signature sidecar. + * - ``JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY`` + - unset / a path + - Ed25519 private key (PEM). Set it on the signing machine only; signing + then writes a version-2 sidecar. + * - ``JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY`` + - unset / a path + - The matching public key. Set it on every machine that executes: it + verifies and cannot sign. + * - ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE`` + - unset / a passphrase + - Passphrase of the private key, when it was created with one. + * - ``JE_AUTOCONTROL_ACCEPT_LEGACY_ACTION_SIGNATURES`` + - unset / ``1`` + - Migration mode: once a public key is configured, HMAC sidecars written + before version 2 are refused unless this is set. Switch it off again + when every file has been re-signed. + +MCP server +========== + +See :doc:`../mcp_server/mcp_server_doc`. + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - Variable + - Values (default first) + - Effect + * - ``JE_AUTOCONTROL_MCP_READONLY`` + - unset / ``1`` + - Only tools marked read-only are offered and callable. + * - ``JE_AUTOCONTROL_MCP_TOOL_MODE`` + - ``full`` / ``progressive`` / ``static`` + - How much of the registry ``tools/list`` offers. An unknown value is an + error rather than ``full``. + * - ``JE_AUTOCONTROL_MCP_TOOL_PROFILE`` + - unset / comma-separated entries + - Tool names and ``category:`` entries of the ``static`` profile. + * - ``JE_AUTOCONTROL_MCP_ALIASES`` + - ``1`` / ``0`` + - Whether the short aliases (``click``, ``screenshot``, …) are registered + beside the ``ac_*`` tools. + * - ``JE_AUTOCONTROL_MCP_TOKEN`` + - unset / a token + - Bearer token of the HTTP transport. Not accepted once RBAC is on. + * - ``JE_AUTOCONTROL_MCP_ALLOWED_ORIGINS`` + - unset / comma-separated origins + - Extra browser origins (exact, e.g. ``https://example.test:8443``) the + HTTP transport accepts. Loopback origins always are. + * - ``JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE`` + - unset / ``1`` + - Destructive tools ask the client for confirmation (MCP elicitation) + before they run. + * - ``JE_AUTOCONTROL_MCP_PATH_ROOTS`` + - unset / directories + - Directories (separated by ``os.pathsep``) every file argument of a tool + must stay inside. Unset: no confinement. + * - ``JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT`` + - unset / ``1`` + - Also accept the roots the MCP client reports through ``roots/list``. + * - ``JE_AUTOCONTROL_MCP_ENV_REF_ALLOW`` + - unset / comma-separated names + - Environment variables ``ac_resolve_ref`` may read (``fnmatch`` patterns + allowed). Unset: no restriction; a value that names nothing allows + none. + * - ``JE_AUTOCONTROL_MCP_AUDIT`` + - unset / a path + - JSON-lines file that receives one record per ``tools/call``. + * - ``JE_AUTOCONTROL_MCP_ERROR_SHOTS`` + - unset / a directory + - A screenshot is saved there each time a tool fails. + * - ``JE_AUTOCONTROL_FAKE_BACKEND`` + - unset / ``1`` + - The MCP server records mouse, keyboard and clipboard calls in memory + instead of performing them, for CI without a display. + +Access control and servers +========================== + +See :doc:`../operations_layer/operations_layer_doc` and +:doc:`../config_sync/config_sync_doc`. + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - Variable + - Values (default first) + - Effect + * - ``JE_AUTOCONTROL_RBAC_USERS`` + - unset / a path + - The user store file. Setting it is what switches roles on for the REST + API and the MCP HTTP transport; unset leaves both on their shared + token. + * - ``JE_AUTOCONTROL_USB_PASSTHROUGH`` + - unset / ``1`` + - Enables USB passthrough opcodes on the remote-desktop channel. + * - ``JE_AUTOCONTROL_CHATOPS_SCRIPT_ROOT`` + - unset / a directory + - The only directory the chat-ops ``run`` command may load action files + from. Unset: the command is refused. + * - ``AC_SIGNALING_SECRET`` + - unset / a secret + - Shared secret of the signaling / config-sync server + (``X-Signaling-Secret``). Read by the server when ``--shared-secret`` + is not given, and by ``config_sync_run`` when ``secret`` is not. + * - ``AC_SIGNALING_CONFIG_DB`` + - unset / a path + - SQLite file the signaling server keeps config-sync buckets in. Unset: + ``~/.je_auto_control/config_sync.sqlite3``. diff --git a/docs/source/Eng/doc/examples/examples_doc.rst b/docs/source/Eng/doc/examples/examples_doc.rst new file mode 100644 index 000000000..d58a65c12 --- /dev/null +++ b/docs/source/Eng/doc/examples/examples_doc.rst @@ -0,0 +1,170 @@ +================= +Runnable Examples +================= + +The ``examples/`` folder of the repository holds one short script per feature. +The six below cover the features added most recently, and each of them accepts +``--validate``: + +.. code-block:: bash + + pip install -e . + python examples/29_config_sync.py --validate + +With ``--validate`` a script exercises the real API against an in-memory, temp +directory or loopback stand-in and exits ``0`` when what it observed is what +the documentation promises. It never moves the pointer, types, captures the +screen, reaches a device, or talks to anything beyond ``127.0.0.1``. CI runs +all six this way (``test_modernization_examples.py``), with every input and +process seam replaced by a tripwire. + +That is evidence the calls are wired the way these pages say. It is not +evidence about hardware: nothing here has touched a Wayland compositor, an +Android device or an iPhone. Where a real run needs something, the script's +own docstring names it. + +.. list-table:: + :header-rows: 1 + :widths: 30 40 30 + + * - Script + - What ``--validate`` does + - Without the flag + * - ``28_wayland_diagnostics.py`` + - Probes four described desktops (GNOME before and after a refused + consent, sway with ``ydotool``, X11 on a Wayland session) and journals + a dry run with ``InputStepLog``. + - Probes this session. Also free of side effects. + * - ``29_config_sync.py`` + - Two machines sync script files through a store in a temp directory: + arrival, server restart, offline queue, a conflict and its resolution. + - Prints the recorded status; syncs this machine only with ``--run``. + * - ``30_mobile_devices.py`` + - Opens an Android session over an offline transport and checks which + ``adb`` commands a setup report, a capture, gestures and typing send. + - Prints a device's setup report; taps only with ``--tap X Y``. + * - ``31_healing_comparison.py`` + - Scores two template-matching versions on five labelled frames drawn in + memory (HiDPI, negative origin, absent, restyled). + - ``--dataset FILE`` evaluates a dataset file. Offline as well. + * - ``32_codegen_from_log.py`` + - Journals a run of variable and flow-control commands, rebuilds it as a + candidate script and dry-runs the result. + - ``--journal FILE`` converts a journal you recorded. + * - ``33_mcp_progressive.py`` + - Drives an in-process MCP server: lists, searches, reads a schema and + enables a tool. Calls no tool that touches the desktop. + - Serves MCP over stdio in progressive mode. + +Where each feature is documented +================================ + +.. list-table:: + :header-rows: 1 + :widths: 24 38 38 + + * - Feature + - Guide + - Entry points + * - Capability and authorisation states + - :doc:`../wayland/wayland_capabilities_doc` + - ``probe_capabilities``, ``AC_probe_capabilities``, + ``ac_probe_capabilities``, the Diagnostics tab + * - Recording without a global hook + - :doc:`../record/record_doc` + - ``InputStepLog``, ``PhysicalRecorder``, ``StopShortcutSession`` + * - Config sync + - :doc:`../config_sync/config_sync_doc` + - ``config_sync_run`` / ``_status`` / ``_resolve`` / ``_full_resync``, + ``AC_config_sync_*``, ``ac_config_sync_*``, the Config Sync tab + * - Mobile device sessions + - :doc:`../mobile/mobile_doc` + - ``open_device``, ``use_device``, ``run_mobile_command``, + ``AC_android_*`` / ``AC_ios_*``, the Mobile tab + * - Healing evaluation and template revisions + - :doc:`../new_features/v2_features_doc` + - ``evaluate_locators``, ``evaluate_healing_dataset``, + ``AC_self_heal_evaluate``, ``propose_template_revision`` + * - Action journal and candidate scripts + - :doc:`../new_features/v5_features_doc` + - ``start_action_journal``, ``generate_candidate_from_log``, + ``AC_journal_*``, ``je_auto_control codegen --from-log`` + * - MCP tool modes + - :doc:`../mcp_server/mcp_server_doc` + - ``--tool-mode``, ``JE_AUTOCONTROL_MCP_TOOL_MODE``, ``ac_tools_*`` + * - Roles and deferred ownership + - :doc:`../operations_layer/operations_layer_doc` + - ``JE_AUTOCONTROL_RBAC_USERS``, ``rbac_add_user``, ``AC_user_*``, + ``je_auto_control users`` + * - Signed action files (Ed25519) + - :doc:`../new_features/v4_features_doc` + - ``create_signing_keypair``, ``sign_action_file``, + ``JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS`` + * - Window shell, navigation, background work + - :doc:`../new_features/v223_features_doc` + - the Actions and View menus, ``JE_AUTOCONTROL_GUI_SETTINGS`` + * - Every environment variable + - :doc:`../configuration/configuration_doc` + - + +Three things the examples show that are easy to miss +==================================================== + +**The executor's options are on the executor object.** The module-level +``je_auto_control.execute_action(actions)`` takes the action list and nothing +else. ``dry_run``, ``raise_on_error`` and ``step_callback`` belong to +``je_auto_control.executor.execute_action``: + +.. code-block:: python + + import je_auto_control as ac + + ac.executor.execute_action(actions, dry_run=True) # resolve, call nothing + ac.executor.execute_action(actions, raise_on_error=True) # stop at the first failure + +**A failed sync backs off.** After a send that did not reach the server, the +next ``config_sync_run`` inside the back-off window (2 s, doubling, capped at +5 minutes) does not contact the server: it reports ``offline`` again with +``error`` set to ``waiting to retry after an earlier failure``. Pass +``wait=True`` to sleep through the back-off instead — that is what a scheduled +sync wants, and a ``cancel`` event ends the wait. + +**Searching a tool does not enable it.** In progressive mode +``ac_tools_search`` and ``ac_tools_schema`` only describe; the session's +``tools/list`` changes when ``ac_tools_enable`` is called, and the client is +told with ``notifications/tools/list_changed``. + +The validation path of each script +================================== + +These are included from the scripts themselves, so they are what CI runs. + +.. literalinclude:: ../../../../../examples/28_wayland_diagnostics.py + :language: python + :pyobject: validate + :caption: examples/28_wayland_diagnostics.py + +.. literalinclude:: ../../../../../examples/29_config_sync.py + :language: python + :pyobject: _walkthrough + :caption: examples/29_config_sync.py + +.. literalinclude:: ../../../../../examples/30_mobile_devices.py + :language: python + :pyobject: validate + :caption: examples/30_mobile_devices.py + +.. literalinclude:: ../../../../../examples/31_healing_comparison.py + :language: python + :pyobject: validate + :caption: examples/31_healing_comparison.py + +.. literalinclude:: ../../../../../examples/32_codegen_from_log.py + :language: python + :pyobject: validate + :caption: examples/32_codegen_from_log.py + +.. literalinclude:: ../../../../../examples/33_mcp_progressive.py + :language: python + :pyobject: validate + :caption: examples/33_mcp_progressive.py diff --git a/docs/source/Eng/doc/new_features/v223_features_doc.rst b/docs/source/Eng/doc/new_features/v223_features_doc.rst index 7751a50f4..03a934dd6 100644 --- a/docs/source/Eng/doc/new_features/v223_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v223_features_doc.rst @@ -117,3 +117,43 @@ process's memory. ``gui_workloads.py`` opens every tab once switches theme while a 5 ms timer ticks; ``event_loop_p95_ms`` is how late those ticks arrive. ``--compare`` sets two reports side by side and refuses when the workload or the environment differs. + +Background work in a tab +------------------------ + +A tab that waits on a network peer or a device starts that work through +``je_auto_control.gui.task_controller`` instead of a bare thread: + +.. code-block:: python + + from je_auto_control.gui.task_controller import CancellationToken, task_controller + + def fetch(url: str, token: CancellationToken) -> dict: + token.raise_if_cancelled() + return download(url, timeout=token.remaining(default=10.0)) + + url = self._url_edit.text() # read the widget first + handle = task_controller().submit( + functools.partial(fetch, url), owner=self, timeout_s=30.0) + handle.result.connect(self._show) # emitted on the GUI thread + handle.error.connect(self._show_error) # the exception object + handle.finished.connect(self._enable_button) # once, whatever happened + +What it adds to a plain worker: + +* **Cancellation reaches the backend.** The work receives a + ``CancellationToken``: it polls ``raise_if_cancelled()`` / ``wait()``, hands + ``remaining()`` to a backend that takes a timeout, and registers how to let + go of what it holds with ``on_cancel()``. ``handle.cancel()`` runs those at + once, on the cancelling thread. +* **Typed outcomes.** ``result`` carries the return value, ``error`` the + exception, ``progress`` whatever ``token.report_progress()`` was given. A + timeout is reported as ``TaskTimeout``, a cancellation as ``TaskCancelled``. +* **Owner death.** The handle is a child of ``owner``. Destroying the tab + cancels the work, and a result that arrives afterwards goes to ``discard=`` + (close the session nobody will use), never to a slot of the dead widget. +* **No worker touches a widget.** Work that is a widget's method, or closes + over one, is refused with ``TaskUsageError`` before it starts. + +This is a GUI-internal API (it imports ``PySide6``) and is not re-exported by +the package facade; the headless functions a tab calls stay usable without it. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 91b261978..5a6c855f3 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -9,6 +9,7 @@ Comprehensive guides for all AutoControl features. :caption: User Guide doc/installation/installation_doc + doc/configuration/configuration_doc doc/mouse/mouse_doc doc/keyboard/keyboard_doc doc/screen/screen_doc @@ -25,6 +26,7 @@ Comprehensive guides for all AutoControl features. doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/config_sync/config_sync_doc + doc/examples/examples_doc doc/create_project/create_project_doc doc/new_features/new_features_doc doc/new_features/v2_features_doc diff --git a/docs/source/Zh/doc/config_sync/config_sync_doc.rst b/docs/source/Zh/doc/config_sync/config_sync_doc.rst index 7e882b1b6..9c357ea7e 100644 --- a/docs/source/Zh/doc/config_sync/config_sync_doc.rst +++ b/docs/source/Zh/doc/config_sync/config_sync_doc.rst @@ -15,7 +15,7 @@ server 重啟後仍在。檔案位置依序取自 ``--config-db PATH``、環境 在第一個 ``/config`` 請求時才開啟,不是啟動時。內嵌 app 時傳 ``create_app(config_store_path=...)``。 -每個 bucket 有 server 指派的 **revision**(第一次寫入為 ``1``,之後每次 commit ``+1``)。 +每個 bucket 有 server 指派的 **revision**\ (第一次寫入為 ``1``,之後每次 commit ``+1``)。 寫入必須指明它所依據的 revision,比較與寫入在同一個 SQLite transaction 內完成: .. code-block:: python @@ -60,8 +60,8 @@ entry id 對應到物件的 mapping。 會以 ``ConfigSyncError: config sync PUT returned HTTP 428`` 失敗且不寫入任何東西; ``fetch`` 照常運作(回覆欄位相同,多一個它們會忽略的 ``version``)。要繼續服務它們, 以 ``--allow-blind-config-writes`` 啟動 server,並接受兩個這種 client 同時 push 仍可能互相覆寫。 -**version-2 client 面對舊 server** 會被明確告知:回覆沒有 revision,``push`` 丟出 -``ConfigSyncError``(「predates revision-checked writes」)—— 請先升級 server。 +**version-2 client 面對舊 server** 會被明確告知:回覆沒有 revision, ``push`` 丟出 +``ConfigSyncError``\ (「predates revision-checked writes」)—— 請先升級 server。 Client ------ @@ -229,7 +229,7 @@ Adapter:同步什麼、什麼留在本機 - ``ac_config_sync_run`` - 同步一次 * - ``AC_config_sync_status`` - - ``ac_config_sync_status``(唯讀) + - ``ac_config_sync_status``\ (唯讀) - 已記錄的狀態;不連網、不建立任何東西 * - ``AC_config_sync_resolve`` - ``ac_config_sync_resolve`` diff --git a/docs/source/Zh/doc/configuration/configuration_doc.rst b/docs/source/Zh/doc/configuration/configuration_doc.rst new file mode 100644 index 000000000..386ad0bbb --- /dev/null +++ b/docs/source/Zh/doc/configuration/configuration_doc.rst @@ -0,0 +1,247 @@ +============ +設定參考 +============ + +AutoControl 讀取的每一個環境變數都列在這裡。每一列會指向說明該功能的頁面； +本頁只說明變數是什麼、接受哪些值，以及未設定時的行為。 + +這裡沒有任何一項是必填的。完全不設定時，AutoControl 會自行選擇平台後端， +伺服器只綁定 ``127.0.0.1``，所有需要主動開啟的功能（簽章強制、RBAC、USB +直通、工具路徑限制）都是關閉的。 + +``test/unit_test/headless/test_modernization_examples.py`` 會拿本頁和程式碼 +比對：套件讀取但本頁沒有列出的變數，中英兩個版本都會讓 CI 失敗。 + +.. contents:: + :local: + :depth: 1 + +平台後端 +======== + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - 變數 + - 值（預設在前） + - 作用 + * - ``JE_AUTOCONTROL_WIN32_BACKEND`` + - ``sendinput`` / ``interception`` + - Windows 的鍵盤與滑鼠後端。``interception`` 使用 Interception 驅動程式； + 找不到驅動或 DLL 時會警告並退回 ``SendInput``。 + * - ``JE_AUTOCONTROL_INTERCEPTION_DLL`` + - 未設定 / 路徑 + - ``interception.dll`` 的完整路徑。未設定時先找 ``PATH``，再找套件旁邊。 + * - ``JE_AUTOCONTROL_INTERCEPTION_KEYBOARD`` + - ``1`` / ``1``–``10`` + - 鍵盤事件要送往的 Interception 裝置編號。 + * - ``JE_AUTOCONTROL_INTERCEPTION_MOUSE`` + - ``11`` / ``11``–``20`` + - 滑鼠事件要送往的 Interception 裝置編號。 + * - ``JE_AUTOCONTROL_LINUX_BACKEND`` + - ``x11`` / ``uinput`` + - X11 的輸入後端。``uinput`` 直接寫入核心事件；``/dev/uinput`` 無法寫入時 + 會警告並退回 XTest。 + * - ``JE_AUTOCONTROL_LINUX_DISPLAY_SERVER`` + - ``auto`` / ``wayland`` / ``x11`` + - 載入哪一個 Linux 後端。``auto`` 讀取 ``XDG_SESSION_TYPE`` 與 + ``WAYLAND_DISPLAY``；在 Wayland 工作階段設成 ``x11`` 只能操作 XWayland + 視窗。 + +Wayland +======= + +請參考 :doc:`../wayland/wayland_capabilities_doc`。``probe_capabilities()`` +可以在沒有任何副作用的情況下看到下列每一項的效果。 + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - 變數 + - 值（預設在前） + - 作用 + * - ``JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND`` + - ``auto`` / ``cli`` + - ``cli`` 透過 ``ydotool`` 送出輸入，完全不向桌面 portal 提出要求。這是 + 啟動前就做好的選擇：使用者拒絕授權後，AutoControl 不會自行切換過去。 + * - ``JE_AUTOCONTROL_WAYLAND_EI_WORKER`` + - 未設定 / ``1`` + - 把 libei 工作階段放到輔助行程裡執行，而不是放在本行程。 + * - ``JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL`` + - ``warn`` / ``flat`` / ``strict`` + - 只影響 ``ydotool`` 路徑的絕對移動（那其實是會被合成器加速的相對位移）。 + ``warn``（未設定時亦同）警告一次後照樣移動；``flat`` 表示加速已關閉， + 不再警告；``strict`` 直接拒絕移動。 + * - ``JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND`` + - 未設定 / 指令列 + - 自訂的截圖指令，以 ``{output}`` 表示 PNG 的輸出路徑。優先於 ``grim``、 + ``gnome-screenshot``、``spectacle`` 與 portal。 + * - ``JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES`` + - 未設定 / 以逗號分隔的路徑 + - ``PhysicalRecorder`` 可以讀取的 ``/dev/input/event*`` 裝置。未設定代表 + 一個都不讀：實體輸入錄製必須逐一指定裝置才會啟用。 + +記錄檔與本機狀態 +================ + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - 變數 + - 值（預設在前） + - 作用 + * - ``JE_AUTOCONTROL_LOG_FILE`` + - 未設定 / 路徑 + - 記錄檔寫入的位置。未設定時為 + ``~/.je_auto_control/logs/AutoControlGUI.log``。指定空裝置 + （``/dev/null``、``NUL``）即可關閉檔案輸出。在寫入第一筆記錄時才讀取， + 而不是在 import 時。 + * - ``JE_AUTOCONTROL_GUI_SETTINGS`` + - 未設定 / 路徑 / ``off`` + - 主視窗保存佈景主題、文字大小、導覽面板與視窗位置的檔案。未設定時為 + ``~/.je_auto_control/gui_settings.ini``。設成 ``off``、``0``、``none``、 + ``false`` 或空字串時，不讀也不寫。 + * - ``JE_AUTOCONTROL_ENV`` + - ``default`` / 名稱 + - 資產庫目前使用的環境（``active_environment()``），讓同一份腳本在 + ``dev`` 與 ``prod`` 讀到不同的值。 + * - ``JE_AUTOCONTROL_REDACTION`` + - ``off`` / ``moderate`` / ``strict`` + - 截圖遮蔽的預設政策。未知的名稱會報錯，不會被當成 ``off``。 + * - ``JE_AUTOCONTROL_REMOTE_DOWNLOAD_DIR`` + - 未設定 / 目錄 + - 遠端桌面檢視端存放主機傳來檔案的位置。未設定時為 + ``~/Downloads/AutoControl``。收到的路徑一律限制在這個目錄內。 + * - ``JE_AUTOCONTROL_PYTEST_ARTIFACTS`` + - 未設定 / 目錄 + - 測試沒有使用 ``autocontrol_screenshot_dir`` fixture 時，pytest 外掛寫入 + 失敗截圖的位置。未設定時為 ``./autocontrol_screenshots``。 + +執行與簽署動作檔 +================ + +請參考 :doc:`../keyword_and_executor/keyword_and_executor_doc` 與 +:doc:`../new_features/v4_features_doc`。 + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - 變數 + - 值（預設在前） + - 作用 + * - ``JE_AUTOCONTROL_ALLOWED_PACKAGES`` + - 未設定 / 以逗號分隔的名稱 + - ``AC_add_package_to_executor`` 可以載入的套件（含子模組），對所有進入點 + 生效。只在行程啟動時讀取一次。 + * - ``JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS`` + - 未設定 / ``1`` + - 所有會執行動作檔的路徑，都拒絕沒有有效簽章檔的動作檔。 + * - ``JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY`` + - 未設定 / 路徑 + - Ed25519 私鑰（PEM）。只在負責簽署的機器上設定；設定後簽署會寫出第 2 版 + 的簽章檔。 + * - ``JE_AUTOCONTROL_ACTION_SIGNING_PUBLIC_KEY`` + - 未設定 / 路徑 + - 對應的公鑰。在每一台執行端設定：它只能驗證，不能簽署。 + * - ``JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE`` + - 未設定 / 通行碼 + - 私鑰建立時若有設定通行碼，在這裡提供。 + * - ``JE_AUTOCONTROL_ACCEPT_LEGACY_ACTION_SIGNATURES`` + - 未設定 / ``1`` + - 遷移模式：設定公鑰之後，第 2 版之前寫出的 HMAC 簽章檔會被拒絕，除非 + 設定了這個變數。所有檔案重新簽署後請再關掉。 + +MCP 伺服器 +========== + +請參考 :doc:`../mcp_server/mcp_server_doc`。 + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - 變數 + - 值（預設在前） + - 作用 + * - ``JE_AUTOCONTROL_MCP_READONLY`` + - 未設定 / ``1`` + - 只提供、也只能呼叫標示為唯讀的工具。 + * - ``JE_AUTOCONTROL_MCP_TOOL_MODE`` + - ``full`` / ``progressive`` / ``static`` + - ``tools/list`` 要提供登錄表的多少內容。未知的值會報錯，不會被當成 + ``full``。 + * - ``JE_AUTOCONTROL_MCP_TOOL_PROFILE`` + - 未設定 / 以逗號分隔的項目 + - ``static`` 模式的工具清單：工具名稱與 ``category:<名稱>``。 + * - ``JE_AUTOCONTROL_MCP_ALIASES`` + - ``1`` / ``0`` + - 是否在 ``ac_*`` 工具之外另外註冊簡短別名（``click``、``screenshot`` …）。 + * - ``JE_AUTOCONTROL_MCP_TOKEN`` + - 未設定 / 權杖 + - HTTP 傳輸的 Bearer 權杖。啟用 RBAC 之後不再接受。 + * - ``JE_AUTOCONTROL_MCP_ALLOWED_ORIGINS`` + - 未設定 / 以逗號分隔的來源 + - HTTP 傳輸額外接受的瀏覽器來源（須完全相符，例如 + ``https://example.test:8443``）。本機來源永遠接受。 + * - ``JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE`` + - 未設定 / ``1`` + - 具破壞性的工具在執行前先向用戶端要求確認（MCP elicitation）。 + * - ``JE_AUTOCONTROL_MCP_PATH_ROOTS`` + - 未設定 / 目錄 + - 工具的每一個檔案參數都必須落在這些目錄內（以 ``os.pathsep`` 分隔）。 + 未設定時不限制。 + * - ``JE_AUTOCONTROL_MCP_PATH_ROOTS_FROM_CLIENT`` + - 未設定 / ``1`` + - 同時接受 MCP 用戶端透過 ``roots/list`` 回報的根目錄。 + * - ``JE_AUTOCONTROL_MCP_ENV_REF_ALLOW`` + - 未設定 / 以逗號分隔的名稱 + - ``ac_resolve_ref`` 可以讀取的環境變數（可用 ``fnmatch`` 樣式）。未設定時 + 不限制；設了卻沒有指名任何變數時，一個都不允許。 + * - ``JE_AUTOCONTROL_MCP_AUDIT`` + - 未設定 / 路徑 + - 每一次 ``tools/call`` 寫入一筆記錄的 JSON-lines 檔案。 + * - ``JE_AUTOCONTROL_MCP_ERROR_SHOTS`` + - 未設定 / 目錄 + - 工具每次失敗時，在這裡存一張截圖。 + * - ``JE_AUTOCONTROL_FAKE_BACKEND`` + - 未設定 / ``1`` + - MCP 伺服器只在記憶體中記錄滑鼠、鍵盤與剪貼簿呼叫而不實際執行，供沒有 + 顯示器的 CI 使用。 + +存取控制與伺服器 +================ + +請參考 :doc:`../operations_layer/operations_layer_doc` 與 +:doc:`../config_sync/config_sync_doc`。 + +.. list-table:: + :header-rows: 1 + :widths: 34 20 46 + + * - 變數 + - 值（預設在前） + - 作用 + * - ``JE_AUTOCONTROL_RBAC_USERS`` + - 未設定 / 路徑 + - 使用者檔案。設定它就等於為 REST API 與 MCP HTTP 傳輸啟用角色；未設定時 + 兩者都沿用共用權杖。 + * - ``JE_AUTOCONTROL_USB_PASSTHROUGH`` + - 未設定 / ``1`` + - 在遠端桌面通道上啟用 USB 直通指令。 + * - ``JE_AUTOCONTROL_CHATOPS_SCRIPT_ROOT`` + - 未設定 / 目錄 + - chat-ops 的 ``run`` 指令唯一可以載入動作檔的目錄。未設定時該指令會被 + 拒絕。 + * - ``AC_SIGNALING_SECRET`` + - 未設定 / 密鑰 + - 信令／設定同步伺服器的共用密鑰（``X-Signaling-Secret``）。伺服器在沒有 + ``--shared-secret`` 時讀取它，``config_sync_run`` 在沒有 ``secret`` + 時也讀取它。 + * - ``AC_SIGNALING_CONFIG_DB`` + - 未設定 / 路徑 + - 信令伺服器存放設定同步資料的 SQLite 檔案。未設定時為 + ``~/.je_auto_control/config_sync.sqlite3``。 diff --git a/docs/source/Zh/doc/examples/examples_doc.rst b/docs/source/Zh/doc/examples/examples_doc.rst new file mode 100644 index 000000000..2d2d675ef --- /dev/null +++ b/docs/source/Zh/doc/examples/examples_doc.rst @@ -0,0 +1,164 @@ +============ +可執行範例 +============ + +儲存庫的 ``examples/`` 資料夾裡，每個功能都有一支簡短的腳本。下列六支涵蓋最近 +新增的功能，而且每一支都接受 ``--validate``： + +.. code-block:: bash + + pip install -e . + python examples/29_config_sync.py --validate + +加上 ``--validate`` 時，腳本會以記憶體、暫存目錄或 loopback 的替身來執行真正的 +API，觀察到的結果與文件所述相符時以 ``0`` 結束。它不會移動游標、不會輸入文字、 +不會擷取畫面、不會連到任何裝置，也不會與 ``127.0.0.1`` 以外的對象通訊。CI 就是 +這樣執行這六支腳本的（``test_modernization_examples.py``），而且每一個輸入與 +行程的接點都換成了絆線。 + +這能證明呼叫的接法與這些頁面所寫的一致，但不能證明硬體行為：這裡沒有任何東西 +碰過 Wayland 合成器、Android 裝置或 iPhone。真正執行時需要什麼，各腳本自己的 +docstring 會寫明。 + +.. list-table:: + :header-rows: 1 + :widths: 30 40 30 + + * - 腳本 + - ``--validate`` 做什麼 + - 不加旗標時 + * - ``28_wayland_diagnostics.py`` + - 探測四個以描述建構的桌面（拒絕授權前後的 GNOME、搭配 ``ydotool`` 的 + sway、Wayland 工作階段上的 X11），並用 ``InputStepLog`` 記錄一次 dry run。 + - 探測目前的工作階段，同樣沒有副作用。 + * - ``29_config_sync.py`` + - 兩台機器透過暫存目錄裡的儲存庫同步腳本檔：送達、伺服器重啟、離線佇列、 + 衝突與衝突的解決。 + - 印出已記錄的狀態；加上 ``--run`` 才會真的同步這台機器。 + * - ``30_mobile_devices.py`` + - 以離線傳輸開啟 Android 工作階段，檢查設定報告、擷取、手勢與輸入各送出 + 哪些 ``adb`` 指令。 + - 印出裝置的設定報告；加上 ``--tap X Y`` 才會點擊。 + * - ``31_healing_comparison.py`` + - 以記憶體中畫出的五張已標註畫面（HiDPI、負座標原點、目標不存在、改版後） + 為兩個版本的樣板比對評分。 + - ``--dataset FILE`` 評估資料集檔案，同樣是離線的。 + * - ``32_codegen_from_log.py`` + - 把一次只含變數與流程控制指令的執行寫入日誌，重建成候選腳本，再 dry run + 一次。 + - ``--journal FILE`` 轉換你自己錄下的日誌。 + * - ``33_mcp_progressive.py`` + - 驅動行程內的 MCP 伺服器：列出、搜尋、讀取 schema、啟用一個工具。不會呼叫 + 任何會碰到桌面的工具。 + - 以 progressive 模式透過 stdio 提供 MCP 服務。 + +各功能的文件位置 +================ + +.. list-table:: + :header-rows: 1 + :widths: 24 38 38 + + * - 功能 + - 說明頁 + - 進入點 + * - 能力與授權狀態 + - :doc:`../wayland/wayland_capabilities_doc` + - ``probe_capabilities``、``AC_probe_capabilities``、 + ``ac_probe_capabilities``、Diagnostics 分頁 + * - 不靠全域掛鉤的錄製 + - :doc:`../record/record_doc` + - ``InputStepLog``、``PhysicalRecorder``、``StopShortcutSession`` + * - 設定同步 + - :doc:`../config_sync/config_sync_doc` + - ``config_sync_run`` / ``_status`` / ``_resolve`` / ``_full_resync``、 + ``AC_config_sync_*``、``ac_config_sync_*``、Config Sync 分頁 + * - 行動裝置工作階段 + - :doc:`../mobile/mobile_doc` + - ``open_device``、``use_device``、``run_mobile_command``、 + ``AC_android_*`` / ``AC_ios_*``、Mobile 分頁 + * - 自我修復評估與樣板修訂 + - :doc:`../new_features/v2_features_doc` + - ``evaluate_locators``、``evaluate_healing_dataset``、 + ``AC_self_heal_evaluate``、``propose_template_revision`` + * - 動作日誌與候選腳本 + - :doc:`../new_features/v5_features_doc` + - ``start_action_journal``、``generate_candidate_from_log``、 + ``AC_journal_*``、``je_auto_control codegen --from-log`` + * - MCP 工具模式 + - :doc:`../mcp_server/mcp_server_doc` + - ``--tool-mode``、``JE_AUTOCONTROL_MCP_TOOL_MODE``、``ac_tools_*`` + * - 角色與延後工作的擁有者 + - :doc:`../operations_layer/operations_layer_doc` + - ``JE_AUTOCONTROL_RBAC_USERS``、``rbac_add_user``、``AC_user_*``、 + ``je_auto_control users`` + * - 動作檔簽章（Ed25519） + - :doc:`../new_features/v4_features_doc` + - ``create_signing_keypair``、``sign_action_file``、 + ``JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS`` + * - 視窗外殼、導覽與背景工作 + - :doc:`../new_features/v223_features_doc` + - Actions 與 View 選單、``JE_AUTOCONTROL_GUI_SETTINGS`` + * - 所有環境變數 + - :doc:`../configuration/configuration_doc` + - + +範例裡容易忽略的三件事 +====================== + +**執行器的選項在執行器物件上。** 模組層級的 +``je_auto_control.execute_action(actions)`` 只接受動作清單。``dry_run``、 +``raise_on_error`` 與 ``step_callback`` 屬於 +``je_auto_control.executor.execute_action``： + +.. code-block:: python + + import je_auto_control as ac + + ac.executor.execute_action(actions, dry_run=True) # 只解析，不呼叫 + ac.executor.execute_action(actions, raise_on_error=True) # 第一個失敗就停 + +**同步失敗後會退避。** 送出失敗之後，在退避時間內（2 秒起、每次加倍、上限 +5 分鐘）再呼叫 ``config_sync_run`` 並不會連線到伺服器：它會再次回報 +``offline``，``error`` 為 ``waiting to retry after an earlier failure``。傳入 +``wait=True`` 則會等到退避結束再送——排程同步需要的就是這個行為，而 ``cancel`` +事件可以中斷等待。 + +**搜尋工具不等於啟用工具。** 在 progressive 模式下，``ac_tools_search`` 與 +``ac_tools_schema`` 只負責描述；呼叫 ``ac_tools_enable`` 之後，該工作階段的 +``tools/list`` 才會改變，並以 ``notifications/tools/list_changed`` 通知用戶端。 + +各腳本的驗證路徑 +================ + +以下內容直接取自腳本本身，也就是 CI 實際執行的程式碼。 + +.. literalinclude:: ../../../../../examples/28_wayland_diagnostics.py + :language: python + :pyobject: validate + :caption: examples/28_wayland_diagnostics.py + +.. literalinclude:: ../../../../../examples/29_config_sync.py + :language: python + :pyobject: _walkthrough + :caption: examples/29_config_sync.py + +.. literalinclude:: ../../../../../examples/30_mobile_devices.py + :language: python + :pyobject: validate + :caption: examples/30_mobile_devices.py + +.. literalinclude:: ../../../../../examples/31_healing_comparison.py + :language: python + :pyobject: validate + :caption: examples/31_healing_comparison.py + +.. literalinclude:: ../../../../../examples/32_codegen_from_log.py + :language: python + :pyobject: validate + :caption: examples/32_codegen_from_log.py + +.. literalinclude:: ../../../../../examples/33_mcp_progressive.py + :language: python + :pyobject: validate + :caption: examples/33_mcp_progressive.py diff --git a/docs/source/Zh/doc/new_features/v223_features_doc.rst b/docs/source/Zh/doc/new_features/v223_features_doc.rst index 57b84df59..3f1940515 100644 --- a/docs/source/Zh/doc/new_features/v223_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v223_features_doc.rst @@ -97,3 +97,42 @@ GUI 設定檔): (每個分頁的 ``first_open_ms``)、再切回去、在搜尋框輸入、切換主題,同時有一個 5 ms 的計時器在跳;``event_loop_p95_ms`` 就是那些 tick 晚到多久。``--compare`` 把兩份 報告並排,工作量或環境不同時拒絕比較。 + +分頁裡的背景工作 +---------------- + +需要等待網路對端或裝置的分頁,透過 ``je_auto_control.gui.task_controller`` +啟動工作,而不是自己開執行緒: + +.. code-block:: python + + from je_auto_control.gui.task_controller import CancellationToken, task_controller + + def fetch(url: str, token: CancellationToken) -> dict: + token.raise_if_cancelled() + return download(url, timeout=token.remaining(default=10.0)) + + url = self._url_edit.text() # 先把 widget 的值讀出來 + handle = task_controller().submit( + functools.partial(fetch, url), owner=self, timeout_s=30.0) + handle.result.connect(self._show) # 在 GUI 執行緒發出 + handle.error.connect(self._show_error) # 例外物件本身 + handle.finished.connect(self._enable_button) # 無論結果如何都只發一次 + +相較於單純的 worker,它多了這些: + +* **取消會傳到後端。** 工作會收到 ``CancellationToken``:以 + ``raise_if_cancelled()`` / ``wait()`` 輪詢、把 ``remaining()`` 交給接受逾時的 + 後端,並用 ``on_cancel()`` 登記如何釋放手上的資源。``handle.cancel()`` 會在 + 呼叫取消的執行緒上立刻執行那些釋放動作。 +* **有型別的結果。** ``result`` 帶回傳值、``error`` 帶例外、``progress`` 帶 + ``token.report_progress()`` 傳入的內容。逾時回報為 ``TaskTimeout``,取消回報為 + ``TaskCancelled``。 +* **擁有者消失。** handle 是 ``owner`` 的子物件。分頁被銷毀時工作會被取消,之後 + 才到的結果交給 ``discard=``\ (關掉已經沒人要用的工作階段),絕不會送到已銷毀 + widget 的 slot。 +* **worker 不碰 widget。** 工作若是 widget 的方法,或閉包裡抓著 widget,會在啟動 + 前以 ``TaskUsageError`` 拒絕。 + +這是 GUI 內部 API(會 import ``PySide6``),套件 facade 不會重新匯出;分頁所呼叫的 +無頭函式不需要它也能使用。 diff --git a/docs/source/Zh/doc/new_features/v2_features_doc.rst b/docs/source/Zh/doc/new_features/v2_features_doc.rst index fab36f0e2..33096e3e3 100644 --- a/docs/source/Zh/doc/new_features/v2_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v2_features_doc.rst @@ -30,7 +30,7 @@ Executor：``AC_self_heal_locate / _click / _log_list / _log_clear``。 MCP：``ac_self_heal_*``。GUI：**Self-Healing** 分頁。 -``screen_region`` 是螢幕座標的 ``[x1, y1, x2, y2]``，**兩種**策略都受它限制； +``screen_region`` 是螢幕座標的 ``[x1, y1, x2, y2]``，\ **兩種**\ 策略都受它限制； 以前只有 VLM 收到它，樣板比對可能在區域之外回報命中。 **找到不等於驗證過。** ``HealOutcome.found`` 只表示某個策略回傳了一個點。 diff --git a/docs/source/Zh/doc/new_features/v5_features_doc.rst b/docs/source/Zh/doc/new_features/v5_features_doc.rst index f4be0193a..e57574f66 100644 --- a/docs/source/Zh/doc/new_features/v5_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v5_features_doc.rst @@ -160,8 +160,8 @@ SQL 查詢僅限單句唯讀的 ``SELECT`` / ``WITH``,以唯讀連線執行,且值一律以 參數綁定(絕不字串拼接)。 -執行器指令:``AC_sql_to_var``(列 / 單列 / 純量存入變數)與 -``AC_assert_db``(對純量查詢以 eq / ne / lt / gt / contains / ... 斷言)。 +執行器指令:``AC_sql_to_var``\ (列 / 單列 / 純量存入變數)與 +``AC_assert_db``\ (對純量查詢以 eq / ne / lt / gt / contains / ... 斷言)。 Email(SMTP) diff --git a/docs/source/Zh/doc/new_features/v81_features_doc.rst b/docs/source/Zh/doc/new_features/v81_features_doc.rst index c9e534116..41356f31d 100644 --- a/docs/source/Zh/doc/new_features/v81_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v81_features_doc.rst @@ -1,5 +1,5 @@ 分層設定解析器 -============ +============== ``json_patch.merge_patch`` 只合併兩份文件,``config_sync`` 逐筆以 version vector 合併(同一筆的並行編輯保留為衝突,見 :doc:`../config_sync/config_sync_doc`), ``AssetStore`` 則是每環境的扁平結構。它們都無法組成一個有序的 ``defaults < file < env < CLI`` 優先序 diff --git a/docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst b/docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst index bcec4267c..23a025ae8 100644 --- a/docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst +++ b/docs/source/Zh/doc/wayland/wayland_capabilities_doc.rst @@ -27,7 +27,7 @@ sender，也不會送出任何事件。 同一份資料在每個入口都拿得到： * JSON 動作：``[["AC_probe_capabilities"]]`` -* MCP 工具：``ac_probe_capabilities``（唯讀） +* MCP 工具：``ac_probe_capabilities``\ （唯讀） * GUI：**診斷** 分頁的第二張表 四項能力各自獨立診斷，因為它們各自獨立地壞： @@ -74,7 +74,7 @@ sender，也不會送出任何事件。 - 不產生副作用就無法判定（例如 macOS 的權限） 每項能力另外帶有 ``backend``（由誰提供）、``desktop_wide``、``detail``、 -``recovery``（英文的處理方式）與 ``recovery_key``（同一段建議在 GUI 語系表的鍵）。 +``recovery``\ （英文的處理方式）與 ``recovery_key``\ （同一段建議在 GUI 語系表的鍵）。 libei 路徑上的 ``restore_token`` 是 ``unsupported``：這個綁定不向 portal 要求 restore token，所以同意是每個行程問一次，重新啟動後不會保留。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 16fccd37a..174c87a1b 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -9,6 +9,7 @@ AutoControl 所有功能的完整使用指南。 :caption: 使用者指南 doc/installation/installation_doc + doc/configuration/configuration_doc doc/mouse/mouse_doc doc/keyboard/keyboard_doc doc/screen/screen_doc @@ -25,6 +26,7 @@ AutoControl 所有功能的完整使用指南。 doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/config_sync/config_sync_doc + doc/examples/examples_doc doc/create_project/create_project_doc doc/new_features/new_features_doc doc/new_features/v2_features_doc diff --git a/examples/28_wayland_diagnostics.py b/examples/28_wayland_diagnostics.py new file mode 100644 index 000000000..d0bd187ab --- /dev/null +++ b/examples/28_wayland_diagnostics.py @@ -0,0 +1,155 @@ +"""Ask what this session can do, and what stands in the way when it cannot. + +``22_wayland_backend.py`` shows which helper programs are installed. This one +asks the question that matters on Wayland: *can input, capture, recording and +the stop key be used right now, through which backend, and if not, what does +the operator do about it?* + + snapshot = ac.probe_capabilities() + for capability in snapshot.capabilities: + print(capability.name, capability.state.value, capability.backend) + if capability.recovery: + print(" ->", capability.recovery) + +**The probe has no side effect.** It reads the environment, looks names up on +``PATH``, asks the loader whether a library exists and reads the authorisation +ledger. It never opens a portal request, never shows a consent dialog and +never sends an event, so it is safe from a health check or an MCP client. The +same data is ``AC_probe_capabilities`` in an action file, the +``ac_probe_capabilities`` MCP tool and the Diagnostics tab. + +States worth recognising: + +``not_requested`` usable; the desktop asks for consent on first use +``needs_permission`` someone refused, or a device node is not readable +``needs_setup`` something has to be installed or configured +``revoked`` the compositor took a granted session away +``compositor_restarted`` the grant came from a compositor that is gone +``unknown`` not determinable without side effects (macOS) + +A refused or revoked consent is **not** worked around: input raises +``WaylandAuthorisationError`` instead of quietly switching to ``ydotool``, +until ``ac.reset_input_authorisation()`` (``AC_reset_input_authorisation``). + +Recording is two different things on Wayland. What *this program* executed +needs no hook at all -- ``InputStepLog`` collects it on every backend. What the +*user* typed needs the kernel's event devices, which is opt-in per device +through ``JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES`` (``PhysicalRecorder``). + +Relevant settings:: + + JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=auto|wayland|x11 + JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=auto|cli # cli = ydotool, never the portal + JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mytool {output}" # your own capture tool + JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES=/dev/input/event3,/dev/input/event7 + JE_AUTOCONTROL_WAYLAND_EI_WORKER=1 # libei in a helper process + JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat|strict # ydotool absolute moves + +``--validate`` describes four desktops from made-up contexts (so it gives the +same answer on Windows, macOS and Linux) and journals a dry run. Without the +flag the script probes the session it is running in, which is equally free of +side effects. +""" +import argparse +import sys +from typing import Dict, List, Optional + +import je_auto_control as ac +# Only needed to *describe* a desktop; probing the real one takes no ledger. +from je_auto_control.linux_wayland.authorisation import ( + INPUT, AuthorisationLedger, AuthorisationState, +) + +_WAYLAND_ENV = {"XDG_SESSION_TYPE": "wayland", "WAYLAND_DISPLAY": "wayland-0", + "XDG_RUNTIME_DIR": "/run/user/1000"} + + +def show(title: str, snapshot: ac.CapabilitySnapshot) -> None: + """Print one snapshot as a small table.""" + print(f"{title}: platform={snapshot.platform} display={snapshot.display_server}" + f" xwayland={snapshot.xwayland}") + for capability in snapshot.capabilities: + scope = "" if capability.desktop_wide else " (XWayland windows only)" + print(f" {capability.name:<13} {capability.state.value:<21} via {capability.backend}{scope}") + if capability.recovery: + print(f" fix: {capability.recovery}") + + +def _context(environ: Dict[str, str], *, tools: tuple = (), libei: bool = False, + bus: bool = True, ledger: Optional[AuthorisationLedger] = None, + loaded: Optional[str] = None) -> ac.BackendContext: + """A desktop described by hand: nothing here reads the real machine.""" + return ac.BackendContext( + platform="linux", environ=environ, + which=lambda name: f"/usr/bin/{name}" if name in tools else None, + library_present=lambda name: libei and name in ("ei", "oeffis"), + session_bus_present=lambda: bus, readable=lambda _path: False, + compositor=lambda _environ: "8:1234:1", + authorisations=ledger if ledger is not None else AuthorisationLedger(), + loaded_backend=loaded) + + +def validate() -> int: + """Probe four described desktops and check each reads the way it should.""" + refused = AuthorisationLedger() + refused.transition(INPUT, AuthorisationState.DECLINED, "the user dismissed the dialog") + desktops = { + "GNOME, portal not asked yet": _context( + _WAYLAND_ENV, tools=("gnome-screenshot",), libei=True), + "GNOME, consent refused": _context( + _WAYLAND_ENV, tools=("gnome-screenshot", "ydotool"), libei=True, ledger=refused), + "sway with ydotool and grim": _context( + {**_WAYLAND_ENV, "JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES": "/dev/input/event3"}, + tools=("ydotool", "grim"), bus=False), + "X11 backend on a Wayland session": _context( + {**_WAYLAND_ENV, "DISPLAY": ":0"}, loaded="x11"), + } + snapshots = {title: ac.probe_capabilities(context) for title, context in desktops.items()} + for title, snapshot in snapshots.items(): + show(title, snapshot) + + # What this program executes can be journalled with no input hook at all. + # A dry run resolves every command without calling it. + log = ac.InputStepLog() + log.run([["AC_set_var", {"name": "greeting", "value": "hello"}], + ["AC_get_var", {"name": "greeting"}]], dry_run=True) + print(f"InputStepLog noted {len(log.steps)} step(s) from a dry run: " + f"{[step[0] for step in log.steps]}") + + expected = { + "GNOME, portal not asked yet": ("not_requested", "libei", True), + # ydotool is installed, and is still not used: a refusal is not a fallback. + "GNOME, consent refused": ("needs_permission", "libei", True), + "sway with ydotool and grim": ("available", "ydotool", True), + "X11 backend on a Wayland session": ("available", "xwayland", False), + } + problems: List[str] = [] + for title, wanted in expected.items(): + found = snapshots[title].input + got = (found.state.value, found.backend, found.desktop_wide) + if got != wanted: + problems.append(f"{title}: input is {got}, expected {wanted}") + if snapshots["sway with ydotool and grim"].get("recording").state.value != "needs_permission": + problems.append("an unreadable recording device should read as needs_permission") + if len(log.steps) != 2: + problems.append(f"expected 2 journalled steps, got {len(log.steps)}") + for problem in problems: + print(f"FAILED: {problem}") + print("validate:", "failed" if problems else "ok") + return 1 if problems else 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Parse the command line; see the module docstring.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--validate", action="store_true", + help="probe described desktops instead of this one") + args = parser.parse_args(argv) + if args.validate: + return validate() + show("this session", ac.probe_capabilities()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/29_config_sync.py b/examples/29_config_sync.py new file mode 100644 index 000000000..882537428 --- /dev/null +++ b/examples/29_config_sync.py @@ -0,0 +1,290 @@ +"""Keep scripts and settings in step across machines, offline included. + +Each user has one bucket on a sync server; every machine merges its changes +into it. One call does a whole cycle -- stage local changes, push what is +queued, merge what came back, apply it locally:: + + report = ac.config_sync_run( + "http://127.0.0.1:8765", "alice", + scripts_dir="~/automation/scripts") # + hotkeys, triggers, address book + print(report["state"], report["revision"], report["pending"], report["conflicts"]) + +The server is the signaling server; it keeps buckets in SQLite, so they +survive a restart:: + + python -m je_auto_control.utils.remote_desktop.signaling_server \\ + --bind 127.0.0.1 --port 8765 --shared-secret "$AC_SIGNALING_SECRET" \\ + --config-db /var/lib/autocontrol/config_sync.sqlite3 # or AC_SIGNALING_CONFIG_DB + +What ``state`` means: + +``synced`` nothing is waiting +``pending`` changes are queued and will be sent on the next run +``offline`` the server was not reached, or an earlier failure is still backing + off (``error`` says which); nothing was lost, the queue is on disk +``conflict`` two machines changed one entry apart -- both versions are kept +``resync_required`` this device was retired; call ``ac.config_sync_full_resync`` + +Things that are deliberate: + +* **No clock decides a merge.** Each entry carries a version vector; a change + made knowing the other wins, two changes made apart are both kept until a + person chooses with ``ac.config_sync_resolve(server, user, section, key, choice)``. +* **A write names the revision it was built on** (wire version 2). The server + refuses it with HTTP 409 when another machine got there first and the client + merges and retries; a bare bucket from an older client is answered 428. +* **Secrets and machine paths do not travel.** A script holding a literal + secret is withheld (use ``${secrets.NAME}``); applying synced data never + enables a hotkey or a trigger. + +The same cycle is ``AC_config_sync_run`` / ``_status`` / ``_resolve`` / +``_full_resync`` in an action file, four MCP tools, and the Config Sync tab. + +``--validate`` plays two machines against a store in a temp directory. Its +server is a few lines of ``http.server`` on 127.0.0.1 speaking the same wire +format over the real ``ConfigStore`` -- a stand-in for the signaling server so +the example needs no extra. Only script files in temp folders are synced; this +machine's hotkeys, triggers and address book are not read. Without the flag +the script prints the recorded status for ``--server`` / ``--user`` (no +network) and syncs only with ``--run``. +""" +import argparse +import json +import sys +import tempfile +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import unquote + +import je_auto_control as ac +# The store's own conflict error is not re-exported by the facade. +from je_auto_control.utils.config_sync import RevisionConflictError + +_USER = "alice" + + +def _handler(store_path: Path) -> type: + """A request handler serving ``GET`` / ``PUT /config/`` from ``store_path``.""" + + class Handler(BaseHTTPRequestHandler): + def _send(self, status: int, body: Dict[str, Any]) -> None: + data = json.dumps(body).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _user(self) -> str: + return unquote(self.path.rsplit("/", 1)[-1]) + + def do_GET(self) -> None: # noqa: N802 # reason: http.server's naming + bucket = ac.ConfigStore(store_path).get(self._user()) + if bucket is None: + self._send(404, {"detail": "no bucket"}) + else: + self._send(200, {**bucket.to_dict(), "version": 2}) + + def do_PUT(self) -> None: # noqa: N802 # reason: http.server's naming + envelope = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + if envelope.get("version") != 2: + self._send(428, {"detail": "a version-2 write is required"}) + return + try: + revision = ac.ConfigStore(store_path).commit( + self._user(), ac.ConfigBucket.from_dict(envelope["bucket"]), + base_revision=envelope["base_revision"], + operation_id=envelope["operation_id"]) + except RevisionConflictError as conflict: + self._send(409, {"detail": "revision conflict", + "revision": conflict.current_revision}) + return + self._send(200, {"ok": True, "revision": revision, "version": 2}) + + def log_message(self, *_args: Any) -> None: + """Keep the example's output to what it prints itself.""" + + return Handler + + +class LoopbackServer: + """The stand-in sync server: start, stop, start again on the same store.""" + + def __init__(self, store_path: Path) -> None: + self._store_path = store_path + self._server: Optional[ThreadingHTTPServer] = None + self.port = 0 + + @property + def url(self) -> str: + """Where clients reach it.""" + return f"http://127.0.0.1:{self.port}" + + def start(self) -> None: + """Listen on 127.0.0.1 (the same port again after a stop).""" + self._server = ThreadingHTTPServer(("127.0.0.1", self.port), _handler(self._store_path)) + self.port = self._server.server_address[1] + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + def stop(self) -> None: + """Stop listening; the store file stays.""" + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + + +class Machine: + """One computer: its own scripts folder, outbox file and device id.""" + + def __init__(self, name: str, root: Path, server: LoopbackServer) -> None: + self.name = name + self.scripts = root / name / "scripts" + self.scripts.mkdir(parents=True) + self._outbox = root / name / "outbox.sqlite3" + self._server = server + + def _options(self) -> Dict[str, Any]: + return {"device_id": self.name, "sections": "scripts", "timeout_s": 2.0, + "scripts_dir": str(self.scripts), "outbox_path": str(self._outbox)} + + def sync(self, wait: bool = False) -> Dict[str, Any]: + """One sync cycle; prints and returns the report.""" + report = ac.config_sync_run(self._server.url, _USER, wait=wait, **self._options()) + print(f" {self.name:<8} {report['state']:<9} revision={report['revision']}" + f" pending={report['pending']} conflicts={report['conflicts']}") + return report + + def status(self) -> Dict[str, Any]: + """The recorded status; no network.""" + return ac.config_sync_status(self._server.url, _USER, str(self._outbox)) + + def resolve(self, key: str, choice: int) -> Dict[str, Any]: + """Keep candidate number ``choice`` of a conflicted script.""" + return ac.config_sync_resolve(self._server.url, _USER, "scripts", key, choice, + **self._options()) + + def write(self, name: str, actions: List[Any]) -> None: + """Save an action script in this machine's folder.""" + (self.scripts / name).write_text(json.dumps(actions), encoding="utf-8") + + def read(self, name: str) -> Optional[List[Any]]: + """The script's actions, or ``None`` when this machine does not have it.""" + path = self.scripts / name + return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None + + +def _walkthrough(root: Path) -> Tuple[List[str], Dict[str, Any]]: + store_path = root / "server" / "config_sync.sqlite3" + server = LoopbackServer(store_path) + server.start() + laptop, desktop = Machine("laptop", root, server), Machine("desktop", root, server) + seen: Dict[str, Any] = {} + try: + print("1. a script written on the laptop reaches the desktop") + laptop.write("login.json", [["AC_set_var", {"name": "user", "value": "alice"}]]) + laptop.sync() + desktop.sync() + seen["arrived"] = desktop.read("login.json") + + print("2. the server restarts; the bucket is still there") + committed = ac.ConfigStore(store_path).revision(_USER) + server.stop() + server.start() + seen["restart"] = (committed, ac.ConfigStore(store_path).revision(_USER)) + + print("3. the laptop works offline; the change waits in its outbox") + server.stop() + laptop.write("report.json", [["AC_set_var", {"name": "mode", "value": "daily"}]]) + seen["offline"] = laptop.sync() + server.start() + # A failed send backs off (2 s, doubling, capped at 5 min). A run inside + # that window does not contact the server and says "offline" again with + # error "waiting to retry after an earlier failure"; wait=True sleeps + # through the back-off instead, which is what a scheduled sync wants. + seen["early"] = laptop.sync() + seen["back_online"] = laptop.sync(wait=True) + + print("4. both machines edit login.json apart: neither edit is dropped") + desktop.sync() + laptop.write("login.json", [["AC_set_var", {"name": "user", "value": "from-laptop"}]]) + desktop.write("login.json", [["AC_set_var", {"name": "user", "value": "from-desktop"}]]) + laptop.sync() + seen["conflict"] = desktop.sync() + details = desktop.status()["conflict_details"] + for detail in details: + origins = [choice["origin"] for choice in detail["choices"]] + print(f" {detail['section']}/{detail['key']}: choose between {origins}") + seen["details"] = details + + print("5. a person picks the laptop's version on the desktop; both converge") + if details: + choices = [choice["origin"] for choice in details[0]["choices"]] + desktop.resolve(details[0]["key"], choices.index("laptop")) + seen["resolved"] = desktop.sync() + laptop.sync() + seen["final"] = (laptop.read("login.json"), desktop.read("login.json")) + finally: + server.stop() + return _check(seen), seen + + +def _check(seen: Dict[str, Any]) -> List[str]: + problems = [] + if seen.get("arrived") != [["AC_set_var", {"name": "user", "value": "alice"}]]: + problems.append("the laptop's script did not reach the desktop") + committed, reopened = seen.get("restart", (0, -1)) + if committed < 1 or committed != reopened: + problems.append(f"revision {committed} became {reopened} across the restart") + if seen.get("offline", {}).get("state") != "offline" or not seen["offline"]["pending"]: + problems.append("an unreachable server should leave the change pending") + if seen.get("back_online", {}).get("state") != "synced": + problems.append("the queued change was not sent once the server came back") + if seen.get("conflict", {}).get("state") != "conflict" or not seen.get("details"): + problems.append("edits made apart should be reported as a conflict") + if seen.get("resolved", {}).get("state") != "synced": + problems.append("the resolved conflict did not sync") + wanted = [["AC_set_var", {"name": "user", "value": "from-laptop"}]] + if seen.get("final") != (wanted, wanted): + problems.append(f"the machines did not converge on the chosen version: {seen.get('final')}") + return problems + + +def validate() -> int: + """Run the two-machine walkthrough in a temp directory and check each step.""" + with tempfile.TemporaryDirectory(prefix="ac_config_sync_") as folder: + problems, _seen = _walkthrough(Path(folder)) + for problem in problems: + print(f"FAILED: {problem}") + print("validate:", "failed" if problems else "ok") + return 1 if problems else 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Parse the command line; see the module docstring.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--validate", action="store_true", + help="two machines, a temp store and a 127.0.0.1 stand-in server") + parser.add_argument("--server", help="sync server URL, e.g. http://127.0.0.1:8765") + parser.add_argument("--user", help="the account whose bucket to use") + parser.add_argument("--scripts-dir", help="also sync the *.json scripts in this folder") + parser.add_argument("--run", action="store_true", + help="REAL SYNC: reads and writes this machine's hotkeys, triggers, " + "address book (and --scripts-dir)") + args = parser.parse_args(argv) + if args.validate: + return validate() + if not (args.server and args.user): + parser.error("give --server and --user, or use --validate") + if args.run: + options = {"scripts_dir": args.scripts_dir} if args.scripts_dir else {} + print(json.dumps(ac.config_sync_run(args.server, args.user, **options), indent=2)) + else: + print(json.dumps(ac.config_sync_status(args.server, args.user), indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/30_mobile_devices.py b/examples/30_mobile_devices.py new file mode 100644 index 000000000..6f4c71887 --- /dev/null +++ b/examples/30_mobile_devices.py @@ -0,0 +1,193 @@ +"""Drive Android and iOS devices through isolated device sessions. + +One :class:`DeviceContext` names one device; :func:`open_device` turns it into +a session that owns its own transport, timeout and cancellation. Nothing in a +session reads a process-wide default, so two workers of a device matrix cannot +reach each other's phone by leaving an argument out:: + + context = ac.DeviceContext("android", "emulator-5554", timeout_s=20) + with ac.open_device(context) as session: + print(ac.device_setup_report(session).to_dict()) # sends no input + frame = session.capture() # a DeviceFrame + x, y = frame.locate_image("login_button.png") # answers in device points + session.perform(ac.Tap(x, y)) + session.type_text("hello") + +iOS is the same with ``ac.DeviceContext("ios", "http://127.0.0.1:8100")`` (a +WebDriverAgent URL). Gestures are ``Tap``, ``LongPress``, ``Swipe``, ``Drag`` +and ``Pinch``; app lifecycle is ``ac.launch_app`` / ``stop_app`` / +``app_state`` / ``wait_for_app``. + +**Check a setup before blaming a script.** ``device_setup_report`` says which +backend and version answered and gives every capability a state -- +``available``, ``needs_permission`` (accept the USB-debugging prompt), +``needs_dependency`` (install the ADB Keyboard IME for non-ASCII text, or the +``uiautomator2`` extra for multi-touch) or ``unsupported`` -- with the reason. + +Every operation is also an ``AC_android_*`` / ``AC_ios_*`` command and an MCP +tool, generated from one table:: + + ac.run_mobile_command("AC_android_tap", {"x": 540, "y": 960, "serial": "emulator-5554"}) + ac.mobile_capability_matrix() # which commands deliver what, per platform + +What has no mobile counterpart (windows, mouse buttons, hotkeys, the desktop +accessibility tree) is listed there too, with the alternative. + +Requirements for a real device: Android needs ``adb`` on ``PATH`` and USB +debugging authorised; iOS needs a running WebDriverAgent and +``pip install je_auto_control[ios]``. Neither can be verified without the +hardware -- ``--validate`` replaces the transport with an offline stand-in +that answers the adb commands the session builds, so it proves which commands +are sent and nothing about a phone. Without the flag the script prints the +setup report of the device you name and sends it no input unless ``--tap`` is +given. +""" +import argparse +import io +import subprocess # nosec B404 # reason: only for the CompletedProcess type; nothing is spawned +import sys +from typing import Any, List, Optional, Sequence + +import je_auto_control as ac +# Subclassed below to build the offline transport; a real run never imports it. +from je_auto_control.android.adb_client import AdbClient + +_SERIAL = "emulator-5554" +_SIZE = (1080, 1920) +_INPUT_COMMANDS = ("input ", "am broadcast", "am start", "monkey ") + + +def _blank_png() -> bytes: + from PIL import Image + buffer = io.BytesIO() + Image.new("RGB", _SIZE, (20, 20, 20)).save(buffer, format="PNG") + return buffer.getvalue() + + +class OfflineAdb(AdbClient): + """The real client with the one method that spawns ``adb`` replaced. + + Every other method -- ``tap``, ``text``, ``screencap_png``, ``version`` -- + is the production code, so the argv recorded here is what a device would + receive. No process is started. + """ + + def __init__(self, serial: str) -> None: + super().__init__(adb_path="adb (offline)", default_serial=serial) + self.sent: List[str] = [] + + def run(self, args: Sequence[str], **_options: Any) -> "subprocess.CompletedProcess[bytes]": + """Answer one adb invocation from canned replies.""" + self.sent.append(" ".join(args)) + return subprocess.CompletedProcess(list(args), 0, self._reply(list(args)), b"") + + def _reply(self, args: List[str]) -> bytes: + if args[:1] == ["devices"]: + return f"List of devices attached\n{self.default_serial}\tdevice\n".encode() + if args[:1] == ["version"]: + return b"Android Debug Bridge version 1.0.41\nVersion 35.0.2-12147458\n" + if args[:2] == ["exec-out", "screencap"]: + return _blank_png() + replies = { + "wm size": f"Physical size: {_SIZE[0]}x{_SIZE[1]}\n", + "dumpsys input": " SurfaceOrientation: 0\n", + "getprop ro.build.version.release": "14\n", + "settings get secure default_input_method": + "com.google.android.inputmethod.latin/.LatinIME\n", + } + command = args[1] if args[:1] == ["shell"] and len(args) > 1 else "" + return next((text for prefix, text in replies.items() + if command.startswith(prefix)), "").encode() + + @property + def input_sent(self) -> List[str]: + """The shell commands that would have acted on a device.""" + return [line[len("shell "):] for line in self.sent + if line.startswith("shell ") and line[len("shell "):].startswith(_INPUT_COMMANDS)] + + +def show_report(report: ac.DeviceSetupReport) -> None: + """Print a setup report: what answered, and each capability's state.""" + print(f"{report.platform} {report.device_id}: backend {report.backend}" + f" {report.backend_version} / OS {report.os_version} -> {report.state}") + for name, capability in report.capabilities.items(): + reason = f" -- {capability.reason}" if capability.reason else "" + print(f" {name:<14} {capability.state}{reason}") + + +def validate() -> int: + """Open a session over the offline transport and check what it would send.""" + adb = OfflineAdb(_SERIAL) + context = ac.DeviceContext("android", _SERIAL, timeout_s=5, label="offline pixel") + with ac.open_device(context, adb=adb) as session: + report = ac.device_setup_report(session) + show_report(report) + probing = list(adb.input_sent) # a probe must not touch the device + frame = session.capture() + print(f"captured {frame.pixel_size} px as {frame.to_dict()['point_size']} points," + f" scale {frame.scale:g}") + session.perform(ac.Tap(540, 960)) + session.perform(ac.Swipe(540, 1500, 540, 500, duration_s=0.2)) + session.type_text("hello") + # A bound session is what address-less mobile commands target. + with ac.use_device(session): + ac.run_mobile_command("AC_android_tap", {"x": 10, "y": 20}) + print("sent to the device:", adb.input_sent) + + matrix = ac.mobile_capability_matrix() + for row in matrix["capabilities"]: + print(f" {row['capability']:<14} android={len(row['android'])} ios={len(row['ios'])} command(s)") + print(f" {len(matrix['desktop_only'])} desktop-only feature(s) documented with an alternative") + + problems = [] + if probing: + problems.append(f"the setup report sent input: {probing}") + if report.state != "available" or report.backend != "adb": + problems.append(f"unexpected setup report: {report.to_dict()}") + wanted = ["input tap 540 960", "input swipe 540 1500 540 500 200", "input tap 10 20"] + missing = [command for command in wanted if command not in adb.input_sent] + if missing: + problems.append(f"not sent: {missing}") + if not any(command.startswith("input text") for command in adb.input_sent): + problems.append("type_text sent no input text command") + if session.connected: + problems.append("the session was not closed on leaving the with block") + for problem in problems: + print(f"FAILED: {problem}") + print("validate:", "failed" if problems else "ok") + return 1 if problems else 0 + + +def real_device(platform: str, device_id: str, tap: Optional[List[int]]) -> int: + """Print the setup report of a real device; tap only when asked to.""" + with ac.open_device(ac.DeviceContext(platform, device_id)) as session: + report = ac.device_setup_report(session) + show_report(report) + if tap is None: + return 0 + if report.state != "available": + print(f"not tapping: input is {report.state} ({report.reason})") + return 1 + session.perform(ac.Tap(tap[0], tap[1])) + print(f"tapped {tap[0]},{tap[1]}") + return 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Parse the command line; see the module docstring.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--validate", action="store_true", + help="use an offline transport; no adb, no device") + parser.add_argument("--platform", choices=("android", "ios"), default="android") + parser.add_argument("--device", default="", + help="adb serial or WebDriverAgent URL (empty: the only one attached)") + parser.add_argument("--tap", nargs=2, type=int, metavar=("X", "Y"), + help="REAL INPUT: tap this point on the device") + args = parser.parse_args(argv) + if args.validate: + return validate() + return real_device(args.platform, args.device, args.tap) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/31_healing_comparison.py b/examples/31_healing_comparison.py new file mode 100644 index 000000000..b8e0ea464 --- /dev/null +++ b/examples/31_healing_comparison.py @@ -0,0 +1,149 @@ +"""Compare two versions of a locator on frames that carry the right answer. + +The heal log (``19_self_healing_locator.py``) records *what* a locator did. It +cannot say whether that was right: a strategy that returns a confident wrong +point looks exactly like one that healed. An evaluation runs every version +over the **same** labelled frames and scores the answers:: + + samples = [ + ac.EvaluationSample("login", frame, expected_box=(100, 60, 140, 84), template=tpl), + ac.EvaluationSample("no-such-button", other_frame, expect_miss=True, template=tpl), + ] + comparison = ac.evaluate_locators(samples, { + "v1": ac.template_match_strategy(threshold=0.9), + "v2": ac.template_match_strategy(threshold=0.9, scales=(1.0, 1.5)), + }) + print(comparison.report("v2").accuracy) # e.g. 4/4 = 1.0 + print(comparison.failures("v1")) # which samples, and how + +How answers are scored: + +* ``correct`` -- the point is inside the sample's ``expected_box``; +* ``false_positive`` -- a point anywhere else, or any point on a sample that + expects a miss. It was located, and it is not a recovery; +* ``miss`` / ``true_negative`` -- nothing returned, where something / nothing + was expected; +* ``unknown`` -- the sample has no label; it counts toward no correctness rate. + +Every rate carries its numerator and denominator, and a rate over nothing has +no value rather than a perfect one. Coordinates are *screen* coordinates: +``origin`` is the screen position of the frame's top-left pixel (negative for +a monitor left of the primary) and ``scale`` is frame pixels per screen unit, +so a HiDPI capture and a second monitor are scored in the space a click uses. + +A strategy is any callable taking a ``LocateRequest`` and returning ``(x, y)`` +or ``None`` -- wrap a VLM call in one to evaluate it the same way. Datasets +can also live in a JSON file (``ac.evaluate_healing_dataset(path)``, +``AC_self_heal_evaluate``, schema version 1) with pass/fail thresholds; +``benchmarks/self_healing/run.py`` is the maintained one. + +Nothing here reads the screen or calls a model. ``--validate`` draws its +frames in memory and checks the scores; ``--dataset FILE`` evaluates a dataset +file of your own. Both are offline. +""" +import argparse +import json +import sys +from typing import List, Optional, Tuple + +import cv2 +import numpy as np + +import je_auto_control as ac +# The threshold gate and the text table live beside the evaluator, not on the facade. +from je_auto_control.utils.self_healing.evaluation import check_thresholds, format_comparison + +_TEMPLATE_SIZE = (40, 24) # width, height in screen units + + +def draw_button(scale: float = 1.0, restyled: bool = False) -> np.ndarray: + """The control being located, as a grayscale image at ``scale``.""" + width, height = (round(side * scale) for side in _TEMPLATE_SIZE) + image = np.tile(np.linspace(70, 190, width, dtype=np.float32), (height, 1)).astype(np.uint8) + cv2.rectangle(image, (0, 0), (width - 1, height - 1), 20, max(1, round(2 * scale))) + cv2.circle(image, (height // 2, height // 2), height // 4, 250, -1) + cv2.rectangle(image, (width // 2, height // 3), (width - round(6 * scale), height // 2), 240, -1) + return 255 - image if restyled else image + + +def draw_frame(at: Optional[Tuple[int, int]], scale: float = 1.0, + restyled: bool = False, seed: int = 7) -> np.ndarray: + """A 320x200 (screen units) noisy frame with the button at ``at``, if any.""" + rng = np.random.default_rng(seed) + frame = rng.integers(105, 135, size=(round(200 * scale), round(320 * scale)), dtype=np.uint8) + if at is not None: + button = draw_button(scale, restyled) + left, top = round(at[0] * scale), round(at[1] * scale) + frame[top:top + button.shape[0], left:left + button.shape[1]] = button + return frame + + +def build_samples() -> List[ac.EvaluationSample]: + """Five labelled frames: the cases a real locator meets.""" + template = draw_button() + width, height = _TEMPLATE_SIZE + + def box(left: int, top: int, origin: Tuple[int, int] = (0, 0)) -> Tuple[int, int, int, int]: + return (origin[0] + left, origin[1] + top, + origin[0] + left + width, origin[1] + top + height) + + return [ + ac.EvaluationSample("plain", draw_frame((100, 60)), box(100, 60), template=template), + # The same screen captured on a 150 % display: 1.5 frame pixels per unit. + ac.EvaluationSample("hidpi-150", draw_frame((100, 60), scale=1.5), box(100, 60), + scale=1.5, template=template), + # A monitor left of the primary: its top-left is at a negative x. + ac.EvaluationSample("left-monitor", draw_frame((30, 120)), box(30, 120, (-320, 0)), + origin=(-320, 0), template=template), + # The button is not on this screen at all: the right answer is a miss. + ac.EvaluationSample("absent", draw_frame(None), expect_miss=True, template=template), + # After a redesign the old template should not be "found" anywhere. + ac.EvaluationSample("restyled", draw_frame((200, 40), restyled=True), + expect_miss=True, template=template), + ] + + +def validate() -> int: + """Score two template versions on the drawn frames and check the result.""" + comparison = ac.evaluate_locators(build_samples(), { + "v1-single-scale": ac.template_match_strategy(threshold=0.9), + "v2-multi-scale": ac.template_match_strategy(threshold=0.9, scales=(1.0, 1.5)), + }) + print(format_comparison(comparison)) + gate = {"v2-multi-scale": {"min_accuracy": 1.0, "max_false_positive": 0.0}} + violations = check_thresholds(comparison, gate) + print("thresholds:", "met" if not violations else violations) + + old, new = comparison.report("v1-single-scale"), comparison.report("v2-multi-scale") + problems = list(violations) + if [row.sample_id for row in comparison.failures("v1-single-scale")] != ["hidpi-150"]: + problems.append("v1 should fail exactly the HiDPI sample") + if comparison.failures("v2-multi-scale"): + problems.append("v2 should get every labelled sample right") + if not (old.accuracy.value or 0) < (new.accuracy.value or 0): + problems.append("v2 should be measurably more accurate than v1") + for problem in problems: + print(f"FAILED: {problem}") + print("validate:", "failed" if problems else "ok") + return 1 if problems else 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Parse the command line; see the module docstring.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--validate", action="store_true", + help="evaluate frames drawn in memory and check the scores") + parser.add_argument("--dataset", help="a dataset JSON file to evaluate (offline)") + args = parser.parse_args(argv) + if args.dataset: + report = ac.evaluate_healing_dataset(args.dataset) + print(json.dumps({key: report[key] for key in ("versions", "violations", "passed")}, + indent=2)) + return 0 if report["passed"] else 1 + if not args.validate: + parser.error("use --validate, or name a dataset with --dataset") + return validate() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/32_codegen_from_log.py b/examples/32_codegen_from_log.py new file mode 100644 index 000000000..49d706559 --- /dev/null +++ b/examples/32_codegen_from_log.py @@ -0,0 +1,143 @@ +"""Turn a run recorded in the action journal into a candidate script. + +The action journal is an opt-in JSONL file (schema version 1) of what the +executor ran: one ``start`` line and one ``end`` line per action, grouped by +``run_id``. :func:`generate_candidate_from_log` reads one run back and builds +a script for a person to review:: + + ac.start_action_journal("run.jsonl", run_id="checkout", session="laptop") + ac.execute_action(actions) + ac.stop_action_journal() + + candidate = ac.generate_candidate_from_log("run.jsonl", run_id="checkout") + print(candidate.code) # pytest by default; target="python" / "robot" + print(candidate.warnings) # what a reviewer has to check + print(candidate.manifest) # where each emitted step came from + +Three things to know before trusting a candidate: + +* It is built from the *inputs* the journal recorded, never from outcomes, and + generating it executes nothing (``manifest["executed"]`` is ``False``). +* A masked secret is not replayable. It comes back as + ``${journal_redacted_N_M}``, which fails as an unknown variable until you + replace it; a ``${secrets.NAME}`` reference in the journal is kept as is. +* ``candidate.observed_path_only`` is true when some control flow could not be + rebuilt and the steps that ran stand in for it. Read ``candidate.warnings``. + +The same thing from a shell:: + + je_auto_control codegen --from-log run.jsonl --run-id checkout -o test_checkout.py + +and from an action file: ``AC_journal_start`` / ``AC_journal_stop`` / +``AC_generate_code_from_journal``. + +Run ``--validate`` to see the whole round trip offline: it records a run made +of variable and flow-control commands only (no mouse, keyboard or screen +command), generates the candidate and dry-runs it. Without the flag the script +converts a journal you name and writes nothing unless ``--output`` is given. +""" +import argparse +import sys +import tempfile +from pathlib import Path +from typing import List, Optional + +import je_auto_control as ac +# The two helpers the CLI uses; the facade exports generate_candidate_from_log only. +from je_auto_control.utils.codegen.journal_import import only_run_id, write_candidate + +#: Variable and flow-control commands only: running these touches no device. +_OFFLINE_RUN = [ + ["AC_set_var", {"name": "attempts", "value": 0}], + ["AC_loop", {"times": 3, "body": [["AC_inc_var", {"name": "attempts"}]]}], + ["AC_if_var", {"name": "attempts", "op": "eq", "value": 3, + "then": [["AC_set_var", {"name": "status", "value": "done"}]], + "else": [["AC_set_var", {"name": "status", "value": "short"}]]}], +] + + +def describe(candidate: ac.CandidateScript) -> None: + """Print what a reviewer looks at first.""" + manifest = candidate.manifest + print(f"run {manifest['run_id']}: {manifest['event_count']} journal event(s)" + f" -> {manifest['action_count']} top-level action(s)") + print(f" executed while generating: {manifest['executed']}") + print(f" observed path only: {candidate.observed_path_only}") + for step in manifest["steps"]: + print(f" journal line {step['line']}: {step['command']}" + f" ({step['mode']}, {step['status']})") + for warning in candidate.warnings: + print(f" ! {warning}") + + +def validate() -> int: + """Record an offline run, rebuild it, and dry-run the result.""" + with tempfile.TemporaryDirectory(prefix="ac_journal_") as folder: + journal = Path(folder) / "run.jsonl" + started = ac.start_action_journal(journal, run_id="example", session="validate") + try: + # The module-level ac.execute_action takes the list only; the options + # (raise_on_error, dry_run, step_callback) are on the executor object. + ac.executor.execute_action(_OFFLINE_RUN, raise_on_error=True) + finally: + stopped = ac.stop_action_journal() + print(f"journal {Path(started['path']).name}: {stopped['events']} event(s) recorded") + + runs = ac.list_journal_runs(journal) + candidate = ac.generate_candidate_from_log(journal, run_id=only_run_id(journal)) + describe(candidate) + + written = write_candidate(candidate, Path(folder) / "test_example.py", + manifest=Path(folder) / "test_example.manifest.json") + compile(Path(written["output"]).read_text(encoding="utf-8"), written["output"], "exec") + # The candidate's action list is what the script replays. A dry run + # resolves every command name and argument without calling anything. + dry = ac.executor.execute_action(candidate.actions, dry_run=True) + + problems = [] + if [run["run_id"] for run in runs] != ["example"]: + problems.append(f"expected one run called 'example', got {runs}") + if candidate.actions != _OFFLINE_RUN: + problems.append("the rebuilt action list differs from what was executed") + if candidate.manifest["executed"] or candidate.manifest["outcomes_used_as_input"]: + problems.append("the manifest says something was executed or an outcome was reused") + if len(dry) != len(_OFFLINE_RUN): + problems.append(f"the dry run covered {len(dry)} of {len(_OFFLINE_RUN)} actions") + for problem in problems: + print(f"FAILED: {problem}") + print("validate:", "failed" if problems else "ok") + return 1 if problems else 0 + + +def convert(journal: str, run_id: Optional[str], target: str, output: Optional[str]) -> int: + """Build a candidate from a journal on disk; print it or write it.""" + chosen = run_id or only_run_id(journal) + candidate = ac.generate_candidate_from_log(journal, run_id=chosen, target=target) + describe(candidate) + if output: + written = write_candidate(candidate, output, manifest=f"{output}.manifest.json") + print(f"wrote {written['output']} and {written['manifest']}") + else: + print(candidate.code) + return 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Parse the command line; see the module docstring.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--validate", action="store_true", + help="record, convert and dry-run an offline run in a temp directory") + parser.add_argument("--journal", help="an existing action journal (.jsonl) to convert") + parser.add_argument("--run-id", help="the run to use (optional when the journal holds one)") + parser.add_argument("--target", default="pytest", choices=("pytest", "python", "robot")) + parser.add_argument("--output", help="write the candidate (and its manifest) here") + args = parser.parse_args(argv) + if args.validate: + return validate() + if not args.journal: + parser.error("name a journal with --journal, or use --validate") + return convert(args.journal, args.run_id, args.target, args.output) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/33_mcp_progressive.py b/examples/33_mcp_progressive.py new file mode 100644 index 000000000..64f4641d3 --- /dev/null +++ b/examples/33_mcp_progressive.py @@ -0,0 +1,148 @@ +"""Give an MCP client a handful of tools instead of several hundred schemas. + +The default (``full``) tool mode answers ``tools/list`` with every registered +tool. Two other modes exist for clients that should not carry all of that: + +* ``progressive`` -- a session starts with five core tools and grows:: + + ac_tools_search find tools by words; returns name, summary, category + ac_tools_schema the full input schema of one tool + ac_tools_enable add tools (or ``category:``) to this session + ac_tools_disable take them out again + ac_tools_state what this session currently has enabled + + Enabling sends the session ``notifications/tools/list_changed``. +* ``static`` -- ``tools/list`` is a fixed profile and nothing can be enabled, + for a client that reads the list once. + +Choose the mode when starting the server:: + + python -m je_auto_control.utils.mcp_server --tool-mode progressive + JE_AUTOCONTROL_MCP_TOOL_MODE=progressive python -m je_auto_control.utils.mcp_server + ac.start_mcp_stdio_server(tool_mode="progressive") + +``JE_AUTOCONTROL_MCP_TOOL_PROFILE`` (comma-separated tool names and +``category:`` entries) sets the static profile. + +Disclosure changes what is *offered*, never what is *allowed*: a tool the +caller's role, read-only mode or the path roots rule out cannot be searched +for or enabled, and an enabled tool still passes every existing check when it +is called. + +``--validate`` drives an in-process server through the same ``handle_line`` +entry point both transports use. It lists, searches, reads a schema and +enables a tool; it never *calls* one that touches the desktop. Without the +flag the script serves MCP over stdio in progressive mode (and blocks, like +``14_mcp_stdio_server.py``). +""" +import argparse +import json +import sys +from typing import Any, Dict, List, Optional + +import je_auto_control as ac + +_QUERY = "screenshot" +_PROTOCOL = "2025-11-25" + + +class _Session: + """A JSON-RPC client talking to one in-process server, no transport.""" + + def __init__(self, server: ac.MCPServer) -> None: + self._server = server + self._next_id = 0 + self.notifications: List[str] = [] + # What a stdio transport does with a server-initiated message. + server.set_notifier(lambda method, _params: self.notifications.append(method)) + + def request(self, method: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Send one request and return its ``result``; raise on a JSON-RPC error.""" + self._next_id += 1 + line = json.dumps({"jsonrpc": "2.0", "id": self._next_id, "method": method, + "params": params or {}}) + reply = json.loads(self._server.handle_line(line)) + if "error" in reply: + raise RuntimeError(f"{method}: {reply['error']}") + return reply["result"] + + def notify(self, method: str) -> None: + """Send a notification (no id, no reply).""" + self._server.handle_line(json.dumps({"jsonrpc": "2.0", "method": method, "params": {}})) + + def tool_names(self) -> List[str]: + """Every tool name ``tools/list`` offers this session, all pages.""" + names: List[str] = [] + cursor = None + while True: + page = self.request("tools/list", {"cursor": cursor} if cursor else {}) + names += [tool["name"] for tool in page["tools"]] + cursor = page.get("nextCursor") + if cursor is None: + return names + + def call(self, name: str, arguments: Dict[str, Any]) -> Any: + """Call a tool and decode its JSON text result.""" + result = self.request("tools/call", {"name": name, "arguments": arguments}) + text = result["content"][0]["text"] + if result.get("isError"): + raise RuntimeError(f"{name}: {text}") + return json.loads(text) + + +def _open(mode: str) -> _Session: + session = _Session(ac.MCPServer(tools=ac.build_default_tool_registry(), tool_mode=mode)) + session.request("initialize", {"protocolVersion": _PROTOCOL, "capabilities": {}}) + session.notify("notifications/initialized") + return session + + +def validate() -> int: + """Compare a full and a progressive session, then grow the progressive one.""" + full = _open("full").tool_names() + session = _open("progressive") + core = session.tool_names() + print(f"full mode offers {len(full)} tools; a progressive session starts with {len(core)}:") + print(" " + ", ".join(core)) + + found = session.call("ac_tools_search", {"query": _QUERY, "limit": 5}) + print(f"search {_QUERY!r} -> {json.dumps(found)[:200]}...") + target = found["tools"][0]["name"] + schema = session.call("ac_tools_schema", {"name": target}) + enabled = session.call("ac_tools_enable", {"names": [target]}) + after = session.tool_names() + state = session.call("ac_tools_state", {}) + print(f"schema of {target}: {sorted(schema)}") + print(f"enabled {target}: {json.dumps(enabled)[:160]}") + print(f"the session now lists {len(after)} tools; notifications: {session.notifications}") + print(f"state: {json.dumps(state)[:200]}") + + problems = [] + if not len(core) < len(full): + problems.append("progressive mode did not start smaller than full mode") + if target in core or target not in after: + problems.append(f"{target} was not added by ac_tools_enable") + if "notifications/tools/list_changed" not in session.notifications: + problems.append("enabling did not notify the session that its list changed") + for problem in problems: + print(f"FAILED: {problem}") + print("validate:", "failed" if problems else "ok") + return 1 if problems else 0 + + +def main(argv: Optional[List[str]] = None) -> int: + """Parse the command line; see the module docstring.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--validate", action="store_true", + help="exercise the progressive tools in-process; call no desktop tool") + args = parser.parse_args(argv) + if args.validate: + return validate() + # Blocks until stdin closes. Every tool is still subject to the server's + # own gates (JE_AUTOCONTROL_MCP_READONLY, roles, path roots, confirmation). + ac.start_mcp_stdio_server(tool_mode="progressive") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/README.md b/examples/README.md index 9faa146ce..3fe29e2dd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -42,15 +42,54 @@ glue between AutoControl's public API and your code. | --- | --- | | [`18_slack_daily_report.py`](18_slack_daily_report.py) | Full daily workflow: pull Slack messages → Anthropic summary → HTML/PDF report → SMTP email, on a cron schedule. Every external service degrades to a stub so the script always runs to completion. | +## Locators, agents and other hosts + +| Script | What it shows | +| --- | --- | +| [`19_self_healing_locator.py`](19_self_healing_locator.py) | Locate with an image template first and fall back to a VLM on a miss; read the heal log. | +| [`20_webrunner_bridge.py`](20_webrunner_bridge.py) | Drive WebRunner (browser automation) from an AutoControl script. | +| [`21_computer_use.py`](21_computer_use.py) | Drive Anthropic Computer-Use against the live screen. | +| [`22_wayland_backend.py`](22_wayland_backend.py) | Run on a Wayland session: backend selection and which helper programs are installed. | +| [`23_cross_host_dag.py`](23_cross_host_dag.py) | Orchestrate a DAG of automation steps across several AutoControl hosts. | +| [`24_multi_viewer_presence.py`](24_multi_viewer_presence.py) | Maintain a presence roster across several connected viewers. | +| [`25_chatops_bot.py`](25_chatops_bot.py) | Stand up a Slack chat-ops bot. | +| [`26_pytest_plugin_and_bdd.py`](26_pytest_plugin_and_bdd.py) | Drive AutoControl from pytest and pytest-bdd. | +| [`27_anchor_locator.py`](27_anchor_locator.py) | Compose locators with spatial relations ("Submit below the Username label"). | + +## Platforms, sync and review tooling (each has `--validate`) + +| Script | What it shows | `--validate` uses | +| --- | --- | --- | +| [`28_wayland_diagnostics.py`](28_wayland_diagnostics.py) | `probe_capabilities()`: which of input, capture, recording and the stop key work right now, through which backend, and what to do when one does not. | Four desktops described by hand; no probe of this machine. | +| [`29_config_sync.py`](29_config_sync.py) | Sync scripts and settings across machines: revision-checked writes, an offline queue, conflicts kept for a person to resolve. | Two machines, a temp-directory store, a stand-in server on 127.0.0.1. | +| [`30_mobile_devices.py`](30_mobile_devices.py) | Isolated Android / iOS device sessions: setup report, capture, gestures, typing, the capability matrix. | An offline `adb` transport; no device. | +| [`31_healing_comparison.py`](31_healing_comparison.py) | Score two versions of a locator on labelled frames: accuracy, false positives, recovery. | Frames drawn in memory. | +| [`32_codegen_from_log.py`](32_codegen_from_log.py) | Record a run in the action journal and turn it into a candidate script for review. | Variable and flow-control commands only; a temp journal. | +| [`33_mcp_progressive.py`](33_mcp_progressive.py) | Progressive MCP tool mode: start with five tools, search, read a schema, enable what is needed. | An in-process server; no desktop tool is called. | + +`--validate` runs the real API against in-memory, temp-directory or loopback +stand-ins and exits 0 when what it saw matches what the script's docstring +says. It never moves the pointer, types, captures the screen, reaches a device +or leaves 127.0.0.1, and CI runs all six that way with every such seam replaced +by a tripwire (`test/unit_test/headless/test_modernization_examples.py`). It is +evidence about how the calls are wired, not about hardware. Without the flag +each script does the real thing, and anything that sends input or changes +this machine's settings sits behind an explicit option (`--tap`, `--run`). + ## Running -Each script is standalone and uses only the package facade -(`import je_auto_control as ac`). After `pip install -e .` from the +Each script is standalone and uses the package facade +(`import je_auto_control as ac`); the few that also import a helper from a +submodule say why next to the import. After `pip install -e .` from the repo root: ``` python examples/01_screenshot_and_click.py ``` +``` +python examples/29_config_sync.py --validate +``` + A few scripts have optional dependencies — the script comments mention which `pip install` brings them in. diff --git a/test/unit_test/headless/_example_guard.py b/test/unit_test/headless/_example_guard.py new file mode 100644 index 000000000..f2dbd7973 --- /dev/null +++ b/test/unit_test/headless/_example_guard.py @@ -0,0 +1,142 @@ +"""Run a script with every way of touching a device replaced by a tripwire. + +``python _example_guard.py SCRIPT [ARGS...]`` runs ``SCRIPT`` as ``__main__`` +after replacing: + +* the platform backend's keyboard, mouse, screen and recorder functions; +* ``PIL.ImageGrab.grab`` (the capture path that is not a backend function); +* ``subprocess.Popen`` -- no ``adb``, ``ydotool``, ``grim`` or anything else; +* socket connections to anything that is not this machine. + +A tripped wire is written to the file named by ``AC_EXAMPLE_GUARD_EFFECTS`` +*and* raised. The file is what a test reads: a script that swallowed the +exception would otherwise look clean. + +This is what lets a test say an example's ``--validate`` path has no device +effect, instead of trusting that it was written carefully. +""" +import json +import os +import runpy +import socket +import subprocess # nosec B404 # reason: patched below so that nothing can be spawned +import sys +import types +from typing import Any, Callable, List + +EFFECTS_ENV = "AC_EXAMPLE_GUARD_EFFECTS" +_LOOPBACK = frozenset({"127.0.0.1", "::1", "localhost"}) +_BACKEND_MODULES = ("keyboard", "mouse", "screen", "keyboard_check", "_keyboard", "_mouse") +_effects: List[str] = [] + + +class DeviceEffect(RuntimeError): + """Something tried to act outside the process.""" + + +def record(kind: str, detail: Any) -> None: + """Note one effect and persist the list, so it survives a swallowed exception.""" + _effects.append(f"{kind}: {detail}") + path = os.environ.get(EFFECTS_ENV) + if path: + with open(path, "w", encoding="utf-8") as handle: + json.dump(_effects, handle) + + +def effects() -> List[str]: + """Every effect recorded in this process.""" + return list(_effects) + + +def _tripwire(kind: str, name: str) -> Callable[..., Any]: + def tripped(*_args: Any, **_kwargs: Any) -> Any: + record(kind, name) + raise DeviceEffect(f"{kind}: {name}") + return tripped + + +def tripwire_module(module: types.ModuleType, kind: str = "backend") -> int: + """Replace every function ``module`` defines; return how many were replaced.""" + replaced = 0 + for name, value in list(vars(module).items()): + if isinstance(value, types.FunctionType) and value.__module__ == module.__name__: + setattr(module, name, _tripwire(kind, f"{module.__name__}.{name}")) + replaced += 1 + return replaced + + +class _TrippedObject: + """Stands in for a backend object: any use of it is an effect.""" + + def __init__(self, name: str) -> None: + self._name = name + + def __getattr__(self, attribute: str) -> Any: + return _tripwire("backend", f"{self._name}.{attribute}") + + +def _is_loopback(address: Any) -> bool: + return isinstance(address, tuple) and bool(address) and str(address[0]) in _LOOPBACK + + +def _guard_network() -> None: + real_connect = socket.socket.connect + + def connect(self: socket.socket, address: Any) -> Any: + if not _is_loopback(address): + record("network", address) + raise DeviceEffect(f"network: {address!r}") + return real_connect(self, address) + + socket.socket.connect = connect # type: ignore[method-assign] + socket.socket.connect_ex = connect # type: ignore[method-assign] + + +def _guard_processes() -> None: + def refuse(_self: Any, args: Any = None, *_rest: Any, **_options: Any) -> None: + record("process", args) + raise DeviceEffect(f"process: {args!r}") + + subprocess.Popen.__init__ = refuse # type: ignore[method-assign] + + +def _guard_backends() -> None: + try: + from je_auto_control.wrapper import platform_wrapper + except Exception: # noqa: BLE001 # reason: no backend loaded means none to guard; the script reports its own import failure + return + for attribute in _BACKEND_MODULES: + module = getattr(platform_wrapper, attribute, None) + if isinstance(module, types.ModuleType): + tripwire_module(module) + if getattr(platform_wrapper, "recorder", None) is not None: + platform_wrapper.recorder = _TrippedObject("recorder") + try: + from PIL import ImageGrab + except ImportError: + return + ImageGrab.grab = _tripwire("capture", "PIL.ImageGrab.grab") + + +def install() -> None: + """Install every tripwire in this process.""" + _guard_backends() + _guard_processes() + _guard_network() + + +def main(argv: List[str]) -> None: + """Install the tripwires, then run ``argv[0]`` as ``__main__`` with the rest as its argv.""" + if not argv: + raise SystemExit("usage: _example_guard.py SCRIPT [ARGS...]") + record_path = os.environ.get(EFFECTS_ENV) + if record_path: + with open(record_path, "w", encoding="utf-8") as handle: + json.dump([], handle) + install() + sys.argv = list(argv) + runpy.run_path(argv[0], run_name="__main__") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/test/unit_test/headless/test_modernization_examples.py b/test/unit_test/headless/test_modernization_examples.py new file mode 100644 index 000000000..aac164bbb --- /dev/null +++ b/test/unit_test/headless/test_modernization_examples.py @@ -0,0 +1,294 @@ +"""The new examples validate offline, and the docs agree with the code they describe. + +Three things a reader relies on and nothing else checked: + +* each example added with the platform / sync / mobile work has a ``--validate`` + path that runs to completion **without touching a device** -- run here as a + child process under ``_example_guard.py``, which replaces every input, + capture, process and network seam with a tripwire; +* every environment variable the package reads is in the configuration + reference, in both languages, and the three READMEs name the same ones; +* ``docs/CAPABILITY_MATRIX.md`` says what ``probe_capabilities()`` and + ``mobile_capability_matrix()`` say. Its two generated blocks are rewritten by + ``python test/unit_test/headless/test_modernization_examples.py --fix``. +""" +import json +import os +import py_compile +import re +import subprocess # nosec B404 # reason: runs this interpreter on repository files, fixed argv, no shell +import sys +import types +from pathlib import Path +from typing import Dict, Iterable, List, Set, Tuple + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[3] +EXAMPLES = REPO_ROOT / "examples" +GUARD = Path(__file__).with_name("_example_guard.py") +MATRIX = REPO_ROOT / "docs" / "CAPABILITY_MATRIX.md" +CONFIG_PAGES = { + "Eng": REPO_ROOT / "docs" / "source" / "Eng" / "doc" / "configuration" / "configuration_doc.rst", + "Zh": REPO_ROOT / "docs" / "source" / "Zh" / "doc" / "configuration" / "configuration_doc.rst", +} +READMES = ("README.md", "README/README_zh-CN.md", "README/README_zh-TW.md") + +NEW_EXAMPLES = ( + "28_wayland_diagnostics.py", "29_config_sync.py", "30_mobile_devices.py", + "31_healing_comparison.py", "32_codegen_from_log.py", "33_mcp_progressive.py", +) + +#: Variables the package reads that ``README.md`` does not mention. The README +#: is an overview and the configuration reference is the complete list, so +#: these are recorded rather than required -- but the set may not grow: a new +#: variable is either mentioned in the README or added here on purpose. +README_UNDOCUMENTED = frozenset({ + "AC_SIGNALING_CONFIG_DB", "AC_SIGNALING_SECRET", + "JE_AUTOCONTROL_ACCEPT_LEGACY_ACTION_SIGNATURES", + "JE_AUTOCONTROL_ACTION_SIGNING_PASSPHRASE", "JE_AUTOCONTROL_ACTION_SIGNING_PRIVATE_KEY", + "JE_AUTOCONTROL_CHATOPS_SCRIPT_ROOT", "JE_AUTOCONTROL_ENV", "JE_AUTOCONTROL_FAKE_BACKEND", + "JE_AUTOCONTROL_GUI_SETTINGS", "JE_AUTOCONTROL_INTERCEPTION_DLL", + "JE_AUTOCONTROL_INTERCEPTION_KEYBOARD", "JE_AUTOCONTROL_INTERCEPTION_MOUSE", + "JE_AUTOCONTROL_MCP_ALLOWED_ORIGINS", "JE_AUTOCONTROL_MCP_AUDIT", + "JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "JE_AUTOCONTROL_MCP_ERROR_SHOTS", + "JE_AUTOCONTROL_MCP_READONLY", "JE_AUTOCONTROL_MCP_TOKEN", "JE_AUTOCONTROL_MCP_TOOL_MODE", + "JE_AUTOCONTROL_MCP_TOOL_PROFILE", "JE_AUTOCONTROL_PYTEST_ARTIFACTS", + "JE_AUTOCONTROL_REDACTION", "JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS", + "JE_AUTOCONTROL_USB_PASSTHROUGH", "JE_AUTOCONTROL_WAYLAND_EI_WORKER", + "JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND", "JE_AUTOCONTROL_WAYLAND_RECORD_DEVICES", +}) + +_ENV_NAME = re.compile(r"\b(?:JE_AUTOCONTROL|AC_SIGNALING)_[A-Z0-9_]+\b") +_BLOCK = "" + + +# --- examples ------------------------------------------------------------- + +def _run_guarded(script: Path, arguments: List[str], tmp_path: Path) -> Tuple[int, List[str], str]: + """Run ``script`` under the guard; ``(exit code, device effects, output)``.""" + effects_file = tmp_path / "effects.json" + env = dict(os.environ) + env.update({"PYTHONPATH": str(REPO_ROOT), "PYTHONIOENCODING": "utf-8", + "AC_EXAMPLE_GUARD_EFFECTS": str(effects_file), + "JE_AUTOCONTROL_LOG_FILE": os.devnull, "JE_AUTOCONTROL_GUI_SETTINGS": "off", + "HOME": str(tmp_path), "USERPROFILE": str(tmp_path)}) + for name in ("JE_AUTOCONTROL_RBAC_USERS", "JE_AUTOCONTROL_MCP_TOOL_MODE", + "JE_AUTOCONTROL_MCP_READONLY", "AC_SIGNALING_SECRET"): + env.pop(name, None) + done = subprocess.run( # nosec B603 # nosemgrep # reason: this interpreter, repository files, no shell + [sys.executable, str(GUARD), str(script), *arguments], + cwd=tmp_path, env=env, capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=180, check=False) + effects = json.loads(effects_file.read_text(encoding="utf-8")) if effects_file.exists() else [ + "the guard never started"] + return done.returncode, effects, done.stdout + done.stderr + + +@pytest.mark.parametrize("name", NEW_EXAMPLES) +def test_examples_compile_and_validate_without_device(name, tmp_path): + script = EXAMPLES / name + py_compile.compile(str(script), cfile=str(tmp_path / "compiled.pyc"), doraise=True) + source = script.read_text(encoding="utf-8") + assert '"--validate"' in source, f"{name} offers no --validate" + code, device_effects_in_validate, output = _run_guarded(script, ["--validate"], tmp_path) + assert device_effects_in_validate == [], output + assert code == 0, output + assert "validate: ok" in output + + +def test_every_numbered_example_is_listed_in_the_examples_readme(): + listed = set(re.findall(r"\[`(\d\d_[a-z0-9_]+\.py)`\]", (EXAMPLES / "README.md").read_text( + encoding="utf-8"))) + on_disk = {path.name for path in EXAMPLES.glob("[0-9][0-9]_*.py")} + assert on_disk - listed == set(), "examples missing from examples/README.md" + assert listed - on_disk == set(), "examples/README.md lists scripts that do not exist" + assert set(NEW_EXAMPLES) <= on_disk + + +def test_the_guard_trips_on_a_backend_function(): + """The guard is only evidence if a replaced function really refuses to run.""" + import _example_guard as guard + module = types.ModuleType("fake_backend") + + def set_position(x, y): + return x, y + + set_position.__module__ = module.__name__ + module.set_position = set_position + module.CONSTANT = 3 + assert guard.tripwire_module(module) == 1 + before = len(guard.effects()) + with pytest.raises(guard.DeviceEffect): + module.set_position(1, 2) + assert guard.effects()[before:] == ["backend: fake_backend.set_position"] + assert module.CONSTANT == 3 + + +@pytest.mark.parametrize("body, kind", [ + # Harmless even if the guard failed: the child only starts `python -c pass`. + ("import subprocess, sys\n" + "try:\n subprocess.run([sys.executable, '-c', 'pass'])\n" + "except Exception:\n pass\n", "process"), + # 192.0.2.0/24 is TEST-NET-1: never routed, so an unguarded attempt goes nowhere. + ("import socket\ns = socket.socket()\ns.settimeout(0.2)\n" + "try:\n s.connect(('192.0.2.1', 9))\nexcept Exception:\n pass\n", "network"), +]) +def test_the_guard_reports_an_effect_the_script_swallowed(body, kind, tmp_path): + script = tmp_path / "probe.py" + script.write_text(body, encoding="utf-8") + code, effects, output = _run_guarded(script, [], tmp_path) + assert code == 0, output + assert len(effects) == 1 and effects[0].startswith(f"{kind}: "), effects + + +def test_the_guard_allows_loopback(tmp_path): + script = tmp_path / "probe.py" + script.write_text( + "import socket\nserver = socket.socket()\nserver.bind(('127.0.0.1', 0))\nserver.listen(1)\n" + "client = socket.socket()\nclient.connect(server.getsockname())\nprint('connected')\n", + encoding="utf-8") + code, effects, output = _run_guarded(script, [], tmp_path) + assert (code, effects) == (0, []) and "connected" in output, output + + +# --- configuration -------------------------------------------------------- + +def _names(paths: Iterable[Path]) -> Set[str]: + found: Set[str] = set() + for path in paths: + found |= set(_ENV_NAME.findall(path.read_text(encoding="utf-8", errors="replace"))) + return found + + +def _names_in_code() -> Set[str]: + sources = [path for path in (REPO_ROOT / "je_auto_control").rglob("*.py") + if "__pycache__" not in path.parts] + return _names([*sources, REPO_ROOT / "je_auto_control_pytest.py"]) + + +def test_readme_configuration_parity(): + in_code = _names_in_code() + english_configuration_keys = _names([CONFIG_PAGES["Eng"]]) + translated_configuration_keys = _names([CONFIG_PAGES["Zh"]]) + assert translated_configuration_keys == english_configuration_keys + assert in_code - english_configuration_keys == set(), ( + "read by the package and missing from the configuration reference (both languages)") + assert english_configuration_keys - in_code == set(), ( + "documented in the configuration reference and read by nothing") + + readme, simplified, traditional = (_names([REPO_ROOT / name]) for name in READMES) + assert simplified == readme and traditional == readme, ( + "the three READMEs name different environment variables") + assert readme - in_code == set(), "README.md names a variable nothing reads" + assert (in_code - readme) - README_UNDOCUMENTED == set(), ( + "a new variable is in neither README.md nor README_UNDOCUMENTED") + + +def test_the_configuration_page_is_reachable_in_both_languages(): + for language, index in (("Eng", "eng_index.rst"), ("Zh", "zh_index.rst")): + toctree = (REPO_ROOT / "docs" / "source" / language / index).read_text(encoding="utf-8") + for page in ("doc/configuration/configuration_doc", "doc/examples/examples_doc"): + assert f" {page}\n" in toctree.replace("\r\n", "\n"), f"{page} not in {index}" + assert (REPO_ROOT / "docs" / "source" / language / f"{page}.rst").is_file() + + +# --- capability matrix ---------------------------------------------------- + +def _probe_rows() -> List[Dict[str, str]]: + """What a probe reports on the platforms where it does not depend on the session.""" + from je_auto_control.wrapper.capabilities import BackendContext, probe_capabilities + rows = [] + for platform in ("win32", "darwin"): + snapshot = probe_capabilities(BackendContext(platform=platform, environ={})) + for capability in snapshot.capabilities: + rows.append({"platform": platform, "capability": capability.name, + "state": capability.state.value, "backend": capability.backend, + "detail": capability.detail}) + return rows + + +def render_probe_block() -> str: + """The generated table of ``probe_capabilities()`` on Windows and macOS.""" + lines = ["| Platform | Capability | State | Backend | Evidence |", "|---|---|---|---|---|"] + for row in _probe_rows(): + evidence = (f"not probed: {row['detail']}" if row["detail"] + else "reported, not probed: nothing to authorise") + lines.append(f"| `{row['platform']}` | `{row['capability']}` | `{row['state']}` " + f"| `{row['backend']}` | {evidence} |") + return "\n".join(lines) + + +def render_mobile_block() -> str: + """The generated tables of ``mobile_capability_matrix()``.""" + from je_auto_control.wrapper.mobile_commands import mobile_capability_matrix + matrix = mobile_capability_matrix() + + def commands(names: List[str]) -> str: + return ", ".join(f"`{name}`" for name in names) or "—" + + lines = ["| Capability | Android commands | iOS commands |", "|---|---|---|"] + lines += [f"| `{row['capability']}` | {commands(row['android'])} | {commands(row['ios'])} |" + for row in matrix["capabilities"]] + lines += ["", "| Desktop-only feature | Why | Use instead |", "|---|---|---|"] + lines += [f"| {row['feature']} | {row['limitation']} | {row['alternative']} |" + for row in matrix["desktop_only"]] + return "\n".join(lines) + + +GENERATED = {"probe-capabilities": render_probe_block, "mobile-matrix": render_mobile_block} + + +def _block(text: str, name: str) -> str: + begin, end = (_BLOCK.format(name=name, edge=edge) for edge in ("begin", "end")) + assert text.count(begin) == 1 and text.count(end) == 1, f"markers of {name} are missing" + return text.split(begin, 1)[1].split(end, 1)[0].strip("\n") + + +def _matrix_text() -> str: + return MATRIX.read_text(encoding="utf-8").replace("\r\n", "\n") + + +@pytest.mark.parametrize("name", sorted(GENERATED)) +def test_matrix_matches_capabilities(name): + assert _block(_matrix_text(), name) == GENERATED[name](), ( + "re-generate: python test/unit_test/headless/test_modernization_examples.py --fix") + + +def test_matrix_names_every_capability_and_state(): + from je_auto_control.linux_wayland.authorisation import CAPTURE, INPUT + from je_auto_control.wrapper import device_context + from je_auto_control.wrapper.capabilities import RECORDING, STOP_SHORTCUT, CapabilityStatus + text = _matrix_text() + wanted = {INPUT, CAPTURE, RECORDING, STOP_SHORTCUT, *device_context.CAPABILITY_NAMES, + *(status.value for status in CapabilityStatus), + device_context.STATE_AVAILABLE, device_context.STATE_NEEDS_PERMISSION, + device_context.STATE_NEEDS_DEPENDENCY, device_context.STATE_UNSUPPORTED} + undocumented_capabilities = {name for name in wanted if f"`{name}`" not in text} + assert undocumented_capabilities == set() + + +def test_matrix_does_not_claim_hardware_for_mobile(): + """The mobile row stays honest until a job drives a real device.""" + row = next(line for line in _matrix_text().splitlines() + if line.startswith("| Android and iOS bridges")) + assert "mocked CI" in row and "hardware" not in row.replace("hardware-unverified", "") + + +def _fix() -> None: + text = _matrix_text() + for name, render in GENERATED.items(): + begin, end = (_BLOCK.format(name=name, edge=edge) for edge in ("begin", "end")) + head, rest = text.split(begin, 1) + _old, tail = rest.split(end, 1) + text = f"{head}{begin}\n{render()}\n{end}{tail}" + MATRIX.write_text(text, encoding="utf-8", newline="\n") + print(f"rewrote the generated blocks of {MATRIX.name}; review the diff") + + +if __name__ == "__main__": + sys.path.insert(0, str(REPO_ROOT)) + if sys.argv[1:] == ["--fix"]: + _fix() + else: + raise SystemExit("usage: test_modernization_examples.py --fix") From b251f1b51af6f723b6fcb9165184d32ca09803d4 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:51:10 +0800 Subject: [PATCH 38/44] Join journal, candidate, device session, config sync and capability reports end to end on fakes Each feature has its own tests; nothing crossed the seams between them. A run journalled on one fake device now has to dry-run and replay on another, a sync has to survive its store being reopened through the functions the Config Sync tab calls, and a capability report has to say for every statement whether it was probed or skipped and why -- and may not claim hardware. --- .../test_modernization_integration.py | 370 ++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 test/unit_test/headless/test_modernization_integration.py diff --git a/test/unit_test/headless/test_modernization_integration.py b/test/unit_test/headless/test_modernization_integration.py new file mode 100644 index 000000000..eb5ec89b5 --- /dev/null +++ b/test/unit_test/headless/test_modernization_integration.py @@ -0,0 +1,370 @@ +"""The features of the platform / sync / mobile round, joined end to end -- on fakes. + +Each feature has its own tests. These three cross the seams between them, +because that is where a set of individually green features can still not add +up to a workflow: + +* a run journalled on one device becomes a candidate script that dry-runs, and + then replays, on another; +* a config sync survives the server's store being reopened, through the same + functions the Config Sync tab calls; +* a capability report says where each of its statements comes from. + +Everything here runs against a fake ``adb`` host, an in-memory wire over a real +SQLite store, and desktops described through ``BackendContext``. That is +evidence about how the pieces fit. It is not evidence about a phone, a +compositor or a network, and the last test holds the reports to saying so. +""" +import json +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +import pytest + +import je_auto_control as ac +from je_auto_control.linux_wayland.authorisation import ( + INPUT, AuthorisationLedger, AuthorisationState, +) +from je_auto_control.utils.config_sync import ( + ConfigBucket, ConfigSyncClient, ConfigSyncConflict, ConfigSyncError, + RevisionConflictError, session as sync_session, +) + +from _mobile_doubles import FakeAdbHost + +_USER = "alice" +_SERVER = "https://sync.invalid" + +_RUN = [ + ["AC_android_tap", {"x": 10, "y": 20}], + ["AC_loop", {"times": 2, "body": [ + ["AC_android_swipe", {"x1": 1, "y1": 2, "x2": 3, "y2": 4}]]}], + ["AC_android_type_text", {"text": "hello"}], +] + + +@pytest.fixture +def adb_host(monkeypatch): + """Two pretend Android devices behind the real ``AdbClient``.""" + host = FakeAdbHost() + host.add("recorded-on") + host.add("replayed-on") + monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", host.run) + monkeypatch.setattr("je_auto_control.android.adb_client.shutil.which", lambda _name: "adb") + return host + + +@pytest.fixture(autouse=True) +def _no_journal_left_running(): + yield + ac.stop_action_journal() + + +# --- journal -> candidate script -> device -------------------------------- + +@dataclass +class ReplayReport: + """What running a candidate did, in the terms its manifest uses.""" + + source_steps: List[str] = field(default_factory=list) + device_effects: List[str] = field(default_factory=list) + + +def _replay(actions: List[Any], host: FakeAdbHost, serial: str, *, dry_run: bool) -> ReplayReport: + report = ReplayReport() + before = len(host.devices[serial].input_calls) + with ac.open_device(ac.DeviceContext("android", serial)) as session, ac.use_device(session): + ac.executor.execute_action( + actions, dry_run=dry_run, raise_on_error=True, + step_callback=lambda action: report.source_steps.append(action[0])) + report.device_effects = host.devices[serial].input_calls[before:] + return report + + +def test_journal_to_script_to_device_result(adb_host, tmp_path): + journal = tmp_path / "run.jsonl" + with ac.open_device(ac.DeviceContext("android", "recorded-on")) as session, \ + ac.use_device(session): + ac.start_action_journal(journal, run_id="checkout", session=session.device_id) + try: + ac.executor.execute_action(_RUN, raise_on_error=True) + finally: + stopped = ac.stop_action_journal() + recorded = list(adb_host.devices["recorded-on"].input_calls) + assert stopped["events"] == 5 and len(recorded) == 4 # the loop body ran twice + + candidate = ac.generate_candidate_from_log(journal, run_id="checkout") + manifest = candidate.manifest + assert manifest["executed"] is False and manifest["outcomes_used_as_input"] is False + assert candidate.observed_path_only is False and candidate.actions == _RUN + assert {event.session for event in ac.read_events(journal, run_id="checkout")} == { + "recorded-on"} + compile(candidate.code, "candidate.py", "exec") + manifest_source_steps = [step["command"] for step in manifest["steps"] if step["emitted"]] + + # A dry run on the second device resolves every step and sends it nothing. + dry = _replay(candidate.actions, adb_host, "replayed-on", dry_run=True) + assert dry.source_steps == manifest_source_steps + assert dry.device_effects == [] + + # Run for real -- still a fake device -- it sends what the first one received. + replay_report = _replay(candidate.actions, adb_host, "replayed-on", dry_run=False) + assert replay_report.source_steps == manifest_source_steps + assert replay_report.device_effects == recorded + # Generating and replaying never reached back to the device it was recorded on. + assert adb_host.devices["recorded-on"].input_calls == recorded + + +def test_each_run_of_a_shared_journal_becomes_its_own_candidate(adb_host, tmp_path): + """One journal file holds many runs; a candidate is built from exactly one of them.""" + journal = tmp_path / "run.jsonl" + with ac.open_device(ac.DeviceContext("android", "recorded-on")) as session, \ + ac.use_device(session): + ac.start_action_journal(journal, run_id="first") + ac.executor.execute_action(_RUN[:1], raise_on_error=True) + ac.stop_action_journal() + ac.start_action_journal(journal, run_id="second") + ac.executor.execute_action(_RUN[2:], raise_on_error=True) + ac.stop_action_journal() + assert [run["run_id"] for run in ac.list_journal_runs(journal)] == ["first", "second"] + assert ac.generate_candidate_from_log(journal, run_id="first").actions == _RUN[:1] + assert ac.generate_candidate_from_log(journal, run_id="second").actions == _RUN[2:] + + +# --- config sync through a store restart ---------------------------------- + +class _Wire: + """``ConfigSyncClient._request`` answered from a ``ConfigStore`` on disk. + + A new store object is built for every request, which is what a restarted + server amounts to: nothing is carried in memory between two requests. + """ + + def __init__(self, store_path: Any) -> None: + self.store_path = store_path + self.requests: List[str] = [] + self.offline = False + + def request(self, client: ConfigSyncClient, method: str, + body: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]: + self.requests.append(method) + if self.offline: + raise ConfigSyncError(f"config sync {method} failed: connection refused") + store = ac.ConfigStore(self.store_path) + if method == "GET": + bucket = store.get(client.user_id) + return None if bucket is None else {**bucket.to_dict(), "version": 2} + assert body is not None and body["version"] == 2, "the client must send a v2 envelope" + try: + revision = store.commit( + client.user_id, ConfigBucket.from_dict(body["bucket"]), + base_revision=body["base_revision"], operation_id=body["operation_id"]) + except RevisionConflictError as conflict: + raise ConfigSyncConflict("behind", conflict.current_revision) from conflict + return {"ok": True, "revision": revision, "version": 2} + + +@pytest.fixture +def wire(tmp_path, monkeypatch): + endpoint = _Wire(tmp_path / "server" / "config_sync.sqlite3") + monkeypatch.setattr( + ConfigSyncClient, "_request", + lambda client, method, body=None: endpoint.request(client, method, body)) + monkeypatch.delenv("AC_SIGNALING_SECRET", raising=False) + return endpoint + + +def _machine(tmp_path, name: str) -> Dict[str, Any]: + scripts = tmp_path / name / "scripts" + scripts.mkdir(parents=True) + return {"device_id": name, "sections": "scripts", "scripts_dir": str(scripts), + "outbox_path": str(tmp_path / name / "outbox.sqlite3")} + + +def test_sync_restart_and_gui_session(wire, tmp_path): + laptop, desktop = _machine(tmp_path, "laptop"), _machine(tmp_path, "desktop") + script = [["AC_set_var", {"name": "user", "value": "alice"}]] + (tmp_path / "laptop" / "scripts" / "login.json").write_text(json.dumps(script), "utf-8") + + first = sync_session.config_sync_run(_SERVER, _USER, **laptop) + assert first["state"] == "synced" and first["pending"] == 0 + committed_revision = ac.ConfigStore(wire.store_path).revision(_USER) + assert committed_revision >= 1 + + # The "restart": a store object that has never seen the file, and a status + # read that touches no network at all. + reopened_store = ac.ConfigStore(wire.store_path) + assert reopened_store.revision(_USER) == committed_revision + requests_before = len(wire.requests) + reopened_sync = sync_session.config_sync_status(_SERVER, _USER, laptop["outbox_path"]) + assert len(wire.requests) == requests_before + assert reopened_sync["revision"] == committed_revision + assert reopened_sync["state"] == "synced" and reopened_sync["pending"] == 0 + + # The other machine gets the script from the reopened store. + second = sync_session.config_sync_run(_SERVER, _USER, **desktop) + assert second["state"] == "synced" + arrived = tmp_path / "desktop" / "scripts" / "login.json" + assert json.loads(arrived.read_text("utf-8")) == script + + # Offline: the change is queued on disk and survives until the server is back. + wire.offline = True + (tmp_path / "laptop" / "scripts" / "report.json").write_text("[]", "utf-8") + offline = sync_session.config_sync_run(_SERVER, _USER, **laptop) + assert offline["state"] == "offline" and offline["pending"] == 1 + assert sync_session.config_sync_status(_SERVER, _USER, laptop["outbox_path"])["pending"] == 1 + wire.offline = False + back = sync_session.config_sync_run(_SERVER, _USER, wait=True, **laptop) + assert back["state"] == "synced" and back["pending"] == 0 + assert ac.ConfigStore(wire.store_path).revision(_USER) > committed_revision + + +def test_the_config_sync_tab_calls_these_same_functions(): + """What this file drives is what the GUI drives -- not a parallel path.""" + pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + from je_auto_control.gui import config_sync_tab + assert config_sync_tab.session is sync_session + source = open(config_sync_tab.__file__, encoding="utf-8").read() + for name in ("config_sync_run", "config_sync_status", "config_sync_resolve", + "config_sync_full_resync"): + assert f"session.{name}(" in source + assert getattr(ac, name) is getattr(sync_session, name) + + +# --- a capability report that states its evidence ------------------------- + +_WAYLAND = {"XDG_SESSION_TYPE": "wayland", "WAYLAND_DISPLAY": "wayland-0", + "XDG_RUNTIME_DIR": "/run/user/1000"} + + +def _desktop(platform: str, environ: Optional[Dict[str, str]] = None, *, tools: tuple = (), + libei: bool = False, ledger: Optional[AuthorisationLedger] = None, + ) -> ac.BackendContext: + return ac.BackendContext( + platform=platform, environ=environ or {}, + which=lambda name: f"/usr/bin/{name}" if name in tools else None, + library_present=lambda name: libei and name in ("ei", "oeffis"), + session_bus_present=lambda: True, readable=lambda _path: True, + compositor=lambda _environ: "8:1:1", + authorisations=ledger if ledger is not None else AuthorisationLedger()) + + +def _desktop_evidence(capability: ac.Capability, platform: str) -> Dict[str, Any]: + """Where one desktop capability's state comes from.""" + if platform.startswith("linux"): + return {"kind": "probed", "reason": "read from the environment, PATH, the loader and " + "the authorisation ledger; no event was sent"} + reason = capability.detail or "the backend has no authorisation to report" + return {"kind": "skipped", "reason": reason} + + +def desktop_report(context: ac.BackendContext) -> Dict[str, Any]: + """A capability report for one described desktop, with its evidence.""" + snapshot = ac.probe_capabilities(context) + return { + "platform": snapshot.platform, "backend": snapshot.display_server, + # CapabilitySnapshot carries no backend version; say so instead of inventing one. + "version": None, "version_reason": "probe_capabilities() reports no backend version", + "transport": "described", "hardware_verified": False, + "capabilities": [{ + "name": capability.name, "state": capability.state.value, + "backend": capability.backend, "usable": capability.usable, + "recovery": capability.recovery, + "evidence": _desktop_evidence(capability, snapshot.platform), + } for capability in snapshot.capabilities], + } + + +def device_report(session: ac.DeviceSession) -> Dict[str, Any]: + """A capability report for one device session, with its evidence.""" + setup = ac.device_setup_report(session) + return { + "platform": setup.platform, "backend": setup.backend, + "version": setup.backend_version, "os_version": setup.os_version, + "transport": "fake", "hardware_verified": False, + "capabilities": [{ + "name": name, "state": capability.state, "backend": setup.backend, + "usable": capability.available, "recovery": capability.reason, + "evidence": ({"kind": "probed", "reason": "reported by the session's setup probe"} + if capability.available else + {"kind": "skipped", "reason": capability.reason}), + } for name, capability in setup.capabilities.items()], + } + + +def _problems(report: Dict[str, Any]) -> List[str]: + """Everything a report states without saying how it knows.""" + label = f"{report['platform']}/{report['backend']}" + problems = [] + if not report["platform"] or not report["backend"]: + problems.append(f"{label}: no platform or backend") + if not report["version"] and not report.get("version_reason"): + problems.append(f"{label}: no version and no reason for its absence") + if report["hardware_verified"]: + problems.append(f"{label}: claims hardware verification from a {report['transport']} run") + for capability in report["capabilities"]: + evidence = capability["evidence"] + where = f"{label}: {capability['name']}" + if evidence["kind"] not in ("probed", "skipped") or not evidence["reason"]: + problems.append(f"{where} has no evidence") + if not capability["backend"]: + problems.append(f"{where} names no backend") + if not capability["usable"] and not capability["recovery"] and evidence["kind"] == "probed": + problems.append(f"{where} is unusable and says neither why nor what to do") + return problems + + +def test_platform_capability_report_has_evidence(adb_host): + refused = AuthorisationLedger() + refused.transition(INPUT, AuthorisationState.DECLINED, "the user dismissed the dialog") + reports = [ + desktop_report(_desktop("win32")), + desktop_report(_desktop("darwin")), + desktop_report(_desktop("linux", {"DISPLAY": ":0"})), + desktop_report(_desktop("linux", _WAYLAND, tools=("grim",), libei=True)), + desktop_report(_desktop("linux", _WAYLAND, tools=("grim", "ydotool"), libei=True, + ledger=refused)), + desktop_report(_desktop("linux", _WAYLAND)), # nothing installed at all + ] + adb_host.add("locked", state="unauthorized") + for serial in ("recorded-on", "locked"): + with ac.open_device(ac.DeviceContext("android", serial)) as session: + before = list(adb_host.devices[serial].input_calls) + reports.append(device_report(session)) + assert adb_host.devices[serial].input_calls == before # a report sends no input + + problems = [problem for report in reports for problem in _problems(report)] + all_verified_capabilities_have_evidence = problems == [] + assert all_verified_capabilities_have_evidence is True, problems + + by_platform = {(report["platform"], report["transport"]): report for report in reports} + # Windows and macOS: reported, and labelled as not probed. + for platform in ("win32", "darwin"): + kinds = {item["evidence"]["kind"] for item in by_platform[(platform, "described")][ + "capabilities"]} + assert kinds == {"skipped"} + assert {item["state"] for item in reports[1]["capabilities"]} == {"unknown"} + # A refused consent is a state with a way out, and ydotool being installed changes nothing. + declined = next(item for item in reports[4]["capabilities"] if item["name"] == "input") + assert (declined["state"], declined["backend"], declined["usable"]) == ( + "needs_permission", "libei", False) + assert "reset_input_authorisation" in declined["recovery"] + # The device reports carry what answered, and the locked one says what to fix. + ready, locked = reports[-2], reports[-1] + assert ready["backend"] == "adb" and ready["version"].startswith("1.0.41") + assert ready["os_version"] == "14" + assert {item["state"] for item in locked["capabilities"]} == {"needs_permission"} + assert all(item["evidence"]["kind"] == "skipped" for item in locked["capabilities"]) + # Nothing in this file touched hardware, and no report says otherwise. + assert not any(report["hardware_verified"] for report in reports) + json.dumps(reports) # a report is plain data + + +def test_a_report_without_evidence_is_caught(): + """The checker above is only worth something if it fails on a bare claim.""" + report = desktop_report(_desktop("linux", _WAYLAND)) + report["capabilities"][0]["evidence"] = {"kind": "probed", "reason": ""} + report["hardware_verified"] = True + found = _problems(report) + assert any("no evidence" in problem for problem in found) + assert any("claims hardware verification" in problem for problem in found) From c685eb3f1208e92d1b90594d84fd8d4c005372bc Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 03:54:15 +0800 Subject: [PATCH 39/44] Record plan H and what writing the examples turned up --- Progress.md | 13 +++++++++++++ docs/updates/2026-10.md | 6 ++++++ docs/updates/README.md | 3 ++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Progress.md b/Progress.md index 6888d2a2e..19d249a0c 100644 --- a/Progress.md +++ b/Progress.md @@ -316,6 +316,19 @@ U-20261009-18 的每一個裝置面呼叫都只對假的 ADB／uiautomator2／WD - **Sphinx 還有 184 個既有警告**（較舊的 `vN_features_doc.rst`、`getting_started/run_in_ci.rst`、`conf.py` 的 `_static` 不存在）；這個月改過的頁面是 0 個。 - **變數範圍在 free-threaded 版**（執行緒繼承 context）沒有測試。 - **Windows 的版面鍵名**（`slash` 等 8 個，依前景視窗的鍵盤配置解析）只在美式配置上實際呼叫過，非美式配置用的是假的 `user32`。 +- **寫範例時發現的缺口**（U-20261009-26）： + - 門面沒匯出使用者需要的名字：`only_run_id`、`write_candidate`、`check_thresholds`、`format_comparison`、`LocateRequest`、`RevisionConflictError`、 + `ConfigStoreError`、`AuthorisationLedger`、`AuthorisationState`、各個具體的 sync adapter、`run_sync`、`DirectoryAssetTransport`、`SCHEMA_VERSION`／`WIRE_VERSION`。 + - `ac.execute_action(actions)` 只收清單；`dry_run`、`raise_on_error`、`step_callback` 只在 `ac.executor.execute_action` 上。 + - 設定同步：送出失敗後的退避期間（2 秒起、加倍、最長 300 秒）內再同步不會連 server，回報 `state="offline"`，剛恢復連線時按「立即同步」看起來像 server 掛了； + `config_sync_status` 的簽章和另外三個不一致；傳 `scripts_dir` 會連本機的熱鍵、觸發器、通訊錄一起同步（只同步腳本要 `sections="scripts"`）； + 裝置第一次同步即使沒有內容也會多一個 revision。 + - `CapabilitySnapshot` 沒有後端版本；win32 上四個能力都是寫死的 `available`、darwin 是寫死的 `unknown`，沒有真的探測。 + - `MOBILE_COMMANDS` 裡 `AC_android_list_devices`、`AC_android_shell`、`AC_android_device_info` 被歸在 `input` 能力下。 + - `mcp_server/audit.py` 的 docstring 說預設寫到 cwd 的 `mcp_audit.jsonl`，建構子在沒設環境變數時看起來是不寫——要對一下。 + - `docs/source/index.rst` 還寫 Wayland 是「via CLI bridges (wtype + ydotool + grim)」。 + - 27 個環境變數只在 Sphinx 的設定頁，不在三份 README（清單在 `test_modernization_examples.py` 的 `README_UNDOCUMENTED`，只准變少）。 + - 計畫 H 裡「裝了 extras 的型別檢查」獨立 job 與 platform-smoke 的補充沒有做。 - **狀態機 `on_enter` 的單一動作寫法**以前一直被拒絕、從沒執行過（已修，U-20261009-24）：依賴「它不會跑」的既有狀態機現在會跑它。 --- diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index bd0e70c05..e8d5a15a1 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -368,3 +368,9 @@ Jeffrey_RPA kept its own `KEY_ALIASES` (40 friendly names) and `_EXTRA_KEY_CODES - **What**: `TranslatableMixin._tr(self, …)` stored the widget in its own registry, so a dropped `UsbPassthroughPromptDialog` waited for the cycle collector; on 3.10 a collection inside `QApplication.topLevelWidgets()` destroys widgets still in the returned list. Reproduced on Windows 3.10.22 with a probe (access violation every time) and gone after the fix. `UsbAcl` changes are serialised per instance (4 threads adding 60 rules kept 43 before). - **Tests**: `test_usb_prompt_lifetime.py` (6), one new ACL test; `PYTHONFAULTHANDLER=1` on the CI test step. - **Open items**: not `#done`: the CI crash itself did not reproduce here; repeated green 3.10 runs on Linux / macOS are the confirmation. + +## U-20261009-26 · 2026-10-09 · Six offline-validated examples, a configuration reference, strict typing for every module added this month, end-to-end tests on fakes · #done #docs #typing #examples #H1 #H2 #H3 + +- **What**: `examples/28`–`33` (Wayland diagnostics, config sync, mobile devices, healing comparison, codegen from a log, MCP progressive), each with `--validate`; `examples/README.md` lists all 33 (it stopped at 18). New Sphinx pages in both languages: `configuration_doc` (all 41 environment variables the package reads; four were in no document before) and `examples_doc`. `docs/CAPABILITY_MATRIX.md` has blocks generated from `probe_capabilities()` and `mobile_capability_matrix()`; `docs/API_LIFECYCLE.md` lists the formats introduced this month. `[[tool.mypy.overrides]]` holds the 83 modules added since v0.0.225 and `je_auto_control_pytest` to `disallow_untyped_defs` + `disallow_any_generics` (57 errors fixed, annotations only); `test/verify/typing_strict_baseline.txt` lists the 1,049 older modules and may only shrink. +- **Tests**: `test_modernization_examples.py` (17: each example runs as a child process with input, capture, process spawn and non-loopback sockets replaced by tripwires), `test_typing_modernization_contract.py` (14), `test_modernization_integration.py` (6). Sphinx: 191 → 181 warnings, 0 in pages changed this month. +- **Open items**: what writing the examples turned up is in `Progress.md` › 雜項 (facade gaps, config-sync back-off reading as offline, capability snapshots without probes on Windows and macOS). 27 environment variables are documented in Sphinx but not in the READMEs. The plan's separate "extras installed" typing job and the platform-smoke additions were not made. diff --git a/docs/updates/README.md b/docs/updates/README.md index 7e2ed2de4..799309515 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261009-26 | 2026-10-09 | Six offline-validated examples, a configuration reference, strict typing for every module added this month, end-to-end tests on fakes | #done #docs #typing #examples #H1 #H2 #H3 | [2026-10](2026-10.md) | | U-20261009-25 | 2026-10-09 | The Python 3.10 headless segfault: a prompt dialog that was its own reference cycle | #bugfix #gui #ci | [2026-10](2026-10.md) | | U-20261009-24 | 2026-10-09 | Follow-ups: USB waits, run-level scopes, load-and-use in one list, layout key names, file splits, Sphinx | #done #followups | [2026-10](2026-10.md) | | U-20261009-23 | 2026-10-09 | RBAC: deferred work runs as its owner, users have a management surface, data tools need read_data | #done #security #rbac | [2026-10](2026-10.md) | @@ -374,7 +375,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-10.md](2026-10.md) | 2026-10 | 38 | +| [2026-10.md](2026-10.md) | 2026-10 | 39 | | [2026-09-e.md](2026-09-e.md) | 2026-09 | 7 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 55 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | From 5627f02286da0952be04cf7a0496a7d6ed0105be Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 04:03:28 +0800 Subject: [PATCH 40/44] Tell the security scanner the journal's masking reason is text, not a credential --- je_auto_control/utils/action_journal/sanitize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/je_auto_control/utils/action_journal/sanitize.py b/je_auto_control/utils/action_journal/sanitize.py index 0a50f9f69..882ea9c03 100644 --- a/je_auto_control/utils/action_journal/sanitize.py +++ b/je_auto_control/utils/action_journal/sanitize.py @@ -21,7 +21,7 @@ from je_auto_control.utils.executor.action_redaction import redact_actions from je_auto_control.utils.script_vars.interpolate import _PLACEHOLDER -REASON_SECRET = "secret masked before the journal was written" +REASON_SECRET = "secret masked before the journal was written" # nosec B105 # reason: explanatory text, not a credential _REASON_TYPE = "not JSON-serialisable ({})" _REASON_FLOAT = "non-finite number ({})" _SCALARS = (str, int, bool, type(None)) From d358f9b68f6d3063c378c009edf7e02d02f7ea1d Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 04:20:25 +0800 Subject: [PATCH 41/44] Make four tests hold on the other runners and settle the analyser findings A deadline compared exactly against 0.1, two waits that were too short for a slow runner, and a GUI fixture that did not skip on an image without libEGL. NaN checks use math.isnan and scale checks math.isclose; the journal's masking reason no longer has a name that reads like a credential. --- architecture_explore.md | 24 +++++++++---------- benchmarks/gui_startup.py | 3 ++- benchmarks/self_healing/run.py | 2 +- benchmarks/self_healing/synthetic.py | 3 ++- examples/30_mobile_devices.py | 3 ++- je_auto_control/gui/tab_registry.py | 2 +- je_auto_control/gui/task_controller.py | 2 +- .../utils/action_journal/sanitize.py | 4 ++-- je_auto_control/utils/action_journal/store.py | 3 ++- .../utils/codegen/journal_import.py | 4 ++-- je_auto_control/utils/config_sync/versions.py | 3 ++- .../utils/self_healing/eval_strategies.py | 3 ++- .../utils/usb/passthrough/_client_requests.py | 3 ++- .../unit_test/headless/test_action_journal.py | 4 ++-- .../headless/test_gui_task_lifecycle.py | 4 +++- .../headless/test_mobile_context_isolation.py | 2 +- .../headless/test_remote_desktop_gui.py | 2 +- .../test_self_healing_eval_surfaces.py | 4 +++- .../headless/test_usb_prompt_lifetime.py | 6 +++++ 19 files changed, 49 insertions(+), 32 deletions(-) diff --git a/architecture_explore.md b/architecture_explore.md index 4cc6fac8d..4e0652f36 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,152 | -| 程式碼總行數 | 179,658 | +| 程式碼總行數 | 179,664 | | `je_auto_control/utils/` 子套件數 | 311 | | `AC_*` 動作指令數（`known_commands()` 實測） | 848 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -272,7 +272,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 25 個套件、約 16,751 行。 +> 25 個套件、約 16,752 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -284,7 +284,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/decision_table/` | 112 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 116 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | | `utils/executor/` | 9,972 | **核心**。`Executor` 指令分派表（848 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | -| `utils/action_journal/` | 718 | 選用的動作日誌：執行邊界記錄 typed 輸入／結果與 parent id，先遮蔽再寫入 JSONL；`codegen.journal_import` 由它產生候選腳本 | +| `utils/action_journal/` | 719 | 選用的動作日誌：執行邊界記錄 typed 輸入／結果與 parent id，先遮蔽再寫入 JSONL；`codegen.journal_import` 由它產生候選腳本 | | `utils/flow_debugger/` | 166 | action list 的單步除錯器與追蹤器 | | `utils/input_macro/` | 462 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | | `utils/json/` | 99 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | @@ -465,7 +465,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.8 元素定位、自我修復與智慧等待 -> 23 個套件、約 5,754 行。 +> 23 個套件、約 5,755 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -488,7 +488,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/observation_delta/` | 122 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | | `utils/screen_state/` | 191 | 語義畫面狀態：快照／差異與結構化畫面描述 | | `utils/scroll_find/` | 103 | 捲動直到目標影像／文字可見 | -| `utils/self_healing/` | 1,669 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | +| `utils/self_healing/` | 1,670 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | | `utils/semantic_recording/` | 498 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | | `utils/settle_detector/` | 79 | 以純函式介面判定 UI 是否已靜止 | | `utils/smart_waits/` | 672 | 智慧等待：以影格差異取代 `time.sleep` | @@ -515,15 +515,15 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 22,996 行。 +> 6 個套件、約 22,998 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/admin/` | 418 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | -| `utils/config_sync/` | 2,910 | 透過訊令伺服器做跨機器設定同步 | +| `utils/config_sync/` | 2,911 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 201 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | | `utils/remote_desktop/` | 13,513 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,946 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/usb/` | 4,947 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 1,008 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 @@ -798,7 +798,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 64 / 127 / 56 / 40 / 28 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | | `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 40 / 29 / 146 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,946 行）與 `utils/usbip/`（1,008 行） +#### `utils/usb/`（4,947 行）與 `utils/usbip/`（1,008 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -1089,7 +1089,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/mcp_server/` | 41 | 20,731 | | `utils/remote_desktop/` | 56 | 13,513 | | `utils/executor/` | 8 | 9,972 | -| `utils/usb/` | 20 | 4,946 | +| `utils/usb/` | 20 | 4,947 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,620 | | `utils/accessibility/` | 14 | 3,143 | | `wrapper/` | 24 | 6,071 | @@ -1105,6 +1105,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 981 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 861 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 714 | 65,451 | -| **總計** | **1,140** | **178,843** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 714 | 65,454 | +| **總計** | **1,140** | **178,847** | diff --git a/benchmarks/gui_startup.py b/benchmarks/gui_startup.py index 915dc60f6..5dda1e384 100644 --- a/benchmarks/gui_startup.py +++ b/benchmarks/gui_startup.py @@ -66,7 +66,8 @@ def _run_child() -> Dict[str, Any]: env.setdefault("QT_QPA_PLATFORM", "offscreen") env[bench.SETTINGS_ENV] = "off" start = time.perf_counter() - done = subprocess.run([sys.executable, os.path.abspath(__file__), "--child"], # nosec B603 # reason: fixed argv + argv = [sys.executable, os.path.abspath(__file__), "--child"] + done = subprocess.run(argv, # nosec B603 # nosemgrep # reason: this file, this interpreter capture_output=True, text=True, env=env, timeout=300, check=False) elapsed = (time.perf_counter() - start) * 1000 if done.returncode != 0: diff --git a/benchmarks/self_healing/run.py b/benchmarks/self_healing/run.py index 371e48751..1b0bb6c31 100644 --- a/benchmarks/self_healing/run.py +++ b/benchmarks/self_healing/run.py @@ -62,7 +62,7 @@ def main(argv: Optional[List[str]] = None) -> int: for violation in payload["violations"]: print(f"THRESHOLD {violation}") if args.json: - Path(args.json).write_text(json.dumps(payload, indent=2, sort_keys=True), + Path(args.json).write_text(json.dumps(payload, indent=2, sort_keys=True), # NOSONAR pythonsecurity:S8707 encoding="utf-8") return 1 if args.check and payload["violations"] else 0 diff --git a/benchmarks/self_healing/synthetic.py b/benchmarks/self_healing/synthetic.py index 4f2e6ba02..0126c44df 100644 --- a/benchmarks/self_healing/synthetic.py +++ b/benchmarks/self_healing/synthetic.py @@ -13,6 +13,7 @@ top-left corner and may be negative (a monitor left of or above the primary). """ import json +import math from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -73,7 +74,7 @@ def _build_sample(entry: Dict[str, Any], template: np.ndarray, for target in entry.get("targets", ()): at_x, at_y = target["at"] element = draw_element(base_width, base_height, target.get("style", STYLE_NORMAL)) - if scale != 1.0: + if not math.isclose(scale, 1.0): # INTER_AREA / INTER_CUBIC on purpose: the strategy under test # resizes with INTER_LINEAR, so a scaled match is close, not exact. element = cv2.resize(element, None, fx=scale, fy=scale, diff --git a/examples/30_mobile_devices.py b/examples/30_mobile_devices.py index 6f4c71887..f69899401 100644 --- a/examples/30_mobile_devices.py +++ b/examples/30_mobile_devices.py @@ -79,7 +79,8 @@ def __init__(self, serial: str) -> None: def run(self, args: Sequence[str], **_options: Any) -> "subprocess.CompletedProcess[bytes]": """Answer one adb invocation from canned replies.""" self.sent.append(" ".join(args)) - return subprocess.CompletedProcess(list(args), 0, self._reply(list(args)), b"") + reply = self._reply(list(args)) # a canned answer; nothing is spawned + return subprocess.CompletedProcess(list(args), 0, reply, b"") # nosemgrep def _reply(self, args: List[str]) -> bytes: if args[:1] == ["devices"]: diff --git a/je_auto_control/gui/tab_registry.py b/je_auto_control/gui/tab_registry.py index 470d72a4f..3fd4ec8b4 100644 --- a/je_auto_control/gui/tab_registry.py +++ b/je_auto_control/gui/tab_registry.py @@ -87,7 +87,7 @@ def release(self) -> bool: dispose = getattr(widget, "dispose", None) try: if callable(dispose): - dispose() + dispose() # pylint: disable=not-callable # reason: guarded by callable() finally: widget.deleteLater() return True diff --git a/je_auto_control/gui/task_controller.py b/je_auto_control/gui/task_controller.py index 0567236df..049732ac3 100644 --- a/je_auto_control/gui/task_controller.py +++ b/je_auto_control/gui/task_controller.py @@ -127,7 +127,7 @@ def report_progress(self, value: object) -> None: if progress is None or self._event.is_set(): return try: - progress(value) + progress(value) # pylint: disable=not-callable # reason: checked for None above except RuntimeError: # reason: the worker object went away with the application pass diff --git a/je_auto_control/utils/action_journal/sanitize.py b/je_auto_control/utils/action_journal/sanitize.py index 882ea9c03..626009185 100644 --- a/je_auto_control/utils/action_journal/sanitize.py +++ b/je_auto_control/utils/action_journal/sanitize.py @@ -21,7 +21,7 @@ from je_auto_control.utils.executor.action_redaction import redact_actions from je_auto_control.utils.script_vars.interpolate import _PLACEHOLDER -REASON_SECRET = "secret masked before the journal was written" # nosec B105 # reason: explanatory text, not a credential +REASON_MASKED = "secret masked before the journal was written" _REASON_TYPE = "not JSON-serialisable ({})" _REASON_FLOAT = "non-finite number ({})" _SCALARS = (str, int, bool, type(None)) @@ -55,7 +55,7 @@ def _is_reference(value: Any) -> bool: def _settle_masked(plain: Any, path: str, notes: Dict[str, str]) -> Any: if _is_reference(plain): return plain - notes[path] = REASON_SECRET + notes[path] = REASON_MASKED return MASK diff --git a/je_auto_control/utils/action_journal/store.py b/je_auto_control/utils/action_journal/store.py index 97c2865ce..a9d81a631 100644 --- a/je_auto_control/utils/action_journal/store.py +++ b/je_auto_control/utils/action_journal/store.py @@ -75,7 +75,8 @@ def _fold(path: Path, run_id: Optional[str]) -> JournalContents: events: Dict[Tuple[str, str], ActionEvent] = {} lines: Dict[str, int] = {} torn: List[int] = [] - with path.open("r", encoding="utf-8", errors="replace") as handle: + # The journal path is the operator's own (start_action_journal / --from-log). + with path.open("r", encoding="utf-8", errors="replace") as handle: # NOSONAR pythonsecurity:S8707 for number, raw in enumerate(handle, start=1): if not raw.strip(): continue diff --git a/je_auto_control/utils/codegen/journal_import.py b/je_auto_control/utils/codegen/journal_import.py index 320c6da1e..07a5bcbf3 100644 --- a/je_auto_control/utils/codegen/journal_import.py +++ b/je_auto_control/utils/codegen/journal_import.py @@ -43,7 +43,7 @@ JournalFormatError, ) from je_auto_control.utils.action_journal.sanitize import ( - REASON_SECRET, replace_unreplayable, + REASON_MASKED, replace_unreplayable, ) from je_auto_control.utils.action_journal.store import ( JournalContents, list_journal_runs, load_journal, @@ -205,7 +205,7 @@ def _unrecoverable(self, event: ActionEvent) -> Optional[str]: return None if name in self._macros else ( f"macro {name!r} is not defined in this run") lost = [path for path, reason in event.unreplayable.items() - if reason != REASON_SECRET] + if reason != REASON_MASKED] if lost: return f"{', '.join(sorted(lost))} could not be stored" return None diff --git a/je_auto_control/utils/config_sync/versions.py b/je_auto_control/utils/config_sync/versions.py index 91b4b5ff1..b6b82b2f6 100644 --- a/je_auto_control/utils/config_sync/versions.py +++ b/je_auto_control/utils/config_sync/versions.py @@ -24,6 +24,7 @@ """ from __future__ import annotations +import math import uuid from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple @@ -168,7 +169,7 @@ def from_dict(cls, key: str, body: Mapping[str, Any], *, _depth: int = 0) -> "Sy def _number(key: str, value: Any) -> float: - if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value: + if isinstance(value, bool) or not isinstance(value, (int, float)) or math.isnan(value): raise ConfigSyncError(f"entry {key!r}: expected a number, got {value!r}") return float(value) diff --git a/je_auto_control/utils/self_healing/eval_strategies.py b/je_auto_control/utils/self_healing/eval_strategies.py index 661a0b76b..90c68c354 100644 --- a/je_auto_control/utils/self_healing/eval_strategies.py +++ b/je_auto_control/utils/self_healing/eval_strategies.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +import math import os from dataclasses import dataclass from pathlib import Path @@ -59,7 +60,7 @@ def _best_match(area: Any, template: Any, scales: Sequence[float], import cv2 best: Tuple[float, Tuple[int, int], Tuple[int, int]] = (-1.0, (0, 0), (0, 0)) for scale in scales: - scaled = template if scale == 1.0 else cv2.resize( + scaled = template if math.isclose(scale, 1.0) else cv2.resize( template, None, fx=scale, fy=scale, interpolation=cv2.INTER_LINEAR) height, width = scaled.shape[:2] if height < 1 or width < 1 or height > area.shape[0] or width > area.shape[1]: diff --git a/je_auto_control/utils/usb/passthrough/_client_requests.py b/je_auto_control/utils/usb/passthrough/_client_requests.py index 6aa60086b..70266001f 100644 --- a/je_auto_control/utils/usb/passthrough/_client_requests.py +++ b/je_auto_control/utils/usb/passthrough/_client_requests.py @@ -7,6 +7,7 @@ from __future__ import annotations import json +import math import threading from dataclasses import dataclass, field from typing import Any, Dict, Optional, Union @@ -83,7 +84,7 @@ def device_timeout_s(body: Dict[str, Any]) -> float: value = body.get("timeout_ms") if isinstance(value, bool) or not isinstance(value, (int, float)): return 0.0 - if value != value or value <= 0: # NaN compares unequal to itself + if math.isnan(value) or value <= 0: return 0.0 return min(float(value), float(MAX_DEVICE_TIMEOUT_MS)) / 1000.0 diff --git a/test/unit_test/headless/test_action_journal.py b/test/unit_test/headless/test_action_journal.py index 3b6e0ea4b..9f08f6d33 100644 --- a/test/unit_test/headless/test_action_journal.py +++ b/test/unit_test/headless/test_action_journal.py @@ -14,7 +14,7 @@ SCHEMA_VERSION, STATUS_ERROR, STATUS_INCOMPLETE, STATUS_OK, ActionEvent, JournalFormatError, ) -from je_auto_control.utils.action_journal.sanitize import REASON_SECRET, sanitise_params +from je_auto_control.utils.action_journal.sanitize import REASON_MASKED, sanitise_params from je_auto_control.utils.action_journal.store import ( ActionJournal, list_journal_runs, load_journal, read_events, ) @@ -91,7 +91,7 @@ def spy(target, line): events = read_events(path) literal, reference = _by_command(events, "AC_fake_step")[:2] assert literal.params == {"user": "ada", "password": "***"} - assert literal.unreplayable == {"params.password": REASON_SECRET} + assert literal.unreplayable == {"params.password": REASON_MASKED} # A reference names the secret without holding it, so it stays replayable. assert reference.params == {"password": "${pw}"} assert reference.unreplayable == {} diff --git a/test/unit_test/headless/test_gui_task_lifecycle.py b/test/unit_test/headless/test_gui_task_lifecycle.py index 867f00a64..d1f62a192 100644 --- a/test/unit_test/headless/test_gui_task_lifecycle.py +++ b/test/unit_test/headless/test_gui_task_lifecycle.py @@ -242,7 +242,9 @@ def work(token): assert _pump(qapp, lambda: bool(errors)) assert isinstance(errors[0], TaskTimeout) and isinstance(errors[0], TimeoutError) assert handle.state == "timed_out" and handle.wait(3.0) - assert network.active == 0 and 0 < deadlines[0] <= 0.1 + # A hair over 0.1 is the clock's rounding (0.10000000000002274 on the + # Windows 3.10 and 3.11 runners), not a longer deadline. + assert network.active == 0 and 0 < deadlines[0] <= 0.1 + 1e-6 _destroy(qapp, owner) diff --git a/test/unit_test/headless/test_mobile_context_isolation.py b/test/unit_test/headless/test_mobile_context_isolation.py index 54fb16f9d..8f0d4658f 100644 --- a/test/unit_test/headless/test_mobile_context_isolation.py +++ b/test/unit_test/headless/test_mobile_context_isolation.py @@ -165,7 +165,7 @@ def test_adb_timeout_is_a_framework_timeout(monkeypatch): import subprocess def slow(cmd, **kwargs): - raise subprocess.TimeoutExpired(cmd, kwargs.get("timeout")) + raise subprocess.TimeoutExpired(cmd, kwargs.get("timeout")) # nosemgrep monkeypatch.setattr("je_auto_control.android.adb_client.subprocess.run", slow) session = open_device(_android("left")) diff --git a/test/unit_test/headless/test_remote_desktop_gui.py b/test/unit_test/headless/test_remote_desktop_gui.py index 944ed0f53..91df64f82 100644 --- a/test/unit_test/headless/test_remote_desktop_gui.py +++ b/test/unit_test/headless/test_remote_desktop_gui.py @@ -52,7 +52,7 @@ def _make_jpeg(width: int = 64, height: int = 48) -> bytes: return buf.getvalue() -def _process_until(app: QApplication, predicate, timeout: float = 3.0, +def _process_until(app: QApplication, predicate, timeout: float = 10.0, interval_ms: int = 20) -> bool: deadline = time.monotonic() + timeout while time.monotonic() < deadline: diff --git a/test/unit_test/headless/test_self_healing_eval_surfaces.py b/test/unit_test/headless/test_self_healing_eval_surfaces.py index a35a26312..3303a9e92 100644 --- a/test/unit_test/headless/test_self_healing_eval_surfaces.py +++ b/test/unit_test/headless/test_self_healing_eval_surfaces.py @@ -158,7 +158,9 @@ def test_context_reaches_the_log_from_executor_and_mcp(tmp_path, monkeypatch): @pytest.fixture def tab(workspace): - pytest.importorskip("PySide6") + # QtWidgets, not the bare package: an image without libEGL imports + # PySide6 and then fails on the widgets module. + pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) from PySide6.QtWidgets import QApplication app = QApplication.instance() or QApplication([]) from je_auto_control.gui.self_healing_tab import SelfHealingTab diff --git a/test/unit_test/headless/test_usb_prompt_lifetime.py b/test/unit_test/headless/test_usb_prompt_lifetime.py index 49bbfede2..6773a0663 100644 --- a/test/unit_test/headless/test_usb_prompt_lifetime.py +++ b/test/unit_test/headless/test_usb_prompt_lifetime.py @@ -145,6 +145,12 @@ def work(worker_index): # No flush of deferred deletes and no collection has run: each dialog went # when its slot returned, on the GUI thread, not later and not elsewhere. assert len(seen) == total + # The last slot can still be unwinding when the last worker is seen dead, + # so the event loop gets a moment; the collector stays off throughout. + settle_until = time.monotonic() + 5.0 + while any(ref() is not None for ref in seen) and time.monotonic() < settle_until: + qapp.processEvents() + time.sleep(0.01) assert [ref for ref in seen if ref() is not None] == [] From 9743a546cc52067231b258a676d8aa39f01a896a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 04:21:58 +0800 Subject: [PATCH 42/44] Count only liboeffis's 'Portal denied ...' as a declined consent The first run against a real liboeffis showed a portal that withholds the EIS descriptor answering 'Error calling ConnectToEIS: Permission denied'. Matching the bare word treated that as the user's refusal and stopped the fallback to ydotool for a failure that is not theirs. --- Progress.md | 14 ++++++++++++-- docker/portal_verify.py | 2 +- je_auto_control/linux_wayland/oeffis.py | 11 +++++++++-- .../headless/test_wayland_capability_states.py | 5 +++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/Progress.md b/Progress.md index 19d249a0c..1ea121678 100644 --- a/Progress.md +++ b/Progress.md @@ -163,8 +163,18 @@ pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version 選用的 libei helper 行程（`JE_AUTOCONTROL_WAYLAND_EI_WORKER=1`，預設關閉）都是在 Windows 上對假物件寫的。要看 CI 的 `portal-verification`、 `eis-verification`、`wayland-verification` 三個 job 的輸出（新增的都是 `info` 行，不改變 job 的結果）： -- **「同意被拒絕就不退回 ydotool」靠 liboeffis 的錯誤文字含 "denied" 來判斷**——沒對真的 liboeffis 驗過；判斷不出來時行為與以前相同（退回 ydotool）。 - 同意對話框放著不回答（逾時）仍會退回 ydotool。 +2026-10-09 在 #510 的 CI 第一次對真的函式庫跑過，結果： + +- **同意被拒絕**：真的 liboeffis 回的是 `Portal denied Start`，規則認得。第一版規則只看有沒有 "denied"，把「portal 不給 EIS descriptor」 + （`Error calling ConnectToEIS: Permission denied`）也當成使用者拒絕；`portal-verification` 的 `*** REVISIT ***` 抓到，已改成只認 `Portal denied …` 開頭。 + 修正後還沒有再跑過那個 job 的結果可看。同意對話框放著不回答（逾時）仍會退回 ydotool。 +- **helper 行程對真的 EIS server 可用**：按鍵、絕對移動、按鈕、捲動都到達，與行程內路徑相同；延遲中位數 0.340 ms（行程內 0.096 ms），p95 0.393 ms。 + 正常關閉時按住的鍵會先放開；helper 被 SIGKILL 時 server 沒收到 key-up，放開交給合成器。 +- **半開交握**：崩潰確定是 `ei_unref`；3 次半開交握行程內漏 9 個 fd，經 helper 漏 3 個，helper 都以 exit status 1 結束、沒有殘留行程。 + 既有的真實函式庫檢查（20/20）在改過的 `oeffis.py`／`libei.py`／`_select_input.py`／`portal.py` 上仍然通過。 + +還沒有實證的： + - 沒有任何腳本從 server 端切斷一個連線中的 client，所以 `EI_EVENT_DISCONNECT` 對應到 `revoked` 沒有實證。 - helper：SIGTERM 時放開按鍵、經 portal（fd 路徑）而非 socket 路徑，沒有 job 會跑到。 - `compositor_identity`、uinput／藍牙裝置在 sysfs 的位置、`struct input_event` 的配置、GlobalShortcuts 的線上格式與哪些桌面有實作、XWayland 的可及範圍： diff --git a/docker/portal_verify.py b/docker/portal_verify.py index 0ec5f55ad..86ccd5c8a 100644 --- a/docker/portal_verify.py +++ b/docker/portal_verify.py @@ -502,7 +502,7 @@ def _measure_classification(label: str, oeffis, expect_declined: bool, """Print how one refusal is classified; never count it. ``_select_input`` stops falling back to ydotool only when the portal - *declined* — recognised from the word "denied" in liboeffis's own error + *declined* — recognised from "Portal denied ..." at the start of liboeffis's own error text, which no fake can vouch for. This prints the real text for every way a request can end, so a wording change shows up here as a banner rather than as a silent change in who is allowed to fall back. diff --git a/je_auto_control/linux_wayland/oeffis.py b/je_auto_control/linux_wayland/oeffis.py index 548bc3896..26dc4226a 100644 --- a/je_auto_control/linux_wayland/oeffis.py +++ b/je_auto_control/linux_wayland/oeffis.py @@ -93,8 +93,15 @@ def __init__(self, message: str, outcome: str, detail: str = "") -> None: @property def declined(self) -> bool: - """Whether the portal answered the request with a refusal.""" - return "denied" in self.detail.lower() + """Whether the portal answered the request with a refusal. + + liboeffis words a dismissed consent as ``Portal denied `` + ("Portal denied Start", measured against the real library). The bare + word is not enough: a portal that withholds the EIS descriptor says + "Error calling ConnectToEIS: Permission denied", which is a failure of + the route, not the user's answer, and has to keep falling back. + """ + return self.detail.strip().lower().startswith("portal denied") def load_symbols() -> Optional[BoundSymbols]: diff --git a/test/unit_test/headless/test_wayland_capability_states.py b/test/unit_test/headless/test_wayland_capability_states.py index fbe3d58bf..7d5266c23 100644 --- a/test/unit_test/headless/test_wayland_capability_states.py +++ b/test/unit_test/headless/test_wayland_capability_states.py @@ -340,9 +340,14 @@ def test_only_a_stated_denial_counts_as_declined(): absent = oeffis_mod.PortalConsentNotGranted( "x", "disconnected", "no such interface") silent = oeffis_mod.PortalConsentNotGranted("x", "timeout") + # What a real liboeffis said when the portal withheld the EIS descriptor + # (portal-verification, 2026-10-09): a failed route, not the user's "no". + withheld = oeffis_mod.PortalConsentNotGranted( + "x", "disconnected", "Error calling ConnectToEIS: Permission denied") assert denied.declined is True assert absent.declined is False assert silent.declined is False + assert withheld.declined is False assert isinstance(denied, oeffis_mod.OeffisUnavailable) From 6a66b4ab26336451f1a41ab7a6cd1dd15b4c92c5 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 04:22:18 +0800 Subject: [PATCH 43/44] Re-measure the map after the oeffis change --- architecture_explore.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/architecture_explore.md b/architecture_explore.md index 4e0652f36..cc6336438 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,152 | -| 程式碼總行數 | 179,664 | +| 程式碼總行數 | 179,671 | | `je_auto_control/utils/` 子套件數 | 311 | | `AC_*` 動作指令數（`known_commands()` 實測） | 848 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -233,7 +233,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 44 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 130 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，23 檔／5,153 行） +#### Linux Wayland（`linux_wayland/`，23 檔／5,160 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -243,7 +243,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `_dbus_client.py` | 24 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | | `_select_input.py` | 224 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | | `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | -| `oeffis.py` | 230 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | +| `oeffis.py` | 237 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | | `libei.py` | 712 | libei 綁定與完整握手（seat 綁定能力 → 由事件取得 device → start_emulating → 每次發送後 frame）。另負責絕對指標的座標空間:讀回裝置的 region,把版面座標映射進去,沒有任何 region 涵蓋就拒絕（libei 對這種移動是靜靜丟掉的）。 | | `mouse.py` | 384 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | | `keyboard.py` | 173 | 鍵盤後端：libei 優先，退回 ydotool／wtype。 | @@ -1097,7 +1097,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/rest_api/` | 8 | 2,078 | | `utils/agent/` | 13 | 2,251 | | `linux_with_x11/` | 19 | 1,281 | -| `linux_wayland/` | 23 | 5,153 | +| `linux_wayland/` | 23 | 5,160 | | `utils/triggers/` | 4 | 1,396 | | `utils/ocr/` | 10 | 1,381 | | `utils/usbip/` | 5 | 1,008 | @@ -1106,5 +1106,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 861 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 714 | 65,454 | -| **總計** | **1,140** | **178,847** | +| **總計** | **1,140** | **178,854** | From e74abc2ca74b906dad600bda345d561c6060cfb7 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 9 Oct 2026 04:49:32 +0800 Subject: [PATCH 44/44] Show the frame that arrives before the remote screen window exists With the connect on a worker, the host's first frame can reach the GUI thread before the connect result does. The viewer tab dropped it, and a host that sends only changed frames never sent another: a still remote screen stayed blank. macOS / Python 3.10 CI hit this on every run. --- architecture_explore.md | 10 +++---- .../gui/remote_desktop/viewer_panel.py | 15 +++++++++- .../headless/test_remote_desktop_gui.py | 30 +++++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/architecture_explore.md b/architecture_explore.md index cc6336438..8888f6a93 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,152 | -| 程式碼總行數 | 179,671 | +| 程式碼總行數 | 179,684 | | `je_auto_control/utils/` 子套件數 | 311 | | `AC_*` 動作指令數（`known_commands()` 實測） | 848 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -963,7 +963,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6），且 | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | -#### 遠端桌面 GUI（`gui/remote_desktop/`，32 檔／7,350 行） +#### 遠端桌面 GUI（`gui/remote_desktop/`，32 檔／7,363 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -972,7 +972,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6），且 | `advanced_group.py` | 92 | 兩個 WebRTC 面板共用的 Advanced STUN／TURN（含選用硬體編碼器）群組，含它寫回面板的 Protocol。 | | `trusted_group.py` | 70 | WebRTC host 面板的信任 viewer 清單群組（移除／清空／匯入／匯出），含它寫回面板的 Protocol。 | | `connection_screen.py` | 749 | Quick Connect —— AnyDesk 風格單畫面入口。 | -| `viewer_panel.py` | 573 | 「控制另一台機器」子分頁。 | +| `viewer_panel.py` | 586 | 「控制另一台機器」子分頁。 | | `webrtc_known_hosts.py` | 346 | TOFU 釘選庫瀏覽器：`KnownHostsDialog` 與帶外釘選用的小表單。由 `webrtc_dialogs` 再匯出。 | | `host_panel.py` | 371 | 「分享這台機器」子分頁。 | | `frame_display.py` | 228 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | @@ -1085,7 +1085,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 119 | 32,109 | +| `gui/` | 119 | 32,122 | | `utils/mcp_server/` | 41 | 20,731 | | `utils/remote_desktop/` | 56 | 13,513 | | `utils/executor/` | 8 | 9,972 | @@ -1106,5 +1106,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 861 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 714 | 65,454 | -| **總計** | **1,140** | **178,854** | +| **總計** | **1,140** | **178,867** | diff --git a/je_auto_control/gui/remote_desktop/viewer_panel.py b/je_auto_control/gui/remote_desktop/viewer_panel.py index 9c7f85259..2f50eaca1 100644 --- a/je_auto_control/gui/remote_desktop/viewer_panel.py +++ b/je_auto_control/gui/remote_desktop/viewer_panel.py @@ -88,6 +88,8 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: # itself stays compact instead of devoting half its height to a # blank frame area. self._screen_window: Optional[RemoteScreenWindow] = None + # The newest frame that arrived before the screen window existed. + self._pending_frame: Optional[QImage] = None self._connect_btn: Optional[QPushButton] = None self._disconnect_btn: Optional[QPushButton] = None self._action_row: Optional[QWidget] = None @@ -290,6 +292,9 @@ def _on_connected(self, viewer) -> None: # operator gets a real workspace and the control panel stays # uncluttered. window = self._ensure_screen_window() + pending, self._pending_frame = self._pending_frame, None + if pending is not None: + window.set_image(pending) window.show() window.raise_() window.activateWindow() @@ -377,6 +382,7 @@ def _ensure_screen_window(self) -> RemoteScreenWindow: def _close_screen_window(self) -> None: window = self._screen_window self._screen_window = None + self._pending_frame = None if window is not None: try: window.closed.disconnect(self._on_screen_window_closed) @@ -406,7 +412,14 @@ def _refresh_status(self) -> None: def _on_frame_main(self, payload: bytes) -> None: image = QImage.fromData(payload, "JPEG") - if image.isNull() or self._screen_window is None: + if image.isNull(): + return + if self._screen_window is None: + # The connect answers on a worker, so the host's first frame can + # be here before _on_connected has opened the window. A host that + # sends only changed frames would never send it again, and a + # still remote screen stayed blank. + self._pending_frame = image return self._screen_window.set_image(image) diff --git a/test/unit_test/headless/test_remote_desktop_gui.py b/test/unit_test/headless/test_remote_desktop_gui.py index 91df64f82..57e4c7017 100644 --- a/test/unit_test/headless/test_remote_desktop_gui.py +++ b/test/unit_test/headless/test_remote_desktop_gui.py @@ -161,3 +161,33 @@ def test_viewer_input_round_trips_to_dispatcher(qapp): registry.disconnect_viewer() host.stop(timeout=1.0) registry._host = None # noqa: SLF001 + + +def test_a_frame_that_beats_the_connect_result_is_shown(qapp): + """The first frame can arrive before the window exists; it must not be lost. + + macOS / Python 3.10 CI lost it every run once the connect moved off the + GUI thread: the fake host sends an unchanging frame, so there was no second. + """ + from je_auto_control.gui.remote_desktop_tab import _ViewerPanel + + panel = _ViewerPanel() + try: + panel._on_frame_main(_make_jpeg()) # noqa: SLF001 # before any window + assert panel._screen_window is None # noqa: SLF001 + assert panel._pending_frame is not None # noqa: SLF001 + shown = [] + window = panel._ensure_screen_window() # noqa: SLF001 + window.show = lambda: shown.append("show") + window.raise_ = lambda: None + window.activateWindow = lambda: None + panel._start_audio_player_if_requested = lambda: None # noqa: SLF001 + import types + fake_viewer = types.SimpleNamespace(connected=True, disconnect=lambda *a, **k: None) + panel._on_connected(fake_viewer) # noqa: SLF001 + assert window.display.has_image() and shown == ["show"] + assert panel._pending_frame is None # noqa: SLF001 + finally: + registry.evict("viewer", by=panel._owner) # noqa: SLF001 + panel._close_screen_window() # noqa: SLF001 + panel.deleteLater()