Merge pull request #45 from AgentX-ai/fix/score-groups

fix score correctness with group

Author: RobinXL

agentx/monitor/client.py

@@ -187,10 +187,17 @@ def _api_root(self) -> str:
             return self._base_url[: -len(suffix)]
         return self._base_url
 
-    def _request(self, method: str, path: str, timeout: int = 30, base: Optional[str] = None, **kwargs) -> Any:
+    def _request(
+        self, method: str, path: str, timeout: int = 30, base: Optional[str] = None, retry: bool = True, **kwargs
+    ) -> Any:
+        # retry=False for non-idempotent judge-spending POSTs (sweep, coherence, portability,
+        # tuning): a client-side timeout must not fire the same LLM-billing work a second time
+        # while the first invocation is still running server-side. Same precedent as
+        # EvaluationsClient._request / analyze_run.
         url = f"{base or self._base_url}{path}"
         last_exc: Optional[Exception] = None
-        for attempt, wait in enumerate([0.0] + _RETRY_BACKOFF):
+        schedule = [0.0] + _RETRY_BACKOFF if retry else [0.0]
+        for attempt, wait in enumerate(schedule):
             if wait:
                 time.sleep(wait)
             try:
@@ -204,7 +211,7 @@ def _request(self, method: str, path: str, timeout: int = 30, base: Optional[str
                 raise AgentXAuthError("Invalid or missing API key")
             if resp.status_code == 422:
                 raise AgentXValidationError(resp.text)
-            if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
+            if retry and resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
                 logger.debug(
                     "Retryable status %d (attempt %d)", resp.status_code, attempt + 1
                 )
@@ -421,7 +428,7 @@ def run_session_coherence_check(self, session_id: str) -> dict:
         button. Raises AgentXMonitorError if the engine has no judge key configured."""
         data = self._request(
             "POST", f"/agent-monitoring/sessions/{session_id}/coherence-check",
-            base=self._api_root(), timeout=180,
+            base=self._api_root(), timeout=180, retry=False,
         )
         return data.get("score", data) if isinstance(data, dict) else data
 
@@ -444,7 +451,7 @@ def run_session_sweep(self) -> dict:
         engines run this automatically every minute; the manual trigger exists for demos,
         tests, and backfills. Returns ``{"judged": n}``."""
         return self._request(
-            "POST", "/agent-monitoring/session-sweep/run", base=self._api_root(), timeout=300
+            "POST", "/agent-monitoring/session-sweep/run", base=self._api_root(), timeout=300, retry=False
         )
 
     # ------------------------------------------------------------------
@@ -457,7 +464,7 @@ def run_model_portability(self, trace_id: str, model_ids: List[str]) -> dict:
         plus judging, so expect tens of seconds."""
         return self._request(
             "POST", f"/agent-monitoring/traces/{trace_id}/portability",
-            base=self._api_root(), json={"modelIds": model_ids}, timeout=300,
+            base=self._api_root(), json={"modelIds": model_ids}, timeout=300, retry=False,
         )
 
     # ------------------------------------------------------------------

agentx/monitor/patterns.py

@@ -114,6 +114,10 @@ def builder(
             agent_ids=agent_ids,
         )
 
+    def delete(self, pattern_id: str) -> None:
+        """Delete a pattern. Its historical signals remain as history."""
+        self._client._request("DELETE", f"/agent-monitoring/patterns/{pattern_id}", base=self._client._api_root())
+
     def get(self, pattern_id: str) -> MonitorPattern:
         return self._client.get_pattern(pattern_id)
 

agentx/monitor/scorer_groups.py

@@ -67,8 +67,8 @@ def create(
         """``members``: [{"kind": "judge"|"pattern"|"custom", "refId": ..., "weight": 1, "gate": False}].
         ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"}.
         Add ``"scope": "session", "idleSeconds": 120`` to score whole multi-turn sessions once
-        idle, instead of each sampled trace.
-        or None for offline-only."""
+        idle, instead of each sampled trace. Pass ``online=None`` (the default) for a group
+        that only grades offline dataset runs."""
         payload: Dict[str, Any] = {"name": name, "members": members}
         if description is not None:
             payload["description"] = description

agentx/monitor/sessions.py

@@ -22,8 +22,9 @@ def spans(self, session_id: str) -> List[dict]:
 
     def scores(self, session_id: str) -> List[dict]:
         """Session-level verdicts, newest first. ``kind`` says who scored: a session-scoped
-        online evaluator (``online-eval:<id>``) or a session-scoped scorer group
-        (``scorer-group:<id>``)."""
+        online evaluator (``online-eval:<id>``), a session-scoped scorer group
+        (``scorer-group:<id>``), or legacy ``"coherence"`` rows written before the Session
+        Baseline Judge existed - branch defensively on unknown kinds."""
         return self._client.list_session_scores(session_id)
 
     def run_sweep(self) -> dict:

tests/test_selfhost_analysis_fallback.py

@@ -217,6 +217,12 @@ def test_the_fallback_request_gets_the_long_analysis_timeout():
     assert kwargs["timeout"] > 60, "a synchronous judge pass needs more than the 30s default"
     assert kwargs["json"]["judges"] == [{"model": "gpt-5.6-luna"}]
 
+    # judges=None must OMIT the key - the engine then scores with its platform default model;
+    # injecting a hosted-only default (the old "gpt-5.5") produced uncallable judges.
+    client.analyze_run(RUN)
+    _, _, kwargs = session.calls[-1]
+    assert "judges" not in kwargs["json"]
+
 
 # ---------------------------------------------------------------------------
 # Only a 404 means "wrong engine"