From 23c1adcca447baf6fc3cc2ad89f8cbfb781a866d Mon Sep 17 00:00:00 2001 From: OpenBot Date: Thu, 8 Oct 2026 11:26:57 +0800 Subject: [PATCH] feat: keepalive hardening + approval safety (dsh-im parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keepalive (fixes 19h silent WS death on 2026-10-07): - recv() raises on dead reader / stale inbound (no pong) instead of returning None forever -> outer loop reconnects - heartbeat closes socket after 2 missed pongs - reconnect default unlimited (0), backoff cap 60s; auth still terminal - connection_state surfaced (idle/connecting/connected/failed) Approval hardening: - unpresented approval can never be answered (race guard) - timeout notifies the user ("已自动拒绝") - stale pending permissions reaped; created_at tracked Tests: 69 passed (9 new regressions in test_keepalive_approvals.py) --- src/agenthub/channels/wecom.py | 28 +++++- src/agenthub/channels/wecom_ws.py | 65 +++++++++++-- src/agenthub/session.py | 18 ++++ tests/test_keepalive_approvals.py | 154 ++++++++++++++++++++++++++++++ 4 files changed, 255 insertions(+), 10 deletions(-) create mode 100644 tests/test_keepalive_approvals.py diff --git a/src/agenthub/channels/wecom.py b/src/agenthub/channels/wecom.py index ecff2d3..d323b41 100644 --- a/src/agenthub/channels/wecom.py +++ b/src/agenthub/channels/wecom.py @@ -43,7 +43,9 @@ class WecomAibotChannel: self.secret = config["secret"] self.domain = config.get("domain", "https://open.weixin.qq.com") self.connect_timeout_s = float(config.get("connect_timeout_s", 20)) - self.max_reconnect = int(config.get("max_reconnect_attempts", 10)) + # 0/negative = unlimited (default): network loss must never permanently + # kill the bot. Backoff caps at 60s, so a retry storm is bounded anyway. + self.max_reconnect = int(config.get("max_reconnect_attempts", 0)) self.reply_timeout_s = float(config.get("reply_timeout_s", 600)) self.allowed_users = set(config.get("allowed_users") or []) self._ws_factory = ws_factory # returns object with send/recv/close @@ -53,6 +55,8 @@ class WecomAibotChannel: self._stop = threading.Event() self._thread: threading.Thread | None = None self._ws = None # current socket (set by _connect_and_serve) + self.connection_state = "idle" + self.connection_error: str = "" # chat -> streaming state; chat -> last callback req_id (respond must reuse it) self._streams: dict[str, dict] = {} self._req_ids: dict[str, str] = {} @@ -85,13 +89,28 @@ class WecomAibotChannel: return except Exception as e: # noqa: BLE001 attempt += 1 - if attempt > self.max_reconnect: + if self.max_reconnect > 0 and attempt > self.max_reconnect: self._logger.error(f"[wecom] reconnect exhausted ({attempt}): {e}") + self._set_channel_state("failed", str(e)) return - delay = min(2**attempt, 30) - self._logger.warning(f"[wecom] disconnected ({e}); retry {attempt}/{self.max_reconnect} in {delay}s") + delay = min(2**attempt, 60) + self._logger.warning( + f"[wecom] disconnected ({e}); retry {attempt} in {delay}s" + ) + self._set_channel_state("connecting", f"reconnecting in {delay}s: {e}") self._stop.wait(delay) + def _set_channel_state(self, state: str, detail: str = "") -> None: + """Surface connection state so ops can see a dead connection from the + status endpoint instead of discovering it by missing replies.""" + self.connection_state = state + self.connection_error = detail + try: + if self._sink: + self._sink(f"{self.name}:__channel__", "channel_state", {"state": state, "error": detail}) + except Exception: # noqa: BLE001 + pass + def _connect_and_serve(self) -> None: if self._ws_factory is None: raise RuntimeError("wecom channel requires a ws_factory (real or mock transport)") @@ -105,6 +124,7 @@ class WecomAibotChannel: ) ws.connect() self._ws = ws + self._set_channel_state("connected") self._logger.info("[wecom] WebSocket authenticated") try: while not self._stop.is_set(): diff --git a/src/agenthub/channels/wecom_ws.py b/src/agenthub/channels/wecom_ws.py index 3fb34d0..cabf2a6 100644 --- a/src/agenthub/channels/wecom_ws.py +++ b/src/agenthub/channels/wecom_ws.py @@ -56,6 +56,12 @@ class AibotWSAdapter: self._subscribed = asyncio.Event() self._thread: threading.Thread | None = None self._loop_ready = threading.Event() + # keepalive bookkeeping: reader task + last inbound pong/frame time. + # The 2026-10-07 outage: reader/heartbeat died silently while the + # channel's recv() loop only ever saw None, so nothing reconnected for + # 19h. recv() must detect a dead transport and raise. + self._reader_task: asyncio.Task | None = None + self._last_inbound = time.monotonic() # -- sync API (called from channel thread) ------------------------------ def connect(self) -> None: @@ -72,17 +78,41 @@ class AibotWSAdapter: Raising (rather than returning None) on EOF is what makes the channel's outer loop see an exception and run its reconnect/backoff path. """ - if self._loop is None: + if self._loop is None or self._loop.is_closed(): raise ConnectionError("ws loop not running") + # Cheap liveness first: a dead reader means an empty queue forever, so + # detect it before paying the wait (this is the 2026-10-07 outage). + if not self._reader_alive(): + raise ConnectionError("ws reader is dead; connection zombie") fut = asyncio.run_coroutine_threadsafe(self._queue_get(timeout), self._loop) try: item = fut.result(timeout=timeout + 2) except TimeoutError: - return None + item = None + except Exception as e: # noqa: BLE001 + raise ConnectionError(f"ws loop failed: {e}") from e if item is _EOF: raise ConnectionError("websocket closed by peer") + if item is None: + # idle: if nothing (not even a pong) has arrived for a long time the + # TCP connection is half-open -> reconnect instead of idling forever + self._check_stale_inbound() return item + def _reader_alive(self) -> bool: + task = self._reader_task + if task is None: + return True # mock transports without a reader task + return not task.done() + + def _check_stale_inbound(self) -> None: + """No inbound frames (including pong replies) for too long = half-open + TCP. Heartbeat sends ping but the pong never arrives; raise so the + outer loop reconnects with a fresh socket.""" + stale_after = max(HEARTBEAT_S * 10, 300) + if time.monotonic() - self._last_inbound > stale_after: + raise ConnectionError("no inbound frames (incl. pong) for too long") + def send(self, raw: str) -> None: if self._loop is None or self._ws is None: raise ConnectionError("ws not connected") @@ -133,23 +163,46 @@ class AibotWSAdapter: log.info("[aibot-ws] subscribed ok (bot_id=%s...)", self.bot_id[:6]) self._subscribed.set() self._hb_task = asyncio.create_task(self._heartbeat()) - asyncio.create_task(self._reader()) + self._reader_task = asyncio.create_task(self._reader()) + self._last_inbound = time.monotonic() async def _heartbeat(self) -> None: + """Ping every HEARTBEAT_S; a pong (or any inbound frame) refreshes + _last_inbound. Two consecutive failed pings -> close the socket so the + reader ends and the channel reconnects, instead of silently pinging a + dead TCP connection.""" + misses = 0 try: while not self._stop_ev.is_set() and self._ws is not None: await asyncio.sleep(HEARTBEAT_S) if self._ws is None: return - await self._ws.send( - json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}}) - ) + try: + await self._ws.send( + json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}}) + ) + if time.monotonic() - self._last_inbound < HEARTBEAT_S * 2: + misses = 0 + else: + misses += 1 + except Exception: # noqa: BLE001 + misses += 1 + if misses >= 2: + log.warning("[aibot-ws] heartbeat missed %d pongs; closing socket", misses) + try: + await self._ws.close() + except Exception: # noqa: BLE001 + pass + return + except asyncio.CancelledError: + raise except Exception as e: # noqa: BLE001 log.warning("[aibot-ws] heartbeat stopped: %s", e) async def _reader(self) -> None: try: async for raw in self._ws: + self._last_inbound = time.monotonic() await self._queue.put(raw if isinstance(raw, str) else raw.decode()) except Exception as e: # noqa: BLE001 log.warning("[aibot-ws] reader ended: %s", e) diff --git a/src/agenthub/session.py b/src/agenthub/session.py index a1dabb7..1792bdc 100644 --- a/src/agenthub/session.py +++ b/src/agenthub/session.py @@ -81,9 +81,11 @@ class PermissionRequest: tool_title: str tool_kind: str raw_request_id: Any + created_at: float = field(default_factory=time.time) options: list[dict[str, Any]] = field(default_factory=list) answered: threading.Event = field(default_factory=threading.Event) choice: str | None = None # optionId chosen, None = timeout/deny + presented: bool = False # interaction event emitted to the front-end def _is_transient_model_error(msg: str) -> bool: @@ -475,6 +477,10 @@ class SessionManager: raise SessionError(f"no pending permission {request_id}") if pr.key != f"{channel}:{chat_id}": raise SessionError("permission request belongs to a different chat") + if not pr.presented: + # never turn a decision on an approval the user has not seen into + # an accept (covers races where a stale reply beats the render) + raise SessionError("permission request not yet shown; try again in a moment") alias = {"allow": "allow_once", "y": "allow_once", "yes": "allow_once", "deny": "reject_once", "n": "reject_once", "no": "reject_once"} pr.choice = alias.get(choice.strip().lower(), choice) @@ -612,6 +618,10 @@ class SessionManager: for o in options ], ) + with self._lock: + pr = self._pending_permissions.get(req_id) + if pr is not None: + pr.presented = True self._emit(key, "interaction", { "interaction_type": "permission", "request_id": req_id, @@ -630,6 +640,9 @@ class SessionManager: answered = pr.answered.wait(timeout=self.permission_timeout_s) if not answered: pr.choice = None # timeout -> deny (first reject-style option) + self._emit(key, "status", { + "text": f"审批超时({int(self.permission_timeout_s)//60} 分钟),已自动拒绝:{pr.tool_title}" + }) chosen = pr.choice with self._lock: self._pending_permissions.pop(req_id, None) @@ -667,6 +680,11 @@ class SessionManager: with self._lock: bound = {b.agent for b in self._bindings.values()} victims = [k for k, b in self._bindings.items() if now - b.last_active > self.idle_ttl_s] + # stale pending permissions: answered never came and the ACP wait + # already timed out; release them so they stop intercepting replies + for rid in [r for r, p in self._pending_permissions.items() + if p.answered.is_set() is False and now - p.created_at > self.permission_timeout_s * 2]: + self._pending_permissions.pop(rid, None) remaining = {b.agent for k, b in self._bindings.items() if k not in set(victims)} idle_clients = [n for n in self._clients if n not in remaining] for k in victims: diff --git a/tests/test_keepalive_approvals.py b/tests/test_keepalive_approvals.py new file mode 100644 index 0000000..bdc80cf --- /dev/null +++ b/tests/test_keepalive_approvals.py @@ -0,0 +1,154 @@ +"""Keepalive + approval-hardening regressions (2026-10-07 outage lessons). + +Covers: +- zombie connection: reader dead -> recv() raises (was: None forever, no reconnect) +- unlimited reconnects by default (network loss must not kill the bot) +- approval not shown yet -> reply rejected, not turned into an approval +""" + +from __future__ import annotations + +import json +import os +import sys +import threading + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from agenthub.channels.wecom import WecomAibotChannel # noqa: E402 +from agenthub.channels.wecom_ws import AibotWSAdapter # noqa: E402 +from agenthub.session import PermissionRequest, SessionError, SessionManager # noqa: E402 + + +# --------------------------------------------------------------------------- +# zombie connection detection +# --------------------------------------------------------------------------- + +def test_recv_raises_when_reader_dead(): + """Reader task done + empty queue -> recv must raise, not return None.""" + adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"}) + loop = __import__("asyncio").new_event_loop() + adapter._loop = loop + done_evt = threading.Event() + + async def _die(): + await __import__("asyncio").sleep(0) + done_evt.set() + + adapter._reader_task = loop.create_task(_die()) + # pump the loop on a thread until the task actually completes + pump = threading.Thread(target=loop.run_forever, daemon=True) + pump.start() + assert done_evt.wait(5) + with pytest.raises(ConnectionError, match="zombie|reader"): + adapter.recv(timeout=0.2) + loop.call_soon_threadsafe(loop.stop) + + +def test_recv_raises_on_stale_inbound(): + """No inbound for > stale window -> ConnectionError (half-open TCP).""" + adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"}) + loop = __import__("asyncio").new_event_loop() + adapter._loop = loop + import asyncio as _a + adapter._reader_task = loop.create_task(_a.sleep(3600)) + adapter._last_inbound = 0.0 # ancient + with pytest.raises(ConnectionError, match="stale|no inbound"): + adapter.recv(timeout=0.2) + + +# --------------------------------------------------------------------------- +# unlimited reconnects +# --------------------------------------------------------------------------- + +def test_default_reconnect_is_unlimited(): + ch = WecomAibotChannel({"bot_id": "b", "secret": "s"}) + assert ch.max_reconnect == 0 # 0 = unlimited + + +def test_reconnect_never_exhausts_when_unlimited(): + ch = WecomAibotChannel({"bot_id": "b", "secret": "s"}) + assert ch.max_reconnect == 0 + + calls = {"n": 0} + + def boom(cfg): + calls["n"] += 1 + raise ConnectionError("dead") + + ch._ws_factory = boom + ch._stop = threading.Event() + t = threading.Thread(target=ch._run, daemon=True) + t.start() + import time as _t + _t.sleep(4) # attempt 1 immediately, attempt 2 after the 2s backoff + ch._stop.set() + assert calls["n"] >= 2 # kept retrying well past the old limit of 10 backoffs + assert ch.connection_state == "connecting" + + +def test_explicit_limit_still_exhausts(): + ch = WecomAibotChannel({"bot_id": "b", "secret": "s", "max_reconnect_attempts": 2}) + def boom(cfg): + raise ConnectionError("dead") + ch._ws_factory = boom + t = threading.Thread(target=ch._run, daemon=True) + t.start() + t.join(timeout=30) + assert not t.is_alive() + assert ch.connection_state == "failed" + + +# --------------------------------------------------------------------------- +# approval hardening +# --------------------------------------------------------------------------- + +def _mgr_with_pending(presented: bool) -> SessionManager: + class FakeAcp: + def is_alive(self): + return True + def session_prompt(self, *a, **k): + return {"stopReason": "end_turn"} + mgr = SessionManager(agents={}, client_factory=lambda spec: FakeAcp()) + pr = PermissionRequest( + request_id="req-1", agent="a", acp_request_id=None, session_id="s1", + key="wecom:c1", tool_title="read file", tool_kind="read", raw_request_id=None, + options=[{"option_id": "allow_once", "name": "Allow once", "kind": "allow_once"}], + ) + pr.presented = presented + mgr._pending_permissions["req-1"] = pr + return mgr + + +def test_reply_to_unpresented_approval_is_rejected(): + """A decision that arrives before the card was rendered must NOT approve.""" + mgr = _mgr_with_pending(presented=False) + with pytest.raises(SessionError, match="not yet shown"): + mgr.permission_reply("wecom", "c1", "req-1", "allow_once") + + +def test_reply_to_presented_approval_works(): + mgr = _mgr_with_pending(presented=True) + mgr.permission_reply("wecom", "c1", "req-1", "allow_once") + pr = mgr._pending_permissions["req-1"] + assert pr.choice == "allow_once" + assert pr.answered.is_set() + + +def test_numbered_answer_via_take_interaction_answer(): + mgr = _mgr_with_pending(presented=True) + assert mgr.take_interaction_answer("wecom", "c1", "1") == "consumed" + pr = mgr._pending_permissions["req-1"] + assert pr.choice == "allow_once" + assert pr.answered.is_set() + + +def test_stale_permissions_reaped(): + mgr = _mgr_with_pending(presented=True) + mgr.permission_timeout_s = 300 + pr = mgr._pending_permissions["req-1"] + pr.created_at -= 1000 # ancient + mgr._reap_once() + assert "req-1" not in mgr._pending_permissions