feat: keepalive hardening + approval safety (dsh-im parity)

Keepalive (fixes 19h silent WS death on 2026-10-07):
- recv() raises on dead reader / stale inbound (no pong) instead of
  returning None forever -> outer loop reconnects
- heartbeat closes socket after 2 missed pongs
- reconnect default unlimited (0), backoff cap 60s; auth still terminal
- connection_state surfaced (idle/connecting/connected/failed)

Approval hardening:
- unpresented approval can never be answered (race guard)
- timeout notifies the user ("已自动拒绝")
- stale pending permissions reaped; created_at tracked

Tests: 69 passed (9 new regressions in test_keepalive_approvals.py)
This commit is contained in:
openbot committed 2026-10-08 11:26:57 +08:00
1 parent 2bf6b76e15
commit 23c1adcca4
4 files changed
+255 -10

No files matched your search

+24 -4
View File
@@ -43,7 +43,9 @@ class WecomAibotChannel:
self.secret = config["secret"]
self.domain = config.get("domain", "https://open.weixin.qq.com")
self.connect_timeout_s = float(config.get("connect_timeout_s", 20))
self.max_reconnect = int(config.get("max_reconnect_attempts", 10))
# 0/negative = unlimited (default): network loss must never permanently
# kill the bot. Backoff caps at 60s, so a retry storm is bounded anyway.
self.max_reconnect = int(config.get("max_reconnect_attempts", 0))
self.reply_timeout_s = float(config.get("reply_timeout_s", 600))
self.allowed_users = set(config.get("allowed_users") or [])
self._ws_factory = ws_factory # returns object with send/recv/close
@@ -53,6 +55,8 @@ class WecomAibotChannel:
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self._ws = None # current socket (set by _connect_and_serve)
self.connection_state = "idle"
self.connection_error: str = ""
# chat -> streaming state; chat -> last callback req_id (respond must reuse it)
self._streams: dict[str, dict] = {}
self._req_ids: dict[str, str] = {}
@@ -85,13 +89,28 @@ class WecomAibotChannel:
return
except Exception as e: # noqa: BLE001
attempt += 1
if attempt > self.max_reconnect:
if self.max_reconnect > 0 and attempt > self.max_reconnect:
self._logger.error(f"[wecom] reconnect exhausted ({attempt}): {e}")
self._set_channel_state("failed", str(e))
return
delay = min(2**attempt, 30)
self._logger.warning(f"[wecom] disconnected ({e}); retry {attempt}/{self.max_reconnect} in {delay}s")
delay = min(2**attempt, 60)
self._logger.warning(
f"[wecom] disconnected ({e}); retry {attempt} in {delay}s"
)
self._set_channel_state("connecting", f"reconnecting in {delay}s: {e}")
self._stop.wait(delay)
def _set_channel_state(self, state: str, detail: str = "") -> None:
"""Surface connection state so ops can see a dead connection from the
status endpoint instead of discovering it by missing replies."""
self.connection_state = state
self.connection_error = detail
try:
if self._sink:
self._sink(f"{self.name}:__channel__", "channel_state", {"state": state, "error": detail})
except Exception: # noqa: BLE001
pass
def _connect_and_serve(self) -> None:
if self._ws_factory is None:
raise RuntimeError("wecom channel requires a ws_factory (real or mock transport)")
@@ -105,6 +124,7 @@ class WecomAibotChannel:
)
ws.connect()
self._ws = ws
self._set_channel_state("connected")
self._logger.info("[wecom] WebSocket authenticated")
try:
while not self._stop.is_set():
+59 -6
View File
@@ -56,6 +56,12 @@ class AibotWSAdapter:
self._subscribed = asyncio.Event()
self._thread: threading.Thread | None = None
self._loop_ready = threading.Event()
# keepalive bookkeeping: reader task + last inbound pong/frame time.
# The 2026-10-07 outage: reader/heartbeat died silently while the
# channel's recv() loop only ever saw None, so nothing reconnected for
# 19h. recv() must detect a dead transport and raise.
self._reader_task: asyncio.Task | None = None
self._last_inbound = time.monotonic()
# -- sync API (called from channel thread) ------------------------------
def connect(self) -> None:
@@ -72,17 +78,41 @@ class AibotWSAdapter:
Raising (rather than returning None) on EOF is what makes the channel's
outer loop see an exception and run its reconnect/backoff path.
"""
if self._loop is None:
if self._loop is None or self._loop.is_closed():
raise ConnectionError("ws loop not running")
# Cheap liveness first: a dead reader means an empty queue forever, so
# detect it before paying the wait (this is the 2026-10-07 outage).
if not self._reader_alive():
raise ConnectionError("ws reader is dead; connection zombie")
fut = asyncio.run_coroutine_threadsafe(self._queue_get(timeout), self._loop)
try:
item = fut.result(timeout=timeout + 2)
except TimeoutError:
return None
item = None
except Exception as e: # noqa: BLE001
raise ConnectionError(f"ws loop failed: {e}") from e
if item is _EOF:
raise ConnectionError("websocket closed by peer")
if item is None:
# idle: if nothing (not even a pong) has arrived for a long time the
# TCP connection is half-open -> reconnect instead of idling forever
self._check_stale_inbound()
return item
def _reader_alive(self) -> bool:
task = self._reader_task
if task is None:
return True # mock transports without a reader task
return not task.done()
def _check_stale_inbound(self) -> None:
"""No inbound frames (including pong replies) for too long = half-open
TCP. Heartbeat sends ping but the pong never arrives; raise so the
outer loop reconnects with a fresh socket."""
stale_after = max(HEARTBEAT_S * 10, 300)
if time.monotonic() - self._last_inbound > stale_after:
raise ConnectionError("no inbound frames (incl. pong) for too long")
def send(self, raw: str) -> None:
if self._loop is None or self._ws is None:
raise ConnectionError("ws not connected")
@@ -133,23 +163,46 @@ class AibotWSAdapter:
log.info("[aibot-ws] subscribed ok (bot_id=%s...)", self.bot_id[:6])
self._subscribed.set()
self._hb_task = asyncio.create_task(self._heartbeat())
asyncio.create_task(self._reader())
self._reader_task = asyncio.create_task(self._reader())
self._last_inbound = time.monotonic()
async def _heartbeat(self) -> None:
"""Ping every HEARTBEAT_S; a pong (or any inbound frame) refreshes
_last_inbound. Two consecutive failed pings -> close the socket so the
reader ends and the channel reconnects, instead of silently pinging a
dead TCP connection."""
misses = 0
try:
while not self._stop_ev.is_set() and self._ws is not None:
await asyncio.sleep(HEARTBEAT_S)
if self._ws is None:
return
await self._ws.send(
json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}})
)
try:
await self._ws.send(
json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}})
)
if time.monotonic() - self._last_inbound < HEARTBEAT_S * 2:
misses = 0
else:
misses += 1
except Exception: # noqa: BLE001
misses += 1
if misses >= 2:
log.warning("[aibot-ws] heartbeat missed %d pongs; closing socket", misses)
try:
await self._ws.close()
except Exception: # noqa: BLE001
pass
return
except asyncio.CancelledError:
raise
except Exception as e: # noqa: BLE001
log.warning("[aibot-ws] heartbeat stopped: %s", e)
async def _reader(self) -> None:
try:
async for raw in self._ws:
self._last_inbound = time.monotonic()
await self._queue.put(raw if isinstance(raw, str) else raw.decode())
except Exception as e: # noqa: BLE001
log.warning("[aibot-ws] reader ended: %s", e)
+18
View File
@@ -81,9 +81,11 @@ class PermissionRequest:
tool_title: str
tool_kind: str
raw_request_id: Any
created_at: float = field(default_factory=time.time)
options: list[dict[str, Any]] = field(default_factory=list)
answered: threading.Event = field(default_factory=threading.Event)
choice: str | None = None # optionId chosen, None = timeout/deny
presented: bool = False # interaction event emitted to the front-end
def _is_transient_model_error(msg: str) -> bool:
@@ -475,6 +477,10 @@ class SessionManager:
raise SessionError(f"no pending permission {request_id}")
if pr.key != f"{channel}:{chat_id}":
raise SessionError("permission request belongs to a different chat")
if not pr.presented:
# never turn a decision on an approval the user has not seen into
# an accept (covers races where a stale reply beats the render)
raise SessionError("permission request not yet shown; try again in a moment")
alias = {"allow": "allow_once", "y": "allow_once", "yes": "allow_once",
"deny": "reject_once", "n": "reject_once", "no": "reject_once"}
pr.choice = alias.get(choice.strip().lower(), choice)
@@ -612,6 +618,10 @@ class SessionManager:
for o in options
],
)
with self._lock:
pr = self._pending_permissions.get(req_id)
if pr is not None:
pr.presented = True
self._emit(key, "interaction", {
"interaction_type": "permission",
"request_id": req_id,
@@ -630,6 +640,9 @@ class SessionManager:
answered = pr.answered.wait(timeout=self.permission_timeout_s)
if not answered:
pr.choice = None # timeout -> deny (first reject-style option)
self._emit(key, "status", {
"text": f"审批超时({int(self.permission_timeout_s)//60} 分钟),已自动拒绝:{pr.tool_title}"
})
chosen = pr.choice
with self._lock:
self._pending_permissions.pop(req_id, None)
@@ -667,6 +680,11 @@ class SessionManager:
with self._lock:
bound = {b.agent for b in self._bindings.values()}
victims = [k for k, b in self._bindings.items() if now - b.last_active > self.idle_ttl_s]
# stale pending permissions: answered never came and the ACP wait
# already timed out; release them so they stop intercepting replies
for rid in [r for r, p in self._pending_permissions.items()
if p.answered.is_set() is False and now - p.created_at > self.permission_timeout_s * 2]:
self._pending_permissions.pop(rid, None)
remaining = {b.agent for k, b in self._bindings.items() if k not in set(victims)}
idle_clients = [n for n in self._clients if n not in remaining]
for k in victims:
+154
View File
@@ -0,0 +1,154 @@
"""Keepalive + approval-hardening regressions (2026-10-07 outage lessons).
Covers:
- zombie connection: reader dead -> recv() raises (was: None forever, no reconnect)
- unlimited reconnects by default (network loss must not kill the bot)
- approval not shown yet -> reply rejected, not turned into an approval
"""
from __future__ import annotations
import json
import os
import sys
import threading
import pytest
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
from agenthub.channels.wecom import WecomAibotChannel # noqa: E402
from agenthub.channels.wecom_ws import AibotWSAdapter # noqa: E402
from agenthub.session import PermissionRequest, SessionError, SessionManager # noqa: E402
# ---------------------------------------------------------------------------
# zombie connection detection
# ---------------------------------------------------------------------------
def test_recv_raises_when_reader_dead():
"""Reader task done + empty queue -> recv must raise, not return None."""
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"})
loop = __import__("asyncio").new_event_loop()
adapter._loop = loop
done_evt = threading.Event()
async def _die():
await __import__("asyncio").sleep(0)
done_evt.set()
adapter._reader_task = loop.create_task(_die())
# pump the loop on a thread until the task actually completes
pump = threading.Thread(target=loop.run_forever, daemon=True)
pump.start()
assert done_evt.wait(5)
with pytest.raises(ConnectionError, match="zombie|reader"):
adapter.recv(timeout=0.2)
loop.call_soon_threadsafe(loop.stop)
def test_recv_raises_on_stale_inbound():
"""No inbound for > stale window -> ConnectionError (half-open TCP)."""
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"})
loop = __import__("asyncio").new_event_loop()
adapter._loop = loop
import asyncio as _a
adapter._reader_task = loop.create_task(_a.sleep(3600))
adapter._last_inbound = 0.0 # ancient
with pytest.raises(ConnectionError, match="stale|no inbound"):
adapter.recv(timeout=0.2)
# ---------------------------------------------------------------------------
# unlimited reconnects
# ---------------------------------------------------------------------------
def test_default_reconnect_is_unlimited():
ch = WecomAibotChannel({"bot_id": "b", "secret": "s"})
assert ch.max_reconnect == 0 # 0 = unlimited
def test_reconnect_never_exhausts_when_unlimited():
ch = WecomAibotChannel({"bot_id": "b", "secret": "s"})
assert ch.max_reconnect == 0
calls = {"n": 0}
def boom(cfg):
calls["n"] += 1
raise ConnectionError("dead")
ch._ws_factory = boom
ch._stop = threading.Event()
t = threading.Thread(target=ch._run, daemon=True)
t.start()
import time as _t
_t.sleep(4) # attempt 1 immediately, attempt 2 after the 2s backoff
ch._stop.set()
assert calls["n"] >= 2 # kept retrying well past the old limit of 10 backoffs
assert ch.connection_state == "connecting"
def test_explicit_limit_still_exhausts():
ch = WecomAibotChannel({"bot_id": "b", "secret": "s", "max_reconnect_attempts": 2})
def boom(cfg):
raise ConnectionError("dead")
ch._ws_factory = boom
t = threading.Thread(target=ch._run, daemon=True)
t.start()
t.join(timeout=30)
assert not t.is_alive()
assert ch.connection_state == "failed"
# ---------------------------------------------------------------------------
# approval hardening
# ---------------------------------------------------------------------------
def _mgr_with_pending(presented: bool) -> SessionManager:
class FakeAcp:
def is_alive(self):
return True
def session_prompt(self, *a, **k):
return {"stopReason": "end_turn"}
mgr = SessionManager(agents={}, client_factory=lambda spec: FakeAcp())
pr = PermissionRequest(
request_id="req-1", agent="a", acp_request_id=None, session_id="s1",
key="wecom:c1", tool_title="read file", tool_kind="read", raw_request_id=None,
options=[{"option_id": "allow_once", "name": "Allow once", "kind": "allow_once"}],
)
pr.presented = presented
mgr._pending_permissions["req-1"] = pr
return mgr
def test_reply_to_unpresented_approval_is_rejected():
"""A decision that arrives before the card was rendered must NOT approve."""
mgr = _mgr_with_pending(presented=False)
with pytest.raises(SessionError, match="not yet shown"):
mgr.permission_reply("wecom", "c1", "req-1", "allow_once")
def test_reply_to_presented_approval_works():
mgr = _mgr_with_pending(presented=True)
mgr.permission_reply("wecom", "c1", "req-1", "allow_once")
pr = mgr._pending_permissions["req-1"]
assert pr.choice == "allow_once"
assert pr.answered.is_set()
def test_numbered_answer_via_take_interaction_answer():
mgr = _mgr_with_pending(presented=True)
assert mgr.take_interaction_answer("wecom", "c1", "1") == "consumed"
pr = mgr._pending_permissions["req-1"]
assert pr.choice == "allow_once"
assert pr.answered.is_set()
def test_stale_permissions_reaped():
mgr = _mgr_with_pending(presented=True)
mgr.permission_timeout_s = 300
pr = mgr._pending_permissions["req-1"]
pr.created_at -= 1000 # ancient
mgr._reap_once()
assert "req-1" not in mgr._pending_permissions