feat: keepalive hardening + approval safety (dsh-im parity)

Keepalive (fixes 19h silent WS death on 2026-10-07):
- recv() raises on dead reader / stale inbound (no pong) instead of
  returning None forever -> outer loop reconnects
- heartbeat closes socket after 2 missed pongs
- reconnect default unlimited (0), backoff cap 60s; auth still terminal
- connection_state surfaced (idle/connecting/connected/failed)

Approval hardening:
- unpresented approval can never be answered (race guard)
- timeout notifies the user ("已自动拒绝")
- stale pending permissions reaped; created_at tracked

Tests: 69 passed (9 new regressions in test_keepalive_approvals.py)
This commit is contained in:
openbot committed 2026-10-08 11:26:57 +08:00
1 parent 2bf6b76e15
commit 23c1adcca4
4 files changed
+255 -10

No files matched your search

+24 -4
View File
@@ -43,7 +43,9 @@ class WecomAibotChannel:
self.secret = config["secret"] self.secret = config["secret"]
self.domain = config.get("domain", "https://open.weixin.qq.com") self.domain = config.get("domain", "https://open.weixin.qq.com")
self.connect_timeout_s = float(config.get("connect_timeout_s", 20)) self.connect_timeout_s = float(config.get("connect_timeout_s", 20))
self.max_reconnect = int(config.get("max_reconnect_attempts", 10)) # 0/negative = unlimited (default): network loss must never permanently
# kill the bot. Backoff caps at 60s, so a retry storm is bounded anyway.
self.max_reconnect = int(config.get("max_reconnect_attempts", 0))
self.reply_timeout_s = float(config.get("reply_timeout_s", 600)) self.reply_timeout_s = float(config.get("reply_timeout_s", 600))
self.allowed_users = set(config.get("allowed_users") or []) self.allowed_users = set(config.get("allowed_users") or [])
self._ws_factory = ws_factory # returns object with send/recv/close self._ws_factory = ws_factory # returns object with send/recv/close
@@ -53,6 +55,8 @@ class WecomAibotChannel:
self._stop = threading.Event() self._stop = threading.Event()
self._thread: threading.Thread | None = None self._thread: threading.Thread | None = None
self._ws = None # current socket (set by _connect_and_serve) self._ws = None # current socket (set by _connect_and_serve)
self.connection_state = "idle"
self.connection_error: str = ""
# chat -> streaming state; chat -> last callback req_id (respond must reuse it) # chat -> streaming state; chat -> last callback req_id (respond must reuse it)
self._streams: dict[str, dict] = {} self._streams: dict[str, dict] = {}
self._req_ids: dict[str, str] = {} self._req_ids: dict[str, str] = {}
@@ -85,13 +89,28 @@ class WecomAibotChannel:
return return
except Exception as e: # noqa: BLE001 except Exception as e: # noqa: BLE001
attempt += 1 attempt += 1
if attempt > self.max_reconnect: if self.max_reconnect > 0 and attempt > self.max_reconnect:
self._logger.error(f"[wecom] reconnect exhausted ({attempt}): {e}") self._logger.error(f"[wecom] reconnect exhausted ({attempt}): {e}")
self._set_channel_state("failed", str(e))
return return
delay = min(2**attempt, 30) delay = min(2**attempt, 60)
self._logger.warning(f"[wecom] disconnected ({e}); retry {attempt}/{self.max_reconnect} in {delay}s") self._logger.warning(
f"[wecom] disconnected ({e}); retry {attempt} in {delay}s"
)
self._set_channel_state("connecting", f"reconnecting in {delay}s: {e}")
self._stop.wait(delay) self._stop.wait(delay)
def _set_channel_state(self, state: str, detail: str = "") -> None:
"""Surface connection state so ops can see a dead connection from the
status endpoint instead of discovering it by missing replies."""
self.connection_state = state
self.connection_error = detail
try:
if self._sink:
self._sink(f"{self.name}:__channel__", "channel_state", {"state": state, "error": detail})
except Exception: # noqa: BLE001
pass
def _connect_and_serve(self) -> None: def _connect_and_serve(self) -> None:
if self._ws_factory is None: if self._ws_factory is None:
raise RuntimeError("wecom channel requires a ws_factory (real or mock transport)") raise RuntimeError("wecom channel requires a ws_factory (real or mock transport)")
@@ -105,6 +124,7 @@ class WecomAibotChannel:
) )
ws.connect() ws.connect()
self._ws = ws self._ws = ws
self._set_channel_state("connected")
self._logger.info("[wecom] WebSocket authenticated") self._logger.info("[wecom] WebSocket authenticated")
try: try:
while not self._stop.is_set(): while not self._stop.is_set():
+59 -6
View File
@@ -56,6 +56,12 @@ class AibotWSAdapter:
self._subscribed = asyncio.Event() self._subscribed = asyncio.Event()
self._thread: threading.Thread | None = None self._thread: threading.Thread | None = None
self._loop_ready = threading.Event() self._loop_ready = threading.Event()
# keepalive bookkeeping: reader task + last inbound pong/frame time.
# The 2026-10-07 outage: reader/heartbeat died silently while the
# channel's recv() loop only ever saw None, so nothing reconnected for
# 19h. recv() must detect a dead transport and raise.
self._reader_task: asyncio.Task | None = None
self._last_inbound = time.monotonic()
# -- sync API (called from channel thread) ------------------------------ # -- sync API (called from channel thread) ------------------------------
def connect(self) -> None: def connect(self) -> None:
@@ -72,17 +78,41 @@ class AibotWSAdapter:
Raising (rather than returning None) on EOF is what makes the channel's Raising (rather than returning None) on EOF is what makes the channel's
outer loop see an exception and run its reconnect/backoff path. outer loop see an exception and run its reconnect/backoff path.
""" """
if self._loop is None: if self._loop is None or self._loop.is_closed():
raise ConnectionError("ws loop not running") raise ConnectionError("ws loop not running")
# Cheap liveness first: a dead reader means an empty queue forever, so
# detect it before paying the wait (this is the 2026-10-07 outage).
if not self._reader_alive():
raise ConnectionError("ws reader is dead; connection zombie")
fut = asyncio.run_coroutine_threadsafe(self._queue_get(timeout), self._loop) fut = asyncio.run_coroutine_threadsafe(self._queue_get(timeout), self._loop)
try: try:
item = fut.result(timeout=timeout + 2) item = fut.result(timeout=timeout + 2)
except TimeoutError: except TimeoutError:
return None item = None
except Exception as e: # noqa: BLE001
raise ConnectionError(f"ws loop failed: {e}") from e
if item is _EOF: if item is _EOF:
raise ConnectionError("websocket closed by peer") raise ConnectionError("websocket closed by peer")
if item is None:
# idle: if nothing (not even a pong) has arrived for a long time the
# TCP connection is half-open -> reconnect instead of idling forever
self._check_stale_inbound()
return item return item
def _reader_alive(self) -> bool:
task = self._reader_task
if task is None:
return True # mock transports without a reader task
return not task.done()
def _check_stale_inbound(self) -> None:
"""No inbound frames (including pong replies) for too long = half-open
TCP. Heartbeat sends ping but the pong never arrives; raise so the
outer loop reconnects with a fresh socket."""
stale_after = max(HEARTBEAT_S * 10, 300)
if time.monotonic() - self._last_inbound > stale_after:
raise ConnectionError("no inbound frames (incl. pong) for too long")
def send(self, raw: str) -> None: def send(self, raw: str) -> None:
if self._loop is None or self._ws is None: if self._loop is None or self._ws is None:
raise ConnectionError("ws not connected") raise ConnectionError("ws not connected")
@@ -133,23 +163,46 @@ class AibotWSAdapter:
log.info("[aibot-ws] subscribed ok (bot_id=%s...)", self.bot_id[:6]) log.info("[aibot-ws] subscribed ok (bot_id=%s...)", self.bot_id[:6])
self._subscribed.set() self._subscribed.set()
self._hb_task = asyncio.create_task(self._heartbeat()) self._hb_task = asyncio.create_task(self._heartbeat())
asyncio.create_task(self._reader()) self._reader_task = asyncio.create_task(self._reader())
self._last_inbound = time.monotonic()
async def _heartbeat(self) -> None: async def _heartbeat(self) -> None:
"""Ping every HEARTBEAT_S; a pong (or any inbound frame) refreshes
_last_inbound. Two consecutive failed pings -> close the socket so the
reader ends and the channel reconnects, instead of silently pinging a
dead TCP connection."""
misses = 0
try: try:
while not self._stop_ev.is_set() and self._ws is not None: while not self._stop_ev.is_set() and self._ws is not None:
await asyncio.sleep(HEARTBEAT_S) await asyncio.sleep(HEARTBEAT_S)
if self._ws is None: if self._ws is None:
return return
await self._ws.send( try:
json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}}) await self._ws.send(
) json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}})
)
if time.monotonic() - self._last_inbound < HEARTBEAT_S * 2:
misses = 0
else:
misses += 1
except Exception: # noqa: BLE001
misses += 1
if misses >= 2:
log.warning("[aibot-ws] heartbeat missed %d pongs; closing socket", misses)
try:
await self._ws.close()
except Exception: # noqa: BLE001
pass
return
except asyncio.CancelledError:
raise
except Exception as e: # noqa: BLE001 except Exception as e: # noqa: BLE001
log.warning("[aibot-ws] heartbeat stopped: %s", e) log.warning("[aibot-ws] heartbeat stopped: %s", e)
async def _reader(self) -> None: async def _reader(self) -> None:
try: try:
async for raw in self._ws: async for raw in self._ws:
self._last_inbound = time.monotonic()
await self._queue.put(raw if isinstance(raw, str) else raw.decode()) await self._queue.put(raw if isinstance(raw, str) else raw.decode())
except Exception as e: # noqa: BLE001 except Exception as e: # noqa: BLE001
log.warning("[aibot-ws] reader ended: %s", e) log.warning("[aibot-ws] reader ended: %s", e)
+18
View File
@@ -81,9 +81,11 @@ class PermissionRequest:
tool_title: str tool_title: str
tool_kind: str tool_kind: str
raw_request_id: Any raw_request_id: Any
created_at: float = field(default_factory=time.time)
options: list[dict[str, Any]] = field(default_factory=list) options: list[dict[str, Any]] = field(default_factory=list)
answered: threading.Event = field(default_factory=threading.Event) answered: threading.Event = field(default_factory=threading.Event)
choice: str | None = None # optionId chosen, None = timeout/deny choice: str | None = None # optionId chosen, None = timeout/deny
presented: bool = False # interaction event emitted to the front-end
def _is_transient_model_error(msg: str) -> bool: def _is_transient_model_error(msg: str) -> bool:
@@ -475,6 +477,10 @@ class SessionManager:
raise SessionError(f"no pending permission {request_id}") raise SessionError(f"no pending permission {request_id}")
if pr.key != f"{channel}:{chat_id}": if pr.key != f"{channel}:{chat_id}":
raise SessionError("permission request belongs to a different chat") raise SessionError("permission request belongs to a different chat")
if not pr.presented:
# never turn a decision on an approval the user has not seen into
# an accept (covers races where a stale reply beats the render)
raise SessionError("permission request not yet shown; try again in a moment")
alias = {"allow": "allow_once", "y": "allow_once", "yes": "allow_once", alias = {"allow": "allow_once", "y": "allow_once", "yes": "allow_once",
"deny": "reject_once", "n": "reject_once", "no": "reject_once"} "deny": "reject_once", "n": "reject_once", "no": "reject_once"}
pr.choice = alias.get(choice.strip().lower(), choice) pr.choice = alias.get(choice.strip().lower(), choice)
@@ -612,6 +618,10 @@ class SessionManager:
for o in options for o in options
], ],
) )
with self._lock:
pr = self._pending_permissions.get(req_id)
if pr is not None:
pr.presented = True
self._emit(key, "interaction", { self._emit(key, "interaction", {
"interaction_type": "permission", "interaction_type": "permission",
"request_id": req_id, "request_id": req_id,
@@ -630,6 +640,9 @@ class SessionManager:
answered = pr.answered.wait(timeout=self.permission_timeout_s) answered = pr.answered.wait(timeout=self.permission_timeout_s)
if not answered: if not answered:
pr.choice = None # timeout -> deny (first reject-style option) pr.choice = None # timeout -> deny (first reject-style option)
self._emit(key, "status", {
"text": f"审批超时({int(self.permission_timeout_s)//60} 分钟),已自动拒绝:{pr.tool_title}"
})
chosen = pr.choice chosen = pr.choice
with self._lock: with self._lock:
self._pending_permissions.pop(req_id, None) self._pending_permissions.pop(req_id, None)
@@ -667,6 +680,11 @@ class SessionManager:
with self._lock: with self._lock:
bound = {b.agent for b in self._bindings.values()} bound = {b.agent for b in self._bindings.values()}
victims = [k for k, b in self._bindings.items() if now - b.last_active > self.idle_ttl_s] victims = [k for k, b in self._bindings.items() if now - b.last_active > self.idle_ttl_s]
# stale pending permissions: answered never came and the ACP wait
# already timed out; release them so they stop intercepting replies
for rid in [r for r, p in self._pending_permissions.items()
if p.answered.is_set() is False and now - p.created_at > self.permission_timeout_s * 2]:
self._pending_permissions.pop(rid, None)
remaining = {b.agent for k, b in self._bindings.items() if k not in set(victims)} remaining = {b.agent for k, b in self._bindings.items() if k not in set(victims)}
idle_clients = [n for n in self._clients if n not in remaining] idle_clients = [n for n in self._clients if n not in remaining]
for k in victims: for k in victims:
+154
View File
@@ -0,0 +1,154 @@
"""Keepalive + approval-hardening regressions (2026-10-07 outage lessons).
Covers:
- zombie connection: reader dead -> recv() raises (was: None forever, no reconnect)
- unlimited reconnects by default (network loss must not kill the bot)
- approval not shown yet -> reply rejected, not turned into an approval
"""
from __future__ import annotations
import json
import os
import sys
import threading
import pytest
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
from agenthub.channels.wecom import WecomAibotChannel # noqa: E402
from agenthub.channels.wecom_ws import AibotWSAdapter # noqa: E402
from agenthub.session import PermissionRequest, SessionError, SessionManager # noqa: E402
# ---------------------------------------------------------------------------
# zombie connection detection
# ---------------------------------------------------------------------------
def test_recv_raises_when_reader_dead():
"""Reader task done + empty queue -> recv must raise, not return None."""
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"})
loop = __import__("asyncio").new_event_loop()
adapter._loop = loop
done_evt = threading.Event()
async def _die():
await __import__("asyncio").sleep(0)
done_evt.set()
adapter._reader_task = loop.create_task(_die())
# pump the loop on a thread until the task actually completes
pump = threading.Thread(target=loop.run_forever, daemon=True)
pump.start()
assert done_evt.wait(5)
with pytest.raises(ConnectionError, match="zombie|reader"):
adapter.recv(timeout=0.2)
loop.call_soon_threadsafe(loop.stop)
def test_recv_raises_on_stale_inbound():
"""No inbound for > stale window -> ConnectionError (half-open TCP)."""
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"})
loop = __import__("asyncio").new_event_loop()
adapter._loop = loop
import asyncio as _a
adapter._reader_task = loop.create_task(_a.sleep(3600))
adapter._last_inbound = 0.0 # ancient
with pytest.raises(ConnectionError, match="stale|no inbound"):
adapter.recv(timeout=0.2)
# ---------------------------------------------------------------------------
# unlimited reconnects
# ---------------------------------------------------------------------------
def test_default_reconnect_is_unlimited():
ch = WecomAibotChannel({"bot_id": "b", "secret": "s"})
assert ch.max_reconnect == 0 # 0 = unlimited
def test_reconnect_never_exhausts_when_unlimited():
ch = WecomAibotChannel({"bot_id": "b", "secret": "s"})
assert ch.max_reconnect == 0
calls = {"n": 0}
def boom(cfg):
calls["n"] += 1
raise ConnectionError("dead")
ch._ws_factory = boom
ch._stop = threading.Event()
t = threading.Thread(target=ch._run, daemon=True)
t.start()
import time as _t
_t.sleep(4) # attempt 1 immediately, attempt 2 after the 2s backoff
ch._stop.set()
assert calls["n"] >= 2 # kept retrying well past the old limit of 10 backoffs
assert ch.connection_state == "connecting"
def test_explicit_limit_still_exhausts():
ch = WecomAibotChannel({"bot_id": "b", "secret": "s", "max_reconnect_attempts": 2})
def boom(cfg):
raise ConnectionError("dead")
ch._ws_factory = boom
t = threading.Thread(target=ch._run, daemon=True)
t.start()
t.join(timeout=30)
assert not t.is_alive()
assert ch.connection_state == "failed"
# ---------------------------------------------------------------------------
# approval hardening
# ---------------------------------------------------------------------------
def _mgr_with_pending(presented: bool) -> SessionManager:
class FakeAcp:
def is_alive(self):
return True
def session_prompt(self, *a, **k):
return {"stopReason": "end_turn"}
mgr = SessionManager(agents={}, client_factory=lambda spec: FakeAcp())
pr = PermissionRequest(
request_id="req-1", agent="a", acp_request_id=None, session_id="s1",
key="wecom:c1", tool_title="read file", tool_kind="read", raw_request_id=None,
options=[{"option_id": "allow_once", "name": "Allow once", "kind": "allow_once"}],
)
pr.presented = presented
mgr._pending_permissions["req-1"] = pr
return mgr
def test_reply_to_unpresented_approval_is_rejected():
"""A decision that arrives before the card was rendered must NOT approve."""
mgr = _mgr_with_pending(presented=False)
with pytest.raises(SessionError, match="not yet shown"):
mgr.permission_reply("wecom", "c1", "req-1", "allow_once")
def test_reply_to_presented_approval_works():
mgr = _mgr_with_pending(presented=True)
mgr.permission_reply("wecom", "c1", "req-1", "allow_once")
pr = mgr._pending_permissions["req-1"]
assert pr.choice == "allow_once"
assert pr.answered.is_set()
def test_numbered_answer_via_take_interaction_answer():
mgr = _mgr_with_pending(presented=True)
assert mgr.take_interaction_answer("wecom", "c1", "1") == "consumed"
pr = mgr._pending_permissions["req-1"]
assert pr.choice == "allow_once"
assert pr.answered.is_set()
def test_stale_permissions_reaped():
mgr = _mgr_with_pending(presented=True)
mgr.permission_timeout_s = 300
pr = mgr._pending_permissions["req-1"]
pr.created_at -= 1000 # ancient
mgr._reap_once()
assert "req-1" not in mgr._pending_permissions