feat: keepalive hardening + approval safety (dsh-im parity)
Keepalive (fixes 19h silent WS death on 2026-10-07):
- recv() raises on dead reader / stale inbound (no pong) instead of
returning None forever -> outer loop reconnects
- heartbeat closes socket after 2 missed pongs
- reconnect default unlimited (0), backoff cap 60s; auth still terminal
- connection_state surfaced (idle/connecting/connected/failed)
Approval hardening:
- unpresented approval can never be answered (race guard)
- timeout notifies the user ("已自动拒绝")
- stale pending permissions reaped; created_at tracked
Tests: 69 passed (9 new regressions in test_keepalive_approvals.py)
This commit is contained in:
1 parent
2bf6b76e15
commit
23c1adcca4
4 files changed
+252
-7
No files matched your search
@@ -43,7 +43,9 @@ class WecomAibotChannel:
|
||||
self.secret = config["secret"]
|
||||
self.domain = config.get("domain", "https://open.weixin.qq.com")
|
||||
self.connect_timeout_s = float(config.get("connect_timeout_s", 20))
|
||||
self.max_reconnect = int(config.get("max_reconnect_attempts", 10))
|
||||
# 0/negative = unlimited (default): network loss must never permanently
|
||||
# kill the bot. Backoff caps at 60s, so a retry storm is bounded anyway.
|
||||
self.max_reconnect = int(config.get("max_reconnect_attempts", 0))
|
||||
self.reply_timeout_s = float(config.get("reply_timeout_s", 600))
|
||||
self.allowed_users = set(config.get("allowed_users") or [])
|
||||
self._ws_factory = ws_factory # returns object with send/recv/close
|
||||
@@ -53,6 +55,8 @@ class WecomAibotChannel:
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._ws = None # current socket (set by _connect_and_serve)
|
||||
self.connection_state = "idle"
|
||||
self.connection_error: str = ""
|
||||
# chat -> streaming state; chat -> last callback req_id (respond must reuse it)
|
||||
self._streams: dict[str, dict] = {}
|
||||
self._req_ids: dict[str, str] = {}
|
||||
@@ -85,13 +89,28 @@ class WecomAibotChannel:
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
attempt += 1
|
||||
if attempt > self.max_reconnect:
|
||||
if self.max_reconnect > 0 and attempt > self.max_reconnect:
|
||||
self._logger.error(f"[wecom] reconnect exhausted ({attempt}): {e}")
|
||||
self._set_channel_state("failed", str(e))
|
||||
return
|
||||
delay = min(2**attempt, 30)
|
||||
self._logger.warning(f"[wecom] disconnected ({e}); retry {attempt}/{self.max_reconnect} in {delay}s")
|
||||
delay = min(2**attempt, 60)
|
||||
self._logger.warning(
|
||||
f"[wecom] disconnected ({e}); retry {attempt} in {delay}s"
|
||||
)
|
||||
self._set_channel_state("connecting", f"reconnecting in {delay}s: {e}")
|
||||
self._stop.wait(delay)
|
||||
|
||||
def _set_channel_state(self, state: str, detail: str = "") -> None:
|
||||
"""Surface connection state so ops can see a dead connection from the
|
||||
status endpoint instead of discovering it by missing replies."""
|
||||
self.connection_state = state
|
||||
self.connection_error = detail
|
||||
try:
|
||||
if self._sink:
|
||||
self._sink(f"{self.name}:__channel__", "channel_state", {"state": state, "error": detail})
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def _connect_and_serve(self) -> None:
|
||||
if self._ws_factory is None:
|
||||
raise RuntimeError("wecom channel requires a ws_factory (real or mock transport)")
|
||||
@@ -105,6 +124,7 @@ class WecomAibotChannel:
|
||||
)
|
||||
ws.connect()
|
||||
self._ws = ws
|
||||
self._set_channel_state("connected")
|
||||
self._logger.info("[wecom] WebSocket authenticated")
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
|
||||
@@ -56,6 +56,12 @@ class AibotWSAdapter:
|
||||
self._subscribed = asyncio.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._loop_ready = threading.Event()
|
||||
# keepalive bookkeeping: reader task + last inbound pong/frame time.
|
||||
# The 2026-10-07 outage: reader/heartbeat died silently while the
|
||||
# channel's recv() loop only ever saw None, so nothing reconnected for
|
||||
# 19h. recv() must detect a dead transport and raise.
|
||||
self._reader_task: asyncio.Task | None = None
|
||||
self._last_inbound = time.monotonic()
|
||||
|
||||
# -- sync API (called from channel thread) ------------------------------
|
||||
def connect(self) -> None:
|
||||
@@ -72,17 +78,41 @@ class AibotWSAdapter:
|
||||
Raising (rather than returning None) on EOF is what makes the channel's
|
||||
outer loop see an exception and run its reconnect/backoff path.
|
||||
"""
|
||||
if self._loop is None:
|
||||
if self._loop is None or self._loop.is_closed():
|
||||
raise ConnectionError("ws loop not running")
|
||||
# Cheap liveness first: a dead reader means an empty queue forever, so
|
||||
# detect it before paying the wait (this is the 2026-10-07 outage).
|
||||
if not self._reader_alive():
|
||||
raise ConnectionError("ws reader is dead; connection zombie")
|
||||
fut = asyncio.run_coroutine_threadsafe(self._queue_get(timeout), self._loop)
|
||||
try:
|
||||
item = fut.result(timeout=timeout + 2)
|
||||
except TimeoutError:
|
||||
return None
|
||||
item = None
|
||||
except Exception as e: # noqa: BLE001
|
||||
raise ConnectionError(f"ws loop failed: {e}") from e
|
||||
if item is _EOF:
|
||||
raise ConnectionError("websocket closed by peer")
|
||||
if item is None:
|
||||
# idle: if nothing (not even a pong) has arrived for a long time the
|
||||
# TCP connection is half-open -> reconnect instead of idling forever
|
||||
self._check_stale_inbound()
|
||||
return item
|
||||
|
||||
def _reader_alive(self) -> bool:
|
||||
task = self._reader_task
|
||||
if task is None:
|
||||
return True # mock transports without a reader task
|
||||
return not task.done()
|
||||
|
||||
def _check_stale_inbound(self) -> None:
|
||||
"""No inbound frames (including pong replies) for too long = half-open
|
||||
TCP. Heartbeat sends ping but the pong never arrives; raise so the
|
||||
outer loop reconnects with a fresh socket."""
|
||||
stale_after = max(HEARTBEAT_S * 10, 300)
|
||||
if time.monotonic() - self._last_inbound > stale_after:
|
||||
raise ConnectionError("no inbound frames (incl. pong) for too long")
|
||||
|
||||
def send(self, raw: str) -> None:
|
||||
if self._loop is None or self._ws is None:
|
||||
raise ConnectionError("ws not connected")
|
||||
@@ -133,23 +163,46 @@ class AibotWSAdapter:
|
||||
log.info("[aibot-ws] subscribed ok (bot_id=%s...)", self.bot_id[:6])
|
||||
self._subscribed.set()
|
||||
self._hb_task = asyncio.create_task(self._heartbeat())
|
||||
asyncio.create_task(self._reader())
|
||||
self._reader_task = asyncio.create_task(self._reader())
|
||||
self._last_inbound = time.monotonic()
|
||||
|
||||
async def _heartbeat(self) -> None:
|
||||
"""Ping every HEARTBEAT_S; a pong (or any inbound frame) refreshes
|
||||
_last_inbound. Two consecutive failed pings -> close the socket so the
|
||||
reader ends and the channel reconnects, instead of silently pinging a
|
||||
dead TCP connection."""
|
||||
misses = 0
|
||||
try:
|
||||
while not self._stop_ev.is_set() and self._ws is not None:
|
||||
await asyncio.sleep(HEARTBEAT_S)
|
||||
if self._ws is None:
|
||||
return
|
||||
try:
|
||||
await self._ws.send(
|
||||
json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}})
|
||||
)
|
||||
if time.monotonic() - self._last_inbound < HEARTBEAT_S * 2:
|
||||
misses = 0
|
||||
else:
|
||||
misses += 1
|
||||
except Exception: # noqa: BLE001
|
||||
misses += 1
|
||||
if misses >= 2:
|
||||
log.warning("[aibot-ws] heartbeat missed %d pongs; closing socket", misses)
|
||||
try:
|
||||
await self._ws.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("[aibot-ws] heartbeat stopped: %s", e)
|
||||
|
||||
async def _reader(self) -> None:
|
||||
try:
|
||||
async for raw in self._ws:
|
||||
self._last_inbound = time.monotonic()
|
||||
await self._queue.put(raw if isinstance(raw, str) else raw.decode())
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("[aibot-ws] reader ended: %s", e)
|
||||
|
||||
@@ -81,9 +81,11 @@ class PermissionRequest:
|
||||
tool_title: str
|
||||
tool_kind: str
|
||||
raw_request_id: Any
|
||||
created_at: float = field(default_factory=time.time)
|
||||
options: list[dict[str, Any]] = field(default_factory=list)
|
||||
answered: threading.Event = field(default_factory=threading.Event)
|
||||
choice: str | None = None # optionId chosen, None = timeout/deny
|
||||
presented: bool = False # interaction event emitted to the front-end
|
||||
|
||||
|
||||
def _is_transient_model_error(msg: str) -> bool:
|
||||
@@ -475,6 +477,10 @@ class SessionManager:
|
||||
raise SessionError(f"no pending permission {request_id}")
|
||||
if pr.key != f"{channel}:{chat_id}":
|
||||
raise SessionError("permission request belongs to a different chat")
|
||||
if not pr.presented:
|
||||
# never turn a decision on an approval the user has not seen into
|
||||
# an accept (covers races where a stale reply beats the render)
|
||||
raise SessionError("permission request not yet shown; try again in a moment")
|
||||
alias = {"allow": "allow_once", "y": "allow_once", "yes": "allow_once",
|
||||
"deny": "reject_once", "n": "reject_once", "no": "reject_once"}
|
||||
pr.choice = alias.get(choice.strip().lower(), choice)
|
||||
@@ -612,6 +618,10 @@ class SessionManager:
|
||||
for o in options
|
||||
],
|
||||
)
|
||||
with self._lock:
|
||||
pr = self._pending_permissions.get(req_id)
|
||||
if pr is not None:
|
||||
pr.presented = True
|
||||
self._emit(key, "interaction", {
|
||||
"interaction_type": "permission",
|
||||
"request_id": req_id,
|
||||
@@ -630,6 +640,9 @@ class SessionManager:
|
||||
answered = pr.answered.wait(timeout=self.permission_timeout_s)
|
||||
if not answered:
|
||||
pr.choice = None # timeout -> deny (first reject-style option)
|
||||
self._emit(key, "status", {
|
||||
"text": f"审批超时({int(self.permission_timeout_s)//60} 分钟),已自动拒绝:{pr.tool_title}"
|
||||
})
|
||||
chosen = pr.choice
|
||||
with self._lock:
|
||||
self._pending_permissions.pop(req_id, None)
|
||||
@@ -667,6 +680,11 @@ class SessionManager:
|
||||
with self._lock:
|
||||
bound = {b.agent for b in self._bindings.values()}
|
||||
victims = [k for k, b in self._bindings.items() if now - b.last_active > self.idle_ttl_s]
|
||||
# stale pending permissions: answered never came and the ACP wait
|
||||
# already timed out; release them so they stop intercepting replies
|
||||
for rid in [r for r, p in self._pending_permissions.items()
|
||||
if p.answered.is_set() is False and now - p.created_at > self.permission_timeout_s * 2]:
|
||||
self._pending_permissions.pop(rid, None)
|
||||
remaining = {b.agent for k, b in self._bindings.items() if k not in set(victims)}
|
||||
idle_clients = [n for n in self._clients if n not in remaining]
|
||||
for k in victims:
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Keepalive + approval-hardening regressions (2026-10-07 outage lessons).
|
||||
|
||||
Covers:
|
||||
- zombie connection: reader dead -> recv() raises (was: None forever, no reconnect)
|
||||
- unlimited reconnects by default (network loss must not kill the bot)
|
||||
- approval not shown yet -> reply rejected, not turned into an approval
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
||||
|
||||
from agenthub.channels.wecom import WecomAibotChannel # noqa: E402
|
||||
from agenthub.channels.wecom_ws import AibotWSAdapter # noqa: E402
|
||||
from agenthub.session import PermissionRequest, SessionError, SessionManager # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# zombie connection detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_recv_raises_when_reader_dead():
|
||||
"""Reader task done + empty queue -> recv must raise, not return None."""
|
||||
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"})
|
||||
loop = __import__("asyncio").new_event_loop()
|
||||
adapter._loop = loop
|
||||
done_evt = threading.Event()
|
||||
|
||||
async def _die():
|
||||
await __import__("asyncio").sleep(0)
|
||||
done_evt.set()
|
||||
|
||||
adapter._reader_task = loop.create_task(_die())
|
||||
# pump the loop on a thread until the task actually completes
|
||||
pump = threading.Thread(target=loop.run_forever, daemon=True)
|
||||
pump.start()
|
||||
assert done_evt.wait(5)
|
||||
with pytest.raises(ConnectionError, match="zombie|reader"):
|
||||
adapter.recv(timeout=0.2)
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
|
||||
|
||||
def test_recv_raises_on_stale_inbound():
|
||||
"""No inbound for > stale window -> ConnectionError (half-open TCP)."""
|
||||
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "ws_url": "ws://127.0.0.1:1"})
|
||||
loop = __import__("asyncio").new_event_loop()
|
||||
adapter._loop = loop
|
||||
import asyncio as _a
|
||||
adapter._reader_task = loop.create_task(_a.sleep(3600))
|
||||
adapter._last_inbound = 0.0 # ancient
|
||||
with pytest.raises(ConnectionError, match="stale|no inbound"):
|
||||
adapter.recv(timeout=0.2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# unlimited reconnects
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_default_reconnect_is_unlimited():
|
||||
ch = WecomAibotChannel({"bot_id": "b", "secret": "s"})
|
||||
assert ch.max_reconnect == 0 # 0 = unlimited
|
||||
|
||||
|
||||
def test_reconnect_never_exhausts_when_unlimited():
|
||||
ch = WecomAibotChannel({"bot_id": "b", "secret": "s"})
|
||||
assert ch.max_reconnect == 0
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def boom(cfg):
|
||||
calls["n"] += 1
|
||||
raise ConnectionError("dead")
|
||||
|
||||
ch._ws_factory = boom
|
||||
ch._stop = threading.Event()
|
||||
t = threading.Thread(target=ch._run, daemon=True)
|
||||
t.start()
|
||||
import time as _t
|
||||
_t.sleep(4) # attempt 1 immediately, attempt 2 after the 2s backoff
|
||||
ch._stop.set()
|
||||
assert calls["n"] >= 2 # kept retrying well past the old limit of 10 backoffs
|
||||
assert ch.connection_state == "connecting"
|
||||
|
||||
|
||||
def test_explicit_limit_still_exhausts():
|
||||
ch = WecomAibotChannel({"bot_id": "b", "secret": "s", "max_reconnect_attempts": 2})
|
||||
def boom(cfg):
|
||||
raise ConnectionError("dead")
|
||||
ch._ws_factory = boom
|
||||
t = threading.Thread(target=ch._run, daemon=True)
|
||||
t.start()
|
||||
t.join(timeout=30)
|
||||
assert not t.is_alive()
|
||||
assert ch.connection_state == "failed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# approval hardening
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _mgr_with_pending(presented: bool) -> SessionManager:
|
||||
class FakeAcp:
|
||||
def is_alive(self):
|
||||
return True
|
||||
def session_prompt(self, *a, **k):
|
||||
return {"stopReason": "end_turn"}
|
||||
mgr = SessionManager(agents={}, client_factory=lambda spec: FakeAcp())
|
||||
pr = PermissionRequest(
|
||||
request_id="req-1", agent="a", acp_request_id=None, session_id="s1",
|
||||
key="wecom:c1", tool_title="read file", tool_kind="read", raw_request_id=None,
|
||||
options=[{"option_id": "allow_once", "name": "Allow once", "kind": "allow_once"}],
|
||||
)
|
||||
pr.presented = presented
|
||||
mgr._pending_permissions["req-1"] = pr
|
||||
return mgr
|
||||
|
||||
|
||||
def test_reply_to_unpresented_approval_is_rejected():
|
||||
"""A decision that arrives before the card was rendered must NOT approve."""
|
||||
mgr = _mgr_with_pending(presented=False)
|
||||
with pytest.raises(SessionError, match="not yet shown"):
|
||||
mgr.permission_reply("wecom", "c1", "req-1", "allow_once")
|
||||
|
||||
|
||||
def test_reply_to_presented_approval_works():
|
||||
mgr = _mgr_with_pending(presented=True)
|
||||
mgr.permission_reply("wecom", "c1", "req-1", "allow_once")
|
||||
pr = mgr._pending_permissions["req-1"]
|
||||
assert pr.choice == "allow_once"
|
||||
assert pr.answered.is_set()
|
||||
|
||||
|
||||
def test_numbered_answer_via_take_interaction_answer():
|
||||
mgr = _mgr_with_pending(presented=True)
|
||||
assert mgr.take_interaction_answer("wecom", "c1", "1") == "consumed"
|
||||
pr = mgr._pending_permissions["req-1"]
|
||||
assert pr.choice == "allow_once"
|
||||
assert pr.answered.is_set()
|
||||
|
||||
|
||||
def test_stale_permissions_reaped():
|
||||
mgr = _mgr_with_pending(presented=True)
|
||||
mgr.permission_timeout_s = 300
|
||||
pr = mgr._pending_permissions["req-1"]
|
||||
pr.created_at -= 1000 # ancient
|
||||
mgr._reap_once()
|
||||
assert "req-1" not in mgr._pending_permissions
|
||||
Reference in new issue
Block a user