fix(wecom): half-open WS detection tests + heartbeat interval config; log card-click event frames
This commit is contained in:
1 parent
719bb66627
commit
0f7adb7966
3 files changed
+151
-4
No files matched your search
@@ -140,7 +140,9 @@ class WecomAibotChannel:
|
||||
msg = json.loads(raw)
|
||||
cmd = msg.get("cmd", "")
|
||||
keys = sorted(msg.keys())
|
||||
if "aibot_msg_callback" in raw or "chatid" in raw:
|
||||
if "aibot_msg_callback" in raw or "chatid" in raw or "aibot_event_callback" in raw:
|
||||
# include event callbacks: card clicks (template_card_event)
|
||||
# carry no chatid and were previously invisible in logs
|
||||
self._logger.info(f"[wecom] frame: {raw[:220]}")
|
||||
try:
|
||||
self._dispatch(msg, ws)
|
||||
|
||||
@@ -48,6 +48,8 @@ class AibotWSAdapter:
|
||||
self.secret = cfg["secret"]
|
||||
self.ws_url = cfg.get("ws_url", WS_URL)
|
||||
self.connect_timeout = float(cfg.get("timeout_s", 20))
|
||||
# heartbeat interval; configurable so tests can shrink it (prod: 30s)
|
||||
self.heartbeat_s = float(cfg.get("heartbeat_s", HEARTBEAT_S))
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._ws: Any = None
|
||||
self._queue: "asyncio.Queue[str | None]" = asyncio.Queue()
|
||||
@@ -109,7 +111,7 @@ class AibotWSAdapter:
|
||||
"""No inbound frames (including pong replies) for too long = half-open
|
||||
TCP. Heartbeat sends ping but the pong never arrives; raise so the
|
||||
outer loop reconnects with a fresh socket."""
|
||||
stale_after = max(HEARTBEAT_S * 10, 300)
|
||||
stale_after = max(self.heartbeat_s * 10, 3 * self.heartbeat_s)
|
||||
if time.monotonic() - self._last_inbound > stale_after:
|
||||
raise ConnectionError("no inbound frames (incl. pong) for too long")
|
||||
|
||||
@@ -174,14 +176,14 @@ class AibotWSAdapter:
|
||||
misses = 0
|
||||
try:
|
||||
while not self._stop_ev.is_set() and self._ws is not None:
|
||||
await asyncio.sleep(HEARTBEAT_S)
|
||||
await asyncio.sleep(self.heartbeat_s)
|
||||
if self._ws is None:
|
||||
return
|
||||
try:
|
||||
await self._ws.send(
|
||||
json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}})
|
||||
)
|
||||
if time.monotonic() - self._last_inbound < HEARTBEAT_S * 2:
|
||||
if time.monotonic() - self._last_inbound < self.heartbeat_s * 2:
|
||||
misses = 0
|
||||
else:
|
||||
misses += 1
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Half-open connection regression: the 2026-10-22 outage was a WeCom WS
|
||||
that stayed ESTABLISHED for 19h while the gateway was gone — agent replies
|
||||
were written into a dead socket and silently lost.
|
||||
|
||||
These tests simulate a gateway that stops answering (no pong, no frames)
|
||||
without closing the TCP connection, and assert the adapter detects the
|
||||
stale link and raises so the channel's reconnect loop can rebuild it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
||||
|
||||
websockets = pytest.importorskip("websockets")
|
||||
|
||||
from agenthub.channels.wecom_ws import AibotWSAdapter # noqa: E402
|
||||
|
||||
|
||||
class MuteAibotServer:
|
||||
"""Aibot gateway that answers the subscribe then goes silent: never
|
||||
replies to ping, never sends frames, never closes the socket — the
|
||||
exact shape of a half-open link from the client's side."""
|
||||
|
||||
def __init__(self):
|
||||
self.port: int | None = None
|
||||
self.pings: list[dict] = []
|
||||
self._ready = threading.Event()
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
def start(self):
|
||||
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||
self._thread.start()
|
||||
assert self._ready.wait(10)
|
||||
|
||||
def _run(self):
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._loop.run_until_complete(self._serve())
|
||||
|
||||
def stop(self):
|
||||
if self._loop and self._loop.is_running():
|
||||
self._loop.call_soon_threadsafe(self._loop.stop)
|
||||
|
||||
async def _serve(self):
|
||||
async def handler(ws):
|
||||
try:
|
||||
async for raw in ws:
|
||||
msg = json.loads(raw)
|
||||
if msg.get("cmd") == "aibot_subscribe":
|
||||
await ws.send(json.dumps(
|
||||
{"headers": msg["headers"], "errcode": 0, "errmsg": "ok"}))
|
||||
elif msg.get("cmd") == "ping":
|
||||
self.pings.append(msg) # record but DO NOT answer
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async with websockets.serve(handler, "127.0.0.1", 0) as server:
|
||||
self.port = server.sockets[0].getsockname()[1]
|
||||
self._ready.set()
|
||||
await asyncio.get_running_loop().create_future()
|
||||
|
||||
# stop() is defined above (clean loop shutdown); no extra teardown needed
|
||||
|
||||
|
||||
def test_stale_halfopen_connection_raises_on_recv():
|
||||
"""No pong for > stale window -> recv() raises ConnectionError so the
|
||||
channel reconnects instead of idling on a dead socket."""
|
||||
srv = MuteAibotServer()
|
||||
srv.start()
|
||||
# heartbeat 1s -> stale window 3s; misses>=2 closes socket around 4s
|
||||
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s",
|
||||
"ws_url": f"ws://127.0.0.1:{srv.port}",
|
||||
"heartbeat_s": 1.0})
|
||||
try:
|
||||
adapter.connect()
|
||||
t0 = time.monotonic()
|
||||
raised = None
|
||||
while time.monotonic() - t0 < 15:
|
||||
try:
|
||||
item = adapter.recv(timeout=1.0)
|
||||
except ConnectionError as e:
|
||||
raised = str(e)
|
||||
break
|
||||
except Exception as e: # noqa: BLE001
|
||||
raised = f"{type(e).__name__}: {e}"
|
||||
break
|
||||
assert raised is not None, (
|
||||
"recv() never raised on a half-open connection — would idle forever"
|
||||
)
|
||||
# the heartbeat must actually have pinged (it was the silence probe)
|
||||
deadline = time.time() + 5
|
||||
while not srv.pings and time.time() < deadline:
|
||||
time.sleep(0.1)
|
||||
assert srv.pings, "heartbeat never sent ping"
|
||||
finally:
|
||||
adapter.close()
|
||||
srv.stop()
|
||||
|
||||
|
||||
def test_heartbeat_two_missed_pongs_closes_socket():
|
||||
"""Two consecutive ping windows without any inbound frame -> the adapter
|
||||
closes the socket itself; recv() then raises (EOF path)."""
|
||||
srv = MuteAibotServer()
|
||||
srv.start()
|
||||
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s",
|
||||
"ws_url": f"ws://127.0.0.1:{srv.port}",
|
||||
"heartbeat_s": 1.0})
|
||||
try:
|
||||
adapter.connect()
|
||||
t0 = time.monotonic()
|
||||
raised = None
|
||||
while time.monotonic() - t0 < 15:
|
||||
try:
|
||||
adapter.recv(timeout=1.0)
|
||||
except ConnectionError as e:
|
||||
raised = str(e)
|
||||
break
|
||||
assert raised is not None
|
||||
# closed-by-heartbeat is the proactive path; EOF/stale both acceptable
|
||||
# but the connection must not survive silently.
|
||||
finally:
|
||||
adapter.close()
|
||||
srv.stop()
|
||||
|
||||
|
||||
def test_stale_check_unit():
|
||||
"""Direct unit check of _check_stale_inbound thresholds."""
|
||||
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "heartbeat_s": 30.0})
|
||||
# fresh connection: no raise
|
||||
adapter._check_stale_inbound()
|
||||
# 400s of silence -> raise
|
||||
adapter._last_inbound = time.monotonic() - 400
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter._check_stale_inbound()
|
||||
Reference in new issue
Block a user