fix(wecom): half-open WS detection tests + heartbeat interval config; log card-click event frames
This commit is contained in:
1 parent
719bb66627
commit
0f7adb7966
3 files changed
+151
-4
No files matched your search
@@ -140,7 +140,9 @@ class WecomAibotChannel:
|
|||||||
msg = json.loads(raw)
|
msg = json.loads(raw)
|
||||||
cmd = msg.get("cmd", "")
|
cmd = msg.get("cmd", "")
|
||||||
keys = sorted(msg.keys())
|
keys = sorted(msg.keys())
|
||||||
if "aibot_msg_callback" in raw or "chatid" in raw:
|
if "aibot_msg_callback" in raw or "chatid" in raw or "aibot_event_callback" in raw:
|
||||||
|
# include event callbacks: card clicks (template_card_event)
|
||||||
|
# carry no chatid and were previously invisible in logs
|
||||||
self._logger.info(f"[wecom] frame: {raw[:220]}")
|
self._logger.info(f"[wecom] frame: {raw[:220]}")
|
||||||
try:
|
try:
|
||||||
self._dispatch(msg, ws)
|
self._dispatch(msg, ws)
|
||||||
|
|||||||
@@ -48,6 +48,8 @@ class AibotWSAdapter:
|
|||||||
self.secret = cfg["secret"]
|
self.secret = cfg["secret"]
|
||||||
self.ws_url = cfg.get("ws_url", WS_URL)
|
self.ws_url = cfg.get("ws_url", WS_URL)
|
||||||
self.connect_timeout = float(cfg.get("timeout_s", 20))
|
self.connect_timeout = float(cfg.get("timeout_s", 20))
|
||||||
|
# heartbeat interval; configurable so tests can shrink it (prod: 30s)
|
||||||
|
self.heartbeat_s = float(cfg.get("heartbeat_s", HEARTBEAT_S))
|
||||||
self._loop: asyncio.AbstractEventLoop | None = None
|
self._loop: asyncio.AbstractEventLoop | None = None
|
||||||
self._ws: Any = None
|
self._ws: Any = None
|
||||||
self._queue: "asyncio.Queue[str | None]" = asyncio.Queue()
|
self._queue: "asyncio.Queue[str | None]" = asyncio.Queue()
|
||||||
@@ -109,7 +111,7 @@ class AibotWSAdapter:
|
|||||||
"""No inbound frames (including pong replies) for too long = half-open
|
"""No inbound frames (including pong replies) for too long = half-open
|
||||||
TCP. Heartbeat sends ping but the pong never arrives; raise so the
|
TCP. Heartbeat sends ping but the pong never arrives; raise so the
|
||||||
outer loop reconnects with a fresh socket."""
|
outer loop reconnects with a fresh socket."""
|
||||||
stale_after = max(HEARTBEAT_S * 10, 300)
|
stale_after = max(self.heartbeat_s * 10, 3 * self.heartbeat_s)
|
||||||
if time.monotonic() - self._last_inbound > stale_after:
|
if time.monotonic() - self._last_inbound > stale_after:
|
||||||
raise ConnectionError("no inbound frames (incl. pong) for too long")
|
raise ConnectionError("no inbound frames (incl. pong) for too long")
|
||||||
|
|
||||||
@@ -174,14 +176,14 @@ class AibotWSAdapter:
|
|||||||
misses = 0
|
misses = 0
|
||||||
try:
|
try:
|
||||||
while not self._stop_ev.is_set() and self._ws is not None:
|
while not self._stop_ev.is_set() and self._ws is not None:
|
||||||
await asyncio.sleep(HEARTBEAT_S)
|
await asyncio.sleep(self.heartbeat_s)
|
||||||
if self._ws is None:
|
if self._ws is None:
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
await self._ws.send(
|
await self._ws.send(
|
||||||
json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}})
|
json.dumps({"cmd": "ping", "headers": {"req_id": uuid.uuid4().hex}})
|
||||||
)
|
)
|
||||||
if time.monotonic() - self._last_inbound < HEARTBEAT_S * 2:
|
if time.monotonic() - self._last_inbound < self.heartbeat_s * 2:
|
||||||
misses = 0
|
misses = 0
|
||||||
else:
|
else:
|
||||||
misses += 1
|
misses += 1
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
"""Half-open connection regression: the 2026-10-22 outage was a WeCom WS
|
||||||
|
that stayed ESTABLISHED for 19h while the gateway was gone — agent replies
|
||||||
|
were written into a dead socket and silently lost.
|
||||||
|
|
||||||
|
These tests simulate a gateway that stops answering (no pong, no frames)
|
||||||
|
without closing the TCP connection, and assert the adapter detects the
|
||||||
|
stale link and raises so the channel's reconnect loop can rebuild it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
||||||
|
|
||||||
|
websockets = pytest.importorskip("websockets")
|
||||||
|
|
||||||
|
from agenthub.channels.wecom_ws import AibotWSAdapter # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class MuteAibotServer:
|
||||||
|
"""Aibot gateway that answers the subscribe then goes silent: never
|
||||||
|
replies to ping, never sends frames, never closes the socket — the
|
||||||
|
exact shape of a half-open link from the client's side."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.port: int | None = None
|
||||||
|
self.pings: list[dict] = []
|
||||||
|
self._ready = threading.Event()
|
||||||
|
self._loop: asyncio.AbstractEventLoop | None = None
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
assert self._ready.wait(10)
|
||||||
|
|
||||||
|
def _run(self):
|
||||||
|
self._loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(self._loop)
|
||||||
|
self._loop.run_until_complete(self._serve())
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
if self._loop and self._loop.is_running():
|
||||||
|
self._loop.call_soon_threadsafe(self._loop.stop)
|
||||||
|
|
||||||
|
async def _serve(self):
|
||||||
|
async def handler(ws):
|
||||||
|
try:
|
||||||
|
async for raw in ws:
|
||||||
|
msg = json.loads(raw)
|
||||||
|
if msg.get("cmd") == "aibot_subscribe":
|
||||||
|
await ws.send(json.dumps(
|
||||||
|
{"headers": msg["headers"], "errcode": 0, "errmsg": "ok"}))
|
||||||
|
elif msg.get("cmd") == "ping":
|
||||||
|
self.pings.append(msg) # record but DO NOT answer
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async with websockets.serve(handler, "127.0.0.1", 0) as server:
|
||||||
|
self.port = server.sockets[0].getsockname()[1]
|
||||||
|
self._ready.set()
|
||||||
|
await asyncio.get_running_loop().create_future()
|
||||||
|
|
||||||
|
# stop() is defined above (clean loop shutdown); no extra teardown needed
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_halfopen_connection_raises_on_recv():
|
||||||
|
"""No pong for > stale window -> recv() raises ConnectionError so the
|
||||||
|
channel reconnects instead of idling on a dead socket."""
|
||||||
|
srv = MuteAibotServer()
|
||||||
|
srv.start()
|
||||||
|
# heartbeat 1s -> stale window 3s; misses>=2 closes socket around 4s
|
||||||
|
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s",
|
||||||
|
"ws_url": f"ws://127.0.0.1:{srv.port}",
|
||||||
|
"heartbeat_s": 1.0})
|
||||||
|
try:
|
||||||
|
adapter.connect()
|
||||||
|
t0 = time.monotonic()
|
||||||
|
raised = None
|
||||||
|
while time.monotonic() - t0 < 15:
|
||||||
|
try:
|
||||||
|
item = adapter.recv(timeout=1.0)
|
||||||
|
except ConnectionError as e:
|
||||||
|
raised = str(e)
|
||||||
|
break
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
raised = f"{type(e).__name__}: {e}"
|
||||||
|
break
|
||||||
|
assert raised is not None, (
|
||||||
|
"recv() never raised on a half-open connection — would idle forever"
|
||||||
|
)
|
||||||
|
# the heartbeat must actually have pinged (it was the silence probe)
|
||||||
|
deadline = time.time() + 5
|
||||||
|
while not srv.pings and time.time() < deadline:
|
||||||
|
time.sleep(0.1)
|
||||||
|
assert srv.pings, "heartbeat never sent ping"
|
||||||
|
finally:
|
||||||
|
adapter.close()
|
||||||
|
srv.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_heartbeat_two_missed_pongs_closes_socket():
|
||||||
|
"""Two consecutive ping windows without any inbound frame -> the adapter
|
||||||
|
closes the socket itself; recv() then raises (EOF path)."""
|
||||||
|
srv = MuteAibotServer()
|
||||||
|
srv.start()
|
||||||
|
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s",
|
||||||
|
"ws_url": f"ws://127.0.0.1:{srv.port}",
|
||||||
|
"heartbeat_s": 1.0})
|
||||||
|
try:
|
||||||
|
adapter.connect()
|
||||||
|
t0 = time.monotonic()
|
||||||
|
raised = None
|
||||||
|
while time.monotonic() - t0 < 15:
|
||||||
|
try:
|
||||||
|
adapter.recv(timeout=1.0)
|
||||||
|
except ConnectionError as e:
|
||||||
|
raised = str(e)
|
||||||
|
break
|
||||||
|
assert raised is not None
|
||||||
|
# closed-by-heartbeat is the proactive path; EOF/stale both acceptable
|
||||||
|
# but the connection must not survive silently.
|
||||||
|
finally:
|
||||||
|
adapter.close()
|
||||||
|
srv.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_check_unit():
|
||||||
|
"""Direct unit check of _check_stale_inbound thresholds."""
|
||||||
|
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "heartbeat_s": 30.0})
|
||||||
|
# fresh connection: no raise
|
||||||
|
adapter._check_stale_inbound()
|
||||||
|
# 400s of silence -> raise
|
||||||
|
adapter._last_inbound = time.monotonic() - 400
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter._check_stale_inbound()
|
||||||
Reference in new issue
Block a user