144 lines
5.0 KiB
Python
144 lines
5.0 KiB
Python
"""Half-open connection regression: the 2026-10-22 outage was a WeCom WS
|
|
that stayed ESTABLISHED for 19h while the gateway was gone — agent replies
|
|
were written into a dead socket and silently lost.
|
|
|
|
These tests simulate a gateway that stops answering (no pong, no frames)
|
|
without closing the TCP connection, and assert the adapter detects the
|
|
stale link and raises so the channel's reconnect loop can rebuild it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
|
|
|
websockets = pytest.importorskip("websockets")
|
|
|
|
from agenthub.channels.wecom_ws import AibotWSAdapter # noqa: E402
|
|
|
|
|
|
class MuteAibotServer:
|
|
"""Aibot gateway that answers the subscribe then goes silent: never
|
|
replies to ping, never sends frames, never closes the socket — the
|
|
exact shape of a half-open link from the client's side."""
|
|
|
|
def __init__(self):
|
|
self.port: int | None = None
|
|
self.pings: list[dict] = []
|
|
self._ready = threading.Event()
|
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
|
|
def start(self):
|
|
self._thread = threading.Thread(target=self._run, daemon=True)
|
|
self._thread.start()
|
|
assert self._ready.wait(10)
|
|
|
|
def _run(self):
|
|
self._loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(self._loop)
|
|
self._loop.run_until_complete(self._serve())
|
|
|
|
def stop(self):
|
|
if self._loop and self._loop.is_running():
|
|
self._loop.call_soon_threadsafe(self._loop.stop)
|
|
|
|
async def _serve(self):
|
|
async def handler(ws):
|
|
try:
|
|
async for raw in ws:
|
|
msg = json.loads(raw)
|
|
if msg.get("cmd") == "aibot_subscribe":
|
|
await ws.send(json.dumps(
|
|
{"headers": msg["headers"], "errcode": 0, "errmsg": "ok"}))
|
|
elif msg.get("cmd") == "ping":
|
|
self.pings.append(msg) # record but DO NOT answer
|
|
except Exception:
|
|
pass
|
|
|
|
async with websockets.serve(handler, "127.0.0.1", 0) as server:
|
|
self.port = server.sockets[0].getsockname()[1]
|
|
self._ready.set()
|
|
await asyncio.get_running_loop().create_future()
|
|
|
|
# stop() is defined above (clean loop shutdown); no extra teardown needed
|
|
|
|
|
|
def test_stale_halfopen_connection_raises_on_recv():
|
|
"""No pong for > stale window -> recv() raises ConnectionError so the
|
|
channel reconnects instead of idling on a dead socket."""
|
|
srv = MuteAibotServer()
|
|
srv.start()
|
|
# heartbeat 1s -> stale window 3s; misses>=2 closes socket around 4s
|
|
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s",
|
|
"ws_url": f"ws://127.0.0.1:{srv.port}",
|
|
"heartbeat_s": 1.0})
|
|
try:
|
|
adapter.connect()
|
|
t0 = time.monotonic()
|
|
raised = None
|
|
while time.monotonic() - t0 < 15:
|
|
try:
|
|
item = adapter.recv(timeout=1.0)
|
|
except ConnectionError as e:
|
|
raised = str(e)
|
|
break
|
|
except Exception as e: # noqa: BLE001
|
|
raised = f"{type(e).__name__}: {e}"
|
|
break
|
|
assert raised is not None, (
|
|
"recv() never raised on a half-open connection — would idle forever"
|
|
)
|
|
# the heartbeat must actually have pinged (it was the silence probe)
|
|
deadline = time.time() + 5
|
|
while not srv.pings and time.time() < deadline:
|
|
time.sleep(0.1)
|
|
assert srv.pings, "heartbeat never sent ping"
|
|
finally:
|
|
adapter.close()
|
|
srv.stop()
|
|
|
|
|
|
def test_heartbeat_two_missed_pongs_closes_socket():
|
|
"""Two consecutive ping windows without any inbound frame -> the adapter
|
|
closes the socket itself; recv() then raises (EOF path)."""
|
|
srv = MuteAibotServer()
|
|
srv.start()
|
|
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s",
|
|
"ws_url": f"ws://127.0.0.1:{srv.port}",
|
|
"heartbeat_s": 1.0})
|
|
try:
|
|
adapter.connect()
|
|
t0 = time.monotonic()
|
|
raised = None
|
|
while time.monotonic() - t0 < 15:
|
|
try:
|
|
adapter.recv(timeout=1.0)
|
|
except ConnectionError as e:
|
|
raised = str(e)
|
|
break
|
|
assert raised is not None
|
|
# closed-by-heartbeat is the proactive path; EOF/stale both acceptable
|
|
# but the connection must not survive silently.
|
|
finally:
|
|
adapter.close()
|
|
srv.stop()
|
|
|
|
|
|
def test_stale_check_unit():
|
|
"""Direct unit check of _check_stale_inbound thresholds."""
|
|
adapter = AibotWSAdapter({"bot_id": "b", "secret": "s", "heartbeat_s": 30.0})
|
|
# fresh connection: no raise
|
|
adapter._check_stale_inbound()
|
|
# 400s of silence -> raise
|
|
adapter._last_inbound = time.monotonic() - 400
|
|
with pytest.raises(ConnectionError):
|
|
adapter._check_stale_inbound()
|