feat(media): image/video/audio/file send+receive across channels
- inbound: wecom file/video frames download+decrypt into InboundMessage.files - session: store non-image attachments under media_in/, reference path in prompt - outbound: workspace sentinel diffs agent cwd after each turn; new files emit media events; wecom uploads via 3-step aibot upload protocol (init/chunk/finish) - >20MB or unsupported channels fall back to text with the server path
This commit is contained in:
1 parent
2b3ed03804
commit
c7497f0c4a
6 files changed
+593
-4
No files matched your search
@@ -14,6 +14,15 @@ from typing import Any, Callable, Protocol
|
|||||||
# inbound message
|
# inbound message
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MediaFile:
|
||||||
|
"""One inbound/outbound media attachment (image/video/audio/file)."""
|
||||||
|
kind: str # "image" | "video" | "audio" | "file"
|
||||||
|
data: bytes = b""
|
||||||
|
filename: str = ""
|
||||||
|
mime: str = "application/octet-stream"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class InboundMessage:
|
class InboundMessage:
|
||||||
channel: str # "wecom" | "wecom-app" | "weixin" | "nats"
|
channel: str # "wecom" | "wecom-app" | "weixin" | "nats"
|
||||||
@@ -22,6 +31,7 @@ class InboundMessage:
|
|||||||
text: str = ""
|
text: str = ""
|
||||||
images: list[bytes] = field(default_factory=list) # raw bytes
|
images: list[bytes] = field(default_factory=list) # raw bytes
|
||||||
image_mime: str = "image/png"
|
image_mime: str = "image/png"
|
||||||
|
files: list[MediaFile] = field(default_factory=list) # all attachments
|
||||||
raw: dict[str, Any] = field(default_factory=dict)
|
raw: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import time
|
|||||||
import uuid
|
import uuid
|
||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
|
|
||||||
from .base import IMChannel, InboundMessage, clean_control_chars, split_message_text, _interaction_text, _command_result_text
|
from .base import IMChannel, InboundMessage, MediaFile, clean_control_chars, split_message_text, _interaction_text, _command_result_text
|
||||||
|
|
||||||
class WecomAibotChannel:
|
class WecomAibotChannel:
|
||||||
name = "wecom"
|
name = "wecom"
|
||||||
@@ -61,6 +61,9 @@ class WecomAibotChannel:
|
|||||||
self._streams: dict[str, dict] = {}
|
self._streams: dict[str, dict] = {}
|
||||||
self._req_ids: dict[str, str] = {}
|
self._req_ids: dict[str, str] = {}
|
||||||
self._stream_lock = threading.Lock()
|
self._stream_lock = threading.Lock()
|
||||||
|
# media upload acks: req_id -> {event, msg}
|
||||||
|
self._pending_acks: dict[str, dict] = {}
|
||||||
|
self._ack_lock = threading.Lock()
|
||||||
# approval card answers: set by runtime, called as
|
# approval card answers: set by runtime, called as
|
||||||
# cb(request_id, option_id, answered_by) -> None
|
# cb(request_id, option_id, answered_by) -> None
|
||||||
self.card_answer_cb: Callable[[str, str, str], None] | None = None
|
self.card_answer_cb: Callable[[str, str, str], None] | None = None
|
||||||
@@ -159,7 +162,9 @@ class WecomAibotChannel:
|
|||||||
"""
|
"""
|
||||||
cmd = msg.get("cmd", "")
|
cmd = msg.get("cmd", "")
|
||||||
if msg.get("errcode") is not None and not cmd:
|
if msg.get("errcode") is not None and not cmd:
|
||||||
return # ack/pong frame
|
# ack/pong frame — route to a waiting media-upload _request() if any
|
||||||
|
self._deliver_ack(msg)
|
||||||
|
return
|
||||||
req_id = (msg.get("headers") or {}).get("req_id", "")
|
req_id = (msg.get("headers") or {}).get("req_id", "")
|
||||||
body = msg.get("body") or msg
|
body = msg.get("body") or msg
|
||||||
msgtype = body.get("msgtype", "text")
|
msgtype = body.get("msgtype", "text")
|
||||||
@@ -179,6 +184,7 @@ class WecomAibotChannel:
|
|||||||
return
|
return
|
||||||
text = ""
|
text = ""
|
||||||
images: list[bytes] = []
|
images: list[bytes] = []
|
||||||
|
files: list[MediaFile] = []
|
||||||
if msgtype == "text":
|
if msgtype == "text":
|
||||||
text = (body.get("text") or {}).get("content", "")
|
text = (body.get("text") or {}).get("content", "")
|
||||||
elif msgtype == "image":
|
elif msgtype == "image":
|
||||||
@@ -195,13 +201,23 @@ class WecomAibotChannel:
|
|||||||
images.append(data)
|
images.append(data)
|
||||||
elif msgtype == "voice":
|
elif msgtype == "voice":
|
||||||
text = (body.get("voice") or {}).get("content", "") # already transcribed
|
text = (body.get("voice") or {}).get("content", "") # already transcribed
|
||||||
|
elif msgtype == "file":
|
||||||
|
finfo = body.get("file") or {}
|
||||||
|
data = self._download_aibot_image(finfo)
|
||||||
|
if data:
|
||||||
|
files.append(MediaFile(kind="file", data=data, filename=str(finfo.get("filename") or "file.bin")))
|
||||||
|
elif msgtype == "video":
|
||||||
|
vinfo = body.get("video") or {}
|
||||||
|
data = self._download_aibot_image(vinfo)
|
||||||
|
if data:
|
||||||
|
files.append(MediaFile(kind="video", data=data, filename=str(vinfo.get("filename") or "video.mp4")))
|
||||||
# remember req_id for streaming replies keyed by chat
|
# remember req_id for streaming replies keyed by chat
|
||||||
if req_id:
|
if req_id:
|
||||||
with self._stream_lock:
|
with self._stream_lock:
|
||||||
self._req_ids[chat_id] = req_id
|
self._req_ids[chat_id] = req_id
|
||||||
text = clean_control_chars(text, max_len=100000).strip() if text else ""
|
text = clean_control_chars(text, max_len=100000).strip() if text else ""
|
||||||
# group chat: only respond when @mentioned is handled by WeCom itself
|
# group chat: only respond when @mentioned is handled by WeCom itself
|
||||||
if not text and not images:
|
if not text and not images and not files:
|
||||||
return
|
return
|
||||||
if self._msg_cb:
|
if self._msg_cb:
|
||||||
self._msg_cb(
|
self._msg_cb(
|
||||||
@@ -211,6 +227,7 @@ class WecomAibotChannel:
|
|||||||
user_id=str(sender),
|
user_id=str(sender),
|
||||||
text=text,
|
text=text,
|
||||||
images=images,
|
images=images,
|
||||||
|
files=files,
|
||||||
raw={"req_id": req_id, "msgid": body.get("msgid"), "chattype": body.get("chattype"), "msg": msg},
|
raw={"req_id": req_id, "msgid": body.get("msgid"), "chattype": body.get("chattype"), "msg": msg},
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -294,6 +311,17 @@ class WecomAibotChannel:
|
|||||||
self.reply(chat_id, _command_result_text(payload))
|
self.reply(chat_id, _command_result_text(payload))
|
||||||
elif kind == "final":
|
elif kind == "final":
|
||||||
self._stream_close(chat_id, str(payload.get("text", "")))
|
self._stream_close(chat_id, str(payload.get("text", "")))
|
||||||
|
elif kind == "media":
|
||||||
|
path = str(payload.get("path", ""))
|
||||||
|
size = int(payload.get("size") or 0)
|
||||||
|
try:
|
||||||
|
sent = self.send_media(chat_id, path) if path else False
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
self._logger.warning(f"[wecom] media send failed: {e}")
|
||||||
|
sent = False
|
||||||
|
if not sent:
|
||||||
|
kb = size // 1024
|
||||||
|
self.reply(chat_id, f"📎 已生成文件({kb} KB): {path}")
|
||||||
elif kind == "permission":
|
elif kind == "permission":
|
||||||
self.reply(chat_id, _interaction_text(payload))
|
self.reply(chat_id, _interaction_text(payload))
|
||||||
# "replace" treated as delta reset for now
|
# "replace" treated as delta reset for now
|
||||||
@@ -462,6 +490,98 @@ class WecomAibotChannel:
|
|||||||
return
|
return
|
||||||
ws.send(raw)
|
ws.send(raw)
|
||||||
|
|
||||||
|
# -- media: 3-step upload protocol (aibot_upload_media_init/chunk/finish) ----
|
||||||
|
UPLOAD_CHUNK_SIZE = 512 * 1024
|
||||||
|
MEDIA_MAX_BYTES = 20 * 1024 * 1024
|
||||||
|
|
||||||
|
def _deliver_ack(self, msg: dict) -> None:
|
||||||
|
"""Route an ack frame to a waiting _request() caller (called from recv loop)."""
|
||||||
|
rid = (msg.get("headers") or {}).get("req_id", "")
|
||||||
|
with self._ack_lock:
|
||||||
|
ev = self._pending_acks.pop(rid, None)
|
||||||
|
if ev is not None:
|
||||||
|
ev["msg"] = msg
|
||||||
|
ev["event"].set()
|
||||||
|
|
||||||
|
def _request(self, cmd: str, body: dict, wait_s: float = 30.0) -> dict | None:
|
||||||
|
"""Send a frame and wait for its errcode ack (matched by req_id)."""
|
||||||
|
rid = uuid.uuid4().hex
|
||||||
|
ev = {"event": threading.Event(), "msg": None}
|
||||||
|
with self._ack_lock:
|
||||||
|
self._pending_acks[rid] = ev
|
||||||
|
self._send({"cmd": cmd, "headers": {"req_id": rid}, "body": body})
|
||||||
|
if not ev["event"].wait(wait_s):
|
||||||
|
with self._ack_lock:
|
||||||
|
self._pending_acks.pop(rid, None)
|
||||||
|
return None
|
||||||
|
return ev["msg"]
|
||||||
|
|
||||||
|
def send_media(self, chat_id: str, path: str) -> bool:
|
||||||
|
"""Upload a local file and send it as an image/video/voice/file message.
|
||||||
|
|
||||||
|
Returns True when sent. Oversize (>20MB) files are refused (the caller
|
||||||
|
should tell the user the server-side path instead).
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
import os as _os
|
||||||
|
|
||||||
|
try:
|
||||||
|
size = _os.path.getsize(path)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
if size > self.MEDIA_MAX_BYTES:
|
||||||
|
self._logger.warning(f"[wecom] media too large, skip upload: {path} ({size}B)")
|
||||||
|
return False
|
||||||
|
name = _os.path.basename(path)
|
||||||
|
ext = _os.path.splitext(name)[1].lower()
|
||||||
|
media_type = ("image" if ext in (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")
|
||||||
|
else "video" if ext in (".mp4", ".mov", ".avi")
|
||||||
|
else "voice" if ext in (".amr", ".mp3", ".wav")
|
||||||
|
else "file")
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
data = fh.read()
|
||||||
|
md5 = hashlib.md5(data).hexdigest()
|
||||||
|
chunks = [data[i:i + self.UPLOAD_CHUNK_SIZE] for i in range(0, len(data), self.UPLOAD_CHUNK_SIZE)] or [b""]
|
||||||
|
|
||||||
|
ack = self._request("aibot_upload_media_init", {
|
||||||
|
"filename": name, "type": media_type, "total_size": size,
|
||||||
|
"total_chunks": len(chunks), "md5": md5,
|
||||||
|
})
|
||||||
|
if not ack or ack.get("errcode") not in (0, None):
|
||||||
|
self._logger.warning(f"[wecom] upload init failed: {ack}")
|
||||||
|
return False
|
||||||
|
upload_id = (ack.get("body") or {}).get("upload_id", "")
|
||||||
|
if not upload_id:
|
||||||
|
self._logger.warning("[wecom] upload init: no upload_id")
|
||||||
|
return False
|
||||||
|
import base64 as _b64
|
||||||
|
for idx, chunk in enumerate(chunks):
|
||||||
|
ack = self._request("aibot_upload_media_chunk", {
|
||||||
|
"upload_id": upload_id, "chunk_index": idx,
|
||||||
|
"total_chunks": len(chunks),
|
||||||
|
"base64_data": _b64.b64encode(chunk).decode("ascii"),
|
||||||
|
})
|
||||||
|
if not ack or ack.get("errcode") not in (0, None):
|
||||||
|
self._logger.warning(f"[wecom] upload chunk {idx} failed: {ack}")
|
||||||
|
return False
|
||||||
|
ack = self._request("aibot_upload_media_finish", {
|
||||||
|
"upload_id": upload_id, "filename": name, "media_type": media_type,
|
||||||
|
})
|
||||||
|
if not ack or ack.get("errcode") not in (0, None):
|
||||||
|
self._logger.warning(f"[wecom] upload finish failed: {ack}")
|
||||||
|
return False
|
||||||
|
media_id = (ack.get("body") or {}).get("media_id", "")
|
||||||
|
if not media_id:
|
||||||
|
self._logger.warning("[wecom] upload finish: no media_id")
|
||||||
|
return False
|
||||||
|
self._send({
|
||||||
|
"cmd": "aibot_send_msg",
|
||||||
|
"headers": {"req_id": uuid.uuid4().hex},
|
||||||
|
"body": {"chatid": chat_id, "chat_type": 0, "msgtype": media_type,
|
||||||
|
media_type: {"media_id": media_id}},
|
||||||
|
})
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
class _AuthFailure(Exception):
|
class _AuthFailure(Exception):
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
"""Media support: workspace sentinel scan for agent-produced files.
|
||||||
|
|
||||||
|
The ACP protocol only streams text updates — there is no "agent produced a
|
||||||
|
file" notification. Outbound media therefore works as a sentinel diff: the
|
||||||
|
session layer snapshots the agent workspace before a turn and scans it after;
|
||||||
|
new files become "media" events that channels deliver (wecom: 3-step upload;
|
||||||
|
other channels: text fallback with the path).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import mimetypes
|
||||||
|
import os
|
||||||
|
|
||||||
|
logger = logging.getLogger("agenthub.media")
|
||||||
|
|
||||||
|
# hard caps: never upload/download anything bigger; keep the scan cheap
|
||||||
|
MEDIA_MAX_BYTES = 20 * 1024 * 1024
|
||||||
|
SCAN_SKIP_DIRS = {".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", "media_in"}
|
||||||
|
SCAN_EXCLUDE_FILES = {".opencode.json", "opencode.json", "AGENTS.md"}
|
||||||
|
MAX_FILES_PER_TURN = 10
|
||||||
|
|
||||||
|
_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".svg"}
|
||||||
|
_VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
|
||||||
|
_AUDIO_EXTS = {".mp3", ".wav", ".amr", ".ogg", ".m4a", ".flac"}
|
||||||
|
|
||||||
|
|
||||||
|
def guess_kind(filename: str) -> str:
|
||||||
|
ext = os.path.splitext(filename)[1].lower()
|
||||||
|
if ext in _IMAGE_EXTS:
|
||||||
|
return "image"
|
||||||
|
if ext in _VIDEO_EXTS:
|
||||||
|
return "video"
|
||||||
|
if ext in _AUDIO_EXTS:
|
||||||
|
return "audio"
|
||||||
|
return "file"
|
||||||
|
|
||||||
|
|
||||||
|
def scan_new_files(cwd: str, seen: dict[str, float]) -> list[dict]:
|
||||||
|
"""Diff cwd against the `seen` snapshot {abspath: size}.
|
||||||
|
|
||||||
|
Returns descriptors for new files and mutates `seen`. Hidden entries and
|
||||||
|
SKIP_DIRS are never reported; oversized files are skipped (logged).
|
||||||
|
"""
|
||||||
|
found: list[dict] = []
|
||||||
|
for root, dirs, names in os.walk(cwd):
|
||||||
|
dirs[:] = [d for d in dirs if not d.startswith(".") and d not in SCAN_SKIP_DIRS]
|
||||||
|
for n in names:
|
||||||
|
if n.startswith(".") or n in SCAN_EXCLUDE_FILES:
|
||||||
|
continue
|
||||||
|
p = os.path.join(root, n)
|
||||||
|
try:
|
||||||
|
st = os.stat(p)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
key = os.path.abspath(p)
|
||||||
|
if key in seen or st.st_size == 0:
|
||||||
|
continue
|
||||||
|
seen[key] = st.st_size
|
||||||
|
if st.st_size > MEDIA_MAX_BYTES:
|
||||||
|
logger.info(f"[media] skip oversize file: {n} ({st.st_size}B)")
|
||||||
|
continue
|
||||||
|
found.append({
|
||||||
|
"path": p,
|
||||||
|
"filename": n,
|
||||||
|
"kind": guess_kind(n),
|
||||||
|
"mime": mimetypes.guess_type(n)[0] or "application/octet-stream",
|
||||||
|
"size": st.st_size,
|
||||||
|
})
|
||||||
|
return found[:MAX_FILES_PER_TURN]
|
||||||
@@ -184,7 +184,9 @@ def _safe_inbound(mgr: SessionManager, ch, m) -> None:
|
|||||||
"(如需指定其他 agent,回复 /agents 查看、/switch <名> 切换)。",
|
"(如需指定其他 agent,回复 /agents 查看、/switch <名> 切换)。",
|
||||||
})
|
})
|
||||||
# 4) normal prompt
|
# 4) normal prompt
|
||||||
mgr.prompt(ch.name, m.chat_id, m.text, images=m.images)
|
mgr.prompt(ch.name, m.chat_id, m.text, images=m.images,
|
||||||
|
files=[{"data": f.data, "filename": f.filename, "mime": f.mime, "kind": f.kind}
|
||||||
|
for f in getattr(m, "files", []) or []])
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
import logging
|
import logging
|
||||||
logging.getLogger("agenthub").error(f"[{ch.name}:{m.chat_id}] inbound failed: {e}")
|
logging.getLogger("agenthub").error(f"[{ch.name}:{m.chat_id}] inbound failed: {e}")
|
||||||
|
|||||||
@@ -156,6 +156,7 @@ class SessionManager:
|
|||||||
self._pending_permissions: dict[str, "PermissionRequest"] = {}
|
self._pending_permissions: dict[str, "PermissionRequest"] = {}
|
||||||
self.permission_timeout_s = 300 # 5min to answer; timeout -> deny
|
self.permission_timeout_s = 300 # 5min to answer; timeout -> deny
|
||||||
self._deltas: dict[str, list[str]] = {} # binding_key -> accumulated turn text
|
self._deltas: dict[str, list[str]] = {} # binding_key -> accumulated turn text
|
||||||
|
self._media_seen: dict[str, dict[str, float]] = {} # agent -> workspace file snapshot
|
||||||
|
|
||||||
self._db: sqlite3.Connection | None = None
|
self._db: sqlite3.Connection | None = None
|
||||||
if db_path:
|
if db_path:
|
||||||
@@ -359,6 +360,7 @@ class SessionManager:
|
|||||||
agent: str | None = None,
|
agent: str | None = None,
|
||||||
cwd: str | None = None,
|
cwd: str | None = None,
|
||||||
images: list[bytes] | None = None,
|
images: list[bytes] | None = None,
|
||||||
|
files: list[dict[str, Any]] | None = None,
|
||||||
timeout_s: float | None = None,
|
timeout_s: float | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Run one turn on the bound (or newly created) session."""
|
"""Run one turn on the bound (or newly created) session."""
|
||||||
@@ -380,11 +382,26 @@ class SessionManager:
|
|||||||
blocks: list[dict[str, Any]] = [{"type": "text", "text": text}]
|
blocks: list[dict[str, Any]] = [{"type": "text", "text": text}]
|
||||||
for img in images or []:
|
for img in images or []:
|
||||||
blocks.append({"type": "image", "data": _b64(img), "mimeType": "image/png"})
|
blocks.append({"type": "image", "data": _b64(img), "mimeType": "image/png"})
|
||||||
|
# non-image attachments (video/audio/file): store under media_in/ and
|
||||||
|
# reference the path — agents read files with their own tools
|
||||||
|
stored_files = self._store_inbound_files(b.agent, files or [])
|
||||||
|
if stored_files:
|
||||||
|
note = "\n".join(f"[用户发送了文件: {p} ({name})]" for p, name in stored_files)
|
||||||
|
blocks[0]["text"] = (blocks[0]["text"] + "\n" + note).strip()
|
||||||
key = f"{channel}:{chat_id}"
|
key = f"{channel}:{chat_id}"
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._deltas[key] = [] # reset accumulated turn text
|
self._deltas[key] = [] # reset accumulated turn text
|
||||||
self._append_text(key, b.agent, b.acp_session_id, {"role": "user", "text": text})
|
self._append_text(key, b.agent, b.acp_session_id, {"role": "user", "text": text})
|
||||||
self._attach_update_relay(b)
|
self._attach_update_relay(b)
|
||||||
|
# outbound media sentinel: snapshot agent workspace before the turn;
|
||||||
|
# after end_turn, new files become "media" events for the channels
|
||||||
|
ws_dir = self._agent_cwd(b.agent) or os.getcwd()
|
||||||
|
seen_before = self._media_seen.setdefault(b.agent, {})
|
||||||
|
try:
|
||||||
|
from .media import scan_new_files
|
||||||
|
scan_new_files(ws_dir, seen_before) # pre-turn snapshot
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.warning(f"[{key}] media pre-scan failed: {e}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
res = client.session_prompt(b.acp_session_id, blocks, timeout=timeout_s)
|
res = client.session_prompt(b.acp_session_id, blocks, timeout=timeout_s)
|
||||||
@@ -409,10 +426,47 @@ class SessionManager:
|
|||||||
with self._lock:
|
with self._lock:
|
||||||
full_text = "".join(self._deltas.get(key, []))
|
full_text = "".join(self._deltas.get(key, []))
|
||||||
self._emit(key, "final", {"text": full_text, "stop_reason": res.get("stopReason")})
|
self._emit(key, "final", {"text": full_text, "stop_reason": res.get("stopReason")})
|
||||||
|
# post-turn sentinel diff: emit newly produced files as media events
|
||||||
|
try:
|
||||||
|
from .media import scan_new_files
|
||||||
|
for ev in scan_new_files(ws_dir, seen_before):
|
||||||
|
self._emit(key, "media", ev)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.warning(f"[{key}] media post-scan failed: {e}")
|
||||||
with self._lock:
|
with self._lock:
|
||||||
b.last_active = time.time()
|
b.last_active = time.time()
|
||||||
return {"stop_reason": res.get("stopReason"), "session_id": b.acp_session_id, "agent": b.agent}
|
return {"stop_reason": res.get("stopReason"), "session_id": b.acp_session_id, "agent": b.agent}
|
||||||
|
|
||||||
|
# -- media ---------------------------------------------------------------
|
||||||
|
def _agent_cwd(self, agent_name: str) -> str | None:
|
||||||
|
"""Agent workspace root: first whitelist entry (expanded) or None."""
|
||||||
|
spec = self.agents.get(agent_name)
|
||||||
|
if spec and spec.cwd_whitelist:
|
||||||
|
return os.path.expanduser(spec.cwd_whitelist[0])
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _media_dir(self, agent: str) -> str:
|
||||||
|
"""Directory where inbound attachments are stored for the agent."""
|
||||||
|
root = self._agent_cwd(agent) or os.path.expanduser("~/agenthub-media")
|
||||||
|
d = os.path.join(root, "media_in")
|
||||||
|
os.makedirs(d, exist_ok=True)
|
||||||
|
return d
|
||||||
|
|
||||||
|
def _store_inbound_files(self, agent: str, files: list[dict[str, Any]]) -> list[tuple[str, str]]:
|
||||||
|
"""Persist inbound attachments; returns [(abs_path, filename)]."""
|
||||||
|
out: list[tuple[str, str]] = []
|
||||||
|
d = self._media_dir(agent)
|
||||||
|
for f in files:
|
||||||
|
name = os.path.basename(str(f.get("filename") or "file.bin")) or "file.bin"
|
||||||
|
path = os.path.join(d, f"{int(time.time()*1000)}_{name}")
|
||||||
|
try:
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(bytes(f.get("data") or b""))
|
||||||
|
out.append((path, name))
|
||||||
|
except OSError as e:
|
||||||
|
log.warning(f"[media] store inbound file failed: {e}")
|
||||||
|
return out
|
||||||
|
|
||||||
# -- permission ---------------------------------------------------------------
|
# -- permission ---------------------------------------------------------------
|
||||||
def pending_interaction(self, channel: str, chat_id: str) -> dict[str, Any] | None:
|
def pending_interaction(self, channel: str, chat_id: str) -> dict[str, Any] | None:
|
||||||
"""The most recent pending interaction for this chat, if any.
|
"""The most recent pending interaction for this chat, if any.
|
||||||
|
|||||||
@@ -0,0 +1,333 @@
|
|||||||
|
"""Media send/receive support (images, video, audio, files).
|
||||||
|
|
||||||
|
Inbound: wecom file/video frames download+decrypt into InboundMessage.files;
|
||||||
|
session layer stores them under media_in/ and tells the agent the path.
|
||||||
|
|
||||||
|
Outbound: after each turn, the session layer diffs the agent's workspace for
|
||||||
|
new files ("media sentinel") and emits "media" events; channels upload and
|
||||||
|
send them (wecom uses the 3-step aibot upload protocol).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from agenthub.channels.base import InboundMessage
|
||||||
|
from agenthub.channels.wecom import WecomAibotChannel
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _channel(ws=None) -> WecomAibotChannel:
|
||||||
|
ch = WecomAibotChannel(
|
||||||
|
{"bot_id": "b", "secret": "s"},
|
||||||
|
ws_factory=lambda cfg: ws,
|
||||||
|
)
|
||||||
|
return ch
|
||||||
|
|
||||||
|
|
||||||
|
class FakeWS:
|
||||||
|
"""Records sent frames; feed acks via pop_sent()."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.sent: list[dict] = []
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
|
||||||
|
def connect(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def send(self, raw: str):
|
||||||
|
with self.lock:
|
||||||
|
self.sent.append(json.loads(raw))
|
||||||
|
|
||||||
|
def recv(self, timeout=1.0):
|
||||||
|
time.sleep(0.01)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _make_media_file(tmp_path, name="out.png", size=600):
|
||||||
|
p = tmp_path / name
|
||||||
|
p.write_bytes(b"\x89PNG" + os.urandom(size))
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# inbound: file / video frames
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_inbound_file_frame_downloaded_and_attached(tmp_path, monkeypatch):
|
||||||
|
ch = _channel()
|
||||||
|
payload = b"PDFDATA" * 10
|
||||||
|
monkeypatch.setattr(ch, "_download_aibot_image", lambda d: payload)
|
||||||
|
got: list[InboundMessage] = []
|
||||||
|
ch.on_message(lambda m: got.append(m))
|
||||||
|
frame = {
|
||||||
|
"cmd": "aibot_msg_callback",
|
||||||
|
"headers": {"req_id": "r1"},
|
||||||
|
"body": {
|
||||||
|
"chatid": "c1",
|
||||||
|
"chattype": 0,
|
||||||
|
"msgid": "m1",
|
||||||
|
"from": {"userid": "u1"},
|
||||||
|
"msgtype": "file",
|
||||||
|
"file": {"url": "https://x/a", "aeskey": "k", "filename": "doc.pdf"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ch._dispatch(frame, None)
|
||||||
|
assert len(got) == 1
|
||||||
|
m = got[0]
|
||||||
|
assert len(m.files) == 1
|
||||||
|
assert m.files[0].data == payload
|
||||||
|
assert m.files[0].filename == "doc.pdf"
|
||||||
|
assert m.files[0].kind == "file"
|
||||||
|
|
||||||
|
|
||||||
|
def test_inbound_video_frame_attached(tmp_path, monkeypatch):
|
||||||
|
ch = _channel()
|
||||||
|
monkeypatch.setattr(ch, "_download_aibot_image", lambda d: b"MP4")
|
||||||
|
got: list[InboundMessage] = []
|
||||||
|
ch.on_message(got.append)
|
||||||
|
frame = {
|
||||||
|
"cmd": "aibot_msg_callback",
|
||||||
|
"headers": {"req_id": "r2"},
|
||||||
|
"body": {
|
||||||
|
"chatid": "c1", "chattype": 0, "msgid": "m2",
|
||||||
|
"from": {"userid": "u1"},
|
||||||
|
"msgtype": "video",
|
||||||
|
"video": {"url": "https://x/v", "aeskey": "k"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ch._dispatch(frame, None)
|
||||||
|
assert got[0].files[0].kind == "video"
|
||||||
|
|
||||||
|
|
||||||
|
def test_inbound_voice_still_transcribed_text():
|
||||||
|
ch = _channel()
|
||||||
|
got: list[InboundMessage] = []
|
||||||
|
ch.on_message(got.append)
|
||||||
|
frame = {
|
||||||
|
"cmd": "aibot_msg_callback",
|
||||||
|
"headers": {"req_id": "r3"},
|
||||||
|
"body": {
|
||||||
|
"chatid": "c1", "chattype": 0, "msgid": "m3",
|
||||||
|
"from": {"userid": "u1"},
|
||||||
|
"msgtype": "voice",
|
||||||
|
"voice": {"content": "帮我查天气"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ch._dispatch(frame, None)
|
||||||
|
assert got[0].text == "帮我查天气"
|
||||||
|
assert got[0].files == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# outbound: wecom 3-step upload + send
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_wecom_send_media_upload_protocol(tmp_path):
|
||||||
|
ws = FakeWS()
|
||||||
|
ch = _channel(ws)
|
||||||
|
ch._ws = ws # pretend connected
|
||||||
|
f = _make_media_file(tmp_path)
|
||||||
|
|
||||||
|
# ack pump: respond to each upload frame with proper ack
|
||||||
|
def pump():
|
||||||
|
deadline = time.time() + 5
|
||||||
|
while time.time() < deadline:
|
||||||
|
for fr in list(ws.sent):
|
||||||
|
cmd = fr.get("cmd", "")
|
||||||
|
if cmd.startswith("aibot_upload_media"):
|
||||||
|
rid = fr["headers"]["req_id"]
|
||||||
|
if cmd.endswith("init"):
|
||||||
|
body = {"errcode": 0, "upload_id": "UP1"}
|
||||||
|
elif cmd.endswith("finish"):
|
||||||
|
body = {"errcode": 0, "media_id": "MEDIA1"}
|
||||||
|
else:
|
||||||
|
body = {"errcode": 0}
|
||||||
|
ch._deliver_ack({"cmd": "", "headers": {"req_id": rid},
|
||||||
|
"errcode": 0, "body": body, **body})
|
||||||
|
with ws.lock:
|
||||||
|
ws.sent = [x for x in ws.sent if x is not fr]
|
||||||
|
time.sleep(0.02)
|
||||||
|
|
||||||
|
t = threading.Thread(target=pump, daemon=True)
|
||||||
|
t.start()
|
||||||
|
ok = ch.send_media("c1", str(f))
|
||||||
|
t.join(timeout=2)
|
||||||
|
|
||||||
|
# pump consumed upload frames; keep a global record via channel outbox trick:
|
||||||
|
# re-run protocol with an ack-pump that records commands before consuming
|
||||||
|
class RecordingWS(FakeWS):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.all_cmds: list[str] = []
|
||||||
|
self.lock2 = threading.Lock()
|
||||||
|
|
||||||
|
def send(self, raw: str):
|
||||||
|
fr = json.loads(raw)
|
||||||
|
with self.lock2:
|
||||||
|
self.all_cmds.append(fr.get("cmd", ""))
|
||||||
|
super().send(raw)
|
||||||
|
|
||||||
|
ws2 = RecordingWS()
|
||||||
|
ch2 = _channel(ws2)
|
||||||
|
ch2._ws = ws2
|
||||||
|
|
||||||
|
def pump2():
|
||||||
|
deadline = time.time() + 5
|
||||||
|
while time.time() < deadline:
|
||||||
|
for fr in list(ws2.sent):
|
||||||
|
cmd = fr.get("cmd", "")
|
||||||
|
if cmd.startswith("aibot_upload_media"):
|
||||||
|
rid = fr["headers"]["req_id"]
|
||||||
|
if cmd.endswith("init"):
|
||||||
|
body = {"errcode": 0, "upload_id": "UP1"}
|
||||||
|
elif cmd.endswith("finish"):
|
||||||
|
body = {"errcode": 0, "media_id": "MEDIA1"}
|
||||||
|
else:
|
||||||
|
body = {"errcode": 0}
|
||||||
|
ch2._deliver_ack({"cmd": "", "headers": {"req_id": rid},
|
||||||
|
"errcode": 0, "body": body, **body})
|
||||||
|
with ws2.lock:
|
||||||
|
ws2.sent = [x for x in ws2.sent if x is not fr]
|
||||||
|
time.sleep(0.02)
|
||||||
|
|
||||||
|
t2 = threading.Thread(target=pump2, daemon=True)
|
||||||
|
t2.start()
|
||||||
|
assert ch2.send_media("c1", str(f))
|
||||||
|
t2.join(timeout=2)
|
||||||
|
|
||||||
|
cmds = list(ws2.all_cmds)
|
||||||
|
assert "aibot_upload_media_init" in cmds
|
||||||
|
assert cmds.count("aibot_upload_media_chunk") == 1
|
||||||
|
assert "aibot_upload_media_finish" in cmds
|
||||||
|
send_frames = [fr for fr in _all_sent(ws2) if fr.get("cmd") == "aibot_send_msg"]
|
||||||
|
assert any(
|
||||||
|
fr["body"].get("msgtype") == "image" and fr["body"]["image"]["media_id"] == "MEDIA1"
|
||||||
|
for fr in send_frames
|
||||||
|
)
|
||||||
|
assert ok
|
||||||
|
|
||||||
|
|
||||||
|
def _all_sent(ws):
|
||||||
|
# frames already consumed by pump are tracked in ch history; simplest: record
|
||||||
|
return ws.sent
|
||||||
|
|
||||||
|
|
||||||
|
def test_wecom_send_media_too_large_downgrades_to_text(tmp_path):
|
||||||
|
ws = FakeWS()
|
||||||
|
ch = _channel(ws)
|
||||||
|
ch._ws = ws
|
||||||
|
f = tmp_path / "big.png"
|
||||||
|
f.write_bytes(b"x" * (21 * 1024 * 1024)) # > 20MB
|
||||||
|
ok = ch.send_media("c1", str(f))
|
||||||
|
assert not ok # rejected, channel should reply text instead (checked in session layer)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# session layer: sentinel scan
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_sentinel_diff_detects_new_files(tmp_path):
|
||||||
|
from agenthub.media import scan_new_files
|
||||||
|
|
||||||
|
base = {}
|
||||||
|
events = scan_new_files(str(tmp_path), base)
|
||||||
|
assert events == []
|
||||||
|
|
||||||
|
f1 = _make_media_file(tmp_path, "a.png")
|
||||||
|
events = scan_new_files(str(tmp_path), base)
|
||||||
|
assert len(events) == 1
|
||||||
|
assert events[0]["filename"] == "a.png"
|
||||||
|
assert events[0]["kind"] == "image"
|
||||||
|
# second scan: no new files
|
||||||
|
assert scan_new_files(str(tmp_path), base) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_sentinel_ignores_hidden_and_oversize(tmp_path):
|
||||||
|
from agenthub.media import scan_new_files, MEDIA_MAX_BYTES
|
||||||
|
|
||||||
|
(tmp_path / ".git").mkdir()
|
||||||
|
(tmp_path / ".git" / "x.png").write_bytes(b"p")
|
||||||
|
big = tmp_path / "big.bin"
|
||||||
|
big.write_bytes(b"z" * (MEDIA_MAX_BYTES + 1))
|
||||||
|
base = {}
|
||||||
|
events = scan_new_files(str(tmp_path), base)
|
||||||
|
assert events == [] # hidden dir ignored; oversize ignored (logged)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# session integration: prompt with files -> blocks + media_in storage
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_prompt_with_file_stores_and_references(tmp_path, monkeypatch):
|
||||||
|
"""prompt() with files stores them under media_in/ and adds a path note."""
|
||||||
|
from agenthub.session import SessionManager
|
||||||
|
import agenthub.session as sess_mod
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
class StubClient:
|
||||||
|
def __init__(self):
|
||||||
|
self._notif_handlers = []
|
||||||
|
self._req_handlers = []
|
||||||
|
|
||||||
|
def is_alive(self):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def register_notification_handler(self, h):
|
||||||
|
self._notif_handlers.append(h)
|
||||||
|
|
||||||
|
def register_request_handler(self, h):
|
||||||
|
self._req_handlers.append(h)
|
||||||
|
|
||||||
|
def session_prompt(self, sid, blocks, timeout=None):
|
||||||
|
captured["blocks"] = blocks
|
||||||
|
return {"stopReason": "end_turn"}
|
||||||
|
|
||||||
|
def session_new(self, cwd=None, mcp_servers=None):
|
||||||
|
return "s9"
|
||||||
|
|
||||||
|
def set_config_option(self, sid, config_id, value):
|
||||||
|
return {"currentValue": value}
|
||||||
|
|
||||||
|
def list_config_options(self, sid):
|
||||||
|
return {"configOptions": []}
|
||||||
|
|
||||||
|
mgr = SessionManager({"opencode": {"cmd": "opencode acp"}}, db_path=str(tmp_path / "s.db"))
|
||||||
|
stub = StubClient()
|
||||||
|
monkeypatch.setattr(mgr, "_client_for", lambda name: stub)
|
||||||
|
monkeypatch.setattr(mgr, "_ensure_session", lambda k, a, cwd=None: (stub, "s9"))
|
||||||
|
|
||||||
|
media_dir = tmp_path / "agent_home"
|
||||||
|
media_dir.mkdir()
|
||||||
|
monkeypatch.setattr(mgr, "_media_dir", lambda agent: str(media_dir))
|
||||||
|
|
||||||
|
f = tmp_path / "in.pdf"
|
||||||
|
f.write_bytes(b"hello pdf")
|
||||||
|
|
||||||
|
# stub workspace scan targets: keep sentinel happy (no whitelist -> cwd fallback)
|
||||||
|
monkeypatch.setattr(mgr, "_agent_cwd", lambda agent: str(tmp_path / "ws"))
|
||||||
|
(tmp_path / "ws").mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
mgr.prompt("wecom", "c1", "看看这个", files=[{"data": f.read_bytes(), "filename": "in.pdf", "mime": "application/pdf"}])
|
||||||
|
|
||||||
|
text = captured["blocks"][0]["text"]
|
||||||
|
assert "用户发送了文件" in text
|
||||||
|
assert "agent_home" in text
|
||||||
|
stored = list(media_dir.glob("*_in.pdf")) or list(media_dir.glob("media_in/*in.pdf"))
|
||||||
|
assert len(stored) == 1
|
||||||
|
assert stored[0].read_bytes() == b"hello pdf"
|
||||||
|
# file also delivered as an ACP content block path note (text only agent-facing)
|
||||||
Reference in new issue
Block a user