agenthub/backend/tests/test_gateway_ws.py

272 lines
12 KiB
Python

"""The connector socket: an agent dials out, the hub drives it from there.
These are the only tests that exercise the real wire format, so they double as
the protocol documentation for anyone writing a connector in another language.
"""
from __future__ import annotations
import threading
import pytest
from starlette.websockets import WebSocketDisconnect
from agenthub.gateway.frames import Frame
from tests.conftest import BASE, register, task_status, wait_for
def hello(offer, **payload) -> dict:
return Frame(
type="event", kind="agent.hello", agent_id=offer["agent_id"], payload={"connector_version": "0.1.0", **payload}
).to_dict()
def event(offer, task_id: str, **payload) -> dict:
return Frame(
type="event",
kind="agent.event",
topic=f"task.{task_id}.event",
agent_id=offer["agent_id"],
task_id=task_id,
payload=payload,
).to_dict()
def dispatch(client, offer) -> dict:
response = client.post(f"{BASE}/tasks", json={"prompt": "do the thing", "agent_id": offer["agent_id"]})
assert response.status_code == 201, response.text
return response.json()
def fire(client, method: str, url: str):
"""Issue a hub request while this thread is busy answering the socket."""
box: dict = {}
def run() -> None:
box["response"] = getattr(client, method)(url)
thread = threading.Thread(target=run, daemon=True)
thread.start()
return box, thread
def test_connector_socket_streams_a_task_end_to_end(hub):
offer = register(hub, "edge-node", driver="connector", deployment_kind="docker", deployment_ref="compose")
agent_id, token = offer["agent_id"], offer["agent_token"]
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={token}") as ws:
greeting = ws.receive_json()
assert greeting["kind"] == "hub.hello"
assert greeting["payload"]["agent_id"] == agent_id
assert greeting["payload"]["ws_url"].endswith("/api/v1/gateway/ws")
ws.send_json(hello(offer, capabilities={"streaming": True}))
assert hub.get(f"{BASE}/gateway/status").json()["connected_agents"] == [agent_id]
agent = hub.get(f"{BASE}/agents/{agent_id}").json()
assert agent["connector"]["connected"] is True and agent["live"] is True
assert agent["deployment_kind"] == "docker"
task = dispatch(hub, offer)
request = ws.receive_json()
assert request["type"] == "req" and request["kind"] == "task.run"
assert request["payload"]["prompt"] == "do the thing"
assert request["payload"]["task_id"] == task["id"]
ws.send_json(event(offer, task["id"], type="delta", text="pon"))
ws.send_json(event(offer, task["id"], type="delta", text="g"))
ws.send_json(event(offer, task["id"], type="completed", text="pong", usage={}))
ws.send_json(
Frame(type="res", kind="task.run", reply_to=request["id"], payload={"ok": True, "text": "pong"}).to_dict()
)
assert wait_for(lambda: hub.get(f"{BASE}/tasks/{task['id']}").json()["status"] == "completed")
assert hub.get(f"{BASE}/gateway/status").json()["connected_agents"] == [agent_id]
# a graceful goodbye over the socket is the same operation as
# POST /gateway/deregister: the channel goes and the agent goes offline
ws.send_json(Frame(type="event", kind="agent.deregister", agent_id=agent_id).to_dict())
assert wait_for(lambda: hub.get(f"{BASE}/gateway/status").json()["connected_agents"] == [])
assert wait_for(lambda: hub.get(f"{BASE}/agents/{agent_id}").json()["status"] == "offline")
finished = hub.get(f"{BASE}/tasks/{task['id']}").json()
assert finished["result"] == "pong"
# every event is stored exactly once, in the order the agent sent it
kinds = [row["kind"] for row in hub.get(f"{BASE}/tasks/{task['id']}/events").json()]
assert kinds == ["status", "delta", "delta", "completed"]
agent = hub.get(f"{BASE}/agents/{agent_id}").json()
assert agent["live"] is False and agent["connector"]["connected"] is False
def test_a_labeled_driver_still_reaches_the_agent_over_its_socket(hub):
"""``--driver nanobot`` says what runs *behind* the connector.
A connector row says the hub must dial out over the socket; dialling the
(absent) base_url used to fail every task.
"""
offer = register(hub, "real-nanobot", driver="nanobot", connection_mode="connector")
agent_id, token = offer["agent_id"], offer["agent_token"]
assert hub.get(f"{BASE}/agents/{agent_id}").json()["connection_mode"] == "connector"
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={token}") as ws:
ws.receive_json()
ws.send_json(hello(offer, capabilities={"streaming": True}))
task = dispatch(hub, offer)
request = ws.receive_json()
assert request["kind"] == "task.run"
assert request["payload"]["provider"] == "" # no override: use our own provider
ws.send_json(
Frame(
type="res", kind="task.run", reply_to=request["id"], payload={"ok": True, "text": "built"}
).to_dict()
)
assert wait_for(lambda: hub.get(f"{BASE}/tasks/{task['id']}").json()["status"] == "completed")
finished = hub.get(f"{BASE}/tasks/{task["id"]}").json()
assert finished["result"] == "built"
events = hub.get(f"{BASE}/tasks/{task["id"]}/events").json()
assert events[0]["payload"]["driver"] == "connector"
def test_connector_can_answer_without_streaming(hub):
offer = register(hub, "quiet-agent", driver="connector")
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={offer['agent_token']}") as ws:
ws.receive_json()
ws.send_json(hello(offer))
task = dispatch(hub, offer)
request = ws.receive_json()
assert request["kind"] == "task.run"
ws.send_json(
Frame(type="res", kind="task.run", reply_to=request["id"], payload={"text": "answer only"}).to_dict()
)
assert wait_for(lambda: task_status(hub, task["id"]) == "completed")
assert hub.get(f"{BASE}/tasks/{task['id']}").json()["result"] == "answer only"
def test_probe_and_describe_travel_down_the_socket(hub):
offer = register(hub, "probed", driver="connector")
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={offer['agent_token']}") as ws:
ws.receive_json()
ws.send_json(hello(offer))
probe, thread = fire(hub, "post", f"{BASE}/agents/{offer['agent_id']}/probe")
ping = ws.receive_json()
assert ping["kind"] == "agent.ping"
ws.send_json(Frame(type="res", kind="agent.ping", reply_to=ping["id"], payload={"ok": True, "rtt_ms": 4.5}).to_dict())
thread.join(20)
assert probe["response"].status_code == 200
body = probe["response"].json()
assert body["ok"] is True and body["health"]["latency_ms"] == 4.5
describe, thread = fire(hub, "post", f"{BASE}/agents/{offer['agent_id']}/describe")
request = ws.receive_json()
assert request["kind"] == "agent.describe"
ws.send_json(
Frame(type="res", kind="agent.describe", reply_to=request["id"], payload={"models": ["nanobot-1"]}).to_dict()
)
thread.join(20)
assert describe["response"].json()["capabilities"]["models"] == ["nanobot-1"]
frames = hub.get(f"{BASE}/gateway/frames?agent_id={offer['agent_id']}").json()
assert {"agent.hello", "agent.ping", "agent.describe"} <= {frame["kind"] for frame in frames}
def test_socket_requires_a_valid_agent_token(hub):
with pytest.raises(WebSocketDisconnect) as failure:
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token=ag_not_a_real_token") as ws:
ws.receive_json()
assert failure.value.code == 4401
def test_console_socket_receives_frames(hub):
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status" ) as ws:
greeting = ws.receive_json()
assert greeting["kind"] == "hub.hello"
assert greeting["payload"]["topics"] == ["agent.*.status"]
offer = register(hub, "watched", driver="connector")
seen = []
for _ in range(6):
frame = ws.receive_json()
if "kind" not in frame: # {"type": "ping"} keep-alive
continue
seen.append((frame["kind"], frame.get("agent_id")))
if frame["kind"] == "agent.changed":
break
assert ("agent.changed", offer["agent_id"]) in seen
ws.send_json({"action": "ping"})
# the pong proves the socket is bidirectional and still alive
for _ in range(6):
frame = ws.receive_json()
if frame.get("type") == "pong":
break
else:
pytest.fail("no pong on the console socket")
def test_cancelling_a_task_reaches_the_agent_over_its_socket(hub):
"""Cancel travels down the same socket as the task: the agent is told to stop."""
offer = register(hub, "slow-edge", driver="connector", deployment_kind="cloud")
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={offer['agent_token']}") as ws:
assert ws.receive_json()["kind"] == "hub.hello"
ws.send_json(hello(offer))
task = dispatch(hub, offer)
run_request = ws.receive_json()
assert run_request["kind"] == "task.run"
box, thread = fire(hub, "post", f"{BASE}/tasks/{task['id']}/cancel")
cancel_request = ws.receive_json()
assert cancel_request["type"] == "req" and cancel_request["kind"] == "task.cancel"
assert cancel_request["payload"]["invocation_id"] == run_request["payload"]["invocation_id"]
ws.send_json(
Frame(type="res", kind="task.cancel", reply_to=cancel_request["id"], payload={"ok": True}).to_dict()
)
thread.join(10)
assert not thread.is_alive() and box["response"].status_code == 200
assert wait_for(lambda: task_status(hub, task["id"]) == "cancelled")
# exactly one cancel RPC per cancel: a connector never sees a duplicate
asked = [row for row in hub.get(f"{BASE}/gateway/frames?kind=task.cancel").json() if row["type"] == "req"]
assert [row["payload"]["invocation_id"] for row in asked] == [run_request["payload"]["invocation_id"]]
kinds = [row["kind"] for row in hub.get(f"{BASE}/tasks/{task['id']}/events").json()]
assert kinds[-1] == "cancelled" and kinds.count("cancelled") == 1
assert hub.get(f"{BASE}/agents/{offer['agent_id']}").json()["live"] is True
# ...and a late answer cannot bring a finished task back to life
ws.send_json(event(offer, task["id"], type="completed", text="too late"))
ws.send_json(
Frame(
type="res", kind="task.run", reply_to=run_request["id"], payload={"ok": True, "text": "too late"}
).to_dict()
)
assert task_status(hub, task["id"]) == "cancelled"
assert hub.get(f"{BASE}/tasks/{task['id']}").json()["result"] != "too late"
kinds = [row["kind"] for row in hub.get(f"{BASE}/tasks/{task['id']}/events").json()]
assert kinds.count("cancelled") == 1 and "completed" not in kinds
def test_console_reconnect_replays_only_the_gap(hub):
"""?since= makes a dropped console socket whole again."""
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status") as ws:
greeting = ws.receive_json()
assert greeting["kind"] == "hub.hello"
since = greeting["payload"]["seq"]
offer = register(hub, "watched", driver="connector")
while True:
frame = ws.receive_json()
if frame.get("kind") == "agent.changed":
break
assert frame["agent_id"] == offer["agent_id"] and frame["seq"] > since
# Reconnecting with ?since= gets the missed frame back, oldest first.
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status&since={since}") as ws:
ws.receive_json() # hub.hello
marker = ws.receive_json()
assert marker["type"] == "replay" and marker["count"] == 1
replayed = ws.receive_json()
assert replayed["kind"] == "agent.changed"
assert replayed["agent_id"] == offer["agent_id"]
# Without it the same agent is simply not delivered a second time.
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status") as ws:
ws.receive_json() # hub.hello
later = register(hub, "later", driver="connector")
while True:
frame = ws.receive_json()
if frame.get("kind") == "agent.changed":
break
assert frame["agent_id"] == later["agent_id"]