272 lines
12 KiB
Python
272 lines
12 KiB
Python
"""The connector socket: an agent dials out, the hub drives it from there.
|
|
|
|
These are the only tests that exercise the real wire format, so they double as
|
|
the protocol documentation for anyone writing a connector in another language.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
|
|
import pytest
|
|
from starlette.websockets import WebSocketDisconnect
|
|
|
|
from agenthub.gateway.frames import Frame
|
|
from tests.conftest import BASE, register, task_status, wait_for
|
|
|
|
|
|
def hello(offer, **payload) -> dict:
|
|
return Frame(
|
|
type="event", kind="agent.hello", agent_id=offer["agent_id"], payload={"connector_version": "0.1.0", **payload}
|
|
).to_dict()
|
|
|
|
|
|
def event(offer, task_id: str, **payload) -> dict:
|
|
return Frame(
|
|
type="event",
|
|
kind="agent.event",
|
|
topic=f"task.{task_id}.event",
|
|
agent_id=offer["agent_id"],
|
|
task_id=task_id,
|
|
payload=payload,
|
|
).to_dict()
|
|
|
|
|
|
def dispatch(client, offer) -> dict:
|
|
response = client.post(f"{BASE}/tasks", json={"prompt": "do the thing", "agent_id": offer["agent_id"]})
|
|
assert response.status_code == 201, response.text
|
|
return response.json()
|
|
|
|
|
|
def fire(client, method: str, url: str):
|
|
"""Issue a hub request while this thread is busy answering the socket."""
|
|
box: dict = {}
|
|
|
|
def run() -> None:
|
|
box["response"] = getattr(client, method)(url)
|
|
|
|
thread = threading.Thread(target=run, daemon=True)
|
|
thread.start()
|
|
return box, thread
|
|
|
|
|
|
def test_connector_socket_streams_a_task_end_to_end(hub):
|
|
offer = register(hub, "edge-node", driver="connector", deployment_kind="docker", deployment_ref="compose")
|
|
agent_id, token = offer["agent_id"], offer["agent_token"]
|
|
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={token}") as ws:
|
|
greeting = ws.receive_json()
|
|
assert greeting["kind"] == "hub.hello"
|
|
assert greeting["payload"]["agent_id"] == agent_id
|
|
assert greeting["payload"]["ws_url"].endswith("/api/v1/gateway/ws")
|
|
ws.send_json(hello(offer, capabilities={"streaming": True}))
|
|
assert hub.get(f"{BASE}/gateway/status").json()["connected_agents"] == [agent_id]
|
|
agent = hub.get(f"{BASE}/agents/{agent_id}").json()
|
|
assert agent["connector"]["connected"] is True and agent["live"] is True
|
|
assert agent["deployment_kind"] == "docker"
|
|
|
|
task = dispatch(hub, offer)
|
|
request = ws.receive_json()
|
|
assert request["type"] == "req" and request["kind"] == "task.run"
|
|
assert request["payload"]["prompt"] == "do the thing"
|
|
assert request["payload"]["task_id"] == task["id"]
|
|
|
|
ws.send_json(event(offer, task["id"], type="delta", text="pon"))
|
|
ws.send_json(event(offer, task["id"], type="delta", text="g"))
|
|
ws.send_json(event(offer, task["id"], type="completed", text="pong", usage={}))
|
|
ws.send_json(
|
|
Frame(type="res", kind="task.run", reply_to=request["id"], payload={"ok": True, "text": "pong"}).to_dict()
|
|
)
|
|
assert wait_for(lambda: hub.get(f"{BASE}/tasks/{task['id']}").json()["status"] == "completed")
|
|
assert hub.get(f"{BASE}/gateway/status").json()["connected_agents"] == [agent_id]
|
|
|
|
# a graceful goodbye over the socket is the same operation as
|
|
# POST /gateway/deregister: the channel goes and the agent goes offline
|
|
ws.send_json(Frame(type="event", kind="agent.deregister", agent_id=agent_id).to_dict())
|
|
assert wait_for(lambda: hub.get(f"{BASE}/gateway/status").json()["connected_agents"] == [])
|
|
assert wait_for(lambda: hub.get(f"{BASE}/agents/{agent_id}").json()["status"] == "offline")
|
|
|
|
finished = hub.get(f"{BASE}/tasks/{task['id']}").json()
|
|
assert finished["result"] == "pong"
|
|
# every event is stored exactly once, in the order the agent sent it
|
|
kinds = [row["kind"] for row in hub.get(f"{BASE}/tasks/{task['id']}/events").json()]
|
|
assert kinds == ["status", "delta", "delta", "completed"]
|
|
agent = hub.get(f"{BASE}/agents/{agent_id}").json()
|
|
assert agent["live"] is False and agent["connector"]["connected"] is False
|
|
|
|
|
|
def test_a_labeled_driver_still_reaches_the_agent_over_its_socket(hub):
|
|
"""``--driver nanobot`` says what runs *behind* the connector.
|
|
|
|
A connector row says the hub must dial out over the socket; dialling the
|
|
(absent) base_url used to fail every task.
|
|
"""
|
|
offer = register(hub, "real-nanobot", driver="nanobot", connection_mode="connector")
|
|
agent_id, token = offer["agent_id"], offer["agent_token"]
|
|
assert hub.get(f"{BASE}/agents/{agent_id}").json()["connection_mode"] == "connector"
|
|
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={token}") as ws:
|
|
ws.receive_json()
|
|
ws.send_json(hello(offer, capabilities={"streaming": True}))
|
|
task = dispatch(hub, offer)
|
|
request = ws.receive_json()
|
|
assert request["kind"] == "task.run"
|
|
assert request["payload"]["provider"] == "" # no override: use our own provider
|
|
ws.send_json(
|
|
Frame(
|
|
type="res", kind="task.run", reply_to=request["id"], payload={"ok": True, "text": "built"}
|
|
).to_dict()
|
|
)
|
|
assert wait_for(lambda: hub.get(f"{BASE}/tasks/{task['id']}").json()["status"] == "completed")
|
|
finished = hub.get(f"{BASE}/tasks/{task["id"]}").json()
|
|
assert finished["result"] == "built"
|
|
events = hub.get(f"{BASE}/tasks/{task["id"]}/events").json()
|
|
assert events[0]["payload"]["driver"] == "connector"
|
|
|
|
|
|
def test_connector_can_answer_without_streaming(hub):
|
|
offer = register(hub, "quiet-agent", driver="connector")
|
|
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={offer['agent_token']}") as ws:
|
|
ws.receive_json()
|
|
ws.send_json(hello(offer))
|
|
task = dispatch(hub, offer)
|
|
request = ws.receive_json()
|
|
assert request["kind"] == "task.run"
|
|
ws.send_json(
|
|
Frame(type="res", kind="task.run", reply_to=request["id"], payload={"text": "answer only"}).to_dict()
|
|
)
|
|
assert wait_for(lambda: task_status(hub, task["id"]) == "completed")
|
|
assert hub.get(f"{BASE}/tasks/{task['id']}").json()["result"] == "answer only"
|
|
|
|
|
|
def test_probe_and_describe_travel_down_the_socket(hub):
|
|
offer = register(hub, "probed", driver="connector")
|
|
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={offer['agent_token']}") as ws:
|
|
ws.receive_json()
|
|
ws.send_json(hello(offer))
|
|
|
|
probe, thread = fire(hub, "post", f"{BASE}/agents/{offer['agent_id']}/probe")
|
|
ping = ws.receive_json()
|
|
assert ping["kind"] == "agent.ping"
|
|
ws.send_json(Frame(type="res", kind="agent.ping", reply_to=ping["id"], payload={"ok": True, "rtt_ms": 4.5}).to_dict())
|
|
thread.join(20)
|
|
assert probe["response"].status_code == 200
|
|
body = probe["response"].json()
|
|
assert body["ok"] is True and body["health"]["latency_ms"] == 4.5
|
|
|
|
describe, thread = fire(hub, "post", f"{BASE}/agents/{offer['agent_id']}/describe")
|
|
request = ws.receive_json()
|
|
assert request["kind"] == "agent.describe"
|
|
ws.send_json(
|
|
Frame(type="res", kind="agent.describe", reply_to=request["id"], payload={"models": ["nanobot-1"]}).to_dict()
|
|
)
|
|
thread.join(20)
|
|
assert describe["response"].json()["capabilities"]["models"] == ["nanobot-1"]
|
|
|
|
frames = hub.get(f"{BASE}/gateway/frames?agent_id={offer['agent_id']}").json()
|
|
assert {"agent.hello", "agent.ping", "agent.describe"} <= {frame["kind"] for frame in frames}
|
|
|
|
|
|
def test_socket_requires_a_valid_agent_token(hub):
|
|
with pytest.raises(WebSocketDisconnect) as failure:
|
|
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token=ag_not_a_real_token") as ws:
|
|
ws.receive_json()
|
|
assert failure.value.code == 4401
|
|
|
|
|
|
def test_console_socket_receives_frames(hub):
|
|
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status" ) as ws:
|
|
greeting = ws.receive_json()
|
|
assert greeting["kind"] == "hub.hello"
|
|
assert greeting["payload"]["topics"] == ["agent.*.status"]
|
|
offer = register(hub, "watched", driver="connector")
|
|
seen = []
|
|
for _ in range(6):
|
|
frame = ws.receive_json()
|
|
if "kind" not in frame: # {"type": "ping"} keep-alive
|
|
continue
|
|
seen.append((frame["kind"], frame.get("agent_id")))
|
|
if frame["kind"] == "agent.changed":
|
|
break
|
|
assert ("agent.changed", offer["agent_id"]) in seen
|
|
ws.send_json({"action": "ping"})
|
|
# the pong proves the socket is bidirectional and still alive
|
|
for _ in range(6):
|
|
frame = ws.receive_json()
|
|
if frame.get("type") == "pong":
|
|
break
|
|
else:
|
|
pytest.fail("no pong on the console socket")
|
|
|
|
|
|
def test_cancelling_a_task_reaches_the_agent_over_its_socket(hub):
|
|
"""Cancel travels down the same socket as the task: the agent is told to stop."""
|
|
offer = register(hub, "slow-edge", driver="connector", deployment_kind="cloud")
|
|
with hub.websocket_connect(f"{BASE}/gateway/ws?agent_token={offer['agent_token']}") as ws:
|
|
assert ws.receive_json()["kind"] == "hub.hello"
|
|
ws.send_json(hello(offer))
|
|
task = dispatch(hub, offer)
|
|
run_request = ws.receive_json()
|
|
assert run_request["kind"] == "task.run"
|
|
|
|
box, thread = fire(hub, "post", f"{BASE}/tasks/{task['id']}/cancel")
|
|
cancel_request = ws.receive_json()
|
|
assert cancel_request["type"] == "req" and cancel_request["kind"] == "task.cancel"
|
|
assert cancel_request["payload"]["invocation_id"] == run_request["payload"]["invocation_id"]
|
|
ws.send_json(
|
|
Frame(type="res", kind="task.cancel", reply_to=cancel_request["id"], payload={"ok": True}).to_dict()
|
|
)
|
|
thread.join(10)
|
|
assert not thread.is_alive() and box["response"].status_code == 200
|
|
|
|
assert wait_for(lambda: task_status(hub, task["id"]) == "cancelled")
|
|
# exactly one cancel RPC per cancel: a connector never sees a duplicate
|
|
asked = [row for row in hub.get(f"{BASE}/gateway/frames?kind=task.cancel").json() if row["type"] == "req"]
|
|
assert [row["payload"]["invocation_id"] for row in asked] == [run_request["payload"]["invocation_id"]]
|
|
kinds = [row["kind"] for row in hub.get(f"{BASE}/tasks/{task['id']}/events").json()]
|
|
assert kinds[-1] == "cancelled" and kinds.count("cancelled") == 1
|
|
assert hub.get(f"{BASE}/agents/{offer['agent_id']}").json()["live"] is True
|
|
|
|
# ...and a late answer cannot bring a finished task back to life
|
|
ws.send_json(event(offer, task["id"], type="completed", text="too late"))
|
|
ws.send_json(
|
|
Frame(
|
|
type="res", kind="task.run", reply_to=run_request["id"], payload={"ok": True, "text": "too late"}
|
|
).to_dict()
|
|
)
|
|
assert task_status(hub, task["id"]) == "cancelled"
|
|
assert hub.get(f"{BASE}/tasks/{task['id']}").json()["result"] != "too late"
|
|
kinds = [row["kind"] for row in hub.get(f"{BASE}/tasks/{task['id']}/events").json()]
|
|
assert kinds.count("cancelled") == 1 and "completed" not in kinds
|
|
|
|
|
|
def test_console_reconnect_replays_only_the_gap(hub):
|
|
"""?since= makes a dropped console socket whole again."""
|
|
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status") as ws:
|
|
greeting = ws.receive_json()
|
|
assert greeting["kind"] == "hub.hello"
|
|
since = greeting["payload"]["seq"]
|
|
offer = register(hub, "watched", driver="connector")
|
|
while True:
|
|
frame = ws.receive_json()
|
|
if frame.get("kind") == "agent.changed":
|
|
break
|
|
assert frame["agent_id"] == offer["agent_id"] and frame["seq"] > since
|
|
|
|
# Reconnecting with ?since= gets the missed frame back, oldest first.
|
|
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status&since={since}") as ws:
|
|
ws.receive_json() # hub.hello
|
|
marker = ws.receive_json()
|
|
assert marker["type"] == "replay" and marker["count"] == 1
|
|
replayed = ws.receive_json()
|
|
assert replayed["kind"] == "agent.changed"
|
|
assert replayed["agent_id"] == offer["agent_id"]
|
|
|
|
# Without it the same agent is simply not delivered a second time.
|
|
with hub.websocket_connect(f"{BASE}/stream?topics=agent.%2A.status") as ws:
|
|
ws.receive_json() # hub.hello
|
|
later = register(hub, "later", driver="connector")
|
|
while True:
|
|
frame = ws.receive_json()
|
|
if frame.get("kind") == "agent.changed":
|
|
break
|
|
assert frame["agent_id"] == later["agent_id"]
|