WeKnora/patches/browserskill/02-gateway-task-controls.patch

721 lines
29 KiB
Diff

diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts
index 830bbbe..1a9a788 100644
--- a/apps/extension/src/entrypoints/background.ts
+++ b/apps/extension/src/entrypoints/background.ts
@@ -4,6 +4,7 @@ import { getAuditEnabled } from "@/lib/audit";
import { attachAuditBridge } from "@/lib/audit-bridge";
import { ConnectionController } from "@/lib/connection-controller";
import { watchDaemonConnection } from "@/lib/daemon-connection-preference";
+import { attachGatewayUI } from "@/lib/gateway-ui";
import { startHeartbeat } from "@/lib/heartbeat";
import {
getConnectionEnabled,
@@ -27,6 +28,8 @@ import {
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
+import { TaskActivity } from "@/lib/task-activity";
+import { captureTaskPreview, taskTarget } from "@/lib/task-preview";
import { attachLongScreenshot } from "@/long-screenshot/background";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler";
@@ -63,7 +66,27 @@ export default defineBackground(() => {
url: __BSK_DAEMON_WS_URL__,
webSocketFactory: (url) => {
if (!connectionPreferenceValid) throw new Error("Connection settings are unavailable");
- return remoteSocket(url, remoteEndpoint);
+ const socket = remoteSocket(url, remoteEndpoint);
+ if (remoteEndpoint)
+ attachGatewayUI(socket, {
+ focus: async (sessionId) => {
+ const tabId = await taskTarget(sessions, sessionId);
+ const task = sessions.get(sessionId);
+ const tab = await chrome.tabs.get(tabId);
+ if (!task || !isAgentControlledTab(task, tabId)) throw new Error("Task unavailable");
+ await chrome.tabs.update(tabId, { active: true });
+ await chrome.windows.update(tab.windowId, { focused: true });
+ return { focused: true };
+ },
+ preview: (sessionId) =>
+ taskActivity.preview(sessionId, () => captureTaskPreview(sessions, cdp, sessionId)),
+ idle: async (sessionId) => {
+ if (!sessions.get(sessionId)?.remote) throw new Error("Task unavailable");
+ await taskActivity.release(sessionId);
+ return { released: true };
+ },
+ });
+ return socket;
},
});
const sessions = new SessionManager({ remote: () => remoteEndpoint !== null });
@@ -72,6 +95,8 @@ export default defineBackground(() => {
});
attachAuditBridge(controller, transport);
const cdp = new ChromiumCdp(undefined, {
+ shouldKeepTaskActive: (tabId) =>
+ sessions.list().some((task) => task.remote && isAgentControlledTab(task, tabId)),
onDocumentChanged: (tabId) => sessions.invalidateTabRefs(tabId),
shouldAutoAcceptDialog: async (tabId) => {
const tab = await chrome.tabs.get(tabId);
@@ -79,6 +104,9 @@ export default defineBackground(() => {
return session !== null && (!session.remote || isAgentControlledTab(session, tabId));
},
});
+ const taskActivity: TaskActivity = new TaskActivity(sessions, (id) =>
+ dispatcher.releaseTaskControl(id),
+ );
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
let overlayGeneration = 0;
const controlModes = new Map<string, OverlayMode>();
@@ -290,6 +318,7 @@ export default defineBackground(() => {
});
void interactionPreferences.readyOrFallback();
const dispatcher = new ToolDispatcher({
+ taskActivity,
interactionPreferences,
transport,
sessions,
diff --git a/apps/extension/src/lib/__tests__/gateway-ui.test.ts b/apps/extension/src/lib/__tests__/gateway-ui.test.ts
new file mode 100644
index 0000000..eda0a0e
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/gateway-ui.test.ts
@@ -0,0 +1,52 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { attachGatewayUI } from "../gateway-ui";
+
+afterEach(() => vi.unstubAllGlobals());
+function fixture() {
+ vi.stubGlobal("WebSocket", { OPEN: 1 });
+ const socket = Object.assign(new EventTarget(), { readyState: 1, send: vi.fn() });
+ const ui = {
+ focus: vi.fn(async () => ({ focused: true })),
+ preview: vi.fn(async () => ({ image_base64: "jpeg" })),
+ idle: vi.fn(async () => ({ released: true })),
+ };
+ attachGatewayUI(socket as unknown as WebSocket, ui);
+ const native = vi.fn();
+ socket.addEventListener("message", native);
+ const request = async (body: unknown) => {
+ socket.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(body) }));
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ };
+ return { socket, ui, native, request };
+}
+describe("gateway UI side channel", () => {
+ it("routes UI requests independently and leaves native RPC untouched", async () => {
+ const f = fixture();
+ for (const name of ["preview", "focus", "idle"] as const) {
+ await f.request({ id: name, method: `gateway.task_${name}`, params: { session_id: "task" } });
+ expect(f.ui[name]).toHaveBeenCalledWith("task");
+ }
+ expect(f.native).not.toHaveBeenCalled();
+ await f.request({ id: "native", method: "tool.snapshot", params: {} });
+ expect(f.native).toHaveBeenCalledOnce();
+ });
+ it("rejects invalid session IDs without invoking browser work", async () => {
+ const f = fixture();
+ await f.request({ id: "bad", method: "gateway.task_focus", params: { session_id: 42 } });
+ expect(f.ui.focus).not.toHaveBeenCalled();
+ expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({
+ error: { code: "invalid_params" },
+ });
+ });
+ it("reports failures and drops late replies on closed sockets", async () => {
+ const f = fixture();
+ f.ui.idle.mockRejectedValueOnce(new Error("unavailable"));
+ await f.request({ id: "idle", method: "gateway.task_idle", params: { session_id: "task" } });
+ expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({
+ error: { code: "task_unavailable" },
+ });
+ f.socket.readyState = 3;
+ await f.request({ id: "late", method: "gateway.task_preview", params: { session_id: "task" } });
+ expect(f.socket.send).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/apps/extension/src/lib/__tests__/task-activity.test.ts b/apps/extension/src/lib/__tests__/task-activity.test.ts
new file mode 100644
index 0000000..7eae50f
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/task-activity.test.ts
@@ -0,0 +1,93 @@
+import { describe, expect, it, vi } from "vitest";
+import type { SessionManager } from "@/session-manager/manager";
+import { TaskActivity } from "../task-activity";
+
+function deferred() {
+ let resolve!: () => void;
+ const promise = new Promise<void>((r) => {
+ resolve = r;
+ });
+ return { promise, resolve };
+}
+function fixture() {
+ const tasks = new Map<string, object>([
+ ["one", { remote: true }],
+ ["two", { remote: true }],
+ ]);
+ const manager = { get: (id: string) => tasks.get(id) } as unknown as SessionManager;
+ const detach = vi.fn(async (_id: string) => {});
+ return { tasks, detach, activity: new TaskActivity(manager, detach) };
+}
+describe("task debugging lifecycle", () => {
+ it("drains an in-flight screenshot before detach and rejects later preview attachment", async () => {
+ const { activity, detach, tasks } = fixture();
+ const shot = deferred();
+ const capture = activity.preview("one", () => shot.promise);
+ const release = activity.release("one");
+ await expect(activity.preview("one", async () => {})).rejects.toThrow("idle");
+ await Promise.resolve();
+ expect(detach).not.toHaveBeenCalled();
+ shot.resolve();
+ await Promise.all([capture, release]);
+ expect(detach).toHaveBeenCalledWith("one");
+ expect(tasks.has("one")).toBe(true);
+ await expect(activity.preview("one", async () => {})).rejects.toThrow("idle");
+ });
+ it("waits for automation even if turn end arrives before the operation begins", async () => {
+ const { activity, detach } = fixture();
+ const op = deferred();
+ const running = activity.run("one", false, () => op.promise);
+ const release = activity.release("one");
+ await Promise.resolve();
+ expect(detach).not.toHaveBeenCalled();
+ op.resolve();
+ await Promise.all([running, release]);
+ expect(detach).toHaveBeenCalledTimes(1);
+ });
+ it("a new turn waits for detach then resumes capture without deleting tabs", async () => {
+ const { activity, detach } = fixture();
+ const gate = deferred();
+ detach.mockImplementationOnce(() => gate.promise);
+ const release = activity.release("one");
+ const op = vi.fn(async () => {});
+ const next = activity.run("one", false, op);
+ await Promise.resolve();
+ expect(op).not.toHaveBeenCalled();
+ gate.resolve();
+ await Promise.all([release, next]);
+ expect(op).toHaveBeenCalledOnce();
+ await expect(activity.preview("one", async () => "frame")).resolves.toBe("frame");
+ });
+ it("stop fences previews before closing tabs", async () => {
+ const { activity, detach } = fixture();
+ await activity.run("one", true, async () => {
+ expect(detach).toHaveBeenCalledWith("one");
+ await expect(activity.preview("one", async () => {})).rejects.toThrow("idle");
+ });
+ await expect(activity.preview("one", async () => {})).rejects.toThrow("idle");
+ });
+ it("failed captures do not prevent release or interfere with another task", async () => {
+ const { activity, detach } = fixture();
+ const capture = activity.preview("one", async () => {
+ throw new Error("failed capture");
+ });
+ const release = activity.release("one");
+ await expect(capture).rejects.toThrow("failed capture");
+ await release;
+ await expect(activity.preview("two", async () => "live")).resolves.toBe("live");
+ expect(detach).toHaveBeenCalledTimes(1);
+ });
+ it("does not resume a deleted session after waiting for release", async () => {
+ const { activity, tasks, detach } = fixture();
+ const gate = deferred();
+ detach.mockImplementationOnce(() => gate.promise);
+ const release = activity.release("one");
+ const op = vi.fn(async () => {});
+ const next = activity.run("one", false, op);
+ tasks.delete("one");
+ gate.resolve();
+ await release;
+ await expect(next).rejects.toThrow("Task ended");
+ expect(op).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/extension/src/lib/__tests__/task-preview.test.ts b/apps/extension/src/lib/__tests__/task-preview.test.ts
new file mode 100644
index 0000000..49d1e24
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/task-preview.test.ts
@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { ChromiumCdp } from "@/browser-driver/chromium-cdp";
+import { runCdpCommand } from "@/browser-driver/command-deadline";
+import type { SessionManager } from "@/session-manager/manager";
+import { TaskActivity } from "../task-activity";
+import { captureTaskPreview } from "../task-preview";
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+});
+function fixture() {
+ const task = {
+ remote: true,
+ agentWindowId: 10,
+ refStore: { documentRevision: () => 1 },
+ agentCreatedTabs: new Set([5]),
+ borrowedTabs: new Map(),
+ };
+ const manager = { get: () => task } as unknown as SessionManager;
+ const cdp = {
+ getAttachmentId: vi.fn(() => "attachment"),
+ trackSessionTab: vi.fn(),
+ hasPendingScreenshot: vi.fn(() => false),
+ ensureAttached: vi.fn(async () => {}),
+ send: vi.fn(async (_tab, method) =>
+ method === "Page.getLayoutMetrics"
+ ? { cssVisualViewport: { pageX: 0, pageY: 0, clientWidth: 1280, clientHeight: 720 } }
+ : { data: btoa("jpeg") },
+ ),
+ } as unknown as ChromiumCdp;
+ const draw = vi.fn();
+ const close = vi.fn();
+ const sizes: number[] = [];
+ vi.stubGlobal("chrome", {
+ tabs: {
+ query: vi.fn(async () => [{ id: 5, windowId: 10, active: true }]),
+ get: vi.fn(async () => ({ id: 5, windowId: 10, title: "task" })),
+ sendMessage: vi.fn(async () => ({ ok: true })),
+ },
+ });
+ vi.stubGlobal(
+ "createImageBitmap",
+ vi.fn(async () => ({ width: 1280, height: 720, close })),
+ );
+ vi.stubGlobal(
+ "OffscreenCanvas",
+ class {
+ width: number;
+ height: number;
+ constructor(w: number, h: number) {
+ this.width = w;
+ this.height = h;
+ sizes.push(w, h);
+ }
+ getContext() {
+ return { drawImage: draw };
+ }
+ async convertToBlob() {
+ return new Blob(["thumbnail"]);
+ }
+ },
+ );
+ return { task, manager, cdp, sizes, close };
+}
+describe("independent task preview", () => {
+ it("keeps control and help overlays visible across repeated preview captures", async () => {
+ const f = fixture();
+ for (let i = 0; i < 3; i++) {
+ await expect(captureTaskPreview(f.manager, f.cdp, "visible")).resolves.toMatchObject({
+ tab_id: 5,
+ image_base64: btoa("thumbnail"),
+ });
+ }
+ expect(f.cdp.send).toHaveBeenCalledTimes(3);
+ expect(chrome.tabs.sendMessage).not.toHaveBeenCalled();
+ });
+
+ it("lets turn-end release finish after a stuck screenshot times out", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ vi.spyOn(console, "debug").mockImplementation(() => {});
+ const f = fixture();
+ const original = vi.mocked(f.cdp.send).getMockImplementation()!;
+ vi.mocked(f.cdp.send).mockImplementation((tab, method, params) =>
+ method === "Page.captureScreenshot"
+ ? runCdpCommand({ tabId: tab }, method, () => new Promise<never>(() => {}))
+ : original(tab, method, params),
+ );
+ const detach = vi.fn(async () => {});
+ const activity = new TaskActivity(f.manager, detach);
+ const frame = activity.preview("stuck", () => captureTaskPreview(f.manager, f.cdp, "stuck"));
+ const checked = expect(frame).rejects.toThrow("Page.captureScreenshot timed out");
+ const released = activity.release("stuck");
+ await vi.advanceTimersByTimeAsync(3_000);
+ await checked;
+ await released;
+ expect(detach).toHaveBeenCalledWith("stuck");
+ expect(vi.getTimerCount()).toBe(0);
+ vi.mocked(f.cdp.send).mockImplementation(original);
+ await expect(
+ activity.run("stuck", false, () => captureTaskPreview(f.manager, f.cdp, "stuck")),
+ ).resolves.toMatchObject({ tab_id: 5 });
+ });
+
+ it("coalesces captures and bounds the physical bitmap on HiDPI screens", async () => {
+ const f = fixture();
+ const first = captureTaskPreview(f.manager, f.cdp, "one");
+ expect(captureTaskPreview(f.manager, f.cdp, "one")).toBe(first);
+ const frame = await first;
+ expect(frame).toMatchObject({ tab_id: 5, format: "jpeg", image_base64: btoa("thumbnail") });
+ expect(f.sizes).toEqual([640, 360]);
+ expect(f.close).toHaveBeenCalled();
+ expect(f.cdp.send).toHaveBeenCalledExactlyOnceWith(5, "Page.captureScreenshot", {
+ format: "jpeg",
+ quality: 50,
+ fromSurface: true,
+ captureBeyondViewport: false,
+ });
+ });
+ it("skips polling before any browser work when Chrome still has a screenshot pending", async () => {
+ const f = fixture();
+ vi.mocked(f.cdp.hasPendingScreenshot).mockReturnValue(true);
+ await expect(captureTaskPreview(f.manager, f.cdp, "pending")).rejects.toThrow("still running");
+ expect(f.cdp.ensureAttached).not.toHaveBeenCalled();
+ expect(f.cdp.send).not.toHaveBeenCalled();
+ });
+ it("does not capture an unowned active user tab", async () => {
+ const f = fixture();
+ f.task.agentCreatedTabs.clear();
+ await expect(captureTaskPreview(f.manager, f.cdp, "empty")).rejects.toThrow(
+ "Task tab unavailable",
+ );
+ expect(f.cdp.send).not.toHaveBeenCalled();
+ });
+ it("discards the result if authorization ends during capture", async () => {
+ const f = fixture();
+ vi.mocked(f.cdp.ensureAttached).mockImplementation(async () => {
+ f.task.agentCreatedTabs.clear();
+ });
+ await expect(captureTaskPreview(f.manager, f.cdp, "ended")).rejects.toThrow(
+ "Task ended during capture",
+ );
+ });
+ it("uses an owned tab when an unauthorized user tab is active", async () => {
+ const f = fixture();
+ vi.mocked(chrome.tabs.query).mockResolvedValue([
+ { id: 9, windowId: 10, active: true },
+ { id: 5, windowId: 10, active: false },
+ ] as chrome.tabs.Tab[]);
+ await expect(captureTaskPreview(f.manager, f.cdp, "user-active")).resolves.toMatchObject({
+ tab_id: 5,
+ });
+ expect(f.cdp.send).toHaveBeenCalledWith(5, "Page.captureScreenshot", expect.anything());
+ });
+ it("discards a frame after the debugger attachment changes", async () => {
+ const f = fixture();
+ vi.mocked(f.cdp.send).mockImplementation(async () => {
+ vi.mocked(f.cdp.getAttachmentId).mockReturnValue("replacement");
+ return { data: btoa("jpeg") } as never;
+ });
+ await expect(captureTaskPreview(f.manager, f.cdp, "replaced")).rejects.toThrow(
+ "Task ended during capture",
+ );
+ });
+});
diff --git a/apps/extension/src/lib/gateway-ui.ts b/apps/extension/src/lib/gateway-ui.ts
new file mode 100644
index 0000000..a020db8
--- /dev/null
+++ b/apps/extension/src/lib/gateway-ui.ts
@@ -0,0 +1,42 @@
+interface GatewayUI {
+ focus(sessionId: string): Promise<unknown>;
+ preview(sessionId: string): Promise<unknown>;
+ idle(sessionId: string): Promise<unknown>;
+}
+
+/** WeKnora's optional UI protocol, attached only to authenticated remote sockets.
+ * Register before WSTransport so these requests bypass native automation. */
+export function attachGatewayUI(socket: WebSocket, ui: GatewayUI): void {
+ socket.addEventListener("message", (event) => {
+ let request: { id?: unknown; method?: unknown; params?: { session_id?: unknown } };
+ try {
+ request = JSON.parse(event.data);
+ if (!request || typeof request !== "object") return;
+ } catch {
+ return;
+ }
+ const methods: Record<string, (sessionId: string) => Promise<unknown>> = {
+ "gateway.task_focus": (id) => ui.focus(id),
+ "gateway.task_preview": (id) => ui.preview(id),
+ "gateway.task_idle": (id) => ui.idle(id),
+ };
+ if (typeof request.method !== "string" || !Object.hasOwn(methods, request.method)) return;
+ event.stopImmediatePropagation();
+ if (typeof request.id !== "string" || !request.id) return;
+ const id = request.id;
+ const sessionId = request.params?.session_id;
+ const send = (body: object) => {
+ if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ id, ...body }));
+ };
+ if (typeof sessionId !== "string" || !sessionId) {
+ send({ error: { code: "invalid_params", message: "session_id is required" } });
+ return;
+ }
+ void Promise.resolve()
+ .then(() => methods[request.method as string]!(sessionId))
+ .then(
+ (result) => send({ result }),
+ () => send({ error: { code: "task_unavailable", message: "Browser task unavailable" } }),
+ );
+ });
+}
diff --git a/apps/extension/src/lib/task-activity.ts b/apps/extension/src/lib/task-activity.ts
new file mode 100644
index 0000000..91bc85a
--- /dev/null
+++ b/apps/extension/src/lib/task-activity.ts
@@ -0,0 +1,86 @@
+import type { SessionManager } from "@/session-manager/manager";
+
+interface Activity {
+ idle: boolean;
+ pending: Set<Promise<unknown>>;
+ releasing?: Promise<void>;
+}
+
+/** Coordinates UI captures with turn-end release and native session stop.
+ * State follows the session object, so deleted sessions cannot retain frames or fences. */
+export class TaskActivity {
+ private readonly states = new WeakMap<object, Activity>();
+ constructor(
+ private readonly manager: SessionManager,
+ private readonly detach: (sessionId: string) => Promise<void>,
+ ) {}
+
+ private state(sessionId: string): Activity | undefined {
+ const task = this.manager.get(sessionId);
+ if (!task?.remote) return undefined;
+ let state = this.states.get(task);
+ if (!state) {
+ state = { idle: false, pending: new Set() };
+ this.states.set(task, state);
+ }
+ return state;
+ }
+
+ private track<T>(state: Activity, run: () => Promise<T>): Promise<T> {
+ const work = Promise.resolve().then(run);
+ state.pending.add(work);
+ void work.then(
+ () => state.pending.delete(work),
+ () => state.pending.delete(work),
+ );
+ return work;
+ }
+
+ preview<T>(sessionId: string, run: () => Promise<T>): Promise<T> {
+ const state = this.state(sessionId);
+ if (!state || state.idle) return Promise.reject(new Error("Task preview is idle"));
+ return this.track(state, run);
+ }
+
+ async run<T>(
+ sessionId: string | undefined,
+ stopping: boolean,
+ run: () => Promise<T>,
+ ): Promise<T> {
+ if (!sessionId) return run();
+ const task = this.manager.get(sessionId);
+ const state = this.state(sessionId);
+ if (!state) return run();
+ if (stopping) {
+ await this.release(sessionId);
+ return run();
+ }
+ if (state.releasing) await state.releasing;
+ if (this.manager.get(sessionId) !== task) throw new Error("Task ended before operation");
+ state.idle = false;
+ return this.track(state, run);
+ }
+
+ release(sessionId: string): Promise<void> {
+ const state = this.state(sessionId);
+ if (!state) return Promise.resolve();
+ if (state.releasing) return state.releasing;
+ state.idle = true;
+ const work = (async () => {
+ // A screenshot's send() can auto-attach. Let every in-flight operation
+ // finish before detaching, and reject new previews throughout release.
+ await Promise.allSettled([...state.pending]);
+ await this.detach(sessionId);
+ })();
+ state.releasing = work;
+ void work.then(
+ () => {
+ state.releasing = undefined;
+ },
+ () => {
+ state.releasing = undefined;
+ },
+ );
+ return work;
+ }
+}
diff --git a/apps/extension/src/lib/task-preview.ts b/apps/extension/src/lib/task-preview.ts
new file mode 100644
index 0000000..38d1917
--- /dev/null
+++ b/apps/extension/src/lib/task-preview.ts
@@ -0,0 +1,103 @@
+import type { ChromiumCdp } from "@/browser-driver/chromium-cdp";
+import { isAgentControlledTab, type SessionManager } from "@/session-manager/manager";
+
+const captures = new Map<string, Promise<unknown>>();
+/** UI-only capture, outside the automation queue (including human-help waits).
+ * Captures only the task's concrete tab; never activates it or reads a user tab. */
+export function captureTaskPreview(
+ manager: SessionManager,
+ cdp: ChromiumCdp,
+ sessionId: string,
+): Promise<unknown> {
+ const pending = captures.get(sessionId);
+ if (pending) return pending;
+ const work = capture(manager, cdp, sessionId).finally(() => captures.delete(sessionId));
+ captures.set(sessionId, work);
+ return work;
+}
+async function capture(manager: SessionManager, cdp: ChromiumCdp, sessionId: string) {
+ const task = manager.get(sessionId);
+ if (!task?.remote) throw new Error("Task unavailable");
+ const tabId = await taskTarget(manager, sessionId);
+ if (tabId === undefined || !isAgentControlledTab(task, tabId))
+ throw new Error("Task tab unavailable");
+ if (cdp.hasPendingScreenshot(tabId))
+ throw new Error("Previous screenshot is still running in Chrome");
+ const tab = await chrome.tabs.get(tabId);
+ cdp.trackSessionTab(sessionId, tabId);
+ await cdp.ensureAttached(tabId);
+ if (manager.get(sessionId) !== task || !isAgentControlledTab(task, tabId))
+ throw new Error("Task ended during capture");
+ const attachment = cdp.getAttachmentId(tabId);
+ const revision = task.refStore.documentRevision(tabId);
+ // UI previews retain overlays: hiding them on every poll makes the user's
+ // control border and help prompts flicker. Agent screenshots hide them separately.
+ // Capture only the viewport and resize the bitmap below without a layout read.
+ const shot = await cdp.send<{ data: string }>(tabId, "Page.captureScreenshot", {
+ format: "jpeg",
+ quality: 50,
+ fromSurface: true,
+ captureBeyondViewport: false,
+ });
+ if (
+ manager.get(sessionId) !== task ||
+ !isAgentControlledTab(task, tabId) ||
+ cdp.getAttachmentId(tabId) !== attachment ||
+ task.refStore.documentRevision(tabId) !== revision
+ )
+ throw new Error("Task ended during capture");
+ // Viewport screenshots may use physical display pixels; HiDPI displays can
+ // return larger bitmaps. Bound the actual encoded image to the preview width.
+ let data = shot.data;
+ const bitmap = await createImageBitmap(
+ new Blob([Uint8Array.from(atob(data), (c) => c.charCodeAt(0))], { type: "image/jpeg" }),
+ );
+ try {
+ if (bitmap.width > 640) {
+ const canvas = new OffscreenCanvas(
+ 640,
+ Math.max(1, Math.round((bitmap.height * 640) / bitmap.width)),
+ );
+ const context = canvas.getContext("2d");
+ if (!context) throw new Error("Preview canvas unavailable");
+ context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+ const jpeg = await canvas.convertToBlob({ type: "image/jpeg", quality: 0.5 });
+ data = btoa(
+ Array.from(new Uint8Array(await jpeg.arrayBuffer()), (b) => String.fromCharCode(b)).join(
+ "",
+ ),
+ );
+ }
+ } finally {
+ bitmap.close();
+ }
+ if (
+ manager.get(sessionId) !== task ||
+ !isAgentControlledTab(task, tabId) ||
+ cdp.getAttachmentId(tabId) !== attachment ||
+ task.refStore.documentRevision(tabId) !== revision
+ )
+ throw new Error("Task ended during capture");
+ return {
+ image_base64: data,
+ format: "jpeg",
+ tab_id: tabId,
+ title: tab.title ?? "",
+ captured_at: new Date().toISOString(),
+ };
+}
+
+/** Use the active owned tab, or another owned tab when the user has selected
+ * an unauthorized page. Window membership alone never grants access. */
+export async function taskTarget(manager: SessionManager, sessionId: string): Promise<number> {
+ const task = manager.get(sessionId);
+ if (!task?.remote) throw new Error("Task unavailable");
+ const tabs = await chrome.tabs.query({ windowId: task.agentWindowId });
+ if (manager.get(sessionId) !== task) throw new Error("Task unavailable");
+ const owned = tabs.filter(
+ (tab) => typeof tab.id === "number" && isAgentControlledTab(task, tab.id),
+ );
+ const tabId = (owned.find((tab) => tab.active) ?? owned[0])?.id;
+ if (tabId === undefined) throw new Error("Task tab unavailable");
+ return tabId;
+}
diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts
index fdac85d..0726f27 100644
--- a/apps/extension/src/tools/dispatcher.ts
+++ b/apps/extension/src/tools/dispatcher.ts
@@ -1,5 +1,6 @@
import type { InteractionPreferenceStore } from "@/lib/interaction-preferences";
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
+import type { TaskActivity } from "@/lib/task-activity";
import { ScreenshotExports } from "@/long-screenshot/exports";
import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport";
@@ -129,6 +130,7 @@ interface HoverLatchScope {
}
export interface DispatcherDeps {
+ taskActivity?: TaskActivity;
transport: Transport;
sessions: SessionManager;
cdp?: DispatcherCdpRunner;
@@ -168,6 +170,7 @@ export interface DispatcherDeps {
* takes the fast path.
*/
export class ToolDispatcher {
+ private readonly taskActivity?: TaskActivity;
private readonly transport: Transport;
private readonly sessions: SessionManager;
private screenshotExports: ScreenshotExports;
@@ -191,6 +194,7 @@ export class ToolDispatcher {
readonly inflightAbortControllers = new Map<string, AbortController>();
constructor(deps: DispatcherDeps) {
+ this.taskActivity = deps.taskActivity;
this.transport = deps.transport;
this.sessions = deps.sessions;
this.screenshotExports = new ScreenshotExports((id) => this.sessions.has(id));
@@ -275,7 +279,14 @@ export class ToolDispatcher {
} catch {
/* The daemon still has the original operation metadata. */
}
- const result = await this.invoke(req, ac.signal);
+ const invoke = () => this.invoke(req, ac.signal);
+ const result = this.taskActivity
+ ? await this.taskActivity.run(
+ (req.params as { session_id?: string } | undefined)?.session_id,
+ req.method === "tool.session_stop",
+ invoke,
+ )
+ : await invoke();
if (isRpcError(result)) {
body = { id: req.id, error: classifyCdpError(result) };
} else {
@@ -854,6 +865,12 @@ export class ToolDispatcher {
);
}
+ async releaseTaskControl(sessionId: string): Promise<void> {
+ await this.releaseHoverLatch(sessionId);
+ this.sessions.get(sessionId)?.refStore.clear();
+ await this.cdp?.detachSession(sessionId);
+ }
+
private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise<void> {
const matchesScope = (entrySessionId: string, entryTabId: number): boolean => {
if (sessionId !== undefined && entrySessionId !== sessionId) return false;