WeKnora/patches/browserskill/02-gateway-task-controls.patch

506 lines
21 KiB
Diff

diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts
index 830bbbe..cb88179 100644
--- a/apps/extension/src/entrypoints/background.ts
+++ b/apps/extension/src/entrypoints/background.ts
@@ -4,6 +4,7 @@ import { getAuditEnabled } from "@/lib/audit";
import { attachAuditBridge } from "@/lib/audit-bridge";
import { ConnectionController } from "@/lib/connection-controller";
import { watchDaemonConnection } from "@/lib/daemon-connection-preference";
+import { attachGatewayUI } from "@/lib/gateway-ui";
import { startHeartbeat } from "@/lib/heartbeat";
import {
getConnectionEnabled,
@@ -27,6 +28,7 @@ import {
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
+import { captureTaskPreview, taskTarget } from "@/lib/task-preview";
import { attachLongScreenshot } from "@/long-screenshot/background";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler";
@@ -63,7 +65,21 @@ export default defineBackground(() => {
url: __BSK_DAEMON_WS_URL__,
webSocketFactory: (url) => {
if (!connectionPreferenceValid) throw new Error("Connection settings are unavailable");
- return remoteSocket(url, remoteEndpoint);
+ const socket = remoteSocket(url, remoteEndpoint);
+ if (remoteEndpoint)
+ attachGatewayUI(socket, {
+ focus: async (sessionId) => {
+ const tabId = await taskTarget(sessions, sessionId);
+ const task = sessions.get(sessionId);
+ const tab = await chrome.tabs.get(tabId);
+ if (!task || !isAgentControlledTab(task, tabId)) throw new Error("Task unavailable");
+ await chrome.tabs.update(tabId, { active: true });
+ await chrome.windows.update(tab.windowId, { focused: true });
+ return { focused: true };
+ },
+ preview: (sessionId) => captureTaskPreview(sessions, cdp, sessionId),
+ });
+ return socket;
},
});
const sessions = new SessionManager({ remote: () => remoteEndpoint !== null });
diff --git a/apps/extension/src/lib/__tests__/gateway-ui.test.ts b/apps/extension/src/lib/__tests__/gateway-ui.test.ts
new file mode 100644
index 0000000..273c15b
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/gateway-ui.test.ts
@@ -0,0 +1,51 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { attachGatewayUI } from "../gateway-ui";
+
+afterEach(() => vi.unstubAllGlobals());
+function fixture() {
+ vi.stubGlobal("WebSocket", { OPEN: 1 });
+ const socket = Object.assign(new EventTarget(), { readyState: 1, send: vi.fn() });
+ const ui = {
+ focus: vi.fn(async () => ({ focused: true })),
+ preview: vi.fn(async () => ({ image_base64: "jpeg" })),
+ };
+ attachGatewayUI(socket as unknown as WebSocket, ui);
+ const native = vi.fn();
+ socket.addEventListener("message", native);
+ const request = async (body: unknown) => {
+ socket.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(body) }));
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ };
+ return { socket, ui, native, request };
+}
+describe("gateway UI side channel", () => {
+ it("routes UI requests independently and leaves native RPC untouched", async () => {
+ const f = fixture();
+ for (const name of ["preview", "focus"] as const) {
+ await f.request({ id: name, method: `gateway.task_${name}`, params: { session_id: "task" } });
+ expect(f.ui[name]).toHaveBeenCalledWith("task");
+ }
+ expect(f.native).not.toHaveBeenCalled();
+ await f.request({ id: "native", method: "tool.snapshot", params: {} });
+ expect(f.native).toHaveBeenCalledOnce();
+ });
+ it("rejects invalid session IDs without invoking browser work", async () => {
+ const f = fixture();
+ await f.request({ id: "bad", method: "gateway.task_focus", params: { session_id: 42 } });
+ expect(f.ui.focus).not.toHaveBeenCalled();
+ expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({
+ error: { code: "invalid_params" },
+ });
+ });
+ it("reports failures and drops late replies on closed sockets", async () => {
+ const f = fixture();
+ f.ui.focus.mockRejectedValueOnce(new Error("unavailable"));
+ await f.request({ id: "focus", method: "gateway.task_focus", params: { session_id: "task" } });
+ expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({
+ error: { code: "task_unavailable" },
+ });
+ f.socket.readyState = 3;
+ await f.request({ id: "late", method: "gateway.task_preview", params: { session_id: "task" } });
+ expect(f.socket.send).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/apps/extension/src/lib/__tests__/task-preview.test.ts b/apps/extension/src/lib/__tests__/task-preview.test.ts
new file mode 100644
index 0000000..3939bae
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/task-preview.test.ts
@@ -0,0 +1,180 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { ChromiumCdp } from "@/browser-driver/chromium-cdp";
+import { runCdpCommand } from "@/browser-driver/command-deadline";
+import type { SessionManager } from "@/session-manager/manager";
+import { captureTaskPreview, withTaskPreviewStop } from "../task-preview";
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+});
+function fixture() {
+ const task = {
+ remote: true,
+ agentWindowId: 10,
+ refStore: { documentRevision: () => 1 },
+ agentCreatedTabs: new Set([5]),
+ borrowedTabs: new Map(),
+ };
+ const manager = { get: () => task } as unknown as SessionManager;
+ const cdp = {
+ getAttachmentId: vi.fn(() => "attachment"),
+ hasPendingScreenshot: vi.fn(() => false),
+ acquireBackgroundExecution: vi.fn(async () => {}),
+ send: vi.fn(async (_tab, method) =>
+ method === "Page.getLayoutMetrics"
+ ? { cssVisualViewport: { pageX: 0, pageY: 0, clientWidth: 1280, clientHeight: 720 } }
+ : { data: btoa("jpeg") },
+ ),
+ } as unknown as ChromiumCdp;
+ const draw = vi.fn();
+ const close = vi.fn();
+ const sizes: number[] = [];
+ vi.stubGlobal("chrome", {
+ tabs: {
+ query: vi.fn(async () => [{ id: 5, windowId: 10, active: true }]),
+ get: vi.fn(async () => ({ id: 5, windowId: 10, title: "task" })),
+ sendMessage: vi.fn(async () => ({ ok: true })),
+ },
+ });
+ vi.stubGlobal(
+ "createImageBitmap",
+ vi.fn(async () => ({ width: 1280, height: 720, close })),
+ );
+ vi.stubGlobal(
+ "OffscreenCanvas",
+ class {
+ width: number;
+ height: number;
+ constructor(w: number, h: number) {
+ this.width = w;
+ this.height = h;
+ sizes.push(w, h);
+ }
+ getContext() {
+ return { drawImage: draw };
+ }
+ async convertToBlob() {
+ return new Blob(["thumbnail"]);
+ }
+ },
+ );
+ return { task, manager, cdp, sizes, close };
+}
+describe("independent task preview", () => {
+ it("keeps control and help overlays visible across repeated preview captures", async () => {
+ const f = fixture();
+ for (let i = 0; i < 3; i++) {
+ await expect(captureTaskPreview(f.manager, f.cdp, "visible")).resolves.toMatchObject({
+ tab_id: 5,
+ image_base64: btoa("thumbnail"),
+ });
+ }
+ expect(f.cdp.acquireBackgroundExecution).toHaveBeenCalledWith("visible", 5);
+ expect(f.cdp.send).toHaveBeenCalledTimes(3);
+ expect(chrome.tabs.sendMessage).not.toHaveBeenCalled();
+ });
+
+ it("lets native stop proceed after a stuck screenshot times out", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ vi.spyOn(console, "debug").mockImplementation(() => {});
+ const f = fixture();
+ const original = vi.mocked(f.cdp.send).getMockImplementation()!;
+ vi.mocked(f.cdp.send).mockImplementation((tab, method, params) =>
+ method === "Page.captureScreenshot"
+ ? runCdpCommand({ tabId: tab }, method, () => new Promise<never>(() => {}))
+ : original(tab, method, params),
+ );
+ const stop = vi.fn(async () => {});
+ const frame = captureTaskPreview(f.manager, f.cdp, "stuck");
+ const checked = expect(frame).rejects.toThrow("Page.captureScreenshot timed out");
+ const released = withTaskPreviewStop(f.manager, "stuck", stop);
+ await expect(captureTaskPreview(f.manager, f.cdp, "stuck")).rejects.toThrow("stopping");
+ expect(stop).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(3_000);
+ await checked;
+ await released;
+ expect(stop).toHaveBeenCalledOnce();
+ expect(vi.getTimerCount()).toBe(0);
+ vi.mocked(f.cdp.send).mockImplementation(original);
+ await expect(
+ captureTaskPreview(f.manager, f.cdp, "stuck"),
+ ).resolves.toMatchObject({ tab_id: 5 });
+ });
+
+ it("blocks previews throughout native cleanup and restores them after a failed stop", async () => {
+ const f = fixture();
+ let reject!: (error: Error) => void;
+ const pending = new Promise<never>((_, fail) => { reject = fail; });
+ const stopping = withTaskPreviewStop(f.manager, "retry", () => pending);
+ const checked = expect(stopping).rejects.toThrow("return failed");
+ await expect(captureTaskPreview(f.manager, f.cdp, "retry")).rejects.toThrow("stopping");
+ expect(f.cdp.acquireBackgroundExecution).not.toHaveBeenCalled();
+ reject(new Error("return failed"));
+ await checked;
+ await expect(captureTaskPreview(f.manager, f.cdp, "retry")).resolves.toMatchObject({ tab_id: 5 });
+ });
+
+ it("coalesces captures and bounds the physical bitmap on HiDPI screens", async () => {
+ const f = fixture();
+ const first = captureTaskPreview(f.manager, f.cdp, "one");
+ expect(captureTaskPreview(f.manager, f.cdp, "one")).toBe(first);
+ const frame = await first;
+ expect(frame).toMatchObject({ tab_id: 5, format: "jpeg", image_base64: btoa("thumbnail") });
+ expect(f.sizes).toEqual([640, 360]);
+ expect(f.close).toHaveBeenCalled();
+ expect(f.cdp.send).toHaveBeenCalledExactlyOnceWith(5, "Page.captureScreenshot", {
+ format: "jpeg",
+ quality: 50,
+ fromSurface: true,
+ captureBeyondViewport: false,
+ });
+ });
+ it("skips polling before any browser work when Chrome still has a screenshot pending", async () => {
+ const f = fixture();
+ vi.mocked(f.cdp.hasPendingScreenshot).mockReturnValue(true);
+ await expect(captureTaskPreview(f.manager, f.cdp, "pending")).rejects.toThrow("still running");
+ expect(f.cdp.acquireBackgroundExecution).not.toHaveBeenCalled();
+ expect(f.cdp.send).not.toHaveBeenCalled();
+ });
+ it("does not capture an unowned active user tab", async () => {
+ const f = fixture();
+ f.task.agentCreatedTabs.clear();
+ await expect(captureTaskPreview(f.manager, f.cdp, "empty")).rejects.toThrow(
+ "Task tab unavailable",
+ );
+ expect(f.cdp.send).not.toHaveBeenCalled();
+ });
+ it("discards the result if authorization ends during capture", async () => {
+ const f = fixture();
+ vi.mocked(f.cdp.acquireBackgroundExecution).mockImplementation(async () => {
+ f.task.agentCreatedTabs.clear();
+ });
+ await expect(captureTaskPreview(f.manager, f.cdp, "ended")).rejects.toThrow(
+ "Task ended during capture",
+ );
+ });
+ it("uses an owned tab when an unauthorized user tab is active", async () => {
+ const f = fixture();
+ vi.mocked(chrome.tabs.query).mockResolvedValue([
+ { id: 9, windowId: 10, active: true },
+ { id: 5, windowId: 10, active: false },
+ ] as chrome.tabs.Tab[]);
+ await expect(captureTaskPreview(f.manager, f.cdp, "user-active")).resolves.toMatchObject({
+ tab_id: 5,
+ });
+ expect(f.cdp.send).toHaveBeenCalledWith(5, "Page.captureScreenshot", expect.anything());
+ });
+ it("discards a frame after the debugger attachment changes", async () => {
+ const f = fixture();
+ vi.mocked(f.cdp.send).mockImplementation(async () => {
+ vi.mocked(f.cdp.getAttachmentId).mockReturnValue("replacement");
+ return { data: btoa("jpeg") } as never;
+ });
+ await expect(captureTaskPreview(f.manager, f.cdp, "replaced")).rejects.toThrow(
+ "Task ended during capture",
+ );
+ });
+});
diff --git a/apps/extension/src/lib/gateway-ui.ts b/apps/extension/src/lib/gateway-ui.ts
new file mode 100644
index 0000000..ef0443f
--- /dev/null
+++ b/apps/extension/src/lib/gateway-ui.ts
@@ -0,0 +1,40 @@
+interface GatewayUI {
+ focus(sessionId: string): Promise<unknown>;
+ preview(sessionId: string): Promise<unknown>;
+}
+
+/** WeKnora's optional UI protocol, attached only to authenticated remote sockets.
+ * Register before WSTransport so these requests bypass native automation. */
+export function attachGatewayUI(socket: WebSocket, ui: GatewayUI): void {
+ socket.addEventListener("message", (event) => {
+ let request: { id?: unknown; method?: unknown; params?: { session_id?: unknown } };
+ try {
+ request = JSON.parse(event.data);
+ if (!request || typeof request !== "object") return;
+ } catch {
+ return;
+ }
+ const methods: Record<string, (sessionId: string) => Promise<unknown>> = {
+ "gateway.task_focus": (id) => ui.focus(id),
+ "gateway.task_preview": (id) => ui.preview(id),
+ };
+ if (typeof request.method !== "string" || !Object.hasOwn(methods, request.method)) return;
+ event.stopImmediatePropagation();
+ if (typeof request.id !== "string" || !request.id) return;
+ const id = request.id;
+ const sessionId = request.params?.session_id;
+ const send = (body: object) => {
+ if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ id, ...body }));
+ };
+ if (typeof sessionId !== "string" || !sessionId) {
+ send({ error: { code: "invalid_params", message: "session_id is required" } });
+ return;
+ }
+ void Promise.resolve()
+ .then(() => methods[request.method as string]!(sessionId))
+ .then(
+ (result) => send({ result }),
+ () => send({ error: { code: "task_unavailable", message: "Browser task unavailable" } }),
+ );
+ });
+}
diff --git a/apps/extension/src/lib/task-preview.ts b/apps/extension/src/lib/task-preview.ts
new file mode 100644
index 0000000..a6d2532
--- /dev/null
+++ b/apps/extension/src/lib/task-preview.ts
@@ -0,0 +1,143 @@
+import type { ChromiumCdp } from "@/browser-driver/chromium-cdp";
+import { isAgentControlledTab, type SessionManager } from "@/session-manager/manager";
+
+interface PreviewState {
+ pending?: Promise<unknown>;
+ stopping: number;
+}
+const previews = new WeakMap<object, PreviewState>();
+function previewState(task: object): PreviewState {
+ let state = previews.get(task);
+ if (!state) {
+ state = { stopping: 0 };
+ previews.set(task, state);
+ }
+ return state;
+}
+
+/** UI-only capture outside the automation queue, including human-help waits. */
+export function captureTaskPreview(
+ manager: SessionManager,
+ cdp: ChromiumCdp,
+ sessionId: string,
+): Promise<unknown> {
+ const task = manager.get(sessionId);
+ if (!task?.remote) return Promise.reject(new Error("Task unavailable"));
+ const state = previewState(task);
+ if (state.stopping) return Promise.reject(new Error("Task is stopping"));
+ if (state.pending) return state.pending;
+ const work = capture(manager, cdp, sessionId).finally(() => {
+ if (state.pending === work) state.pending = undefined;
+ });
+ state.pending = work;
+ return work;
+}
+
+/** Drain side-channel screenshots before the official stop handler detaches
+ * and closes tabs. No turn-end detach or custom session state is introduced. */
+export async function withTaskPreviewStop<T>(
+ manager: SessionManager,
+ sessionId: string | undefined,
+ stop: () => Promise<T>,
+): Promise<T> {
+ const task = sessionId ? manager.get(sessionId) : undefined;
+ if (!task?.remote) return stop();
+ const state = previewState(task);
+ state.stopping++;
+ try {
+ await state.pending?.catch(() => {});
+ return await stop();
+ } finally {
+ // A failed native stop can be retried; it must not permanently disable previews.
+ state.stopping--;
+ }
+}
+
+async function capture(manager: SessionManager, cdp: ChromiumCdp, sessionId: string) {
+ const task = manager.get(sessionId);
+ if (!task?.remote) throw new Error("Task unavailable");
+ const tabId = await taskTarget(manager, sessionId);
+ if (tabId === undefined || !isAgentControlledTab(task, tabId))
+ throw new Error("Task tab unavailable");
+ if (cdp.hasPendingScreenshot(tabId))
+ throw new Error("Previous screenshot is still running in Chrome");
+ const tab = await chrome.tabs.get(tabId);
+ // Preview bypasses the automation dispatcher, so acquire the same explicit
+ // control policy before reading a background page. Native session stop drains
+ // this capture before releasing the claim and closing tabs.
+ await cdp.acquireBackgroundExecution(sessionId, tabId);
+ if (manager.get(sessionId) !== task || !isAgentControlledTab(task, tabId))
+ throw new Error("Task ended during capture");
+ const attachment = cdp.getAttachmentId(tabId);
+ const revision = task.refStore.documentRevision(tabId);
+ // UI previews retain overlays: hiding them on every poll makes the user's
+ // control border and help prompts flicker. Agent screenshots hide them separately.
+ // Capture only the viewport and resize the bitmap below without a layout read.
+ const shot = await cdp.send<{ data: string }>(tabId, "Page.captureScreenshot", {
+ format: "jpeg",
+ quality: 50,
+ fromSurface: true,
+ captureBeyondViewport: false,
+ });
+ if (
+ manager.get(sessionId) !== task ||
+ !isAgentControlledTab(task, tabId) ||
+ cdp.getAttachmentId(tabId) !== attachment ||
+ task.refStore.documentRevision(tabId) !== revision
+ )
+ throw new Error("Task ended during capture");
+ // Viewport screenshots may use physical display pixels; HiDPI displays can
+ // return larger bitmaps. Bound the actual encoded image to the preview width.
+ let data = shot.data;
+ const bitmap = await createImageBitmap(
+ new Blob([Uint8Array.from(atob(data), (c) => c.charCodeAt(0))], { type: "image/jpeg" }),
+ );
+ try {
+ if (bitmap.width > 640) {
+ const canvas = new OffscreenCanvas(
+ 640,
+ Math.max(1, Math.round((bitmap.height * 640) / bitmap.width)),
+ );
+ const context = canvas.getContext("2d");
+ if (!context) throw new Error("Preview canvas unavailable");
+ context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+ const jpeg = await canvas.convertToBlob({ type: "image/jpeg", quality: 0.5 });
+ data = btoa(
+ Array.from(new Uint8Array(await jpeg.arrayBuffer()), (b) => String.fromCharCode(b)).join(
+ "",
+ ),
+ );
+ }
+ } finally {
+ bitmap.close();
+ }
+ if (
+ manager.get(sessionId) !== task ||
+ !isAgentControlledTab(task, tabId) ||
+ cdp.getAttachmentId(tabId) !== attachment ||
+ task.refStore.documentRevision(tabId) !== revision
+ )
+ throw new Error("Task ended during capture");
+ return {
+ image_base64: data,
+ format: "jpeg",
+ tab_id: tabId,
+ title: tab.title ?? "",
+ captured_at: new Date().toISOString(),
+ };
+}
+
+/** Use the active owned tab, or another owned tab when the user has selected
+ * an unauthorized page. Window membership alone never grants access. */
+export async function taskTarget(manager: SessionManager, sessionId: string): Promise<number> {
+ const task = manager.get(sessionId);
+ if (!task?.remote) throw new Error("Task unavailable");
+ const tabs = await chrome.tabs.query({ windowId: task.agentWindowId });
+ if (manager.get(sessionId) !== task) throw new Error("Task unavailable");
+ const owned = tabs.filter(
+ (tab) => typeof tab.id === "number" && isAgentControlledTab(task, tab.id),
+ );
+ const tabId = (owned.find((tab) => tab.active) ?? owned[0])?.id;
+ if (tabId === undefined) throw new Error("Task tab unavailable");
+ return tabId;
+}
diff --git a/apps/extension/src/tools/session.ts b/apps/extension/src/tools/session.ts
index 4e0220b..ce13827 100644
--- a/apps/extension/src/tools/session.ts
+++ b/apps/extension/src/tools/session.ts
@@ -1,3 +1,4 @@
+import { withTaskPreviewStop } from "@/lib/task-preview";
import {
DEFAULT_INTERACTION_PREFERENCES,
type InteractionPreferenceStore,
@@ -196,6 +197,14 @@ export async function handleSessionStop(
manager: SessionManager,
params: SessionStopParams,
deps: SessionStopDeps = {},
+): Promise<SessionStopResult | RpcError> {
+ return withTaskPreviewStop(manager, params?.session_id, () => stopSession(manager, params, deps));
+}
+
+async function stopSession(
+ manager: SessionManager,
+ params: SessionStopParams,
+ deps: SessionStopDeps,
): Promise<SessionStopResult | RpcError> {
if (!params?.session_id) {
return {