506 lines
21 KiB
Diff
506 lines
21 KiB
Diff
diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts
|
|
index 830bbbe..cb88179 100644
|
|
--- a/apps/extension/src/entrypoints/background.ts
|
|
+++ b/apps/extension/src/entrypoints/background.ts
|
|
@@ -4,6 +4,7 @@ import { getAuditEnabled } from "@/lib/audit";
|
|
import { attachAuditBridge } from "@/lib/audit-bridge";
|
|
import { ConnectionController } from "@/lib/connection-controller";
|
|
import { watchDaemonConnection } from "@/lib/daemon-connection-preference";
|
|
+import { attachGatewayUI } from "@/lib/gateway-ui";
|
|
import { startHeartbeat } from "@/lib/heartbeat";
|
|
import {
|
|
getConnectionEnabled,
|
|
@@ -27,6 +28,7 @@ import {
|
|
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
|
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
|
|
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
|
+import { captureTaskPreview, taskTarget } from "@/lib/task-preview";
|
|
import { attachLongScreenshot } from "@/long-screenshot/background";
|
|
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
|
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
|
@@ -63,7 +65,21 @@ export default defineBackground(() => {
|
|
url: __BSK_DAEMON_WS_URL__,
|
|
webSocketFactory: (url) => {
|
|
if (!connectionPreferenceValid) throw new Error("Connection settings are unavailable");
|
|
- return remoteSocket(url, remoteEndpoint);
|
|
+ const socket = remoteSocket(url, remoteEndpoint);
|
|
+ if (remoteEndpoint)
|
|
+ attachGatewayUI(socket, {
|
|
+ focus: async (sessionId) => {
|
|
+ const tabId = await taskTarget(sessions, sessionId);
|
|
+ const task = sessions.get(sessionId);
|
|
+ const tab = await chrome.tabs.get(tabId);
|
|
+ if (!task || !isAgentControlledTab(task, tabId)) throw new Error("Task unavailable");
|
|
+ await chrome.tabs.update(tabId, { active: true });
|
|
+ await chrome.windows.update(tab.windowId, { focused: true });
|
|
+ return { focused: true };
|
|
+ },
|
|
+ preview: (sessionId) => captureTaskPreview(sessions, cdp, sessionId),
|
|
+ });
|
|
+ return socket;
|
|
},
|
|
});
|
|
const sessions = new SessionManager({ remote: () => remoteEndpoint !== null });
|
|
diff --git a/apps/extension/src/lib/__tests__/gateway-ui.test.ts b/apps/extension/src/lib/__tests__/gateway-ui.test.ts
|
|
new file mode 100644
|
|
index 0000000..273c15b
|
|
--- /dev/null
|
|
+++ b/apps/extension/src/lib/__tests__/gateway-ui.test.ts
|
|
@@ -0,0 +1,51 @@
|
|
+import { afterEach, describe, expect, it, vi } from "vitest";
|
|
+import { attachGatewayUI } from "../gateway-ui";
|
|
+
|
|
+afterEach(() => vi.unstubAllGlobals());
|
|
+function fixture() {
|
|
+ vi.stubGlobal("WebSocket", { OPEN: 1 });
|
|
+ const socket = Object.assign(new EventTarget(), { readyState: 1, send: vi.fn() });
|
|
+ const ui = {
|
|
+ focus: vi.fn(async () => ({ focused: true })),
|
|
+ preview: vi.fn(async () => ({ image_base64: "jpeg" })),
|
|
+ };
|
|
+ attachGatewayUI(socket as unknown as WebSocket, ui);
|
|
+ const native = vi.fn();
|
|
+ socket.addEventListener("message", native);
|
|
+ const request = async (body: unknown) => {
|
|
+ socket.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(body) }));
|
|
+ await new Promise((resolve) => setTimeout(resolve, 0));
|
|
+ };
|
|
+ return { socket, ui, native, request };
|
|
+}
|
|
+describe("gateway UI side channel", () => {
|
|
+ it("routes UI requests independently and leaves native RPC untouched", async () => {
|
|
+ const f = fixture();
|
|
+ for (const name of ["preview", "focus"] as const) {
|
|
+ await f.request({ id: name, method: `gateway.task_${name}`, params: { session_id: "task" } });
|
|
+ expect(f.ui[name]).toHaveBeenCalledWith("task");
|
|
+ }
|
|
+ expect(f.native).not.toHaveBeenCalled();
|
|
+ await f.request({ id: "native", method: "tool.snapshot", params: {} });
|
|
+ expect(f.native).toHaveBeenCalledOnce();
|
|
+ });
|
|
+ it("rejects invalid session IDs without invoking browser work", async () => {
|
|
+ const f = fixture();
|
|
+ await f.request({ id: "bad", method: "gateway.task_focus", params: { session_id: 42 } });
|
|
+ expect(f.ui.focus).not.toHaveBeenCalled();
|
|
+ expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({
|
|
+ error: { code: "invalid_params" },
|
|
+ });
|
|
+ });
|
|
+ it("reports failures and drops late replies on closed sockets", async () => {
|
|
+ const f = fixture();
|
|
+ f.ui.focus.mockRejectedValueOnce(new Error("unavailable"));
|
|
+ await f.request({ id: "focus", method: "gateway.task_focus", params: { session_id: "task" } });
|
|
+ expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({
|
|
+ error: { code: "task_unavailable" },
|
|
+ });
|
|
+ f.socket.readyState = 3;
|
|
+ await f.request({ id: "late", method: "gateway.task_preview", params: { session_id: "task" } });
|
|
+ expect(f.socket.send).toHaveBeenCalledTimes(1);
|
|
+ });
|
|
+});
|
|
diff --git a/apps/extension/src/lib/__tests__/task-preview.test.ts b/apps/extension/src/lib/__tests__/task-preview.test.ts
|
|
new file mode 100644
|
|
index 0000000..3939bae
|
|
--- /dev/null
|
|
+++ b/apps/extension/src/lib/__tests__/task-preview.test.ts
|
|
@@ -0,0 +1,180 @@
|
|
+import { afterEach, describe, expect, it, vi } from "vitest";
|
|
+import type { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
|
+import { runCdpCommand } from "@/browser-driver/command-deadline";
|
|
+import type { SessionManager } from "@/session-manager/manager";
|
|
+import { captureTaskPreview, withTaskPreviewStop } from "../task-preview";
|
|
+
|
|
+afterEach(() => {
|
|
+ vi.unstubAllGlobals();
|
|
+ vi.useRealTimers();
|
|
+ vi.restoreAllMocks();
|
|
+});
|
|
+function fixture() {
|
|
+ const task = {
|
|
+ remote: true,
|
|
+ agentWindowId: 10,
|
|
+ refStore: { documentRevision: () => 1 },
|
|
+ agentCreatedTabs: new Set([5]),
|
|
+ borrowedTabs: new Map(),
|
|
+ };
|
|
+ const manager = { get: () => task } as unknown as SessionManager;
|
|
+ const cdp = {
|
|
+ getAttachmentId: vi.fn(() => "attachment"),
|
|
+ hasPendingScreenshot: vi.fn(() => false),
|
|
+ acquireBackgroundExecution: vi.fn(async () => {}),
|
|
+ send: vi.fn(async (_tab, method) =>
|
|
+ method === "Page.getLayoutMetrics"
|
|
+ ? { cssVisualViewport: { pageX: 0, pageY: 0, clientWidth: 1280, clientHeight: 720 } }
|
|
+ : { data: btoa("jpeg") },
|
|
+ ),
|
|
+ } as unknown as ChromiumCdp;
|
|
+ const draw = vi.fn();
|
|
+ const close = vi.fn();
|
|
+ const sizes: number[] = [];
|
|
+ vi.stubGlobal("chrome", {
|
|
+ tabs: {
|
|
+ query: vi.fn(async () => [{ id: 5, windowId: 10, active: true }]),
|
|
+ get: vi.fn(async () => ({ id: 5, windowId: 10, title: "task" })),
|
|
+ sendMessage: vi.fn(async () => ({ ok: true })),
|
|
+ },
|
|
+ });
|
|
+ vi.stubGlobal(
|
|
+ "createImageBitmap",
|
|
+ vi.fn(async () => ({ width: 1280, height: 720, close })),
|
|
+ );
|
|
+ vi.stubGlobal(
|
|
+ "OffscreenCanvas",
|
|
+ class {
|
|
+ width: number;
|
|
+ height: number;
|
|
+ constructor(w: number, h: number) {
|
|
+ this.width = w;
|
|
+ this.height = h;
|
|
+ sizes.push(w, h);
|
|
+ }
|
|
+ getContext() {
|
|
+ return { drawImage: draw };
|
|
+ }
|
|
+ async convertToBlob() {
|
|
+ return new Blob(["thumbnail"]);
|
|
+ }
|
|
+ },
|
|
+ );
|
|
+ return { task, manager, cdp, sizes, close };
|
|
+}
|
|
+describe("independent task preview", () => {
|
|
+ it("keeps control and help overlays visible across repeated preview captures", async () => {
|
|
+ const f = fixture();
|
|
+ for (let i = 0; i < 3; i++) {
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "visible")).resolves.toMatchObject({
|
|
+ tab_id: 5,
|
|
+ image_base64: btoa("thumbnail"),
|
|
+ });
|
|
+ }
|
|
+ expect(f.cdp.acquireBackgroundExecution).toHaveBeenCalledWith("visible", 5);
|
|
+ expect(f.cdp.send).toHaveBeenCalledTimes(3);
|
|
+ expect(chrome.tabs.sendMessage).not.toHaveBeenCalled();
|
|
+ });
|
|
+
|
|
+ it("lets native stop proceed after a stuck screenshot times out", async () => {
|
|
+ vi.useFakeTimers();
|
|
+ vi.spyOn(console, "warn").mockImplementation(() => {});
|
|
+ vi.spyOn(console, "debug").mockImplementation(() => {});
|
|
+ const f = fixture();
|
|
+ const original = vi.mocked(f.cdp.send).getMockImplementation()!;
|
|
+ vi.mocked(f.cdp.send).mockImplementation((tab, method, params) =>
|
|
+ method === "Page.captureScreenshot"
|
|
+ ? runCdpCommand({ tabId: tab }, method, () => new Promise<never>(() => {}))
|
|
+ : original(tab, method, params),
|
|
+ );
|
|
+ const stop = vi.fn(async () => {});
|
|
+ const frame = captureTaskPreview(f.manager, f.cdp, "stuck");
|
|
+ const checked = expect(frame).rejects.toThrow("Page.captureScreenshot timed out");
|
|
+ const released = withTaskPreviewStop(f.manager, "stuck", stop);
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "stuck")).rejects.toThrow("stopping");
|
|
+ expect(stop).not.toHaveBeenCalled();
|
|
+ await vi.advanceTimersByTimeAsync(3_000);
|
|
+ await checked;
|
|
+ await released;
|
|
+ expect(stop).toHaveBeenCalledOnce();
|
|
+ expect(vi.getTimerCount()).toBe(0);
|
|
+ vi.mocked(f.cdp.send).mockImplementation(original);
|
|
+ await expect(
|
|
+ captureTaskPreview(f.manager, f.cdp, "stuck"),
|
|
+ ).resolves.toMatchObject({ tab_id: 5 });
|
|
+ });
|
|
+
|
|
+ it("blocks previews throughout native cleanup and restores them after a failed stop", async () => {
|
|
+ const f = fixture();
|
|
+ let reject!: (error: Error) => void;
|
|
+ const pending = new Promise<never>((_, fail) => { reject = fail; });
|
|
+ const stopping = withTaskPreviewStop(f.manager, "retry", () => pending);
|
|
+ const checked = expect(stopping).rejects.toThrow("return failed");
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "retry")).rejects.toThrow("stopping");
|
|
+ expect(f.cdp.acquireBackgroundExecution).not.toHaveBeenCalled();
|
|
+ reject(new Error("return failed"));
|
|
+ await checked;
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "retry")).resolves.toMatchObject({ tab_id: 5 });
|
|
+ });
|
|
+
|
|
+ it("coalesces captures and bounds the physical bitmap on HiDPI screens", async () => {
|
|
+ const f = fixture();
|
|
+ const first = captureTaskPreview(f.manager, f.cdp, "one");
|
|
+ expect(captureTaskPreview(f.manager, f.cdp, "one")).toBe(first);
|
|
+ const frame = await first;
|
|
+ expect(frame).toMatchObject({ tab_id: 5, format: "jpeg", image_base64: btoa("thumbnail") });
|
|
+ expect(f.sizes).toEqual([640, 360]);
|
|
+ expect(f.close).toHaveBeenCalled();
|
|
+ expect(f.cdp.send).toHaveBeenCalledExactlyOnceWith(5, "Page.captureScreenshot", {
|
|
+ format: "jpeg",
|
|
+ quality: 50,
|
|
+ fromSurface: true,
|
|
+ captureBeyondViewport: false,
|
|
+ });
|
|
+ });
|
|
+ it("skips polling before any browser work when Chrome still has a screenshot pending", async () => {
|
|
+ const f = fixture();
|
|
+ vi.mocked(f.cdp.hasPendingScreenshot).mockReturnValue(true);
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "pending")).rejects.toThrow("still running");
|
|
+ expect(f.cdp.acquireBackgroundExecution).not.toHaveBeenCalled();
|
|
+ expect(f.cdp.send).not.toHaveBeenCalled();
|
|
+ });
|
|
+ it("does not capture an unowned active user tab", async () => {
|
|
+ const f = fixture();
|
|
+ f.task.agentCreatedTabs.clear();
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "empty")).rejects.toThrow(
|
|
+ "Task tab unavailable",
|
|
+ );
|
|
+ expect(f.cdp.send).not.toHaveBeenCalled();
|
|
+ });
|
|
+ it("discards the result if authorization ends during capture", async () => {
|
|
+ const f = fixture();
|
|
+ vi.mocked(f.cdp.acquireBackgroundExecution).mockImplementation(async () => {
|
|
+ f.task.agentCreatedTabs.clear();
|
|
+ });
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "ended")).rejects.toThrow(
|
|
+ "Task ended during capture",
|
|
+ );
|
|
+ });
|
|
+ it("uses an owned tab when an unauthorized user tab is active", async () => {
|
|
+ const f = fixture();
|
|
+ vi.mocked(chrome.tabs.query).mockResolvedValue([
|
|
+ { id: 9, windowId: 10, active: true },
|
|
+ { id: 5, windowId: 10, active: false },
|
|
+ ] as chrome.tabs.Tab[]);
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "user-active")).resolves.toMatchObject({
|
|
+ tab_id: 5,
|
|
+ });
|
|
+ expect(f.cdp.send).toHaveBeenCalledWith(5, "Page.captureScreenshot", expect.anything());
|
|
+ });
|
|
+ it("discards a frame after the debugger attachment changes", async () => {
|
|
+ const f = fixture();
|
|
+ vi.mocked(f.cdp.send).mockImplementation(async () => {
|
|
+ vi.mocked(f.cdp.getAttachmentId).mockReturnValue("replacement");
|
|
+ return { data: btoa("jpeg") } as never;
|
|
+ });
|
|
+ await expect(captureTaskPreview(f.manager, f.cdp, "replaced")).rejects.toThrow(
|
|
+ "Task ended during capture",
|
|
+ );
|
|
+ });
|
|
+});
|
|
diff --git a/apps/extension/src/lib/gateway-ui.ts b/apps/extension/src/lib/gateway-ui.ts
|
|
new file mode 100644
|
|
index 0000000..ef0443f
|
|
--- /dev/null
|
|
+++ b/apps/extension/src/lib/gateway-ui.ts
|
|
@@ -0,0 +1,40 @@
|
|
+interface GatewayUI {
|
|
+ focus(sessionId: string): Promise<unknown>;
|
|
+ preview(sessionId: string): Promise<unknown>;
|
|
+}
|
|
+
|
|
+/** WeKnora's optional UI protocol, attached only to authenticated remote sockets.
|
|
+ * Register before WSTransport so these requests bypass native automation. */
|
|
+export function attachGatewayUI(socket: WebSocket, ui: GatewayUI): void {
|
|
+ socket.addEventListener("message", (event) => {
|
|
+ let request: { id?: unknown; method?: unknown; params?: { session_id?: unknown } };
|
|
+ try {
|
|
+ request = JSON.parse(event.data);
|
|
+ if (!request || typeof request !== "object") return;
|
|
+ } catch {
|
|
+ return;
|
|
+ }
|
|
+ const methods: Record<string, (sessionId: string) => Promise<unknown>> = {
|
|
+ "gateway.task_focus": (id) => ui.focus(id),
|
|
+ "gateway.task_preview": (id) => ui.preview(id),
|
|
+ };
|
|
+ if (typeof request.method !== "string" || !Object.hasOwn(methods, request.method)) return;
|
|
+ event.stopImmediatePropagation();
|
|
+ if (typeof request.id !== "string" || !request.id) return;
|
|
+ const id = request.id;
|
|
+ const sessionId = request.params?.session_id;
|
|
+ const send = (body: object) => {
|
|
+ if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ id, ...body }));
|
|
+ };
|
|
+ if (typeof sessionId !== "string" || !sessionId) {
|
|
+ send({ error: { code: "invalid_params", message: "session_id is required" } });
|
|
+ return;
|
|
+ }
|
|
+ void Promise.resolve()
|
|
+ .then(() => methods[request.method as string]!(sessionId))
|
|
+ .then(
|
|
+ (result) => send({ result }),
|
|
+ () => send({ error: { code: "task_unavailable", message: "Browser task unavailable" } }),
|
|
+ );
|
|
+ });
|
|
+}
|
|
diff --git a/apps/extension/src/lib/task-preview.ts b/apps/extension/src/lib/task-preview.ts
|
|
new file mode 100644
|
|
index 0000000..a6d2532
|
|
--- /dev/null
|
|
+++ b/apps/extension/src/lib/task-preview.ts
|
|
@@ -0,0 +1,143 @@
|
|
+import type { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
|
+import { isAgentControlledTab, type SessionManager } from "@/session-manager/manager";
|
|
+
|
|
+interface PreviewState {
|
|
+ pending?: Promise<unknown>;
|
|
+ stopping: number;
|
|
+}
|
|
+const previews = new WeakMap<object, PreviewState>();
|
|
+function previewState(task: object): PreviewState {
|
|
+ let state = previews.get(task);
|
|
+ if (!state) {
|
|
+ state = { stopping: 0 };
|
|
+ previews.set(task, state);
|
|
+ }
|
|
+ return state;
|
|
+}
|
|
+
|
|
+/** UI-only capture outside the automation queue, including human-help waits. */
|
|
+export function captureTaskPreview(
|
|
+ manager: SessionManager,
|
|
+ cdp: ChromiumCdp,
|
|
+ sessionId: string,
|
|
+): Promise<unknown> {
|
|
+ const task = manager.get(sessionId);
|
|
+ if (!task?.remote) return Promise.reject(new Error("Task unavailable"));
|
|
+ const state = previewState(task);
|
|
+ if (state.stopping) return Promise.reject(new Error("Task is stopping"));
|
|
+ if (state.pending) return state.pending;
|
|
+ const work = capture(manager, cdp, sessionId).finally(() => {
|
|
+ if (state.pending === work) state.pending = undefined;
|
|
+ });
|
|
+ state.pending = work;
|
|
+ return work;
|
|
+}
|
|
+
|
|
+/** Drain side-channel screenshots before the official stop handler detaches
|
|
+ * and closes tabs. No turn-end detach or custom session state is introduced. */
|
|
+export async function withTaskPreviewStop<T>(
|
|
+ manager: SessionManager,
|
|
+ sessionId: string | undefined,
|
|
+ stop: () => Promise<T>,
|
|
+): Promise<T> {
|
|
+ const task = sessionId ? manager.get(sessionId) : undefined;
|
|
+ if (!task?.remote) return stop();
|
|
+ const state = previewState(task);
|
|
+ state.stopping++;
|
|
+ try {
|
|
+ await state.pending?.catch(() => {});
|
|
+ return await stop();
|
|
+ } finally {
|
|
+ // A failed native stop can be retried; it must not permanently disable previews.
|
|
+ state.stopping--;
|
|
+ }
|
|
+}
|
|
+
|
|
+async function capture(manager: SessionManager, cdp: ChromiumCdp, sessionId: string) {
|
|
+ const task = manager.get(sessionId);
|
|
+ if (!task?.remote) throw new Error("Task unavailable");
|
|
+ const tabId = await taskTarget(manager, sessionId);
|
|
+ if (tabId === undefined || !isAgentControlledTab(task, tabId))
|
|
+ throw new Error("Task tab unavailable");
|
|
+ if (cdp.hasPendingScreenshot(tabId))
|
|
+ throw new Error("Previous screenshot is still running in Chrome");
|
|
+ const tab = await chrome.tabs.get(tabId);
|
|
+ // Preview bypasses the automation dispatcher, so acquire the same explicit
|
|
+ // control policy before reading a background page. Native session stop drains
|
|
+ // this capture before releasing the claim and closing tabs.
|
|
+ await cdp.acquireBackgroundExecution(sessionId, tabId);
|
|
+ if (manager.get(sessionId) !== task || !isAgentControlledTab(task, tabId))
|
|
+ throw new Error("Task ended during capture");
|
|
+ const attachment = cdp.getAttachmentId(tabId);
|
|
+ const revision = task.refStore.documentRevision(tabId);
|
|
+ // UI previews retain overlays: hiding them on every poll makes the user's
|
|
+ // control border and help prompts flicker. Agent screenshots hide them separately.
|
|
+ // Capture only the viewport and resize the bitmap below without a layout read.
|
|
+ const shot = await cdp.send<{ data: string }>(tabId, "Page.captureScreenshot", {
|
|
+ format: "jpeg",
|
|
+ quality: 50,
|
|
+ fromSurface: true,
|
|
+ captureBeyondViewport: false,
|
|
+ });
|
|
+ if (
|
|
+ manager.get(sessionId) !== task ||
|
|
+ !isAgentControlledTab(task, tabId) ||
|
|
+ cdp.getAttachmentId(tabId) !== attachment ||
|
|
+ task.refStore.documentRevision(tabId) !== revision
|
|
+ )
|
|
+ throw new Error("Task ended during capture");
|
|
+ // Viewport screenshots may use physical display pixels; HiDPI displays can
|
|
+ // return larger bitmaps. Bound the actual encoded image to the preview width.
|
|
+ let data = shot.data;
|
|
+ const bitmap = await createImageBitmap(
|
|
+ new Blob([Uint8Array.from(atob(data), (c) => c.charCodeAt(0))], { type: "image/jpeg" }),
|
|
+ );
|
|
+ try {
|
|
+ if (bitmap.width > 640) {
|
|
+ const canvas = new OffscreenCanvas(
|
|
+ 640,
|
|
+ Math.max(1, Math.round((bitmap.height * 640) / bitmap.width)),
|
|
+ );
|
|
+ const context = canvas.getContext("2d");
|
|
+ if (!context) throw new Error("Preview canvas unavailable");
|
|
+ context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
|
|
+ const jpeg = await canvas.convertToBlob({ type: "image/jpeg", quality: 0.5 });
|
|
+ data = btoa(
|
|
+ Array.from(new Uint8Array(await jpeg.arrayBuffer()), (b) => String.fromCharCode(b)).join(
|
|
+ "",
|
|
+ ),
|
|
+ );
|
|
+ }
|
|
+ } finally {
|
|
+ bitmap.close();
|
|
+ }
|
|
+ if (
|
|
+ manager.get(sessionId) !== task ||
|
|
+ !isAgentControlledTab(task, tabId) ||
|
|
+ cdp.getAttachmentId(tabId) !== attachment ||
|
|
+ task.refStore.documentRevision(tabId) !== revision
|
|
+ )
|
|
+ throw new Error("Task ended during capture");
|
|
+ return {
|
|
+ image_base64: data,
|
|
+ format: "jpeg",
|
|
+ tab_id: tabId,
|
|
+ title: tab.title ?? "",
|
|
+ captured_at: new Date().toISOString(),
|
|
+ };
|
|
+}
|
|
+
|
|
+/** Use the active owned tab, or another owned tab when the user has selected
|
|
+ * an unauthorized page. Window membership alone never grants access. */
|
|
+export async function taskTarget(manager: SessionManager, sessionId: string): Promise<number> {
|
|
+ const task = manager.get(sessionId);
|
|
+ if (!task?.remote) throw new Error("Task unavailable");
|
|
+ const tabs = await chrome.tabs.query({ windowId: task.agentWindowId });
|
|
+ if (manager.get(sessionId) !== task) throw new Error("Task unavailable");
|
|
+ const owned = tabs.filter(
|
|
+ (tab) => typeof tab.id === "number" && isAgentControlledTab(task, tab.id),
|
|
+ );
|
|
+ const tabId = (owned.find((tab) => tab.active) ?? owned[0])?.id;
|
|
+ if (tabId === undefined) throw new Error("Task tab unavailable");
|
|
+ return tabId;
|
|
+}
|
|
diff --git a/apps/extension/src/tools/session.ts b/apps/extension/src/tools/session.ts
|
|
index 4e0220b..ce13827 100644
|
|
--- a/apps/extension/src/tools/session.ts
|
|
+++ b/apps/extension/src/tools/session.ts
|
|
@@ -1,3 +1,4 @@
|
|
+import { withTaskPreviewStop } from "@/lib/task-preview";
|
|
import {
|
|
DEFAULT_INTERACTION_PREFERENCES,
|
|
type InteractionPreferenceStore,
|
|
@@ -196,6 +197,14 @@ export async function handleSessionStop(
|
|
manager: SessionManager,
|
|
params: SessionStopParams,
|
|
deps: SessionStopDeps = {},
|
|
+): Promise<SessionStopResult | RpcError> {
|
|
+ return withTaskPreviewStop(manager, params?.session_id, () => stopSession(manager, params, deps));
|
|
+}
|
|
+
|
|
+async function stopSession(
|
|
+ manager: SessionManager,
|
|
+ params: SessionStopParams,
|
|
+ deps: SessionStopDeps,
|
|
): Promise<SessionStopResult | RpcError> {
|
|
if (!params?.session_id) {
|
|
return {
|