From 3910cfb1172bc5c170d9f888fe223a43246490d4 Mon Sep 17 00:00:00 2001 From: lyingbug Date: Sat, 12 Sep 2026 09:32:26 +0800 Subject: [PATCH] fix(browser): complete task cancellation, screenshots, and result display (#3206) --- docs/browser-skill-integration.md | 6 + frontend/src/i18n/locales/en-US.ts | 6 + frontend/src/i18n/locales/ja-JP.ts | 6 + frontend/src/i18n/locales/ko-KR.ts | 6 + frontend/src/i18n/locales/ru-RU.ts | 6 + frontend/src/i18n/locales/zh-CN.ts | 6 + frontend/src/utils/browserToolDisplay.test.ts | 39 +++- frontend/src/utils/browserToolDisplay.ts | 54 +++++- .../chat/components/BrowserTaskPreview.vue | 17 +- .../components/BrowserToolDetails.test.ts | 15 ++ .../chat/components/BrowserToolDetails.vue | 9 +- internal/agent/engine.go | 5 +- internal/agent/tool_images.go | 48 +++++ internal/agent/tool_images_test.go | 55 ++++++ internal/agent/tools/browserskill_image.go | 53 +++++ .../agent/tools/browserskill_image_test.go | 36 ++++ internal/agent/tools/browserskill_prompt.go | 3 + internal/agent/tools/browserskill_result.go | 14 ++ .../agent/tools/browserskill_result_test.go | 16 ++ internal/agent/tools/browserskill_schema.go | 4 +- .../agent/tools/browserskill_schema_test.go | 3 +- internal/agent/tools/persist.go | 4 + internal/agent/tools/persist_test.go | 18 ++ internal/application/service/agent_service.go | 10 +- .../application/service/session_agent_qa.go | 1 + internal/browserskill/extension_test.go | 78 ++++++++ internal/browserskill/manager.go | 181 +++++++++++------- internal/browserskill/result.go | 31 +++ internal/browserskill/result_test.go | 25 +++ internal/browserskill/shared_test.go | 104 ++++++++++ internal/browserskill/stop.go | 89 +++++++++ internal/browserskill/stop_test.go | 105 ++++++++++ internal/types/agent.go | 5 +- scripts/test_browserskill_extension.mjs | 18 +- 34 files changed, 971 insertions(+), 105 deletions(-) create mode 100644 internal/agent/tool_images.go create mode 100644 internal/agent/tool_images_test.go create mode 100644 internal/agent/tools/browserskill_image.go create mode 100644 internal/agent/tools/browserskill_image_test.go create mode 100644 internal/browserskill/result.go create mode 100644 internal/browserskill/result_test.go create mode 100644 internal/browserskill/stop.go create mode 100644 internal/browserskill/stop_test.go diff --git a/docs/browser-skill-integration.md b/docs/browser-skill-integration.md index 829b6347..62525392 100644 --- a/docs/browser-skill-integration.md +++ b/docs/browser-skill-integration.md @@ -36,6 +36,12 @@ WeKnora 使用 Tencent/BrowserSkill 的官方 daemon 和扩展执行浏览器操 导航默认等待 `domcontentloaded`,避免已经可读的页面被慢图片等资源拖住;需要完整加载或网络空闲时可显式指定 `wait_until: "load"` 或 `"networkidle"`。文档就绪不代表异步应用内容已加载,操作前仍需观察目标内容是否出现。 +导航返回 `reached: "timeout"` 时,即使 RPC 成功返回,也会标记该次操作未完成并保留任务现场。模型应先重新观察页面,再决定下一步,不能因等待超时自动重放导航。用户点击“结束任务”时会先阻止后续命令、取消正在执行的操作(包括创建任务),再清理任务标签;归还标签失败或清理超时会保留暂停状态,可再次点击结束重试。 + +模型可调用 `{"method":"screenshot"}` 截取当前视口,或使用最近观察中的元素引用,例如 `{"method":"screenshot","ref":"e3"}` 截取元素区域;可选 `tab_id` 限定到任务内标签。截图通过独立图片内容传给支持视觉的对话模型;非视觉模型使用已配置的图片描述模型,未配置或描述失败时明确告知模型无法查看图片。Base64 不进入工具文本,结果卡片和历史消息保留截图。预览的小尺寸 JPEG 仅用于用户查看,与模型截图分开。 + +结果卡片展示控制台消息与堆栈、网络请求状态与错误、脚本返回值及具体失败原因;较长结果会截断并提示。小预览展示最近动作、动作耗时、已知页面地址和最近错误;页面地址去掉账号、查询参数和片段。耗时在动作执行期间更新,结束后固定。 + 模型调用采用扁平参数,例如 `{"method":"navigate","url":"https://example.com"}`、`{"method":"click","ref":"e3"}`、`{"method":"observe"}`。Schema 不再提供 `params` 字段,旧的嵌套格式直接拒绝。服务端校验所选动作的必填字段、字段类型及定位条件,再将参数转换为 BrowserSkill 原生 RPC 信封;会话和设备 ID 始终由服务端绑定。 `wait_ms` 的外层参数为 `duration_ms`,范围 0–10000 毫秒;工具描述、参数 schema 和执行前校验保持一致。命令超时或中断后暂停任务,不自动重放点击/提交。前端显示具体浏览器动作与可读错误,结果卡片分别展示网页内容、截图或标签列表,不展示原始协议 JSON。标签组统一显示 WeKnora,不再附带内部任务 ID。 diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 21b0063c..f4e395dc 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1,5 +1,11 @@ export default { localBrowser: { + captureScreenshot: "Capture screenshot", + navigationIncomplete: "Navigation did not reach the requested loading phase. Check the current page.", + noEntries: "No entries returned.", + stopping: "Ending task…", + elapsedSeconds: "{seconds} s", + searchInstructionsTitle: "Browser search instructions", searchInstructionsDescription: "Set your preferred search engine and search URL.", searchInstructionsHint: "Applies to your next request. Leave empty to use the default.", diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts index b95c8875..66b824d9 100644 --- a/frontend/src/i18n/locales/ja-JP.ts +++ b/frontend/src/i18n/locales/ja-JP.ts @@ -1,5 +1,11 @@ export default { localBrowser: { + captureScreenshot: "スクリーンショットを取得", + navigationIncomplete: "ページの読み込みが指定の段階に達しませんでした。現在のページを確認してください。", + noEntries: "記録はありません。", + stopping: "タスクを終了中…", + elapsedSeconds: "{seconds} 秒", + searchInstructionsTitle: "ブラウザー検索の指示", searchInstructionsDescription: "使用する検索エンジンと検索 URL を設定します。", searchInstructionsHint: "保存後、次のリクエストから適用。空欄は既定値を使用します。", diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index ec0be958..c5775373 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1,5 +1,11 @@ export default { localBrowser: { + captureScreenshot: "스크린샷 캡처", + navigationIncomplete: "페이지가 요청한 로드 단계에 도달하지 못했습니다. 현재 페이지를 확인하세요.", + noEntries: "반환된 기록이 없습니다.", + stopping: "작업 종료 중…", + elapsedSeconds: "{seconds}초", + searchInstructionsTitle: "브라우저 검색 지침", searchInstructionsDescription: "선호하는 검색 엔진과 검색 URL을 설정합니다.", searchInstructionsHint: "저장 후 다음 요청부터 적용됩니다. 비워 두면 기본값을 사용합니다.", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 99950a1b..db6a51f2 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1,5 +1,11 @@ export default { localBrowser: { + captureScreenshot: "Сделать снимок страницы", + navigationIncomplete: "Страница не достигла нужного этапа загрузки. Проверьте её состояние.", + noEntries: "Записи отсутствуют.", + stopping: "Завершение задачи…", + elapsedSeconds: "{seconds} с", + searchInstructionsTitle: "Инструкции поиска в браузере", searchInstructionsDescription: "Укажите предпочитаемую поисковую систему и URL поиска.", searchInstructionsHint: "Применяется со следующего запроса. Пустое поле — настройки по умолчанию.", diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 236662f4..24e3d18a 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1,5 +1,11 @@ export default { localBrowser: { + captureScreenshot: "截取网页画面", + navigationIncomplete: "导航未达到目标加载阶段,请检查当前页面。", + noEntries: "没有返回记录。", + stopping: "正在结束任务…", + elapsedSeconds: "{seconds} 秒", + searchInstructionsTitle: "浏览器搜索指令", searchInstructionsDescription: "设置你偏好的搜索引擎和搜索地址。", searchInstructionsHint: "保存后下次请求生效,留空使用默认指令。", diff --git a/frontend/src/utils/browserToolDisplay.test.ts b/frontend/src/utils/browserToolDisplay.test.ts index a451fa4a..1ff67fb9 100644 --- a/frontend/src/utils/browserToolDisplay.test.ts +++ b/frontend/src/utils/browserToolDisplay.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { browserToolTitle, browserToolSummary, browserToolContent } from './browserToolDisplay' +import { browserToolTitle, browserToolSummary, browserToolContent, browserToolIncomplete, browserActionLabel } from './browserToolDisplay' const t = (key: string) => key test('browser steps expose actions and host without URL credentials/query', () => { @@ -8,6 +8,43 @@ test('browser steps expose actions and host without URL credentials/query', () = assert.equal(title,'localBrowser.local · localBrowser.openPage · example.com…') assert.equal(browserToolTitle(t,{arguments:{method:'wait_ms'},success:false}),'localBrowser.local · localBrowser.waitPage · localBrowser.actionFailed') }) + +test('navigation timeout is incomplete even in historical success records', () => { + const event = { arguments:{method:'navigate'}, success:true, output:{reached:'timeout',error_text:'Timed out waiting for load'} } + assert.equal(browserToolIncomplete(event), true) + assert.equal(browserToolSummary(t,event), 'localBrowser.navigationIncomplete') + assert.match(browserToolTitle(t,event), /actionFailed/) + assert.equal(browserToolContent(event).error, 'Timed out waiting for load') + assert.equal(browserToolIncomplete({...event,arguments:{method:'evaluate'}}), false) +}) + +test('diagnostic and evaluation results remain visible and bounded', () => { + const logs = browserToolContent({arguments:{method:'console'},success:true,output:{entries:[{ + level:'error',text:'API failed '}]}}) + assert.match(logs,/API <script>/) + assert.doesNotMatch(logs,/