WeKnora/internal/agent/tools/web_fetch.go

565 lines
15 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

package tools
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/chromedp/chromedp"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
const (
webFetchTimeout = 60 * time.Second
webFetchMaxChars = 100000
)
type webFetchParams struct {
URL string
Prompt string
}
type webFetchItemResult struct {
output string
data map[string]interface{}
err error
}
// WebFetchTool 拉取网页内容并根据提示执行总结
type WebFetchTool struct {
BaseTool
client *http.Client
chatModel chat.Chat
}
// NewWebFetchTool 创建 web_fetch 工具实例
func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool {
description := `Fetch web content from previously discovered URLs and analyze it with an LLM.
## 流程
- 接收一个或多个 {url, prompt} 组合
- 抓取网页内容并转换为 Markdown 文本
- 使用提示词调用小模型进行分析与总结(若模型可用)
- 返回总结结果与原始内容片段
## 使用场景
- 对 web_search 返回的链接做进一步深入阅读
- 从网页中提取结构化信息或关键信息
## 参数
- items (必填): 数组,元素为 { "url": "...", "prompt": "..." }
`
return &WebFetchTool{
BaseTool: NewBaseTool("web_fetch", description),
client: &http.Client{
Timeout: webFetchTimeout,
},
chatModel: chatModel,
}
}
// Parameters 返回工具参数的 JSON Schema
func (t *WebFetchTool) Parameters() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"items": map[string]interface{}{
"type": "array",
"description": "批量抓取任务,每项包含 url 与 prompt",
"items": map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"url": map[string]interface{}{
"type": "string",
"description": "待抓取的网页 URL,需来自 web_search 结果",
},
"prompt": map[string]interface{}{
"type": "string",
"description": "分析该网页内容时使用的提示词",
},
},
"required": []string{"url", "prompt"},
},
"minItems": 1,
},
},
"required": []string{"items"},
}
}
// Execute 执行 web_fetch 工具
func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) {
logger.Infof(ctx, "[Tool][WebFetch] Execute started")
rawItems, ok := args["items"]
if !ok {
logger.Errorf(ctx, "[Tool][WebFetch] 参数缺失: items")
return &types.ToolResult{
Success: false,
Error: "missing required parameter: items",
}, nil
}
itemsSlice, ok := rawItems.([]interface{})
if !ok || len(itemsSlice) == 0 {
logger.Errorf(ctx, "[Tool][WebFetch] items 解析失败或为空: %#v", rawItems)
return &types.ToolResult{
Success: false,
Error: "'items' must be a non-empty array",
}, nil
}
results := make([]*webFetchItemResult, len(itemsSlice))
var wg sync.WaitGroup
wg.Add(len(itemsSlice))
for idx := range itemsSlice {
i := idx
itemAny := itemsSlice[i]
params := t.parseParams(itemAny)
go func(index int, p webFetchParams) {
defer wg.Done()
if err := t.validateParams(p); err != nil {
results[index] = &webFetchItemResult{
err: err,
data: map[string]interface{}{
"url": p.URL,
"prompt": p.Prompt,
"error": err.Error(),
},
output: fmt.Sprintf("URL: %s\n错误: %v\n\n", p.URL, err),
}
return
}
output, data, err := t.executeFetch(ctx, p)
results[index] = &webFetchItemResult{
output: output,
data: data,
err: err,
}
}(i, params)
}
wg.Wait()
var builder strings.Builder
builder.WriteString("=== Web Fetch Results ===\n\n")
aggregated := make([]map[string]interface{}, 0, len(results))
success := true
var firstErr error
for idx, res := range results {
if res == nil {
success = false
if firstErr == nil {
firstErr = fmt.Errorf("fetch item %d returned nil", idx)
}
builder.WriteString(fmt.Sprintf("#%d: 无结果(内部错误)\n\n", idx+1))
continue
}
builder.WriteString(fmt.Sprintf("#%d:\n%s", idx+1, res.output))
if !strings.HasSuffix(res.output, "\n") {
builder.WriteString("\n")
}
builder.WriteString("\n")
if res.data != nil {
aggregated = append(aggregated, res.data)
}
if res.err != nil {
success = false
if firstErr == nil {
firstErr = res.err
}
}
}
data := map[string]interface{}{
"results": aggregated,
"count": len(aggregated),
"display_type": "web_fetch_results",
}
logger.Infof(ctx, "[Tool][WebFetch] Completed with success=%v, items=%d", success, len(aggregated))
return &types.ToolResult{
Success: success,
Output: builder.String(),
Data: data,
Error: func() string {
if firstErr != nil {
return firstErr.Error()
}
return ""
}(),
}, nil
}
func (t *WebFetchTool) parseParams(item interface{}) webFetchParams {
params := webFetchParams{}
if m, ok := item.(map[string]interface{}); ok {
if v, ok := m["url"].(string); ok {
params.URL = strings.TrimSpace(v)
}
if v, ok := m["prompt"].(string); ok {
params.Prompt = strings.TrimSpace(v)
}
}
return params
}
func (t *WebFetchTool) validateParams(p webFetchParams) error {
if p.URL == "" {
return fmt.Errorf("url is required")
}
if p.Prompt == "" {
return fmt.Errorf("prompt is required")
}
if !strings.HasPrefix(p.URL, "http://") && !strings.HasPrefix(p.URL, "https://") {
return fmt.Errorf("invalid URL format")
}
return nil
}
func (t *WebFetchTool) executeFetch(ctx context.Context, params webFetchParams) (string, map[string]interface{}, error) {
logger.Infof(ctx, "[Tool][WebFetch] Fetching URL: %s", params.URL)
finalURL := t.normalizeGitHubURL(params.URL)
htmlContent, method, err := t.fetchHTMLContent(ctx, finalURL)
if err != nil {
logger.Errorf(ctx, "[Tool][WebFetch] 获取页面失败 url=%s err=%v", finalURL, err)
return fmt.Sprintf("URL: %s\n错误: %v\n", params.URL, err),
map[string]interface{}{
"url": params.URL,
"prompt": params.Prompt,
"error": err.Error(),
}, err
}
textContent := t.convertHTMLToText(htmlContent)
resultData := map[string]interface{}{
"url": params.URL,
"prompt": params.Prompt,
"raw_content": textContent,
"content_length": len(textContent),
"method": method,
}
var summary string
var summaryErr error
summary, summaryErr = t.processWithLLM(ctx, params, textContent)
if summaryErr != nil {
logger.Warnf(ctx, "[Tool][WebFetch] LLM 处理失败 url=%s err=%v", params.URL, summaryErr)
} else if summary != "" {
resultData["summary"] = summary
}
output := t.buildOutputText(params, textContent, summary, summaryErr)
return output, resultData, summaryErr
}
func (t *WebFetchTool) normalizeGitHubURL(source string) string {
if strings.Contains(source, "github.com") && strings.Contains(source, "/blob/") {
source = strings.Replace(source, "github.com", "raw.githubusercontent.com", 1)
source = strings.Replace(source, "/blob/", "/", 1)
}
return source
}
func (t *WebFetchTool) processWithLLM(ctx context.Context, params webFetchParams, content string) (string, error) {
if t.chatModel == nil {
return "", fmt.Errorf("chat model not available for web_fetch")
}
systemMessage := "你是一名擅长阅读网页内容的智能助手,请根据提供的网页文本回答用户需求,严禁编造未在文本中出现的信息。"
userTemplate := `用户请求:
%s
网页内容:
%s`
messages := []chat.Message{
{
Role: "system",
Content: systemMessage,
},
{
Role: "user",
Content: fmt.Sprintf(userTemplate, params.Prompt, content),
},
}
response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{
Temperature: 0.3,
MaxTokens: 1024,
})
if err != nil {
return "", err
}
return strings.TrimSpace(response.Content), nil
}
func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, summary string, summaryErr error) string {
var builder strings.Builder
builder.WriteString(fmt.Sprintf("URL: %s\n", params.URL))
builder.WriteString(fmt.Sprintf("Prompt: %s\n", params.Prompt))
if summaryErr == nil && summary != "" {
builder.WriteString("Summary:\n")
builder.WriteString(summary)
builder.WriteString("\n")
} else {
builder.WriteString("Content Preview:\n")
builder.WriteString(content)
builder.WriteString("\n")
}
return builder.String()
}
func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (string, string, error) {
html, err := t.fetchWithChromedp(ctx, targetURL)
if err == nil && strings.TrimSpace(html) != "" {
return html, "chromedp", nil
}
if err != nil {
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取失败 url=%s err=%v,尝试直接请求", targetURL, err)
}
html, httpErr := t.fetchWithHTTP(ctx, targetURL)
if httpErr != nil {
if err != nil {
return "", "", fmt.Errorf("chromedp error: %v; http error: %w", err, httpErr)
}
return "", "", httpErr
}
return html, "http", nil
}
func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) (string, error) {
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", targetURL)
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.Flag("headless", true),
chromedp.Flag("disable-setuid-sandbox", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-gpu", true),
chromedp.Flag("disable-blink-features", "AutomationControlled"),
chromedp.Flag("disable-features", "VizDisplayCompositor"),
chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
)
allocCtx, cancel := chromedp.NewExecAllocator(ctx, opts...)
defer cancel()
ctx, cancel = chromedp.NewContext(allocCtx)
defer cancel()
ctx, cancel = context.WithTimeout(ctx, webFetchTimeout)
defer cancel()
var html string
err := chromedp.Run(ctx,
chromedp.Navigate(targetURL),
chromedp.WaitReady("body", chromedp.ByQuery),
chromedp.OuterHTML("html", &html),
)
if err != nil {
return "", fmt.Errorf("chromedp run failed: %w", err)
}
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取成功 url=%s", targetURL)
return html, nil
}
func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (string, error) {
resp, err := t.fetchWithTimeout(ctx, targetURL)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("request failed with status %d %s", resp.StatusCode, resp.Status)
}
limitedReader := io.LimitReader(resp.Body, webFetchMaxChars*2)
htmlBytes, err := io.ReadAll(limitedReader)
if err != nil {
return "", fmt.Errorf("failed to read response body: %w", err)
}
return string(htmlBytes), nil
}
func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WebFetchTool/1.0)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
req.Header.Set("Cache-Control", "no-cache")
return t.client.Do(req)
}
func (t *WebFetchTool) convertHTMLToText(html string) string {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
if err != nil {
return t.basicTextExtraction(html)
}
doc.Find("script, style, nav, footer, header").Remove()
var markdown strings.Builder
doc.Find("body").Each(func(i int, body *goquery.Selection) {
t.processNode(body, &markdown)
})
result := markdown.String()
result = regexp.MustCompile(`\n{3,}`).ReplaceAllString(result, "\n\n")
return strings.TrimSpace(result)
}
func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Builder) {
s.Contents().Each(func(i int, node *goquery.Selection) {
nodeName := goquery.NodeName(node)
switch nodeName {
case "h1", "h2", "h3", "h4", "h5", "h6":
headerLevel := int(nodeName[1] - '0')
markdown.WriteString("\n")
markdown.WriteString(strings.Repeat("#", headerLevel))
markdown.WriteString(" ")
markdown.WriteString(strings.TrimSpace(node.Text()))
markdown.WriteString("\n\n")
case "p":
t.processNode(node, markdown)
markdown.WriteString("\n\n")
case "a":
href, exists := node.Attr("href")
text := strings.TrimSpace(node.Text())
if exists && text != "" {
markdown.WriteString("[")
markdown.WriteString(text)
markdown.WriteString("](")
markdown.WriteString(href)
markdown.WriteString(")")
} else if text != "" {
markdown.WriteString(text)
}
case "img":
src, _ := node.Attr("src")
alt, _ := node.Attr("alt")
if src != "" {
markdown.WriteString("![")
markdown.WriteString(alt)
markdown.WriteString("](")
markdown.WriteString(src)
markdown.WriteString(")\n\n")
}
case "ul", "ol":
markdown.WriteString("\n")
isOrdered := nodeName == "ol"
node.Find("li").Each(func(idx int, li *goquery.Selection) {
if isOrdered {
markdown.WriteString(fmt.Sprintf("%d. ", idx+1))
} else {
markdown.WriteString("- ")
}
markdown.WriteString(strings.TrimSpace(li.Text()))
markdown.WriteString("\n")
})
markdown.WriteString("\n")
case "br":
markdown.WriteString("\n")
case "code":
parent := node.Parent()
if goquery.NodeName(parent) == "pre" {
markdown.WriteString("\n```\n")
markdown.WriteString(node.Text())
markdown.WriteString("\n```\n\n")
} else {
markdown.WriteString("`")
markdown.WriteString(node.Text())
markdown.WriteString("`")
}
case "blockquote":
lines := strings.Split(strings.TrimSpace(node.Text()), "\n")
for _, line := range lines {
markdown.WriteString("> ")
markdown.WriteString(strings.TrimSpace(line))
markdown.WriteString("\n")
}
markdown.WriteString("\n")
case "strong", "b":
markdown.WriteString("**")
markdown.WriteString(strings.TrimSpace(node.Text()))
markdown.WriteString("**")
case "em", "i":
markdown.WriteString("*")
markdown.WriteString(strings.TrimSpace(node.Text()))
markdown.WriteString("*")
case "hr":
markdown.WriteString("\n---\n\n")
case "table":
markdown.WriteString("\n")
node.Find("tr").Each(func(idx int, tr *goquery.Selection) {
tr.Find("th, td").Each(func(i int, cell *goquery.Selection) {
markdown.WriteString("| ")
markdown.WriteString(strings.TrimSpace(cell.Text()))
markdown.WriteString(" ")
})
markdown.WriteString("|\n")
if idx == 0 {
tr.Find("th").Each(func(i int, _ *goquery.Selection) {
markdown.WriteString("|---")
})
markdown.WriteString("|\n")
}
})
markdown.WriteString("\n")
case "#text":
text := node.Text()
if strings.TrimSpace(text) != "" {
markdown.WriteString(text)
}
default:
t.processNode(node, markdown)
}
})
}
func (t *WebFetchTool) basicTextExtraction(html string) string {
re := regexp.MustCompile(`<[^>]*>`)
text := re.ReplaceAllString(html, " ")
text = regexp.MustCompile(`\s+`).ReplaceAllString(text, " ")
return strings.TrimSpace(text)
}