WeKnora/internal/agent/tools/web_fetch.go

578 lines
16 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

package tools
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/chromedp/chromedp"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
const (
webFetchTimeout = 60 * time.Second
webFetchMaxChars = 100000
)
type webFetchParams struct {
URL string
Prompt string
}
type webFetchItemResult struct {
output string
data map[string]interface{}
err error
}
// WebFetchTool 拉取网页内容并根据提示执行总结
type WebFetchTool struct {
BaseTool
client *http.Client
chatModel chat.Chat
}
// NewWebFetchTool 创建 web_fetch 工具实例
func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool {
description := `Fetch detailed web content from previously discovered URLs and analyze it with an LLM.
## Usage
- Receive one or more {url, prompt} combinations
- Fetch web page content and convert to Markdown text
- Use prompt to call small model for analysis and summary (if model is available)
- Return summary result and original content fragment
## When to Use
- **MANDATORY**: After web_search returns results, if content is truncated or incomplete, use web_fetch to get full page content
- When web_search snippet is insufficient for answering the question
`
return &WebFetchTool{
BaseTool: NewBaseTool("web_fetch", description),
client: &http.Client{
Timeout: webFetchTimeout,
},
chatModel: chatModel,
}
}
// Parameters 返回工具参数的 JSON Schema
func (t *WebFetchTool) Parameters() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"items": map[string]interface{}{
"type": "array",
"description": "批量抓取任务,每项包含 url 与 prompt",
"items": map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"url": map[string]interface{}{
"type": "string",
"description": "待抓取的网页 URL,需来自 web_search 结果",
},
"prompt": map[string]interface{}{
"type": "string",
"description": "分析该网页内容时使用的提示词",
},
},
"required": []string{"url", "prompt"},
},
"minItems": 1,
},
},
"required": []string{"items"},
}
}
// Execute 执行 web_fetch 工具
func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) {
logger.Infof(ctx, "[Tool][WebFetch] Execute started")
rawItems, ok := args["items"]
if !ok {
logger.Errorf(ctx, "[Tool][WebFetch] 参数缺失: items")
return &types.ToolResult{
Success: false,
Error: "missing required parameter: items",
}, nil
}
itemsSlice, ok := rawItems.([]interface{})
if !ok || len(itemsSlice) == 0 {
logger.Errorf(ctx, "[Tool][WebFetch] items 解析失败或为空: %#v", rawItems)
return &types.ToolResult{
Success: false,
Error: "'items' must be a non-empty array",
}, nil
}
results := make([]*webFetchItemResult, len(itemsSlice))
var wg sync.WaitGroup
wg.Add(len(itemsSlice))
for idx := range itemsSlice {
i := idx
itemAny := itemsSlice[i]
params := t.parseParams(itemAny)
go func(index int, p webFetchParams) {
defer wg.Done()
if err := t.validateParams(p); err != nil {
results[index] = &webFetchItemResult{
err: err,
data: map[string]interface{}{
"url": p.URL,
"prompt": p.Prompt,
"error": err.Error(),
},
output: fmt.Sprintf("URL: %s\n错误: %v\n\n", p.URL, err),
}
return
}
output, data, err := t.executeFetch(ctx, p)
results[index] = &webFetchItemResult{
output: output,
data: data,
err: err,
}
}(i, params)
}
wg.Wait()
var builder strings.Builder
builder.WriteString("=== Web Fetch Results ===\n\n")
aggregated := make([]map[string]interface{}, 0, len(results))
success := true
var firstErr error
for idx, res := range results {
if res == nil {
success = false
if firstErr == nil {
firstErr = fmt.Errorf("fetch item %d returned nil", idx)
}
builder.WriteString(fmt.Sprintf("#%d: 无结果(内部错误)\n\n", idx+1))
continue
}
builder.WriteString(fmt.Sprintf("#%d:\n%s", idx+1, res.output))
if !strings.HasSuffix(res.output, "\n") {
builder.WriteString("\n")
}
builder.WriteString("\n")
if res.data != nil {
aggregated = append(aggregated, res.data)
}
if res.err != nil {
success = false
if firstErr == nil {
firstErr = res.err
}
}
}
// Add guidance for next steps
builder.WriteString("\n=== Next Steps ===\n")
if len(aggregated) > 0 {
builder.WriteString("- ✅ Full page content has been fetched and analyzed.\n")
builder.WriteString("- Evaluate if the content is sufficient to answer the question completely.\n")
builder.WriteString("- Synthesize information from all fetched pages for comprehensive answers.\n")
if !success {
builder.WriteString("- ⚠️ Some URLs failed to fetch. Use available content or try alternative sources.\n")
}
} else {
builder.WriteString("- ❌ No content was successfully fetched. Consider:\n")
builder.WriteString(" - Verify URLs are accessible\n")
builder.WriteString(" - Try alternative sources from web_search results\n")
builder.WriteString(" - Check if information can be found in knowledge base instead\n")
}
data := map[string]interface{}{
"results": aggregated,
"count": len(aggregated),
"display_type": "web_fetch_results",
}
logger.Infof(ctx, "[Tool][WebFetch] Completed with success=%v, items=%d", success, len(aggregated))
return &types.ToolResult{
Success: success,
Output: builder.String(),
Data: data,
Error: func() string {
if firstErr != nil {
return firstErr.Error()
}
return ""
}(),
}, nil
}
func (t *WebFetchTool) parseParams(item interface{}) webFetchParams {
params := webFetchParams{}
if m, ok := item.(map[string]interface{}); ok {
if v, ok := m["url"].(string); ok {
params.URL = strings.TrimSpace(v)
}
if v, ok := m["prompt"].(string); ok {
params.Prompt = strings.TrimSpace(v)
}
}
return params
}
func (t *WebFetchTool) validateParams(p webFetchParams) error {
if p.URL == "" {
return fmt.Errorf("url is required")
}
if p.Prompt == "" {
return fmt.Errorf("prompt is required")
}
if !strings.HasPrefix(p.URL, "http://") && !strings.HasPrefix(p.URL, "https://") {
return fmt.Errorf("invalid URL format")
}
return nil
}
func (t *WebFetchTool) executeFetch(ctx context.Context, params webFetchParams) (string, map[string]interface{}, error) {
logger.Infof(ctx, "[Tool][WebFetch] Fetching URL: %s", params.URL)
finalURL := t.normalizeGitHubURL(params.URL)
htmlContent, method, err := t.fetchHTMLContent(ctx, finalURL)
if err != nil {
logger.Errorf(ctx, "[Tool][WebFetch] 获取页面失败 url=%s err=%v", finalURL, err)
return fmt.Sprintf("URL: %s\n错误: %v\n", params.URL, err),
map[string]interface{}{
"url": params.URL,
"prompt": params.Prompt,
"error": err.Error(),
}, err
}
textContent := t.convertHTMLToText(htmlContent)
resultData := map[string]interface{}{
"url": params.URL,
"prompt": params.Prompt,
"raw_content": textContent,
"content_length": len(textContent),
"method": method,
}
var summary string
var summaryErr error
summary, summaryErr = t.processWithLLM(ctx, params, textContent)
if summaryErr != nil {
logger.Warnf(ctx, "[Tool][WebFetch] LLM 处理失败 url=%s err=%v", params.URL, summaryErr)
} else if summary != "" {
resultData["summary"] = summary
}
output := t.buildOutputText(params, textContent, summary, summaryErr)
return output, resultData, summaryErr
}
func (t *WebFetchTool) normalizeGitHubURL(source string) string {
if strings.Contains(source, "github.com") && strings.Contains(source, "/blob/") {
source = strings.Replace(source, "github.com", "raw.githubusercontent.com", 1)
source = strings.Replace(source, "/blob/", "/", 1)
}
return source
}
func (t *WebFetchTool) processWithLLM(ctx context.Context, params webFetchParams, content string) (string, error) {
if t.chatModel == nil {
return "", fmt.Errorf("chat model not available for web_fetch")
}
systemMessage := "你是一名擅长阅读网页内容的智能助手,请根据提供的网页文本回答用户需求,严禁编造未在文本中出现的信息。"
userTemplate := `用户请求:
%s
网页内容:
%s`
messages := []chat.Message{
{
Role: "system",
Content: systemMessage,
},
{
Role: "user",
Content: fmt.Sprintf(userTemplate, params.Prompt, content),
},
}
response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{
Temperature: 0.3,
MaxTokens: 1024,
})
if err != nil {
return "", err
}
return strings.TrimSpace(response.Content), nil
}
func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, summary string, summaryErr error) string {
var builder strings.Builder
builder.WriteString(fmt.Sprintf("URL: %s\n", params.URL))
builder.WriteString(fmt.Sprintf("Prompt: %s\n", params.Prompt))
if summaryErr == nil && summary != "" {
builder.WriteString("Summary:\n")
builder.WriteString(summary)
builder.WriteString("\n")
} else {
builder.WriteString("Content Preview:\n")
builder.WriteString(content)
builder.WriteString("\n")
}
return builder.String()
}
func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (string, string, error) {
html, err := t.fetchWithChromedp(ctx, targetURL)
if err == nil && strings.TrimSpace(html) != "" {
return html, "chromedp", nil
}
if err != nil {
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取失败 url=%s err=%v,尝试直接请求", targetURL, err)
}
html, httpErr := t.fetchWithHTTP(ctx, targetURL)
if httpErr != nil {
if err != nil {
return "", "", fmt.Errorf("chromedp error: %v; http error: %w", err, httpErr)
}
return "", "", httpErr
}
return html, "http", nil
}
func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) (string, error) {
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", targetURL)
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.Flag("headless", true),
chromedp.Flag("disable-setuid-sandbox", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-gpu", true),
chromedp.Flag("disable-blink-features", "AutomationControlled"),
chromedp.Flag("disable-features", "VizDisplayCompositor"),
chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
)
allocCtx, cancel := chromedp.NewExecAllocator(ctx, opts...)
defer cancel()
ctx, cancel = chromedp.NewContext(allocCtx)
defer cancel()
ctx, cancel = context.WithTimeout(ctx, webFetchTimeout)
defer cancel()
var html string
err := chromedp.Run(ctx,
chromedp.Navigate(targetURL),
chromedp.WaitReady("body", chromedp.ByQuery),
chromedp.OuterHTML("html", &html),
)
if err != nil {
return "", fmt.Errorf("chromedp run failed: %w", err)
}
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取成功 url=%s", targetURL)
return html, nil
}
func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (string, error) {
resp, err := t.fetchWithTimeout(ctx, targetURL)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("request failed with status %d %s", resp.StatusCode, resp.Status)
}
limitedReader := io.LimitReader(resp.Body, webFetchMaxChars*2)
htmlBytes, err := io.ReadAll(limitedReader)
if err != nil {
return "", fmt.Errorf("failed to read response body: %w", err)
}
return string(htmlBytes), nil
}
func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WebFetchTool/1.0)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
req.Header.Set("Cache-Control", "no-cache")
return t.client.Do(req)
}
func (t *WebFetchTool) convertHTMLToText(html string) string {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
if err != nil {
return t.basicTextExtraction(html)
}
doc.Find("script, style, nav, footer, header").Remove()
var markdown strings.Builder
doc.Find("body").Each(func(i int, body *goquery.Selection) {
t.processNode(body, &markdown)
})
result := markdown.String()
result = regexp.MustCompile(`\n{3,}`).ReplaceAllString(result, "\n\n")
return strings.TrimSpace(result)
}
func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Builder) {
s.Contents().Each(func(i int, node *goquery.Selection) {
nodeName := goquery.NodeName(node)
switch nodeName {
case "h1", "h2", "h3", "h4", "h5", "h6":
headerLevel := int(nodeName[1] - '0')
markdown.WriteString("\n")
markdown.WriteString(strings.Repeat("#", headerLevel))
markdown.WriteString(" ")
markdown.WriteString(strings.TrimSpace(node.Text()))
markdown.WriteString("\n\n")
case "p":
t.processNode(node, markdown)
markdown.WriteString("\n\n")
case "a":
href, exists := node.Attr("href")
text := strings.TrimSpace(node.Text())
if exists && text != "" {
markdown.WriteString("[")
markdown.WriteString(text)
markdown.WriteString("](")
markdown.WriteString(href)
markdown.WriteString(")")
} else if text != "" {
markdown.WriteString(text)
}
case "img":
src, _ := node.Attr("src")
alt, _ := node.Attr("alt")
if src != "" {
markdown.WriteString("![")
markdown.WriteString(alt)
markdown.WriteString("](")
markdown.WriteString(src)
markdown.WriteString(")\n\n")
}
case "ul", "ol":
markdown.WriteString("\n")
isOrdered := nodeName == "ol"
node.Find("li").Each(func(idx int, li *goquery.Selection) {
if isOrdered {
markdown.WriteString(fmt.Sprintf("%d. ", idx+1))
} else {
markdown.WriteString("- ")
}
markdown.WriteString(strings.TrimSpace(li.Text()))
markdown.WriteString("\n")
})
markdown.WriteString("\n")
case "br":
markdown.WriteString("\n")
case "code":
parent := node.Parent()
if goquery.NodeName(parent) == "pre" {
markdown.WriteString("\n```\n")
markdown.WriteString(node.Text())
markdown.WriteString("\n```\n\n")
} else {
markdown.WriteString("`")
markdown.WriteString(node.Text())
markdown.WriteString("`")
}
case "blockquote":
lines := strings.Split(strings.TrimSpace(node.Text()), "\n")
for _, line := range lines {
markdown.WriteString("> ")
markdown.WriteString(strings.TrimSpace(line))
markdown.WriteString("\n")
}
markdown.WriteString("\n")
case "strong", "b":
markdown.WriteString("**")
markdown.WriteString(strings.TrimSpace(node.Text()))
markdown.WriteString("**")
case "em", "i":
markdown.WriteString("*")
markdown.WriteString(strings.TrimSpace(node.Text()))
markdown.WriteString("*")
case "hr":
markdown.WriteString("\n---\n\n")
case "table":
markdown.WriteString("\n")
node.Find("tr").Each(func(idx int, tr *goquery.Selection) {
tr.Find("th, td").Each(func(i int, cell *goquery.Selection) {
markdown.WriteString("| ")
markdown.WriteString(strings.TrimSpace(cell.Text()))
markdown.WriteString(" ")
})
markdown.WriteString("|\n")
if idx == 0 {
tr.Find("th").Each(func(i int, _ *goquery.Selection) {
markdown.WriteString("|---")
})
markdown.WriteString("|\n")
}
})
markdown.WriteString("\n")
case "#text":
text := node.Text()
if strings.TrimSpace(text) != "" {
markdown.WriteString(text)
}
default:
t.processNode(node, markdown)
}
})
}
func (t *WebFetchTool) basicTextExtraction(html string) string {
re := regexp.MustCompile(`<[^>]*>`)
text := re.ReplaceAllString(html, " ")
text = regexp.MustCompile(`\s+`).ReplaceAllString(text, " ")
return strings.TrimSpace(text)
}