diff --git a/docs/docs.go b/docs/docs.go index 913b994b..401360e1 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -22885,7 +22885,8 @@ const docTemplate = `{ "zhipu", "exa", "metaso", - "bocha" + "bocha", + "serply" ], "x-enum-varnames": [ "WebSearchProviderTypeBrave", @@ -22900,7 +22901,8 @@ const docTemplate = `{ "WebSearchProviderTypeZhipu", "WebSearchProviderTypeExa", "WebSearchProviderTypeMetaso", - "WebSearchProviderTypeBocha" + "WebSearchProviderTypeBocha", + "WebSearchProviderTypeSerply" ] }, "github_com_Tencent_WeKnora_internal_types.WikiConfig": { diff --git a/docs/swagger.json b/docs/swagger.json index 4150f135..170fab22 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -22878,7 +22878,8 @@ "zhipu", "exa", "metaso", - "bocha" + "bocha", + "serply" ], "x-enum-varnames": [ "WebSearchProviderTypeBrave", @@ -22893,7 +22894,8 @@ "WebSearchProviderTypeZhipu", "WebSearchProviderTypeExa", "WebSearchProviderTypeMetaso", - "WebSearchProviderTypeBocha" + "WebSearchProviderTypeBocha", + "WebSearchProviderTypeSerply" ] }, "github_com_Tencent_WeKnora_internal_types.WikiConfig": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c654366d..35975aae 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -4427,6 +4427,7 @@ definitions: - exa - metaso - bocha + - serply type: string x-enum-varnames: - WebSearchProviderTypeBrave @@ -4442,6 +4443,7 @@ definitions: - WebSearchProviderTypeExa - WebSearchProviderTypeMetaso - WebSearchProviderTypeBocha + - WebSearchProviderTypeSerply github_com_Tencent_WeKnora_internal_types.WikiConfig: properties: content_instructions: diff --git a/frontend/src/api/web-search-provider.ts b/frontend/src/api/web-search-provider.ts index 56289090..425c4013 100644 --- a/frontend/src/api/web-search-provider.ts +++ b/frontend/src/api/web-search-provider.ts @@ -5,7 +5,7 @@ export interface WebSearchProviderEntity { id?: string tenant_id?: number name: string - provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' + provider: 'brave' | 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' | 'bocha' | 'serply' description?: string parameters: { // api_key is never returned by the server in this shape; it lives behind diff --git a/internal/agent/tools/web_search.go b/internal/agent/tools/web_search.go index a32ef682..55154399 100644 --- a/internal/agent/tools/web_search.go +++ b/internal/agent/tools/web_search.go @@ -31,8 +31,9 @@ var webSearchTool = BaseTool{ - Use web_fetch with a returned page ID to read the source when snippets leave gaps. User-supplied URLs can be fetched directly without searching first. - count optionally selects fewer results within the configured maximum. country and freshness require a - provider with filter support (Brave); unsupported providers return an error rather than ignore filters. - Omit country to use the provider default (Brave: US). ALL requests worldwide results when the provider supports it. + provider with filter support (Brave, Serply); unsupported providers return an error rather than ignore filters. + Omit country to use the provider default (Brave: US, Serply: Google default). ALL requests worldwide results + when the provider supports it. - content=true fetches readable excerpts for the first 3 results in parallel (5,000 characters each). Additional hits keep search snippets; use web_fetch to read them. Full saved page addresses can be read with read_file. Page failures retain the search evidence. @@ -48,8 +49,8 @@ var webSearchTool = BaseTool{ type WebSearchInput struct { Query string `json:"query" jsonschema:"Search query string"` Count *int `json:"count,omitempty" jsonschema:"1 to configured maximum (at most 20)"` - Country string `json:"country,omitempty" jsonschema:"Two-letter code or ALL; omit for provider default; requires Brave"` - Freshness string `json:"freshness,omitempty" jsonschema:"pd/pw/pm/py or YYYY-MM-DDtoYYYY-MM-DD (Brave)"` + Country string `json:"country,omitempty" jsonschema:"Two-letter code or ALL (Brave, Serply); omit for default"` + Freshness string `json:"freshness,omitempty" jsonschema:"pd/pw/pm/py; Brave also accepts YYYY-MM-DDtoYYYY-MM-DD"` Content bool `json:"content,omitempty" jsonschema:"Fetch page excerpts; default false"` } diff --git a/internal/application/service/web_search.go b/internal/application/service/web_search.go index 88b97e5e..7e1c4938 100644 --- a/internal/application/service/web_search.go +++ b/internal/application/service/web_search.go @@ -79,7 +79,7 @@ func (s *WebSearchService) Search( filtered, ok := searchProvider.(interfaces.FilteredWebSearchProvider) if !ok { return nil, fmt.Errorf("provider %s does not support country/freshness filters; "+ - "omit them or select Brave", searchProvider.Name()) + "omit them or select Brave or Serply", searchProvider.Name()) } results, err = filtered.SearchWithFilters(ctx, query, config.MaxResults, config.IncludeDate, config.Filters) } else { diff --git a/internal/application/service/web_search_provider.go b/internal/application/service/web_search_provider.go index a64627fc..336b2201 100644 --- a/internal/application/service/web_search_provider.go +++ b/internal/application/service/web_search_provider.go @@ -141,7 +141,8 @@ func isValidProviderType(provider types.WebSearchProviderType) bool { types.WebSearchProviderTypeMetaso, types.WebSearchProviderTypeZhipu, types.WebSearchProviderTypeExa, - types.WebSearchProviderTypeBocha: + types.WebSearchProviderTypeBocha, + types.WebSearchProviderTypeSerply: return true default: return false @@ -155,6 +156,10 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ if strings.TrimSpace(params.APIKey) == "" { return fmt.Errorf("API key is required for Brave provider") } + case types.WebSearchProviderTypeSerply: + if strings.TrimSpace(params.APIKey) == "" { + return fmt.Errorf("API key is required for Serply provider") + } case types.WebSearchProviderTypeBing: if params.APIKey == "" { return fmt.Errorf("API key is required for Bing provider") diff --git a/internal/application/service/web_search_provider_test.go b/internal/application/service/web_search_provider_test.go index ef29042a..b5d9da58 100644 --- a/internal/application/service/web_search_provider_test.go +++ b/internal/application/service/web_search_provider_test.go @@ -83,3 +83,17 @@ func TestValidateProviderParametersBocha(t *testing.T) { t.Fatal("Bocha provider type is not accepted") } } + +func TestValidateProviderParametersSerply(t *testing.T) { + valid := types.WebSearchProviderParameters{APIKey: "serply-test"} + if err := validateProviderParameters(types.WebSearchProviderTypeSerply, valid); err != nil { + t.Fatalf("valid Serply parameters rejected: %v", err) + } + if !isValidProviderType(types.WebSearchProviderTypeSerply) { + t.Fatal("Serply provider type is not accepted") + } + blank := types.WebSearchProviderParameters{APIKey: " "} + if err := validateProviderParameters(types.WebSearchProviderTypeSerply, blank); err == nil { + t.Fatal("blank Serply API key was accepted") + } +} diff --git a/internal/container/container.go b/internal/container/container.go index 7471ff01..14352e59 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -1652,6 +1652,7 @@ func registerWebSearchProviders(registry *infra_web_search.Registry) { registry.Register("metaso", infra_web_search.NewMetasoProvider) registry.Register("bocha", infra_web_search.NewBochaProvider) registry.Register("brave", infra_web_search.NewBraveProvider) + registry.Register("serply", infra_web_search.NewSerplyProvider) } // registerIMService registers adapter factories, loads enabled channels, and diff --git a/internal/infrastructure/web_search/serply.go b/internal/infrastructure/web_search/serply.go new file mode 100644 index 00000000..67791d94 --- /dev/null +++ b/internal/infrastructure/web_search/serply.go @@ -0,0 +1,130 @@ +package web_search + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +const serplySearchURL = "https://api.serply.io/v1/search" + +// Serply returns at most one Google results page per call, so larger requests +// only waste credits without adding rows. +const maxSerplyResults = 10 + +// serplyFreshness maps the shared pd/pw/pm/py filter values onto Google's tbs parameter. +var serplyFreshness = map[string]string{"pd": "qdr:d", "pw": "qdr:w", "pm": "qdr:m", "py": "qdr:y"} + +// SerplyProvider queries the Serply Google search API. +type SerplyProvider struct { + client *http.Client + apiKey string +} + +// NewSerplyProvider creates a guarded, credential-scoped Serply search client. +func NewSerplyProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) { + if strings.TrimSpace(params.APIKey) == "" { + return nil, fmt.Errorf("API key is required for Serply provider") + } + client, err := NewSearchHTTPClient(30*time.Second, params.ProxyURL) + if err != nil { + return nil, err + } + // The official endpoint does not need redirects. Never forward the API key + // to a redirect destination, even another public host. + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + return &SerplyProvider{client: client, apiKey: strings.TrimSpace(params.APIKey)}, nil +} + +// Name returns the provider registry identifier. +func (p *SerplyProvider) Name() string { return "serply" } + +// Search applies the default region and result limit. +func (p *SerplyProvider) Search( + ctx context.Context, query string, maxResults int, includeDate bool, +) ([]*types.WebSearchResult, error) { + return p.SearchWithFilters(ctx, query, maxResults, includeDate, types.WebSearchFilters{}) +} + +// SearchWithFilters sends country (gl) and freshness (tbs) to the official Serply endpoint. +func (p *SerplyProvider) SearchWithFilters( + ctx context.Context, query string, maxResults int, _ bool, filters types.WebSearchFilters, +) ([]*types.WebSearchResult, error) { + if err := filters.Validate(); err != nil { + return nil, err + } + if strings.TrimSpace(query) == "" { + return nil, fmt.Errorf("query is empty") + } + if maxResults <= 0 { + maxResults = 5 + } + params := url.Values{"q": {query}, "num": {strconv.Itoa(min(maxResults, maxSerplyResults))}} + // Without gl Google picks the region itself, so ALL simply omits it. + if country := strings.ToLower(filters.Country); country != "" && country != "all" { + params.Set("gl", country) + } + if filters.Freshness != "" { + tbs, ok := serplyFreshness[filters.Freshness] + if !ok { + return nil, fmt.Errorf("serply freshness must be pd, pw, pm, or py; date ranges are not supported") + } + params.Set("tbs", tbs) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, serplySearchURL+"?"+params.Encode(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "WeKnora/1.0") + req.Header.Set("X-Api-Key", p.apiKey) + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("serply search request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("serply search returned HTTP %d", resp.StatusCode) + } + const maxResponse = 4 << 20 + body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse+1)) + if err != nil { + return nil, fmt.Errorf("read Serply response: %w", err) + } + if len(body) > maxResponse { + return nil, fmt.Errorf("serply response exceeds %d bytes", maxResponse) + } + var response struct { + Results []struct { + Title string `json:"title"` + Link string `json:"link"` + Description string `json:"description"` + } `json:"results"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode Serply response: %w", err) + } + // num is an upper bound on the Google page size, not an exact count, so trim here. + results := make([]*types.WebSearchResult, 0, min(maxResults, len(response.Results))) + for _, row := range response.Results { + if row.Link == "" { + continue + } + results = append(results, &types.WebSearchResult{ + Title: row.Title, URL: row.Link, Snippet: row.Description, Source: p.Name(), + }) + if len(results) == maxResults { + break + } + } + return results, nil +} diff --git a/internal/infrastructure/web_search/serply_test.go b/internal/infrastructure/web_search/serply_test.go new file mode 100644 index 00000000..eccaf092 --- /dev/null +++ b/internal/infrastructure/web_search/serply_test.go @@ -0,0 +1,87 @@ +package web_search + +import ( + "context" + "io" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +type serplyTransport func(*http.Request) (*http.Response, error) + +func (f serplyTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestSerplySearchMapsOptionsAndTrimsResults(t *testing.T) { + p := &SerplyProvider{apiKey: "test-key", client: &http.Client{Transport: serplyTransport( + func(r *http.Request) (*http.Response, error) { + require.Equal(t, serplySearchURL, r.URL.Scheme+"://"+r.URL.Host+r.URL.Path) + require.Equal(t, http.MethodGet, r.Method) + require.Equal(t, "test-key", r.Header.Get("X-Api-Key")) + require.Equal(t, "WeKnora/1.0", r.Header.Get("User-Agent")) + require.Equal(t, "rust & go", r.URL.Query().Get("q")) + require.Equal(t, "2", r.URL.Query().Get("num")) + require.Equal(t, "de", r.URL.Query().Get("gl")) + require.Equal(t, "qdr:w", r.URL.Query().Get("tbs")) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"results":[ + {"title":"One","link":"https://example.com/one","description":"Snippet","position":1}, + {"title":"No link","description":"dropped"}, + {"title":"Two","link":"https://example.com/two"}, + {"title":"Extra","link":"https://example.com/extra"}]}`))}, nil + }, + )}} + results, err := p.SearchWithFilters(t.Context(), "rust & go", 2, false, + types.WebSearchFilters{Country: "DE", Freshness: "pw"}) + require.NoError(t, err) + require.Len(t, results, 2) + require.Equal(t, "Snippet", results[0].Snippet) + require.Equal(t, "https://example.com/two", results[1].URL) + require.Equal(t, "serply", results[1].Source) +} + +func TestSerplyDefaultsLimitsAndErrors(t *testing.T) { + for _, tc := range []struct { + requested int + num string + }{{0, "5"}, {99, "10"}} { + transport := serplyTransport(func(r *http.Request) (*http.Response, error) { + require.Equal(t, tc.num, r.URL.Query().Get("num")) + require.False(t, r.URL.Query().Has("gl")) + require.False(t, r.URL.Query().Has("tbs")) + return &http.Response{ + StatusCode: 401, Body: io.NopCloser(strings.NewReader(`{"detail":"secret upstream diagnostics"}`)), + }, nil + }) + p := &SerplyProvider{client: &http.Client{Transport: transport}} + _, err := p.Search(context.Background(), "query", tc.requested, false) + require.ErrorContains(t, err, "HTTP 401") + require.NotContains(t, err.Error(), "secret") + } + _, err := NewSerplyProvider(types.WebSearchProviderParameters{}) + require.ErrorContains(t, err, "API key") + provider, err := NewSerplyProvider(types.WebSearchProviderParameters{APIKey: "test"}) + require.NoError(t, err) + require.ErrorIs(t, provider.(*SerplyProvider).client.CheckRedirect(nil, nil), http.ErrUseLastResponse) +} + +func TestSerplyOmitsRegionForALLAndRejectsDateRanges(t *testing.T) { + var got url.Values + p := &SerplyProvider{apiKey: "k", client: &http.Client{Transport: serplyTransport( + func(r *http.Request) (*http.Response, error) { + got = r.URL.Query() + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"results":[]}`))}, nil + }, + )}} + _, err := p.SearchWithFilters(t.Context(), "query", 1, false, types.WebSearchFilters{Country: "ALL"}) + require.NoError(t, err) + require.False(t, got.Has("gl")) + got = nil + _, err = p.SearchWithFilters(t.Context(), "query", 1, false, + types.WebSearchFilters{Freshness: "2026-01-01to2026-02-01"}) + require.ErrorContains(t, err, "date ranges are not supported") + require.Nil(t, got, "no request is sent for an unsupported filter") +} diff --git a/internal/types/web_search_provider.go b/internal/types/web_search_provider.go index 228b6fd5..8934f0e6 100644 --- a/internal/types/web_search_provider.go +++ b/internal/types/web_search_provider.go @@ -29,6 +29,7 @@ const ( WebSearchProviderTypeExa WebSearchProviderType = "exa" WebSearchProviderTypeMetaso WebSearchProviderType = "metaso" WebSearchProviderTypeBocha WebSearchProviderType = "bocha" + WebSearchProviderTypeSerply WebSearchProviderType = "serply" ) // WebSearchProviderEntity represents a configured web search provider instance for a workspace. @@ -180,6 +181,11 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo { Description: "Brave Search API (supports country and freshness filters)", DocsURL: "https://api-dashboard.search.brave.com/app/keys", }, + { + ID: "serply", Name: "Serply", RequiresAPIKey: true, SupportsProxy: true, + Description: "Serply Google search API (supports country and freshness filters)", + DocsURL: "https://serply.io/docs", + }, { ID: "duckduckgo", Name: "DuckDuckGo", diff --git a/internal/types/web_search_provider_test.go b/internal/types/web_search_provider_test.go index 7da74d0d..d1a7e1f0 100644 --- a/internal/types/web_search_provider_test.go +++ b/internal/types/web_search_provider_test.go @@ -100,3 +100,20 @@ func TestGetWebSearchProviderTypesIncludesBocha(t *testing.T) { t.Fatalf("unexpected Bocha summary metadata: %+v", summary) } } + +func TestGetWebSearchProviderTypesIncludesSerply(t *testing.T) { + var serply *WebSearchProviderTypeInfo + providerTypes := GetWebSearchProviderTypes() + for i := range providerTypes { + if providerTypes[i].ID == string(WebSearchProviderTypeSerply) { + serply = &providerTypes[i] + break + } + } + if serply == nil { + t.Fatal("Serply provider type not found") + } + if !serply.RequiresAPIKey || !serply.SupportsProxy || serply.RequiresEngineID || serply.RequiresBaseURL { + t.Fatalf("unexpected Serply metadata: %+v", serply) + } +} diff --git a/website-docs/03-features/11-web-search.md b/website-docs/03-features/11-web-search.md index c48dacc4..5f0b36d8 100644 --- a/website-docs/03-features/11-web-search.md +++ b/website-docs/03-features/11-web-search.md @@ -39,8 +39,9 @@ registry.Register("brave", infra_web_search.NewBraveProvider) | Exa | `exa.go` | 是 | `https://api.exa.ai/search` | 默认 highlights,可用 extra_config.include_text 获取正文 | | 博查 Bocha | `bocha.go` | 是 | `https://api.bochaai.com/v1/web-search` | extra_config.freshness、summary | | Brave Search | `brave.go` | 是 | `https://api.search.brave.com/res/v1/web/search` | 支持按次传 country/freshness | +| Serply | `serply.go` | 是 | `https://api.serply.io/v1/search` | Google 结果;支持按次传 country/freshness(仅 pd/pw/pm/py) | -在「设置 → 网络搜索」选择提供商、填写 API Key 并测试,然后在智能体中选择该配置。当前注册 13 个引擎;实际结果数仍受智能体最大结果数约束。 +在「设置 → 网络搜索」选择提供商、填写 API Key 并测试,然后在智能体中选择该配置。当前注册 14 个引擎;实际结果数仍受智能体最大结果数约束。 | 提供商附加配置 | 值 | | --- | --- | @@ -49,6 +50,7 @@ registry.Register("brave", infra_web_search.NewBraveProvider) | Bocha freshness | noLimit(默认)、oneDay、oneWeek、oneMonth、oneYear | | Bocha summary | 字符串布尔值,决定是否请求摘要 | | Brave 按次过滤 | country/freshness 是 web_search 工具参数,见下文;与 Bocha 固定配置的字段取值不同 | +| Serply 按次过滤 | 同 Brave;country 映射为 Google 的 gl,freshness 只接受 pd/pw/pm/py,不支持日期区间 | 除 SearXNG 外,所有引擎端点均硬编码、租户不可配置——这是防 SSRF 的第一道措施(源码注释:`Not configurable by tenants — prevents SSRF`)。 @@ -100,7 +102,7 @@ flowchart TD ``` - `count` 指定结果数量,范围是 1 到当前 Agent 配置的最大结果数(最多 20);省略时沿用现有 Agent 默认值。 -- `country` / `freshness` 通过新增的 Brave 提供商生效。地区接受两字母代码或 `ALL`,时效接受 `pd` / `pw` / `pm` / `py` 或 `YYYY-MM-DDtoYYYY-MM-DD`。省略 `country` 时不向 Brave 传该参数(Brave 自身默认 US);显式 `ALL` 表示全球结果。其它提供商暂不支持这些过滤,显式传入时返回错误,不会静默忽略。参数取值参见 [Brave 官方 API 文档](https://api-dashboard.search.brave.com/api-reference/web/search/get)。 +- `country` / `freshness` 通过 Brave 与 Serply 提供商生效。地区接受两字母代码或 `ALL`,时效接受 `pd` / `pw` / `pm` / `py` 或 `YYYY-MM-DDtoYYYY-MM-DD`。省略 `country` 时不向 Brave 传该参数(Brave 自身默认 US);显式 `ALL` 表示全球结果。Serply 的 `country` 映射为 Google 的 `gl`(省略或 `ALL` 时不传 `gl`,由 Google 决定地区),`freshness` 只接受 `pd` / `pw` / `pm` / `py`。其它提供商暂不支持这些过滤,显式传入时返回错误,不会静默忽略。参数取值参见 [Brave 官方 API 文档](https://api-dashboard.search.brave.com/api-reference/web/search/get)。 - 在联网搜索设置中新建 Brave Search 配置并填写 API Key,可使用现有代理配置;API Key 沿用加密存储和独立凭据接口。 - `content` 默认关闭。设为 `true` 时,并行抓取前 3 条结果的正文(整批 15 秒预算,每页最多 5,000 字符摘录);其余结果保留搜索摘要,需用 `web_fetch` 继续读页。抓取失败仍保留摘要;完整正文地址通过 `full_output_path` 返回。搜索和独立 `web_fetch` 共用本轮快照,短超时不会取消正在进行的共享抓取。 - Brave 的相对 `age` 原样保留,避免把“2 days ago”伪造为精确发布日期。 diff --git a/website-docs/04-api/02-api-infra.md b/website-docs/04-api/02-api-infra.md index 76f7d44b..efbe98e0 100644 --- a/website-docs/04-api/02-api-infra.md +++ b/website-docs/04-api/02-api-infra.md @@ -182,7 +182,7 @@ curl -X PUT $BASE/api/v1/storage-backends/sb-1/default -H "Authorization: Bearer ## Web 搜索(/api/v1/web-search 与 /api/v1/web-search-providers) -当前注册 13 个搜索提供商,包括 Metaso、Exa、Bocha、Brave。各自的 api_key 与 extra_config 参数见[联网搜索](../03-features/11-web-search.md)。 +当前注册 14 个搜索提供商,包括 Metaso、Exa、Bocha、Brave、Serply。各自的 api_key 与 extra_config 参数见[联网搜索](../03-features/11-web-search.md)。 ### GET /api/v1/web-search/providers