feat: add Exa WebSearch (#2617)
* feat: add Exa WebSearch * Fix test for Exa config options validation * feat: add exa connection testmain
parent
06eeb95b4e
commit
5eaabf6bbb
|
|
@ -5,7 +5,7 @@ export interface WebSearchProviderEntity {
|
|||
id?: string
|
||||
tenant_id?: number
|
||||
name: string
|
||||
provider: 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso'
|
||||
provider: 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa'
|
||||
description?: string
|
||||
parameters: {
|
||||
// api_key is never returned by the server in this shape; it lives behind
|
||||
|
|
|
|||
|
|
@ -138,7 +138,8 @@ func isValidProviderType(provider types.WebSearchProviderType) bool {
|
|||
types.WebSearchProviderTypeSearxng,
|
||||
types.WebSearchProviderTypeKeenable,
|
||||
types.WebSearchProviderTypeMetaso,
|
||||
types.WebSearchProviderTypeZhipu:
|
||||
types.WebSearchProviderTypeZhipu,
|
||||
types.WebSearchProviderTypeExa:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
|
|
@ -171,6 +172,10 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ
|
|||
if params.APIKey == "" {
|
||||
return fmt.Errorf("API key is required for Baidu provider")
|
||||
}
|
||||
case types.WebSearchProviderTypeExa:
|
||||
if params.APIKey == "" {
|
||||
return fmt.Errorf("API key is required for Exa provider")
|
||||
}
|
||||
case types.WebSearchProviderTypeZhipu:
|
||||
if err := infra_web_search.ValidateZhipuParameters(params); err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -31,6 +31,20 @@ func TestIsValidProviderTypeIncludesZhipu(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestValidateProviderParametersExa(t *testing.T) {
|
||||
valid := types.WebSearchProviderParameters{APIKey: "exa-test"}
|
||||
if err := validateProviderParameters(types.WebSearchProviderTypeExa, valid); err != nil {
|
||||
t.Fatalf("valid Exa parameters rejected: %v", err)
|
||||
}
|
||||
if !isValidProviderType(types.WebSearchProviderTypeExa) {
|
||||
t.Fatal("Exa provider type is not accepted")
|
||||
}
|
||||
|
||||
if err := validateProviderParameters(types.WebSearchProviderTypeExa, types.WebSearchProviderParameters{}); err == nil {
|
||||
t.Fatal("missing Exa API key was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateProviderParametersMetaso(t *testing.T) {
|
||||
valid := types.WebSearchProviderParameters{
|
||||
APIKey: "mk-test",
|
||||
|
|
|
|||
|
|
@ -1593,6 +1593,7 @@ func registerWebSearchProviders(registry *infra_web_search.Registry) {
|
|||
registry.Register("searxng", infra_web_search.NewSearxngProvider)
|
||||
registry.Register("keenable", infra_web_search.NewKeenableProvider)
|
||||
registry.Register("zhipu", infra_web_search.NewZhipuProvider)
|
||||
registry.Register("exa", infra_web_search.NewExaProvider)
|
||||
registry.Register("metaso", infra_web_search.NewMetasoProvider)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,181 @@
|
|||
package web_search
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultExaSearchURL = "https://api.exa.ai/search"
|
||||
defaultExaTimeout = 15 * time.Second
|
||||
defaultExaResults = 5
|
||||
maxExaResults = 100
|
||||
maxExaResponseBytes = 2 << 20
|
||||
maxExaContentRunes = 12000
|
||||
)
|
||||
|
||||
// ExaProvider implements web search using Exa's official Search API.
|
||||
type ExaProvider struct {
|
||||
client *http.Client
|
||||
baseURL string
|
||||
apiKey string
|
||||
includeText bool
|
||||
}
|
||||
|
||||
// NewExaProvider creates an Exa provider from tenant-specific parameters.
|
||||
func NewExaProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
|
||||
apiKey := strings.TrimSpace(params.APIKey)
|
||||
if apiKey == "" {
|
||||
return nil, fmt.Errorf("API key is required for Exa provider")
|
||||
}
|
||||
client, err := NewSearchHTTPClient(defaultExaTimeout, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ExaProvider{
|
||||
client: client,
|
||||
baseURL: defaultExaSearchURL,
|
||||
apiKey: apiKey,
|
||||
includeText: parseExaBool(params.ExtraConfig, "include_text"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider type identifier.
|
||||
func (p *ExaProvider) Name() string { return "exa" }
|
||||
|
||||
// Search performs a web search through Exa's official Search API.
|
||||
func (p *ExaProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
maxResults int,
|
||||
includeDate bool,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
query = strings.TrimSpace(query)
|
||||
if query == "" {
|
||||
return nil, fmt.Errorf("query is empty")
|
||||
}
|
||||
if maxResults <= 0 {
|
||||
maxResults = defaultExaResults
|
||||
}
|
||||
if maxResults > maxExaResults {
|
||||
maxResults = maxExaResults
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(exaSearchRequest{
|
||||
Query: query,
|
||||
NumResults: maxResults,
|
||||
Contents: exaContents{
|
||||
Highlights: true,
|
||||
Text: p.includeText,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal Exa request: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.baseURL, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Exa request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-api-key", p.apiKey)
|
||||
|
||||
logger.Infof(ctx, "[WebSearch][Exa] query=%q maxResults=%d url=%s", query, maxResults, p.baseURL)
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute Exa request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxExaResponseBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read Exa response: %w", err)
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
logger.Warnf(ctx, "[WebSearch][Exa] API returned status %d: %s", resp.StatusCode, string(body))
|
||||
return nil, fmt.Errorf("exa API returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var data exaSearchResponse
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal Exa response: %w", err)
|
||||
}
|
||||
if data.Error != "" {
|
||||
return nil, fmt.Errorf("exa API error: %s", data.Error)
|
||||
}
|
||||
|
||||
results := make([]*types.WebSearchResult, 0, len(data.Results))
|
||||
for _, item := range data.Results {
|
||||
if len(results) >= maxResults {
|
||||
break
|
||||
}
|
||||
snippet := strings.TrimSpace(strings.Join(item.Highlights, "\n"))
|
||||
content := truncateExaText(strings.TrimSpace(item.Text), maxExaContentRunes)
|
||||
if snippet == "" {
|
||||
snippet = truncateExaText(content, 500)
|
||||
}
|
||||
result := &types.WebSearchResult{Title: item.Title, URL: item.URL, Snippet: snippet, Content: content, Source: "exa"}
|
||||
if includeDate && item.PublishedDate != "" {
|
||||
if publishedAt, err := time.Parse(time.RFC3339, item.PublishedDate); err == nil {
|
||||
result.PublishedAt = &publishedAt
|
||||
}
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
logger.Infof(ctx, "[WebSearch][Exa] returned %d results", len(results))
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func parseExaBool(config map[string]string, key string) bool {
|
||||
v, err := strconv.ParseBool(strings.TrimSpace(config[key]))
|
||||
return err == nil && v
|
||||
}
|
||||
|
||||
func truncateExaText(value string, maxRunes int) string {
|
||||
if maxRunes <= 0 {
|
||||
return ""
|
||||
}
|
||||
if utf8.RuneCountInString(value) <= maxRunes {
|
||||
return value
|
||||
}
|
||||
byteIndex := 0
|
||||
for runeIndex := 0; runeIndex < maxRunes && byteIndex < len(value); runeIndex++ {
|
||||
_, size := utf8.DecodeRuneInString(value[byteIndex:])
|
||||
byteIndex += size
|
||||
}
|
||||
return value[:byteIndex]
|
||||
}
|
||||
|
||||
type exaSearchRequest struct {
|
||||
Query string `json:"query"`
|
||||
NumResults int `json:"numResults"`
|
||||
Contents exaContents `json:"contents"`
|
||||
}
|
||||
|
||||
type exaContents struct {
|
||||
Highlights bool `json:"highlights"`
|
||||
Text bool `json:"text,omitempty"`
|
||||
}
|
||||
|
||||
type exaSearchResponse struct {
|
||||
Results []exaResult `json:"results"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type exaResult struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
PublishedDate string `json:"publishedDate,omitempty"`
|
||||
Highlights []string `json:"highlights,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
package web_search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestExaProviderSearchMapping(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/search" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if got := r.Header.Get("x-api-key"); got != "exa-test" {
|
||||
t.Fatalf("x-api-key = %q", got)
|
||||
}
|
||||
var request exaSearchRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if request.Query != "hello" || request.NumResults != 2 ||
|
||||
!request.Contents.Highlights || !request.Contents.Text {
|
||||
t.Fatalf("request body = %+v", request)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(exaSearchResponse{Results: []exaResult{
|
||||
{
|
||||
Title: "One",
|
||||
URL: "https://example.com/1",
|
||||
Highlights: []string{"first", "second"},
|
||||
Text: "body",
|
||||
PublishedDate: "2026-05-01T00:00:00Z",
|
||||
},
|
||||
{Title: "Two", URL: "https://example.com/2"},
|
||||
{Title: "Three", URL: "https://example.com/3"},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := &ExaProvider{client: srv.Client(), baseURL: srv.URL + "/search", apiKey: "exa-test", includeText: true}
|
||||
results, err := p.Search(context.Background(), "hello", 2, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(results) != 2 || results[0].Snippet != "first\nsecond" ||
|
||||
results[0].Content != "body" || results[0].Source != "exa" {
|
||||
t.Fatalf("unexpected results: %+v", results)
|
||||
}
|
||||
if results[0].PublishedAt == nil || !results[0].PublishedAt.Equal(time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("unexpected date: %v", results[0].PublishedAt)
|
||||
}
|
||||
if results[1].Snippet != "" {
|
||||
t.Fatalf("unexpected empty-result fallback: %+v", results[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExaProviderSupportsConnectionTestProbe(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/search" {
|
||||
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if got := r.Header.Get("x-api-key"); got != "exa-test" {
|
||||
t.Fatalf("x-api-key = %q", got)
|
||||
}
|
||||
var request exaSearchRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if request.Query != "test" || request.NumResults != 1 {
|
||||
t.Fatalf("connection test request = %+v", request)
|
||||
}
|
||||
if !request.Contents.Highlights || request.Contents.Text {
|
||||
t.Fatalf("connection test should request highlights without page text: %+v", request.Contents)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(exaSearchResponse{Results: []exaResult{
|
||||
{Title: "Exa", URL: "https://exa.ai/"},
|
||||
}})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := &ExaProvider{
|
||||
client: srv.Client(),
|
||||
baseURL: srv.URL + "/search",
|
||||
apiKey: "exa-test",
|
||||
}
|
||||
results, err := p.Search(context.Background(), "test", 1, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("len(results) = %d, want 1", len(results))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExaProviderValidationAndStatus(t *testing.T) {
|
||||
if _, err := NewExaProvider(types.WebSearchProviderParameters{}); err == nil {
|
||||
t.Fatal("expected missing API key error")
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = w.Write([]byte(`{"error":"rate limited"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
p := &ExaProvider{client: srv.Client(), baseURL: srv.URL, apiKey: "key"}
|
||||
if _, err := p.Search(context.Background(), "q", 1, false); err == nil {
|
||||
t.Fatal("expected status error")
|
||||
}
|
||||
if _, err := p.Search(context.Background(), " ", 1, false); err == nil {
|
||||
t.Fatal("expected empty query error")
|
||||
}
|
||||
}
|
||||
|
|
@ -35,6 +35,10 @@ func EmptyTestResultsError(providerType string, provider any) error {
|
|||
return fmt.Errorf(
|
||||
"keenable returned 0 results; keyless requests are rate-limited, so verify network connectivity or set an API key to lift the cap",
|
||||
)
|
||||
case types.WebSearchProviderTypeExa:
|
||||
return fmt.Errorf(
|
||||
"exa returned 0 results; verify the API key, account quota, network connectivity, and proxy settings",
|
||||
)
|
||||
default:
|
||||
return fmt.Errorf("search returned 0 results, please verify your API key and configuration")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,6 +66,20 @@ func TestEmptyTestResultsError_Keenable(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestEmptyTestResultsError_Exa(t *testing.T) {
|
||||
err := EmptyTestResultsError(string(types.WebSearchProviderTypeExa), nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
msg := err.Error()
|
||||
if !strings.Contains(msg, "exa returned 0 results") {
|
||||
t.Fatalf("unexpected message: %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "API key") || !strings.Contains(msg, "quota") || !strings.Contains(msg, "proxy") {
|
||||
t.Fatalf("exa message should include actionable connection checks: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyTestResultsError_APIKeyProviders(t *testing.T) {
|
||||
err := EmptyTestResultsError(string(types.WebSearchProviderTypeBing), nil)
|
||||
if err == nil {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const (
|
|||
WebSearchProviderTypeSearxng WebSearchProviderType = "searxng"
|
||||
WebSearchProviderTypeKeenable WebSearchProviderType = "keenable"
|
||||
WebSearchProviderTypeZhipu WebSearchProviderType = "zhipu"
|
||||
WebSearchProviderTypeExa WebSearchProviderType = "exa"
|
||||
WebSearchProviderTypeMetaso WebSearchProviderType = "metaso"
|
||||
)
|
||||
|
||||
|
|
@ -302,5 +303,26 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
|
|||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "exa",
|
||||
Name: "Exa",
|
||||
RequiresAPIKey: true,
|
||||
SupportsProxy: true,
|
||||
Description: "Exa Search API for AI applications (requires API key)",
|
||||
DocsURL: "https://docs.exa.ai/",
|
||||
ConfigFields: []WebSearchProviderConfigField{
|
||||
{
|
||||
Key: "include_text",
|
||||
Label: "Include text",
|
||||
Type: "select",
|
||||
Default: "false",
|
||||
Description: "Include page text in the unified result Content field.",
|
||||
Options: []WebSearchProviderConfigFieldOption{
|
||||
{Label: "Enabled", Value: "true"},
|
||||
{Label: "Disabled", Value: "false"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,32 @@ func TestGetWebSearchProviderTypesIncludesZhipuConfig(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestGetWebSearchProviderTypesIncludesExa(t *testing.T) {
|
||||
var exa *WebSearchProviderTypeInfo
|
||||
providerTypes := GetWebSearchProviderTypes()
|
||||
for i := range providerTypes {
|
||||
if providerTypes[i].ID == string(WebSearchProviderTypeExa) {
|
||||
exa = &providerTypes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if exa == nil {
|
||||
t.Fatal("Exa provider metadata is missing")
|
||||
}
|
||||
if !exa.RequiresAPIKey || !exa.SupportsProxy {
|
||||
t.Fatalf("unexpected Exa capability metadata: %+v", exa)
|
||||
}
|
||||
if len(exa.ConfigFields) != 1 {
|
||||
t.Fatalf("len(ConfigFields) = %d, want 1", len(exa.ConfigFields))
|
||||
}
|
||||
field := exa.ConfigFields[0]
|
||||
if field.Key != "include_text" || field.Type != "select" || field.Default != "false" {
|
||||
t.Fatalf("unexpected Exa config metadata: %+v", field)
|
||||
}
|
||||
if len(field.Options) != 2 || field.Options[0].Value != "true" || field.Options[1].Value != "false" {
|
||||
t.Fatalf("unexpected Exa config options: %+v", field.Options)
|
||||
}
|
||||
}
|
||||
func TestGetWebSearchProviderTypesIncludesMetaso(t *testing.T) {
|
||||
var metaso *WebSearchProviderTypeInfo
|
||||
providerTypes := GetWebSearchProviderTypes()
|
||||
|
|
|
|||
Loading…
Reference in New Issue