From 5eaabf6bbbc97a3669a29bed565994b76f4ac784 Mon Sep 17 00:00:00 2001 From: Syt3s Date: Tue, 11 Aug 2026 17:05:52 +0800 Subject: [PATCH] feat: add Exa WebSearch (#2617) * feat: add Exa WebSearch * Fix test for Exa config options validation * feat: add exa connection test --- frontend/src/api/web-search-provider.ts | 2 +- .../service/web_search_provider.go | 7 +- .../service/web_search_provider_test.go | 14 ++ internal/container/container.go | 1 + internal/infrastructure/web_search/exa.go | 181 ++++++++++++++++++ .../infrastructure/web_search/exa_test.go | 117 +++++++++++ .../infrastructure/web_search/test_errors.go | 4 + .../web_search/test_errors_test.go | 14 ++ internal/types/web_search_provider.go | 22 +++ internal/types/web_search_provider_test.go | 26 +++ 10 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 internal/infrastructure/web_search/exa.go create mode 100644 internal/infrastructure/web_search/exa_test.go diff --git a/frontend/src/api/web-search-provider.ts b/frontend/src/api/web-search-provider.ts index d44937ae..03cd0690 100644 --- a/frontend/src/api/web-search-provider.ts +++ b/frontend/src/api/web-search-provider.ts @@ -5,7 +5,7 @@ export interface WebSearchProviderEntity { id?: string tenant_id?: number name: string - provider: 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' + provider: 'bing' | 'google' | 'duckduckgo' | 'tavily' | 'ollama' | 'baidu' | 'searxng' | 'keenable' | 'zhipu' | 'metaso' | 'exa' description?: string parameters: { // api_key is never returned by the server in this shape; it lives behind diff --git a/internal/application/service/web_search_provider.go b/internal/application/service/web_search_provider.go index 237d2946..cb472f4d 100644 --- a/internal/application/service/web_search_provider.go +++ b/internal/application/service/web_search_provider.go @@ -138,7 +138,8 @@ func isValidProviderType(provider types.WebSearchProviderType) bool { types.WebSearchProviderTypeSearxng, types.WebSearchProviderTypeKeenable, types.WebSearchProviderTypeMetaso, - types.WebSearchProviderTypeZhipu: + types.WebSearchProviderTypeZhipu, + types.WebSearchProviderTypeExa: return true default: return false @@ -171,6 +172,10 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ if params.APIKey == "" { return fmt.Errorf("API key is required for Baidu provider") } + case types.WebSearchProviderTypeExa: + if params.APIKey == "" { + return fmt.Errorf("API key is required for Exa provider") + } case types.WebSearchProviderTypeZhipu: if err := infra_web_search.ValidateZhipuParameters(params); err != nil { return err diff --git a/internal/application/service/web_search_provider_test.go b/internal/application/service/web_search_provider_test.go index 02aa325b..14b3b8fe 100644 --- a/internal/application/service/web_search_provider_test.go +++ b/internal/application/service/web_search_provider_test.go @@ -31,6 +31,20 @@ func TestIsValidProviderTypeIncludesZhipu(t *testing.T) { } } +func TestValidateProviderParametersExa(t *testing.T) { + valid := types.WebSearchProviderParameters{APIKey: "exa-test"} + if err := validateProviderParameters(types.WebSearchProviderTypeExa, valid); err != nil { + t.Fatalf("valid Exa parameters rejected: %v", err) + } + if !isValidProviderType(types.WebSearchProviderTypeExa) { + t.Fatal("Exa provider type is not accepted") + } + + if err := validateProviderParameters(types.WebSearchProviderTypeExa, types.WebSearchProviderParameters{}); err == nil { + t.Fatal("missing Exa API key was accepted") + } +} + func TestValidateProviderParametersMetaso(t *testing.T) { valid := types.WebSearchProviderParameters{ APIKey: "mk-test", diff --git a/internal/container/container.go b/internal/container/container.go index a0449966..daef8152 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -1593,6 +1593,7 @@ func registerWebSearchProviders(registry *infra_web_search.Registry) { registry.Register("searxng", infra_web_search.NewSearxngProvider) registry.Register("keenable", infra_web_search.NewKeenableProvider) registry.Register("zhipu", infra_web_search.NewZhipuProvider) + registry.Register("exa", infra_web_search.NewExaProvider) registry.Register("metaso", infra_web_search.NewMetasoProvider) } diff --git a/internal/infrastructure/web_search/exa.go b/internal/infrastructure/web_search/exa.go new file mode 100644 index 00000000..5e045209 --- /dev/null +++ b/internal/infrastructure/web_search/exa.go @@ -0,0 +1,181 @@ +package web_search + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +const ( + defaultExaSearchURL = "https://api.exa.ai/search" + defaultExaTimeout = 15 * time.Second + defaultExaResults = 5 + maxExaResults = 100 + maxExaResponseBytes = 2 << 20 + maxExaContentRunes = 12000 +) + +// ExaProvider implements web search using Exa's official Search API. +type ExaProvider struct { + client *http.Client + baseURL string + apiKey string + includeText bool +} + +// NewExaProvider creates an Exa provider from tenant-specific parameters. +func NewExaProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) { + apiKey := strings.TrimSpace(params.APIKey) + if apiKey == "" { + return nil, fmt.Errorf("API key is required for Exa provider") + } + client, err := NewSearchHTTPClient(defaultExaTimeout, params.ProxyURL) + if err != nil { + return nil, err + } + return &ExaProvider{ + client: client, + baseURL: defaultExaSearchURL, + apiKey: apiKey, + includeText: parseExaBool(params.ExtraConfig, "include_text"), + }, nil +} + +// Name returns the provider type identifier. +func (p *ExaProvider) Name() string { return "exa" } + +// Search performs a web search through Exa's official Search API. +func (p *ExaProvider) Search( + ctx context.Context, + query string, + maxResults int, + includeDate bool, +) ([]*types.WebSearchResult, error) { + query = strings.TrimSpace(query) + if query == "" { + return nil, fmt.Errorf("query is empty") + } + if maxResults <= 0 { + maxResults = defaultExaResults + } + if maxResults > maxExaResults { + maxResults = maxExaResults + } + + bodyBytes, err := json.Marshal(exaSearchRequest{ + Query: query, + NumResults: maxResults, + Contents: exaContents{ + Highlights: true, + Text: p.includeText, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal Exa request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.baseURL, bytes.NewReader(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("failed to create Exa request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-api-key", p.apiKey) + + logger.Infof(ctx, "[WebSearch][Exa] query=%q maxResults=%d url=%s", query, maxResults, p.baseURL) + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to execute Exa request: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, maxExaResponseBytes)) + if err != nil { + return nil, fmt.Errorf("failed to read Exa response: %w", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + logger.Warnf(ctx, "[WebSearch][Exa] API returned status %d: %s", resp.StatusCode, string(body)) + return nil, fmt.Errorf("exa API returned status %d: %s", resp.StatusCode, string(body)) + } + + var data exaSearchResponse + if err := json.Unmarshal(body, &data); err != nil { + return nil, fmt.Errorf("failed to unmarshal Exa response: %w", err) + } + if data.Error != "" { + return nil, fmt.Errorf("exa API error: %s", data.Error) + } + + results := make([]*types.WebSearchResult, 0, len(data.Results)) + for _, item := range data.Results { + if len(results) >= maxResults { + break + } + snippet := strings.TrimSpace(strings.Join(item.Highlights, "\n")) + content := truncateExaText(strings.TrimSpace(item.Text), maxExaContentRunes) + if snippet == "" { + snippet = truncateExaText(content, 500) + } + result := &types.WebSearchResult{Title: item.Title, URL: item.URL, Snippet: snippet, Content: content, Source: "exa"} + if includeDate && item.PublishedDate != "" { + if publishedAt, err := time.Parse(time.RFC3339, item.PublishedDate); err == nil { + result.PublishedAt = &publishedAt + } + } + results = append(results, result) + } + logger.Infof(ctx, "[WebSearch][Exa] returned %d results", len(results)) + return results, nil +} + +func parseExaBool(config map[string]string, key string) bool { + v, err := strconv.ParseBool(strings.TrimSpace(config[key])) + return err == nil && v +} + +func truncateExaText(value string, maxRunes int) string { + if maxRunes <= 0 { + return "" + } + if utf8.RuneCountInString(value) <= maxRunes { + return value + } + byteIndex := 0 + for runeIndex := 0; runeIndex < maxRunes && byteIndex < len(value); runeIndex++ { + _, size := utf8.DecodeRuneInString(value[byteIndex:]) + byteIndex += size + } + return value[:byteIndex] +} + +type exaSearchRequest struct { + Query string `json:"query"` + NumResults int `json:"numResults"` + Contents exaContents `json:"contents"` +} + +type exaContents struct { + Highlights bool `json:"highlights"` + Text bool `json:"text,omitempty"` +} + +type exaSearchResponse struct { + Results []exaResult `json:"results"` + Error string `json:"error,omitempty"` +} + +type exaResult struct { + Title string `json:"title"` + URL string `json:"url"` + PublishedDate string `json:"publishedDate,omitempty"` + Highlights []string `json:"highlights,omitempty"` + Text string `json:"text,omitempty"` +} diff --git a/internal/infrastructure/web_search/exa_test.go b/internal/infrastructure/web_search/exa_test.go new file mode 100644 index 00000000..23544c15 --- /dev/null +++ b/internal/infrastructure/web_search/exa_test.go @@ -0,0 +1,117 @@ +package web_search + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestExaProviderSearchMapping(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/search" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + if got := r.Header.Get("x-api-key"); got != "exa-test" { + t.Fatalf("x-api-key = %q", got) + } + var request exaSearchRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + if request.Query != "hello" || request.NumResults != 2 || + !request.Contents.Highlights || !request.Contents.Text { + t.Fatalf("request body = %+v", request) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(exaSearchResponse{Results: []exaResult{ + { + Title: "One", + URL: "https://example.com/1", + Highlights: []string{"first", "second"}, + Text: "body", + PublishedDate: "2026-05-01T00:00:00Z", + }, + {Title: "Two", URL: "https://example.com/2"}, + {Title: "Three", URL: "https://example.com/3"}, + }}) + })) + defer srv.Close() + + p := &ExaProvider{client: srv.Client(), baseURL: srv.URL + "/search", apiKey: "exa-test", includeText: true} + results, err := p.Search(context.Background(), "hello", 2, true) + if err != nil { + t.Fatal(err) + } + if len(results) != 2 || results[0].Snippet != "first\nsecond" || + results[0].Content != "body" || results[0].Source != "exa" { + t.Fatalf("unexpected results: %+v", results) + } + if results[0].PublishedAt == nil || !results[0].PublishedAt.Equal(time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC)) { + t.Fatalf("unexpected date: %v", results[0].PublishedAt) + } + if results[1].Snippet != "" { + t.Fatalf("unexpected empty-result fallback: %+v", results[1]) + } +} + +func TestExaProviderSupportsConnectionTestProbe(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/search" { + t.Fatalf("request = %s %s", r.Method, r.URL.Path) + } + if got := r.Header.Get("x-api-key"); got != "exa-test" { + t.Fatalf("x-api-key = %q", got) + } + var request exaSearchRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + if request.Query != "test" || request.NumResults != 1 { + t.Fatalf("connection test request = %+v", request) + } + if !request.Contents.Highlights || request.Contents.Text { + t.Fatalf("connection test should request highlights without page text: %+v", request.Contents) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(exaSearchResponse{Results: []exaResult{ + {Title: "Exa", URL: "https://exa.ai/"}, + }}) + })) + defer srv.Close() + + p := &ExaProvider{ + client: srv.Client(), + baseURL: srv.URL + "/search", + apiKey: "exa-test", + } + results, err := p.Search(context.Background(), "test", 1, false) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("len(results) = %d, want 1", len(results)) + } +} + +func TestExaProviderValidationAndStatus(t *testing.T) { + if _, err := NewExaProvider(types.WebSearchProviderParameters{}); err == nil { + t.Fatal("expected missing API key error") + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"rate limited"}`)) + })) + defer srv.Close() + p := &ExaProvider{client: srv.Client(), baseURL: srv.URL, apiKey: "key"} + if _, err := p.Search(context.Background(), "q", 1, false); err == nil { + t.Fatal("expected status error") + } + if _, err := p.Search(context.Background(), " ", 1, false); err == nil { + t.Fatal("expected empty query error") + } +} diff --git a/internal/infrastructure/web_search/test_errors.go b/internal/infrastructure/web_search/test_errors.go index 919005c9..74cdbd7d 100644 --- a/internal/infrastructure/web_search/test_errors.go +++ b/internal/infrastructure/web_search/test_errors.go @@ -35,6 +35,10 @@ func EmptyTestResultsError(providerType string, provider any) error { return fmt.Errorf( "keenable returned 0 results; keyless requests are rate-limited, so verify network connectivity or set an API key to lift the cap", ) + case types.WebSearchProviderTypeExa: + return fmt.Errorf( + "exa returned 0 results; verify the API key, account quota, network connectivity, and proxy settings", + ) default: return fmt.Errorf("search returned 0 results, please verify your API key and configuration") } diff --git a/internal/infrastructure/web_search/test_errors_test.go b/internal/infrastructure/web_search/test_errors_test.go index 8540b85e..7433fc2d 100644 --- a/internal/infrastructure/web_search/test_errors_test.go +++ b/internal/infrastructure/web_search/test_errors_test.go @@ -66,6 +66,20 @@ func TestEmptyTestResultsError_Keenable(t *testing.T) { } } +func TestEmptyTestResultsError_Exa(t *testing.T) { + err := EmptyTestResultsError(string(types.WebSearchProviderTypeExa), nil) + if err == nil { + t.Fatal("expected error") + } + msg := err.Error() + if !strings.Contains(msg, "exa returned 0 results") { + t.Fatalf("unexpected message: %q", msg) + } + if !strings.Contains(msg, "API key") || !strings.Contains(msg, "quota") || !strings.Contains(msg, "proxy") { + t.Fatalf("exa message should include actionable connection checks: %q", msg) + } +} + func TestEmptyTestResultsError_APIKeyProviders(t *testing.T) { err := EmptyTestResultsError(string(types.WebSearchProviderTypeBing), nil) if err == nil { diff --git a/internal/types/web_search_provider.go b/internal/types/web_search_provider.go index 737e780f..0a21cfbc 100644 --- a/internal/types/web_search_provider.go +++ b/internal/types/web_search_provider.go @@ -24,6 +24,7 @@ const ( WebSearchProviderTypeSearxng WebSearchProviderType = "searxng" WebSearchProviderTypeKeenable WebSearchProviderType = "keenable" WebSearchProviderTypeZhipu WebSearchProviderType = "zhipu" + WebSearchProviderTypeExa WebSearchProviderType = "exa" WebSearchProviderTypeMetaso WebSearchProviderType = "metaso" ) @@ -302,5 +303,26 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo { }, }, }, + { + ID: "exa", + Name: "Exa", + RequiresAPIKey: true, + SupportsProxy: true, + Description: "Exa Search API for AI applications (requires API key)", + DocsURL: "https://docs.exa.ai/", + ConfigFields: []WebSearchProviderConfigField{ + { + Key: "include_text", + Label: "Include text", + Type: "select", + Default: "false", + Description: "Include page text in the unified result Content field.", + Options: []WebSearchProviderConfigFieldOption{ + {Label: "Enabled", Value: "true"}, + {Label: "Disabled", Value: "false"}, + }, + }, + }, + }, } } diff --git a/internal/types/web_search_provider_test.go b/internal/types/web_search_provider_test.go index 072648c0..43fa4d90 100644 --- a/internal/types/web_search_provider_test.go +++ b/internal/types/web_search_provider_test.go @@ -31,6 +31,32 @@ func TestGetWebSearchProviderTypesIncludesZhipuConfig(t *testing.T) { } } +func TestGetWebSearchProviderTypesIncludesExa(t *testing.T) { + var exa *WebSearchProviderTypeInfo + providerTypes := GetWebSearchProviderTypes() + for i := range providerTypes { + if providerTypes[i].ID == string(WebSearchProviderTypeExa) { + exa = &providerTypes[i] + break + } + } + if exa == nil { + t.Fatal("Exa provider metadata is missing") + } + if !exa.RequiresAPIKey || !exa.SupportsProxy { + t.Fatalf("unexpected Exa capability metadata: %+v", exa) + } + if len(exa.ConfigFields) != 1 { + t.Fatalf("len(ConfigFields) = %d, want 1", len(exa.ConfigFields)) + } + field := exa.ConfigFields[0] + if field.Key != "include_text" || field.Type != "select" || field.Default != "false" { + t.Fatalf("unexpected Exa config metadata: %+v", field) + } + if len(field.Options) != 2 || field.Options[0].Value != "true" || field.Options[1].Value != "false" { + t.Fatalf("unexpected Exa config options: %+v", field.Options) + } +} func TestGetWebSearchProviderTypesIncludesMetaso(t *testing.T) { var metaso *WebSearchProviderTypeInfo providerTypes := GetWebSearchProviderTypes()