249 lines
7.8 KiB
Go
249 lines
7.8 KiB
Go
// Package conformance checks that a plugin speaks the extension protocol:
|
|
// it answers the discovery endpoints, rejects what it must reject, and
|
|
// answers every endpoint it claims with either a well-formed output or a
|
|
// well-formed protocol error. It does not judge the answers themselves; a
|
|
// plugin's own tests do that.
|
|
package conformance
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"regexp"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/pluginsdk/client"
|
|
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
|
)
|
|
|
|
// Result is one check.
|
|
type Result struct {
|
|
Name string `json:"name"`
|
|
Passed bool `json:"passed"`
|
|
Detail string `json:"detail,omitempty"`
|
|
}
|
|
|
|
// Report is every check against one plugin.
|
|
type Report struct {
|
|
Plugin string `json:"plugin"`
|
|
Results []Result `json:"results"`
|
|
}
|
|
|
|
// Passed reports whether every check passed.
|
|
func (r Report) Passed() bool {
|
|
for _, res := range r.Results {
|
|
if !res.Passed {
|
|
return false
|
|
}
|
|
}
|
|
return len(r.Results) > 0
|
|
}
|
|
|
|
// Target is the plugin under test.
|
|
type Target struct {
|
|
// Client reaches the plugin with valid credentials.
|
|
Client *client.Client
|
|
// Unauthenticated reaches it without credentials; nil skips the check
|
|
// (a plugin served without authentication in a test harness).
|
|
Unauthenticated *client.Client
|
|
// Raw posts arbitrary bytes, for malformed-request checks; nil skips.
|
|
Raw func(ctx context.Context, path string, body []byte) (*http.Response, error)
|
|
// Timeout bounds each call (default 30s).
|
|
Timeout time.Duration
|
|
}
|
|
|
|
var semver = regexp.MustCompile(`^\d+\.\d+\.\d+([-+].*)?$`)
|
|
|
|
// Run checks a plugin.
|
|
func Run(ctx context.Context, t Target) Report {
|
|
if t.Timeout <= 0 {
|
|
t.Timeout = 30 * time.Second
|
|
}
|
|
var rep Report
|
|
check := func(name string, fn func(ctx context.Context) error) {
|
|
cctx, cancel := context.WithTimeout(ctx, t.Timeout)
|
|
defer cancel()
|
|
res := Result{Name: name, Passed: true}
|
|
if err := fn(cctx); err != nil {
|
|
res.Passed, res.Detail = false, err.Error()
|
|
}
|
|
rep.Results = append(rep.Results, res)
|
|
}
|
|
|
|
var m *pluginapi.Manifest
|
|
check("manifest", func(ctx context.Context) error {
|
|
var err error
|
|
if m, err = t.Client.Manifest(ctx); err != nil {
|
|
return err
|
|
}
|
|
rep.Plugin = m.ID + "@" + m.Version
|
|
switch {
|
|
case m.ID == "":
|
|
return fmt.Errorf("manifest has no id")
|
|
case !semver.MatchString(m.Version):
|
|
return fmt.Errorf("version %q is not semantic", m.Version)
|
|
case m.APIVersion != pluginapi.APIVersion:
|
|
return fmt.Errorf("apiVersion %q, want %q", m.APIVersion, pluginapi.APIVersion)
|
|
case len(m.Contributes) == 0:
|
|
return fmt.Errorf("manifest lists no contributions")
|
|
}
|
|
return nil
|
|
})
|
|
check("health", func(ctx context.Context) error { return t.Client.Health(ctx) })
|
|
if t.Unauthenticated != nil {
|
|
check("rejects unauthenticated calls", func(ctx context.Context) error {
|
|
return wantCode(t.Unauthenticated.Health(ctx), pluginapi.CodeUnauthorized)
|
|
})
|
|
}
|
|
check("unknown endpoint answers not_found", func(ctx context.Context) error {
|
|
return wantCode(
|
|
t.Client.Call(ctx, "/v1/no-such-endpoint", pluginapi.Envelope{}, nil, nil),
|
|
pluginapi.CodeNotFound,
|
|
)
|
|
})
|
|
check("unknown contribution answers not_found", func(ctx context.Context) error {
|
|
return wantCode(t.Client.Call(ctx, pluginapi.SearchPath("no-such-provider"), pluginapi.Envelope{},
|
|
pluginapi.SearchInput{Query: "x"}, nil), pluginapi.CodeNotFound)
|
|
})
|
|
if t.Raw != nil {
|
|
check("malformed envelope answers bad_request", func(ctx context.Context) error {
|
|
resp, err := t.Raw(ctx, "/v1/config/validate", []byte("{not json"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = resp.Body.Close() }()
|
|
return wantErrorBody(resp, pluginapi.CodeBadRequest)
|
|
})
|
|
}
|
|
check("config/validate answers", func(ctx context.Context) error {
|
|
return protocolAnswer(t.Client.Call(ctx, "/v1/config/validate", envelope(), nil, nil))
|
|
})
|
|
if m == nil {
|
|
return rep
|
|
}
|
|
for _, id := range m.Contributes["webSearch"] {
|
|
check("webSearch/"+id+" search answers", func(ctx context.Context) error {
|
|
var out pluginapi.SearchOutput
|
|
err := t.Client.Call(
|
|
ctx,
|
|
pluginapi.SearchPath(id),
|
|
envelope(),
|
|
pluginapi.SearchInput{Query: "weknora", MaxResults: 3},
|
|
&out,
|
|
)
|
|
if err == nil && out.Results == nil {
|
|
return fmt.Errorf("output has no results array")
|
|
}
|
|
return protocolAnswer(err)
|
|
})
|
|
}
|
|
for _, id := range m.Contributes["parsers"] {
|
|
check("parsers/"+id+" parse answers", func(ctx context.Context) error {
|
|
var out pluginapi.ParseOutput
|
|
err := t.Client.Call(ctx, pluginapi.ParsePath(id), envelope(), pluginapi.ParseInput{
|
|
FileName: "conformance.txt", FileType: "txt", Content: []byte("WeKnora conformance check\n"),
|
|
}, &out)
|
|
return protocolAnswer(err)
|
|
})
|
|
}
|
|
for _, id := range m.Contributes["connectors"] {
|
|
check("connectors/"+id+" validate answers", func(ctx context.Context) error {
|
|
return protocolAnswer(t.Client.Call(ctx, pluginapi.ConnectorValidatePath(id), envelope(), nil, nil))
|
|
})
|
|
check("connectors/"+id+" list-resources answers", func(ctx context.Context) error {
|
|
var out pluginapi.ListResourcesOutput
|
|
err := t.Client.Call(
|
|
ctx,
|
|
pluginapi.ConnectorListResourcesPath(id),
|
|
envelope(),
|
|
pluginapi.ListResourcesInput{},
|
|
&out,
|
|
)
|
|
if err == nil && out.Resources == nil {
|
|
return fmt.Errorf("output has no resources array")
|
|
}
|
|
return protocolAnswer(err)
|
|
})
|
|
check("connectors/"+id+" resolve-ancestors answers", func(ctx context.Context) error {
|
|
var out pluginapi.ResolveAncestorsOutput
|
|
err := t.Client.Call(ctx, pluginapi.ConnectorResolveAncestorsPath(id), envelope(),
|
|
pluginapi.ResolveAncestorsInput{ResourceIDs: []string{}}, &out)
|
|
if err == nil && out.Ancestors == nil {
|
|
return fmt.Errorf("output has no ancestors array")
|
|
}
|
|
return protocolAnswer(err)
|
|
})
|
|
check("connectors/"+id+" fetch stream terminates", func(ctx context.Context) error {
|
|
_, err := t.Client.Stream(ctx, pluginapi.ConnectorFetchPath(id), envelope(),
|
|
pluginapi.FetchInput{Mode: pluginapi.FetchFull}, func(ev pluginapi.Event) error {
|
|
switch ev.Type {
|
|
case pluginapi.EventItem, pluginapi.EventCheckpoint, pluginapi.EventProgress, pluginapi.EventLog:
|
|
return nil
|
|
}
|
|
return fmt.Errorf("unknown event type %q", ev.Type)
|
|
})
|
|
return protocolAnswer(err)
|
|
})
|
|
}
|
|
return rep
|
|
}
|
|
|
|
// envelope is a call with empty configuration: plugins must answer it with
|
|
// an output or a protocol error, never a crash or garbage.
|
|
func envelope() pluginapi.Envelope {
|
|
return pluginapi.Envelope{Context: pluginapi.Context{TenantID: 1, RequestID: "conformance", Locale: "en-US"}}
|
|
}
|
|
|
|
// protocolAnswer accepts success or a protocol error the plugin chose, and
|
|
// fails transport failures and internal errors: a plugin must answer an
|
|
// empty configuration with invalid_config, not by crashing.
|
|
func protocolAnswer(err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
var te *client.TransportError
|
|
if errors.As(err, &te) {
|
|
return fmt.Errorf("no protocol answer: %v", te)
|
|
}
|
|
e, ok := pluginapi.AsError(err)
|
|
if !ok {
|
|
return err
|
|
}
|
|
if e.Code == pluginapi.CodeInternal {
|
|
return fmt.Errorf("answered internal: %s", e.Message)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func wantCode(err error, code pluginapi.ErrorCode) error {
|
|
e, ok := pluginapi.AsError(err)
|
|
if !ok {
|
|
return fmt.Errorf("want a %s protocol error, got %v", code, err)
|
|
}
|
|
if e.Code != code {
|
|
return fmt.Errorf("want %s, got %s: %s", code, e.Code, e.Message)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func wantErrorBody(resp *http.Response, code pluginapi.ErrorCode) error {
|
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
|
var body pluginapi.ErrorBody
|
|
if err := json.Unmarshal(b, &body); err != nil {
|
|
return fmt.Errorf("HTTP %d without a protocol error body: %s", resp.StatusCode, b)
|
|
}
|
|
if resp.StatusCode != code.HTTPStatus() || body.Error.Code != code {
|
|
return fmt.Errorf(
|
|
"want HTTP %d %s, got HTTP %d %s",
|
|
code.HTTPStatus(),
|
|
code,
|
|
resp.StatusCode,
|
|
body.Error.Code,
|
|
)
|
|
}
|
|
return nil
|
|
}
|