WeKnora/internal/infrastructure/docparser/engine_registry.go

250 lines
8.4 KiB
Go

package docparser
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// EngineRegistration is what every locally registered parser engine provides:
// the metadata the engine list shows, and the reader that does the parsing.
// Remote-only engines (e.g. markitdown) live in the Python docreader, are
// discovered through its ListEngines RPC, and never register here — the
// registry routes them to the docreader client by default.
type EngineRegistration interface {
Name() string
Description() string
FileTypes(docreaderConnected bool) []string
CheckAvailable(docreaderConnected bool, overrides map[string]string) (available bool, reason string)
// NewReader builds the reader for one parse request. Returning an error
// means this engine cannot serve the request (missing credentials,
// unreachable service); the caller reports it rather than silently
// parsing with something else.
NewReader(ctx context.Context, deps ReaderDeps) (interfaces.DocReader, error)
}
// ReaderDeps carries everything an engine may need to build its reader but
// cannot construct itself: tenant configuration, tenant credentials, and the
// shared docreader connection.
type ReaderDeps struct {
// Overrides holds tenant-level engine configuration (service endpoints,
// API keys), as produced by ParserEngineConfig.ToOverridesMap.
Overrides map[string]string
// Remote is the docreader client. Nil when the service is not connected.
Remote interfaces.DocReader
// WeKnoraCloudCredentials resolves the tenant's WeKnora Cloud
// credentials. It is a function rather than a value because resolving
// them can hit the database, which most engines never need. Nil, or a
// nil return, means the tenant has not configured them.
WeKnoraCloudCredentials func(ctx context.Context) *types.WeKnoraCloudCredentials
}
// enginesMu guards the registries: plugin engines come and go while the
// server runs.
var enginesMu sync.RWMutex
// localEngines holds all locally registered parser engines, in registration
// order — which is also the order the engine list is shown in.
var localEngines []EngineRegistration
// pluginEngines holds engines installed plugins provide, by qualified name
// ("acme.ocr/ocr"). They are kept apart from localEngines so the builtin
// plugin catalog never describes them as builtins.
var pluginEngines = map[string]EngineRegistration{}
// PluginEngineInfo is what a plugin engine adds to the engine list.
type PluginEngineInfo interface {
PluginID() string
// DisplayNames localizes the engine's name, keyed by locale.
DisplayNames() map[string]string
}
// RegisterEngine adds an engine to the local registry. Called from init().
func RegisterEngine(e EngineRegistration) {
enginesMu.Lock()
defer enginesMu.Unlock()
localEngines = append(localEngines, e)
}
// RegisterPluginEngine adds or replaces a plugin's engine. It refuses a name
// a builtin engine uses.
func RegisterPluginEngine(e EngineRegistration) error {
enginesMu.Lock()
defer enginesMu.Unlock()
for _, builtin := range localEngines {
if builtin.Name() == e.Name() {
return fmt.Errorf("parser engine %s already exists", e.Name())
}
}
pluginEngines[e.Name()] = e
return nil
}
// UnregisterPluginEngine removes a plugin's engine.
func UnregisterPluginEngine(name string) {
enginesMu.Lock()
defer enginesMu.Unlock()
delete(pluginEngines, name)
}
// Engines returns the builtin engines in registration order.
func Engines() []EngineRegistration {
enginesMu.RLock()
defer enginesMu.RUnlock()
out := make([]EngineRegistration, len(localEngines))
copy(out, localEngines)
return out
}
// pluginEngineList returns the plugin engines sorted by name.
func pluginEngineList() []EngineRegistration {
enginesMu.RLock()
defer enginesMu.RUnlock()
out := make([]EngineRegistration, 0, len(pluginEngines))
for _, e := range pluginEngines {
out = append(out, e)
}
sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() })
return out
}
// PluginFileTypes returns the file types plugin engines parse, so uploads
// of those types are accepted.
func PluginFileTypes() []string {
var out []string
for _, e := range pluginEngineList() {
out = append(out, e.FileTypes(true)...)
}
return out
}
// lookupEngine returns the registered engine with this name.
func lookupEngine(name string) (EngineRegistration, bool) {
enginesMu.RLock()
defer enginesMu.RUnlock()
for _, engine := range localEngines {
if engine.Name() == name {
return engine, true
}
}
e, ok := pluginEngines[name]
return e, ok
}
// isPluginEngineName reports whether a name is a qualified plugin
// contribution ID, which no builtin or docreader engine uses.
func isPluginEngineName(name string) bool { return strings.Contains(name, "/") }
// NewReader builds the reader for an engine.
//
// An empty engine name means "no explicit choice": simple formats are handled
// in Go and everything else goes to the docreader service. An unknown name is
// routed to the docreader too, so engines that only exist in the Python
// service keep working without a Go-side registration.
func NewReader(
ctx context.Context, engine, fileType string, isURL bool, deps ReaderDeps,
) (interfaces.DocReader, error) {
if registration, ok := lookupEngine(engine); ok {
return registration.NewReader(ctx, deps)
}
if isPluginEngineName(engine) {
// Its plugin was uninstalled or disabled: the Python docreader does
// not know it either, so say so instead of sending it there.
return nil, errEngineUnavailable(engine, "its plugin is not installed or not running")
}
if engine == "" && !isURL && IsSimpleFormat(fileType) {
return &SimpleFormatReader{}, nil
}
return remoteReader(deps)
}
// remoteReader returns the docreader client, or an error when the service is
// not connected — a nil interface value here would panic at the call site.
func remoteReader(deps ReaderDeps) (interfaces.DocReader, error) {
if deps.Remote == nil {
return nil, errNotConnected
}
return deps.Remote, nil
}
// ListAllEngines returns the merged engine list: locally registered engines
// plus engines discovered from the remote docreader via ListEngines RPC.
//
// Merge rules:
// - Local engines are always included, with Go-side availability checks.
// - For a remote engine whose name matches a local one, the remote's
// file_types and description take precedence (the remote service is
// authoritative for its own capabilities).
// - Remote engines not present locally are appended as-is, enabling
// auto-discovery of newly added docreader engines without Go changes.
func ListAllEngines(
docreaderConnected bool, overrides map[string]string, remoteEngines []types.ParserEngineInfo,
) []types.ParserEngineInfo {
remoteMap := make(map[string]types.ParserEngineInfo, len(remoteEngines))
for _, re := range remoteEngines {
remoteMap[re.Name] = re
}
locals := Engines()
seen := make(map[string]bool, len(locals))
result := make([]types.ParserEngineInfo, 0, len(locals)+len(remoteEngines))
for _, e := range locals {
name := e.Name()
seen[name] = true
fileTypes := e.FileTypes(docreaderConnected)
description := e.Description()
if re, ok := remoteMap[name]; ok {
if len(re.FileTypes) > 0 {
fileTypes = re.FileTypes
}
if re.Description != "" {
description = re.Description
}
}
available, reason := e.CheckAvailable(docreaderConnected, overrides)
result = append(result, types.ParserEngineInfo{
Name: name,
Description: description,
FileTypes: fileTypes,
Available: available,
UnavailableReason: reason,
})
}
for _, re := range remoteEngines {
if seen[re.Name] {
continue
}
result = append(result, re)
}
for _, e := range pluginEngineList() {
available, reason := e.CheckAvailable(docreaderConnected, overrides)
info := types.ParserEngineInfo{
Name: e.Name(), Description: e.Description(), FileTypes: e.FileTypes(docreaderConnected),
Available: available, UnavailableReason: reason,
}
if meta, ok := e.(PluginEngineInfo); ok {
info.PluginID, info.DisplayNames = meta.PluginID(), meta.DisplayNames()
}
result = append(result, info)
}
return result
}
// errEngineUnavailable reports an engine that is registered but cannot run for
// this tenant or this build.
func errEngineUnavailable(engine, reason string) error {
return fmt.Errorf("parser engine %q is unavailable: %s", engine, reason)
}