153 lines
4.8 KiB
Go
153 lines
4.8 KiB
Go
package activate
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
|
"github.com/Tencent/WeKnora/internal/plugin/manifest"
|
|
"github.com/Tencent/WeKnora/internal/plugin/reconcile"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
|
)
|
|
|
|
// parseTimeout bounds one plugin parse when the caller set no deadline.
|
|
const parseTimeout = 10 * time.Minute
|
|
|
|
// Parsers registers code plugins' document parsers as parser engines under
|
|
// their qualified IDs, next to the builtin engines and the docreader's.
|
|
type Parsers struct {
|
|
iv *Invoker
|
|
|
|
mu sync.Mutex
|
|
registered map[string][]string
|
|
}
|
|
|
|
// NewParsers creates the parser activator.
|
|
func NewParsers(iv *Invoker) *Parsers {
|
|
return &Parsers{iv: iv, registered: map[string][]string{}}
|
|
}
|
|
|
|
// Name implements reconcile.Activator.
|
|
func (a *Parsers) Name() string { return "parsers" }
|
|
|
|
// Activate implements reconcile.Activator: all of a plugin's parsers or none.
|
|
func (a *Parsers) Activate(_ context.Context, l *reconcile.Loaded) error {
|
|
var ids []string
|
|
for _, c := range l.Manifest.Contributes[manifest.PointParsers] {
|
|
e := &pluginEngine{
|
|
iv: a.iv, m: l.Manifest, local: c.ID, name: manifest.QualifiedID(l.Manifest.ID, c.ID),
|
|
description: c.Description.Default, names: displayNames(c.Name), fileTypes: c.FileTypes,
|
|
}
|
|
if err := docparser.RegisterPluginEngine(e); err != nil {
|
|
for _, done := range ids {
|
|
docparser.UnregisterPluginEngine(done)
|
|
}
|
|
return fmt.Errorf("parser %s: %w", c.ID, err)
|
|
}
|
|
ids = append(ids, e.name)
|
|
}
|
|
a.mu.Lock()
|
|
a.registered[l.Manifest.ID] = ids
|
|
a.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// Deactivate implements reconcile.Activator.
|
|
func (a *Parsers) Deactivate(_ context.Context, pluginID string) error {
|
|
a.mu.Lock()
|
|
ids := a.registered[pluginID]
|
|
delete(a.registered, pluginID)
|
|
a.mu.Unlock()
|
|
for _, id := range ids {
|
|
docparser.UnregisterPluginEngine(id)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func displayNames(t manifest.LocalizedText) map[string]string {
|
|
out := map[string]string{}
|
|
for k, v := range t.Locales {
|
|
out[k] = v
|
|
}
|
|
if t.Default != "" {
|
|
out["default"] = t.Default
|
|
}
|
|
return out
|
|
}
|
|
|
|
// pluginEngine is a plugin parser as a docparser engine.
|
|
type pluginEngine struct {
|
|
iv *Invoker
|
|
m *manifest.Manifest
|
|
local, name string
|
|
description string
|
|
names map[string]string
|
|
fileTypes []string
|
|
}
|
|
|
|
var (
|
|
_ docparser.EngineRegistration = (*pluginEngine)(nil)
|
|
_ docparser.PluginEngineInfo = (*pluginEngine)(nil)
|
|
)
|
|
|
|
func (e *pluginEngine) Name() string { return e.name }
|
|
func (e *pluginEngine) Description() string { return e.description }
|
|
func (e *pluginEngine) FileTypes(bool) []string { return e.fileTypes }
|
|
func (e *pluginEngine) PluginID() string { return e.m.ID }
|
|
func (e *pluginEngine) DisplayNames() map[string]string { return e.names }
|
|
func (e *pluginEngine) CheckAvailable(bool, map[string]string) (bool, string) {
|
|
if _, err := e.iv.clients.Client(e.m.ID); err != nil {
|
|
return false, err.Error()
|
|
}
|
|
return true, ""
|
|
}
|
|
|
|
// NewReader implements docparser.EngineRegistration. deps.Overrides carry
|
|
// other engines' credentials and are deliberately not passed on; the
|
|
// plugin's own settings travel in its system and tenant configuration.
|
|
func (e *pluginEngine) NewReader(context.Context, docparser.ReaderDeps) (interfaces.DocReader, error) {
|
|
return &remoteParser{engine: e}, nil
|
|
}
|
|
|
|
// remoteParser reads one document through the plugin.
|
|
type remoteParser struct{ engine *pluginEngine }
|
|
|
|
// Read implements interfaces.DocReader. A retryable plugin failure (the
|
|
// plugin is down, rate limited) is returned as an error so the task is
|
|
// retried; any other failure is final and reported in the result.
|
|
func (r *remoteParser) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
|
ctx, cancel := withDefaultTimeout(ctx, parseTimeout)
|
|
defer cancel()
|
|
in := pluginapi.ParseInput{
|
|
FileName: req.FileName, FileType: strings.ToLower(strings.TrimPrefix(req.FileType, ".")),
|
|
Content: req.FileContent, URL: req.URL, Title: req.Title,
|
|
}
|
|
var out pluginapi.ParseOutput
|
|
err := r.engine.iv.Call(ctx, r.engine.m, pluginapi.ParsePath(r.engine.local), nil, in, &out)
|
|
if err != nil {
|
|
var pe *pluginapi.Error
|
|
if errors.As(err, &pe) && !pe.Retryable {
|
|
return &types.ReadResult{Error: err.Error()}, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
res := &types.ReadResult{MarkdownContent: out.Markdown, Metadata: out.Metadata}
|
|
for _, img := range out.Images {
|
|
if img.OriginalRef == "" || len(img.Data) == 0 {
|
|
continue
|
|
}
|
|
res.ImageRefs = append(res.ImageRefs, types.ImageRef{
|
|
Filename: path.Base(img.OriginalRef), OriginalRef: img.OriginalRef,
|
|
MimeType: img.MimeType, ImageData: img.Data,
|
|
})
|
|
}
|
|
return res, nil
|
|
}
|