WeKnora/internal/application/service/tenant_skill_transcript_tes...

268 lines
9.3 KiB
Go

package service
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/require"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// transcriptStreams records what the installer projects onto the replayable
// stream, in order, so a test can state the shape the console will read.
type transcriptStreams struct {
events []interfaces.StreamEvent
err error
}
func (s *transcriptStreams) AppendEvent(
_ context.Context, _, _ string, evt interfaces.StreamEvent,
) error {
s.events = append(s.events, evt)
return s.err
}
func (s *transcriptStreams) GetEvents(
_ context.Context, _, _ string, from int,
) ([]interfaces.StreamEvent, int, error) {
return s.events[from:], len(s.events), nil
}
func (s *transcriptStreams) types() []types.ResponseType {
out := make([]types.ResponseType, 0, len(s.events))
for _, evt := range s.events {
out = append(out, evt.Type)
}
return out
}
// transcriptMessages is the durable half. Only the two calls the transcript
// makes are implemented; the rest panic so an accidental new dependency on the
// repository surfaces as a test failure rather than a silent nil write.
type transcriptMessages struct {
interfaces.MessageRepository
created []*types.Message
updated []*types.Message
err error
}
func (m *transcriptMessages) CreateMessage(
_ context.Context, msg *types.Message,
) (*types.Message, error) {
m.created = append(m.created, msg)
return msg, m.err
}
func (m *transcriptMessages) UpdateMessage(_ context.Context, msg *types.Message) error {
m.updated = append(m.updated, msg)
return m.err
}
func newTranscriptForTest(t *testing.T) (
*installTranscript, *event.EventBus, *transcriptStreams, *transcriptMessages,
) {
t.Helper()
bus := event.NewEventBus()
streams := &transcriptStreams{}
messages := &transcriptMessages{}
tr := newInstallTranscript(context.Background(), bus, streams, messages, "sess-1", "msg-1")
tr.Subscribe()
return tr, bus, streams, messages
}
func TestInstallTranscriptProjectsTheAgentsWork(t *testing.T) {
_, bus, streams, _ := newTranscriptForTest(t)
ctx := context.Background()
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "t-1", Type: event.EventAgentThought,
Data: event.AgentThoughtData{Content: "check for uv", Done: false},
}))
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "c-1", Type: event.EventAgentToolCall,
Data: event.AgentToolCallData{
ToolName: "shell_exec", ToolCallID: "call-1",
Arguments: map[string]any{"command": "uv --version"},
},
}))
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "r-1", Type: event.EventAgentToolResult,
Data: event.AgentToolResultData{
ToolName: "shell_exec", ToolCallID: "call-1", Success: true, Output: "uv 0.4.0",
},
}))
require.Equal(t, []types.ResponseType{
types.ResponseTypeThinking,
types.ResponseTypeToolCall,
types.ResponseTypeToolResult,
}, streams.types())
call := streams.events[1]
require.Equal(t, "shell_exec", call.Data["tool_name"])
require.Equal(t, "call-1", call.Data["tool_call_id"])
}
// A failed command is the single most useful thing in an install transcript,
// so it must reach the stream as an error the console already knows how to
// render, not be dropped for having Success=false.
func TestInstallTranscriptReportsAFailedCommandAsAnError(t *testing.T) {
_, bus, streams, _ := newTranscriptForTest(t)
require.NoError(t, bus.Emit(context.Background(), event.Event{
ID: "r-1", Type: event.EventAgentToolResult,
Data: event.AgentToolResultData{
ToolName: "shell_exec", ToolCallID: "call-1",
Success: false, Error: "exit status 1",
},
}))
require.Equal(t, []types.ResponseType{types.ResponseTypeError}, streams.types())
require.Equal(t, "exit status 1", streams.events[0].Content)
require.Equal(t, false, streams.events[0].Data["success"])
}
func TestInstallTranscriptPersistsTheAnswerAndStepsOnComplete(t *testing.T) {
tr, bus, streams, messages := newTranscriptForTest(t)
ctx := context.Background()
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "a-1", Type: event.EventAgentFinalAnswer,
Data: event.AgentFinalAnswerData{Content: "installed ", Done: false},
}))
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "a-1", Type: event.EventAgentFinalAnswer,
Data: event.AgentFinalAnswerData{Content: "pdf-tools", Done: true},
}))
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "done", Type: event.EventAgentComplete,
Data: event.AgentCompleteData{
MessageID: "msg-1", TotalDurationMs: 4200, TotalSteps: 3,
},
}))
tr.Finish(ctx, nil)
require.Equal(t, types.ResponseTypeComplete, streams.types()[len(streams.types())-1])
require.Len(t, messages.updated, 1)
saved := messages.updated[0]
require.Equal(t, "installed pdf-tools", saved.Content)
require.True(t, saved.IsCompleted)
require.EqualValues(t, 4200, saved.AgentDurationMs)
}
// The engine routes a round's plain assistant prose through
// EventAgentFinalAnswer, so every "now I'll check X" preamble arrives as an
// answer chunk. Only the prose of the round that ends the run is the answer.
// Without retracting the preambles the persisted record is every round's
// commentary glued end to end, which is unreadable — and is what the console
// renders as the install's result.
func TestInstallTranscriptRetractsPreamblesSupersededByAToolCall(t *testing.T) {
tr, bus, _, messages := newTranscriptForTest(t)
ctx := context.Background()
emitAnswer := func(id, content string, done bool) {
require.NoError(t, bus.Emit(ctx, event.Event{
ID: id, Type: event.EventAgentFinalAnswer,
Data: event.AgentFinalAnswerData{Content: content, Done: done},
}))
}
emitAnswer("a-1", "The script needs duckduckgo-search, so I'll create the venv:", true)
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "c-1", Type: event.EventAgentToolCall,
Data: event.AgentToolCallData{
ToolName: "shell_exec", ToolCallID: "call-1",
Arguments: map[string]any{"command": "uv venv .venv"},
},
}))
emitAnswer("a-2", "Installed successfully. Now verifying:", true)
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "c-2", Type: event.EventAgentToolCall,
Data: event.AgentToolCallData{
ToolName: "shell_exec", ToolCallID: "call-2",
Arguments: map[string]any{"command": "python scripts/search.py --help"},
},
}))
emitAnswer("a-3", "Installed duckduckgo-search into the skill venv.", true)
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "done", Type: event.EventAgentComplete,
Data: event.AgentCompleteData{MessageID: "msg-1", TotalDurationMs: 900},
}))
tr.Finish(ctx, nil)
require.Len(t, messages.updated, 1)
require.Equal(t,
"Installed duckduckgo-search into the skill venv.",
messages.updated[0].Content,
"only the final round's prose is the answer; earlier preambles were retracted by their tool calls",
)
}
// An install that dies before the engine emits complete is exactly the run
// someone will come looking for, so Finish must close the record itself.
func TestInstallTranscriptRecordsAFailedRun(t *testing.T) {
tr, _, streams, messages := newTranscriptForTest(t)
tr.Finish(context.Background(), errors.New("installer agent failed: boom"))
require.Equal(t, []types.ResponseType{
types.ResponseTypeError,
types.ResponseTypeComplete,
}, streams.types())
require.Len(t, messages.updated, 1)
require.Contains(t, messages.updated[0].Content, "boom")
require.True(t, messages.updated[0].IsCompleted)
}
// The transcript is a troubleshooting aid. A broken Redis or a failing write
// must not be able to fail an otherwise good install.
func TestInstallTranscriptSwallowsItsOwnFailures(t *testing.T) {
bus := event.NewEventBus()
streams := &transcriptStreams{err: errors.New("redis down")}
messages := &transcriptMessages{err: errors.New("db down")}
tr := newInstallTranscript(context.Background(), bus, streams, messages, "sess-1", "msg-1")
tr.Subscribe()
require.NoError(t, bus.Emit(context.Background(), event.Event{
ID: "t-1", Type: event.EventAgentThought,
Data: event.AgentThoughtData{Content: "hi", Done: true},
}))
require.NotPanics(t, func() { tr.Finish(context.Background(), nil) })
}
func TestInstallTranscriptCreateSeedsTheAssistantRow(t *testing.T) {
streams := &transcriptStreams{}
messages := &transcriptMessages{}
tr := newInstallTranscript(
context.Background(), event.NewEventBus(), streams, messages, "sess-1", "msg-1",
)
require.NoError(t, tr.Create(context.Background(), "install pdf-tools"))
require.Len(t, messages.created, 2)
require.Equal(t, "user", messages.created[0].Role)
require.Equal(t, "install pdf-tools", messages.created[0].Content)
require.Equal(t, "assistant", messages.created[1].Role)
require.Equal(t, "msg-1", messages.created[1].ID)
require.Equal(t, "sess-1", messages.created[1].SessionID)
}
// A console that attaches to a running install reads the event log and nothing
// else, so the prompt has to be in the log or the run shows up as an agent
// answering a question nobody can see.
func TestInstallTranscriptCreateOpensTheLogWithThePrompt(t *testing.T) {
streams := &transcriptStreams{}
tr := newInstallTranscript(
context.Background(), event.NewEventBus(), streams, &transcriptMessages{}, "sess-1", "msg-1",
)
require.NoError(t, tr.Create(context.Background(), "install pdf-tools"))
require.Equal(t, []types.ResponseType{types.ResponseTypeInstallPrompt}, streams.types())
require.Equal(t, "install pdf-tools", streams.events[0].Content)
}