fix(chunking): align preview with parent-child splitting / 修复预览与父子分块不一致

main
xiaowang 2026-08-05 14:31:45 +08:00 committed by lyingbug
parent b191201f44
commit d2888e9565
8 changed files with 153 additions and 36 deletions

View File

@ -65,6 +65,9 @@ export interface PreviewChunkingRequest {
chunk_size: number
chunk_overlap: number
separators: string[]
enable_parent_child?: boolean
parent_chunk_size?: number
child_chunk_size?: number
strategy?: string
token_limit?: number
languages?: string[]

View File

@ -224,6 +224,9 @@ interface Props {
chunkSize: number
chunkOverlap: number
separators: string[]
enableParentChild: boolean
parentChunkSize: number
childChunkSize: number
strategy?: string
tokenLimit?: number
languages?: string[]
@ -285,6 +288,9 @@ const runPreview = async () => {
chunk_size: props.config.chunkSize,
chunk_overlap: props.config.chunkOverlap,
separators: props.config.separators,
enable_parent_child: props.config.enableParentChild,
parent_chunk_size: props.config.parentChunkSize,
child_chunk_size: props.config.childChunkSize,
strategy: props.config.strategy ?? '',
token_limit: props.config.tokenLimit ?? 0,
languages: props.config.languages ?? []

View File

@ -328,6 +328,9 @@ const debugConfig = computed(() => ({
chunkSize: localChunkSize.value,
chunkOverlap: localChunkOverlap.value,
separators: localSeparators.value,
enableParentChild: localEnableParentChild.value,
parentChunkSize: localParentChunkSize.value,
childChunkSize: localChildChunkSize.value,
strategy: localStrategy.value,
tokenLimit: localTokenLimit.value,
languages: localLanguages.value

View File

@ -247,27 +247,7 @@ func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterCo
// splitter, so parent-child chunks would silently lose heading alignment and
// ContextHeader breadcrumbs regardless of the configured strategy.
func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) {
parentSize := cc.ParentChunkSize
if parentSize <= 0 {
parentSize = 4096
}
childSize := cc.ChildChunkSize
if childSize <= 0 {
childSize = 384
}
parent = chunker.SplitterConfig{
ChunkSize: parentSize,
ChunkOverlap: base.ChunkOverlap, // reuse configured overlap for parents
Separators: base.Separators,
Strategy: base.Strategy,
}
child = chunker.SplitterConfig{
ChunkSize: childSize,
ChunkOverlap: childSize / 5, // ~20% overlap for child chunks
Separators: base.Separators,
Strategy: base.Strategy,
}
return
return chunker.DeriveParentChildConfigs(base, cc.ParentChunkSize, cc.ChildChunkSize)
}
// processChunks processes chunks and creates embeddings for knowledge content

View File

@ -51,15 +51,18 @@ type PreviewChunkingRequest struct {
// PreviewChunkingPayload mirrors the snake_case JSON the rest of the API
// uses for ChunkingConfig fields. We don't reuse types.ChunkingConfig
// directly because it carries a lot of unrelated fields (parser engine
// rules, parent-child sizes, etc.) that the preview path doesn't need.
// directly because it carries unrelated parser rules and table-processing
// metadata that the preview path does not use.
type PreviewChunkingPayload struct {
ChunkSize int `json:"chunk_size"`
ChunkOverlap int `json:"chunk_overlap"`
Separators []string `json:"separators"`
Strategy string `json:"strategy"`
TokenLimit int `json:"token_limit"`
Languages []string `json:"languages"`
ChunkSize int `json:"chunk_size"`
ChunkOverlap int `json:"chunk_overlap"`
Separators []string `json:"separators"`
EnableParentChild bool `json:"enable_parent_child"`
ParentChunkSize int `json:"parent_chunk_size"`
ChildChunkSize int `json:"child_chunk_size"`
Strategy string `json:"strategy"`
TokenLimit int `json:"token_limit"`
Languages []string `json:"languages"`
}
// PreviewChunkResult describes one chunk emitted during preview.
@ -162,7 +165,23 @@ func PreviewChunking(c *gin.Context) {
}
resCh := make(chan splitResult, 1)
go func() {
chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg)
var chunks []chunker.Chunk
var diag *chunker.Diagnostics
if req.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := chunker.DeriveParentChildConfigs(
cfg,
req.ChunkingConfig.ParentChunkSize,
req.ChunkingConfig.ChildChunkSize,
)
result, parentDiag := chunker.SplitParentChildWithDiagnostics(req.Text, parentCfg, childCfg)
chunks = make([]chunker.Chunk, len(result.Children))
for i, child := range result.Children {
chunks[i] = child.Chunk
}
diag = parentDiag
} else {
chunks, diag = chunker.SplitWithDiagnostics(req.Text, cfg)
}
resCh <- splitResult{chunks: chunks, diag: diag}
}()

View File

@ -197,3 +197,44 @@ func TestPreviewChunking_ChunkTruncation(t *testing.T) {
t.Errorf("stats.truncated_to should reflect ORIGINAL count > %d, got %v", previewMaxChunks, truncated)
}
}
func TestPreviewChunking_ParentChildMatchesIngestion(t *testing.T) {
text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12)
payload := PreviewChunkingPayload{
ChunkSize: 300,
ChunkOverlap: 30,
Separators: []string{"\n\n", "\n"},
EnableParentChild: true,
ParentChunkSize: 300,
ChildChunkSize: 100,
Strategy: chunker.StrategyHeading,
}
w, parsed := postPreview(t, PreviewChunkingRequest{Text: text, ChunkingConfig: payload})
if w.Code != http.StatusOK {
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
}
base := chunker.SplitterConfig{
ChunkSize: payload.ChunkSize,
ChunkOverlap: payload.ChunkOverlap,
Separators: payload.Separators,
Strategy: payload.Strategy,
}
parentCfg, childCfg := chunker.DeriveParentChildConfigs(base, payload.ParentChunkSize, payload.ChildChunkSize)
want := chunker.SplitParentChild(text, parentCfg, childCfg)
data := parsed["data"].(map[string]any)
got := data["chunks"].([]any)
if len(got) != len(want.Children) {
t.Fatalf("chunk count: got %d want %d", len(got), len(want.Children))
}
for i, child := range want.Children {
previewChunk := got[i].(map[string]any)
if previewChunk["content"] != child.Content {
t.Errorf("chunk %d content differs", i)
}
if int(previewChunk["start"].(float64)) != child.Start || int(previewChunk["end"].(float64)) != child.End {
t.Errorf("chunk %d span: got %v-%v want %d-%d", i, previewChunk["start"], previewChunk["end"], child.Start, child.End)
}
}
}

View File

@ -133,15 +133,33 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic
// is bounded by O(sum(parent_size)) ≈ O(N) total, which is the same
// order as the original parent profiling pass.
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
if text == "" {
return ParentChildResult{}
}
result, _ := splitParentChild(text, parentCfg, childCfg, false)
return result
}
// SplitParentChildWithDiagnostics is the diagnostics-aware form of
// SplitParentChild. The result is identical to SplitParentChild; diagnostics
// describe the strategy selected for the parent split over the full document.
// It is intended for debug surfaces rather than the ingestion hot path.
func SplitParentChildWithDiagnostics(text string, parentCfg, childCfg SplitterConfig) (ParentChildResult, *Diagnostics) {
return splitParentChild(text, parentCfg, childCfg, true)
}
func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagnostics bool) (ParentChildResult, *Diagnostics) {
parentCfg = ensureDefaults(parentCfg)
childCfg = ensureDefaults(childCfg)
parents := Split(text, parentCfg)
var (
parents []Chunk
diag *Diagnostics
)
if withDiagnostics {
parents, diag = SplitWithDiagnostics(text, parentCfg)
} else {
parents = Split(text, parentCfg)
}
if len(parents) == 0 {
return ParentChildResult{}
return ParentChildResult{}, diag
}
var newParents []Chunk
@ -164,7 +182,32 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
childSeq++
}
}
return ParentChildResult{Parents: newParents, Children: children}
return ParentChildResult{Parents: newParents, Children: children}, diag
}
// DeriveParentChildConfigs produces the exact parent and child splitter
// configurations used by knowledge ingestion. Keeping this here lets preview
// and ingestion remain in lockstep as the parent-child defaults evolve.
func DeriveParentChildConfigs(base SplitterConfig, parentSize, childSize int) (parent, child SplitterConfig) {
if parentSize <= 0 {
parentSize = 4096
}
if childSize <= 0 {
childSize = 384
}
parent = SplitterConfig{
ChunkSize: parentSize,
ChunkOverlap: base.ChunkOverlap,
Separators: base.Separators,
Strategy: base.Strategy,
}
child = SplitterConfig{
ChunkSize: childSize,
ChunkOverlap: childSize / 5,
Separators: base.Separators,
Strategy: base.Strategy,
}
return
}
// mergeBreadcrumbs combines the parent and child heading breadcrumbs into a

View File

@ -96,3 +96,25 @@ func TestSplitWithDiagnostics_ProfileNilForExplicit(t *testing.T) {
})
}
}
func TestSplitParentChildWithDiagnostics_MatchesSplitParentChild(t *testing.T) {
text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12)
parentCfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{"\n\n", "\n"}, Strategy: StrategyHeading}
childCfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyHeading}
want := SplitParentChild(text, parentCfg, childCfg)
got, diag := SplitParentChildWithDiagnostics(text, parentCfg, childCfg)
if len(got.Parents) != len(want.Parents) || len(got.Children) != len(want.Children) {
t.Fatalf("parent-child result differs: got %d parents/%d children, want %d parents/%d children",
len(got.Parents), len(got.Children), len(want.Parents), len(want.Children))
}
for i := range want.Children {
if got.Children[i] != want.Children[i] {
t.Errorf("child %d differs:\n got: %+v\n want: %+v", i, got.Children[i], want.Children[i])
}
}
if diag == nil || diag.SelectedTier == "" {
t.Fatal("expected diagnostics for parent split")
}
}