fix(chunking): align preview with parent-child splitting / 修复预览与父子分块不一致
parent
b191201f44
commit
d2888e9565
|
|
@ -65,6 +65,9 @@ export interface PreviewChunkingRequest {
|
|||
chunk_size: number
|
||||
chunk_overlap: number
|
||||
separators: string[]
|
||||
enable_parent_child?: boolean
|
||||
parent_chunk_size?: number
|
||||
child_chunk_size?: number
|
||||
strategy?: string
|
||||
token_limit?: number
|
||||
languages?: string[]
|
||||
|
|
|
|||
|
|
@ -224,6 +224,9 @@ interface Props {
|
|||
chunkSize: number
|
||||
chunkOverlap: number
|
||||
separators: string[]
|
||||
enableParentChild: boolean
|
||||
parentChunkSize: number
|
||||
childChunkSize: number
|
||||
strategy?: string
|
||||
tokenLimit?: number
|
||||
languages?: string[]
|
||||
|
|
@ -285,6 +288,9 @@ const runPreview = async () => {
|
|||
chunk_size: props.config.chunkSize,
|
||||
chunk_overlap: props.config.chunkOverlap,
|
||||
separators: props.config.separators,
|
||||
enable_parent_child: props.config.enableParentChild,
|
||||
parent_chunk_size: props.config.parentChunkSize,
|
||||
child_chunk_size: props.config.childChunkSize,
|
||||
strategy: props.config.strategy ?? '',
|
||||
token_limit: props.config.tokenLimit ?? 0,
|
||||
languages: props.config.languages ?? []
|
||||
|
|
|
|||
|
|
@ -328,6 +328,9 @@ const debugConfig = computed(() => ({
|
|||
chunkSize: localChunkSize.value,
|
||||
chunkOverlap: localChunkOverlap.value,
|
||||
separators: localSeparators.value,
|
||||
enableParentChild: localEnableParentChild.value,
|
||||
parentChunkSize: localParentChunkSize.value,
|
||||
childChunkSize: localChildChunkSize.value,
|
||||
strategy: localStrategy.value,
|
||||
tokenLimit: localTokenLimit.value,
|
||||
languages: localLanguages.value
|
||||
|
|
|
|||
|
|
@ -247,27 +247,7 @@ func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterCo
|
|||
// splitter, so parent-child chunks would silently lose heading alignment and
|
||||
// ContextHeader breadcrumbs regardless of the configured strategy.
|
||||
func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) {
|
||||
parentSize := cc.ParentChunkSize
|
||||
if parentSize <= 0 {
|
||||
parentSize = 4096
|
||||
}
|
||||
childSize := cc.ChildChunkSize
|
||||
if childSize <= 0 {
|
||||
childSize = 384
|
||||
}
|
||||
parent = chunker.SplitterConfig{
|
||||
ChunkSize: parentSize,
|
||||
ChunkOverlap: base.ChunkOverlap, // reuse configured overlap for parents
|
||||
Separators: base.Separators,
|
||||
Strategy: base.Strategy,
|
||||
}
|
||||
child = chunker.SplitterConfig{
|
||||
ChunkSize: childSize,
|
||||
ChunkOverlap: childSize / 5, // ~20% overlap for child chunks
|
||||
Separators: base.Separators,
|
||||
Strategy: base.Strategy,
|
||||
}
|
||||
return
|
||||
return chunker.DeriveParentChildConfigs(base, cc.ParentChunkSize, cc.ChildChunkSize)
|
||||
}
|
||||
|
||||
// processChunks processes chunks and creates embeddings for knowledge content
|
||||
|
|
|
|||
|
|
@ -51,15 +51,18 @@ type PreviewChunkingRequest struct {
|
|||
|
||||
// PreviewChunkingPayload mirrors the snake_case JSON the rest of the API
|
||||
// uses for ChunkingConfig fields. We don't reuse types.ChunkingConfig
|
||||
// directly because it carries a lot of unrelated fields (parser engine
|
||||
// rules, parent-child sizes, etc.) that the preview path doesn't need.
|
||||
// directly because it carries unrelated parser rules and table-processing
|
||||
// metadata that the preview path does not use.
|
||||
type PreviewChunkingPayload struct {
|
||||
ChunkSize int `json:"chunk_size"`
|
||||
ChunkOverlap int `json:"chunk_overlap"`
|
||||
Separators []string `json:"separators"`
|
||||
Strategy string `json:"strategy"`
|
||||
TokenLimit int `json:"token_limit"`
|
||||
Languages []string `json:"languages"`
|
||||
ChunkSize int `json:"chunk_size"`
|
||||
ChunkOverlap int `json:"chunk_overlap"`
|
||||
Separators []string `json:"separators"`
|
||||
EnableParentChild bool `json:"enable_parent_child"`
|
||||
ParentChunkSize int `json:"parent_chunk_size"`
|
||||
ChildChunkSize int `json:"child_chunk_size"`
|
||||
Strategy string `json:"strategy"`
|
||||
TokenLimit int `json:"token_limit"`
|
||||
Languages []string `json:"languages"`
|
||||
}
|
||||
|
||||
// PreviewChunkResult describes one chunk emitted during preview.
|
||||
|
|
@ -162,7 +165,23 @@ func PreviewChunking(c *gin.Context) {
|
|||
}
|
||||
resCh := make(chan splitResult, 1)
|
||||
go func() {
|
||||
chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg)
|
||||
var chunks []chunker.Chunk
|
||||
var diag *chunker.Diagnostics
|
||||
if req.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := chunker.DeriveParentChildConfigs(
|
||||
cfg,
|
||||
req.ChunkingConfig.ParentChunkSize,
|
||||
req.ChunkingConfig.ChildChunkSize,
|
||||
)
|
||||
result, parentDiag := chunker.SplitParentChildWithDiagnostics(req.Text, parentCfg, childCfg)
|
||||
chunks = make([]chunker.Chunk, len(result.Children))
|
||||
for i, child := range result.Children {
|
||||
chunks[i] = child.Chunk
|
||||
}
|
||||
diag = parentDiag
|
||||
} else {
|
||||
chunks, diag = chunker.SplitWithDiagnostics(req.Text, cfg)
|
||||
}
|
||||
resCh <- splitResult{chunks: chunks, diag: diag}
|
||||
}()
|
||||
|
||||
|
|
|
|||
|
|
@ -197,3 +197,44 @@ func TestPreviewChunking_ChunkTruncation(t *testing.T) {
|
|||
t.Errorf("stats.truncated_to should reflect ORIGINAL count > %d, got %v", previewMaxChunks, truncated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewChunking_ParentChildMatchesIngestion(t *testing.T) {
|
||||
text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12)
|
||||
payload := PreviewChunkingPayload{
|
||||
ChunkSize: 300,
|
||||
ChunkOverlap: 30,
|
||||
Separators: []string{"\n\n", "\n"},
|
||||
EnableParentChild: true,
|
||||
ParentChunkSize: 300,
|
||||
ChildChunkSize: 100,
|
||||
Strategy: chunker.StrategyHeading,
|
||||
}
|
||||
w, parsed := postPreview(t, PreviewChunkingRequest{Text: text, ChunkingConfig: payload})
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
base := chunker.SplitterConfig{
|
||||
ChunkSize: payload.ChunkSize,
|
||||
ChunkOverlap: payload.ChunkOverlap,
|
||||
Separators: payload.Separators,
|
||||
Strategy: payload.Strategy,
|
||||
}
|
||||
parentCfg, childCfg := chunker.DeriveParentChildConfigs(base, payload.ParentChunkSize, payload.ChildChunkSize)
|
||||
want := chunker.SplitParentChild(text, parentCfg, childCfg)
|
||||
|
||||
data := parsed["data"].(map[string]any)
|
||||
got := data["chunks"].([]any)
|
||||
if len(got) != len(want.Children) {
|
||||
t.Fatalf("chunk count: got %d want %d", len(got), len(want.Children))
|
||||
}
|
||||
for i, child := range want.Children {
|
||||
previewChunk := got[i].(map[string]any)
|
||||
if previewChunk["content"] != child.Content {
|
||||
t.Errorf("chunk %d content differs", i)
|
||||
}
|
||||
if int(previewChunk["start"].(float64)) != child.Start || int(previewChunk["end"].(float64)) != child.End {
|
||||
t.Errorf("chunk %d span: got %v-%v want %d-%d", i, previewChunk["start"], previewChunk["end"], child.Start, child.End)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -133,15 +133,33 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic
|
|||
// is bounded by O(sum(parent_size)) ≈ O(N) total, which is the same
|
||||
// order as the original parent profiling pass.
|
||||
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
||||
if text == "" {
|
||||
return ParentChildResult{}
|
||||
}
|
||||
result, _ := splitParentChild(text, parentCfg, childCfg, false)
|
||||
return result
|
||||
}
|
||||
|
||||
// SplitParentChildWithDiagnostics is the diagnostics-aware form of
|
||||
// SplitParentChild. The result is identical to SplitParentChild; diagnostics
|
||||
// describe the strategy selected for the parent split over the full document.
|
||||
// It is intended for debug surfaces rather than the ingestion hot path.
|
||||
func SplitParentChildWithDiagnostics(text string, parentCfg, childCfg SplitterConfig) (ParentChildResult, *Diagnostics) {
|
||||
return splitParentChild(text, parentCfg, childCfg, true)
|
||||
}
|
||||
|
||||
func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagnostics bool) (ParentChildResult, *Diagnostics) {
|
||||
parentCfg = ensureDefaults(parentCfg)
|
||||
childCfg = ensureDefaults(childCfg)
|
||||
|
||||
parents := Split(text, parentCfg)
|
||||
var (
|
||||
parents []Chunk
|
||||
diag *Diagnostics
|
||||
)
|
||||
if withDiagnostics {
|
||||
parents, diag = SplitWithDiagnostics(text, parentCfg)
|
||||
} else {
|
||||
parents = Split(text, parentCfg)
|
||||
}
|
||||
if len(parents) == 0 {
|
||||
return ParentChildResult{}
|
||||
return ParentChildResult{}, diag
|
||||
}
|
||||
|
||||
var newParents []Chunk
|
||||
|
|
@ -164,7 +182,32 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
|
|||
childSeq++
|
||||
}
|
||||
}
|
||||
return ParentChildResult{Parents: newParents, Children: children}
|
||||
return ParentChildResult{Parents: newParents, Children: children}, diag
|
||||
}
|
||||
|
||||
// DeriveParentChildConfigs produces the exact parent and child splitter
|
||||
// configurations used by knowledge ingestion. Keeping this here lets preview
|
||||
// and ingestion remain in lockstep as the parent-child defaults evolve.
|
||||
func DeriveParentChildConfigs(base SplitterConfig, parentSize, childSize int) (parent, child SplitterConfig) {
|
||||
if parentSize <= 0 {
|
||||
parentSize = 4096
|
||||
}
|
||||
if childSize <= 0 {
|
||||
childSize = 384
|
||||
}
|
||||
parent = SplitterConfig{
|
||||
ChunkSize: parentSize,
|
||||
ChunkOverlap: base.ChunkOverlap,
|
||||
Separators: base.Separators,
|
||||
Strategy: base.Strategy,
|
||||
}
|
||||
child = SplitterConfig{
|
||||
ChunkSize: childSize,
|
||||
ChunkOverlap: childSize / 5,
|
||||
Separators: base.Separators,
|
||||
Strategy: base.Strategy,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// mergeBreadcrumbs combines the parent and child heading breadcrumbs into a
|
||||
|
|
|
|||
|
|
@ -96,3 +96,25 @@ func TestSplitWithDiagnostics_ProfileNilForExplicit(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitParentChildWithDiagnostics_MatchesSplitParentChild(t *testing.T) {
|
||||
text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12)
|
||||
parentCfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{"\n\n", "\n"}, Strategy: StrategyHeading}
|
||||
childCfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyHeading}
|
||||
|
||||
want := SplitParentChild(text, parentCfg, childCfg)
|
||||
got, diag := SplitParentChildWithDiagnostics(text, parentCfg, childCfg)
|
||||
|
||||
if len(got.Parents) != len(want.Parents) || len(got.Children) != len(want.Children) {
|
||||
t.Fatalf("parent-child result differs: got %d parents/%d children, want %d parents/%d children",
|
||||
len(got.Parents), len(got.Children), len(want.Parents), len(want.Children))
|
||||
}
|
||||
for i := range want.Children {
|
||||
if got.Children[i] != want.Children[i] {
|
||||
t.Errorf("child %d differs:\n got: %+v\n want: %+v", i, got.Children[i], want.Children[i])
|
||||
}
|
||||
}
|
||||
if diag == nil || diag.SelectedTier == "" {
|
||||
t.Fatal("expected diagnostics for parent split")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue