diff --git a/frontend/src/types/chunker.ts b/frontend/src/types/chunker.ts index 7065fa51..7cf53659 100644 --- a/frontend/src/types/chunker.ts +++ b/frontend/src/types/chunker.ts @@ -65,6 +65,9 @@ export interface PreviewChunkingRequest { chunk_size: number chunk_overlap: number separators: string[] + enable_parent_child?: boolean + parent_chunk_size?: number + child_chunk_size?: number strategy?: string token_limit?: number languages?: string[] diff --git a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue index 13909ae8..bb869a3e 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue @@ -224,6 +224,9 @@ interface Props { chunkSize: number chunkOverlap: number separators: string[] + enableParentChild: boolean + parentChunkSize: number + childChunkSize: number strategy?: string tokenLimit?: number languages?: string[] @@ -285,6 +288,9 @@ const runPreview = async () => { chunk_size: props.config.chunkSize, chunk_overlap: props.config.chunkOverlap, separators: props.config.separators, + enable_parent_child: props.config.enableParentChild, + parent_chunk_size: props.config.parentChunkSize, + child_chunk_size: props.config.childChunkSize, strategy: props.config.strategy ?? '', token_limit: props.config.tokenLimit ?? 0, languages: props.config.languages ?? [] diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index df321e56..3ec5a9ee 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -328,6 +328,9 @@ const debugConfig = computed(() => ({ chunkSize: localChunkSize.value, chunkOverlap: localChunkOverlap.value, separators: localSeparators.value, + enableParentChild: localEnableParentChild.value, + parentChunkSize: localParentChunkSize.value, + childChunkSize: localChildChunkSize.value, strategy: localStrategy.value, tokenLimit: localTokenLimit.value, languages: localLanguages.value diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 2a1cfd16..3574dc6d 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -247,27 +247,7 @@ func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterCo // splitter, so parent-child chunks would silently lose heading alignment and // ContextHeader breadcrumbs regardless of the configured strategy. func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfig) (parent, child chunker.SplitterConfig) { - parentSize := cc.ParentChunkSize - if parentSize <= 0 { - parentSize = 4096 - } - childSize := cc.ChildChunkSize - if childSize <= 0 { - childSize = 384 - } - parent = chunker.SplitterConfig{ - ChunkSize: parentSize, - ChunkOverlap: base.ChunkOverlap, // reuse configured overlap for parents - Separators: base.Separators, - Strategy: base.Strategy, - } - child = chunker.SplitterConfig{ - ChunkSize: childSize, - ChunkOverlap: childSize / 5, // ~20% overlap for child chunks - Separators: base.Separators, - Strategy: base.Strategy, - } - return + return chunker.DeriveParentChildConfigs(base, cc.ParentChunkSize, cc.ChildChunkSize) } // processChunks processes chunks and creates embeddings for knowledge content diff --git a/internal/handler/chunker_debug.go b/internal/handler/chunker_debug.go index b7f7e818..1be0dee5 100644 --- a/internal/handler/chunker_debug.go +++ b/internal/handler/chunker_debug.go @@ -51,15 +51,18 @@ type PreviewChunkingRequest struct { // PreviewChunkingPayload mirrors the snake_case JSON the rest of the API // uses for ChunkingConfig fields. We don't reuse types.ChunkingConfig -// directly because it carries a lot of unrelated fields (parser engine -// rules, parent-child sizes, etc.) that the preview path doesn't need. +// directly because it carries unrelated parser rules and table-processing +// metadata that the preview path does not use. type PreviewChunkingPayload struct { - ChunkSize int `json:"chunk_size"` - ChunkOverlap int `json:"chunk_overlap"` - Separators []string `json:"separators"` - Strategy string `json:"strategy"` - TokenLimit int `json:"token_limit"` - Languages []string `json:"languages"` + ChunkSize int `json:"chunk_size"` + ChunkOverlap int `json:"chunk_overlap"` + Separators []string `json:"separators"` + EnableParentChild bool `json:"enable_parent_child"` + ParentChunkSize int `json:"parent_chunk_size"` + ChildChunkSize int `json:"child_chunk_size"` + Strategy string `json:"strategy"` + TokenLimit int `json:"token_limit"` + Languages []string `json:"languages"` } // PreviewChunkResult describes one chunk emitted during preview. @@ -162,7 +165,23 @@ func PreviewChunking(c *gin.Context) { } resCh := make(chan splitResult, 1) go func() { - chunks, diag := chunker.SplitWithDiagnostics(req.Text, cfg) + var chunks []chunker.Chunk + var diag *chunker.Diagnostics + if req.ChunkingConfig.EnableParentChild { + parentCfg, childCfg := chunker.DeriveParentChildConfigs( + cfg, + req.ChunkingConfig.ParentChunkSize, + req.ChunkingConfig.ChildChunkSize, + ) + result, parentDiag := chunker.SplitParentChildWithDiagnostics(req.Text, parentCfg, childCfg) + chunks = make([]chunker.Chunk, len(result.Children)) + for i, child := range result.Children { + chunks[i] = child.Chunk + } + diag = parentDiag + } else { + chunks, diag = chunker.SplitWithDiagnostics(req.Text, cfg) + } resCh <- splitResult{chunks: chunks, diag: diag} }() diff --git a/internal/handler/chunker_debug_test.go b/internal/handler/chunker_debug_test.go index 0210667a..5deb197c 100644 --- a/internal/handler/chunker_debug_test.go +++ b/internal/handler/chunker_debug_test.go @@ -197,3 +197,44 @@ func TestPreviewChunking_ChunkTruncation(t *testing.T) { t.Errorf("stats.truncated_to should reflect ORIGINAL count > %d, got %v", previewMaxChunks, truncated) } } + +func TestPreviewChunking_ParentChildMatchesIngestion(t *testing.T) { + text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12) + payload := PreviewChunkingPayload{ + ChunkSize: 300, + ChunkOverlap: 30, + Separators: []string{"\n\n", "\n"}, + EnableParentChild: true, + ParentChunkSize: 300, + ChildChunkSize: 100, + Strategy: chunker.StrategyHeading, + } + w, parsed := postPreview(t, PreviewChunkingRequest{Text: text, ChunkingConfig: payload}) + if w.Code != http.StatusOK { + t.Fatalf("status %d body=%s", w.Code, w.Body.String()) + } + + base := chunker.SplitterConfig{ + ChunkSize: payload.ChunkSize, + ChunkOverlap: payload.ChunkOverlap, + Separators: payload.Separators, + Strategy: payload.Strategy, + } + parentCfg, childCfg := chunker.DeriveParentChildConfigs(base, payload.ParentChunkSize, payload.ChildChunkSize) + want := chunker.SplitParentChild(text, parentCfg, childCfg) + + data := parsed["data"].(map[string]any) + got := data["chunks"].([]any) + if len(got) != len(want.Children) { + t.Fatalf("chunk count: got %d want %d", len(got), len(want.Children)) + } + for i, child := range want.Children { + previewChunk := got[i].(map[string]any) + if previewChunk["content"] != child.Content { + t.Errorf("chunk %d content differs", i) + } + if int(previewChunk["start"].(float64)) != child.Start || int(previewChunk["end"].(float64)) != child.End { + t.Errorf("chunk %d span: got %v-%v want %d-%d", i, previewChunk["start"], previewChunk["end"], child.Start, child.End) + } + } +} diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index 014f99c0..a3dba815 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -133,15 +133,33 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic // is bounded by O(sum(parent_size)) ≈ O(N) total, which is the same // order as the original parent profiling pass. func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult { - if text == "" { - return ParentChildResult{} - } + result, _ := splitParentChild(text, parentCfg, childCfg, false) + return result +} + +// SplitParentChildWithDiagnostics is the diagnostics-aware form of +// SplitParentChild. The result is identical to SplitParentChild; diagnostics +// describe the strategy selected for the parent split over the full document. +// It is intended for debug surfaces rather than the ingestion hot path. +func SplitParentChildWithDiagnostics(text string, parentCfg, childCfg SplitterConfig) (ParentChildResult, *Diagnostics) { + return splitParentChild(text, parentCfg, childCfg, true) +} + +func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagnostics bool) (ParentChildResult, *Diagnostics) { parentCfg = ensureDefaults(parentCfg) childCfg = ensureDefaults(childCfg) - parents := Split(text, parentCfg) + var ( + parents []Chunk + diag *Diagnostics + ) + if withDiagnostics { + parents, diag = SplitWithDiagnostics(text, parentCfg) + } else { + parents = Split(text, parentCfg) + } if len(parents) == 0 { - return ParentChildResult{} + return ParentChildResult{}, diag } var newParents []Chunk @@ -164,7 +182,32 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi childSeq++ } } - return ParentChildResult{Parents: newParents, Children: children} + return ParentChildResult{Parents: newParents, Children: children}, diag +} + +// DeriveParentChildConfigs produces the exact parent and child splitter +// configurations used by knowledge ingestion. Keeping this here lets preview +// and ingestion remain in lockstep as the parent-child defaults evolve. +func DeriveParentChildConfigs(base SplitterConfig, parentSize, childSize int) (parent, child SplitterConfig) { + if parentSize <= 0 { + parentSize = 4096 + } + if childSize <= 0 { + childSize = 384 + } + parent = SplitterConfig{ + ChunkSize: parentSize, + ChunkOverlap: base.ChunkOverlap, + Separators: base.Separators, + Strategy: base.Strategy, + } + child = SplitterConfig{ + ChunkSize: childSize, + ChunkOverlap: childSize / 5, + Separators: base.Separators, + Strategy: base.Strategy, + } + return } // mergeBreadcrumbs combines the parent and child heading breadcrumbs into a diff --git a/internal/infrastructure/chunker/strategy_diagnostics_test.go b/internal/infrastructure/chunker/strategy_diagnostics_test.go index 82bc8aa0..45a53483 100644 --- a/internal/infrastructure/chunker/strategy_diagnostics_test.go +++ b/internal/infrastructure/chunker/strategy_diagnostics_test.go @@ -96,3 +96,25 @@ func TestSplitWithDiagnostics_ProfileNilForExplicit(t *testing.T) { }) } } + +func TestSplitParentChildWithDiagnostics_MatchesSplitParentChild(t *testing.T) { + text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12) + parentCfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{"\n\n", "\n"}, Strategy: StrategyHeading} + childCfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Separators: []string{"\n\n", "\n"}, Strategy: StrategyHeading} + + want := SplitParentChild(text, parentCfg, childCfg) + got, diag := SplitParentChildWithDiagnostics(text, parentCfg, childCfg) + + if len(got.Parents) != len(want.Parents) || len(got.Children) != len(want.Children) { + t.Fatalf("parent-child result differs: got %d parents/%d children, want %d parents/%d children", + len(got.Parents), len(got.Children), len(want.Parents), len(want.Children)) + } + for i := range want.Children { + if got.Children[i] != want.Children[i] { + t.Errorf("child %d differs:\n got: %+v\n want: %+v", i, got.Children[i], want.Children[i]) + } + } + if diag == nil || diag.SelectedTier == "" { + t.Fatal("expected diagnostics for parent split") + } +}