diff --git a/docs/code-slimming-audit.md b/docs/code-slimming-audit.md index 1f37a8e1..93aec044 100644 --- a/docs/code-slimming-audit.md +++ b/docs/code-slimming-audit.md @@ -86,6 +86,17 @@ - 新增 15 项真实 composable 行为测试,覆盖两个入口、接口路由、缓存、引用抽屉、请求竞争、计时和 Vue 挂载/卸载生命周期。 - `scripts/verify_frontend_pr.sh` 通过:948 项测试通过,1 项原有浏览器场景测试跳过,类型检查和生产构建通过。嵌入消息 template/style 与本批基线逐字节一致;未做浏览器视觉验收,未修改或测试后端。 +## 第五批已实施:Go 未引用私有函数 + +基于第四批合入后的 `d645334d0`,逐包核实原有 16 个候选,全部确认无调用;同时删除仅被候选使用的 `collectTableAliases` 与 `escapeDoubleQuotes`。共移除 18 个私有函数和 3 个失效 import,生产 Go 源码净减少 235 行。 + +- 使用 Go AST 检查候选所属 12 个包目录中的 894 个 Go 文件,包括测试、带构建标签和平台后缀的源码;删除集合以外没有对应标识符引用。全仓搜索同时检查文档、脚本和特殊链接引用。 +- 对修改文件逐函数比较:453 个保留函数的签名与函数体完全一致。Qdrant 的 `tokenizeQuery`、Agent 工具包的 `formatFileSize`、JSON 解析包的 `formatValue` 等同名活跃实现继续保留。 +- 更新分块配置相关注释和开发文档,指向仍在使用的 `buildSplitterConfigFromChunking` / `NormalizeSplitterConfig`;迁移历史注释保留旧名称。 +- 13 个受影响包执行 `go test -count=1`:12 个包通过,微信适配包无测试文件但编译通过;3,529 项顶层测试通过、1 项原有测试跳过,计入子测试为 5,274 项通过。跳过项为需要可替换连接探测或真实后端的向量存储创建测试。 +- `integration,e2b_integration,docker_integration` 标签下的 sandbox/tools 测试代码编译通过,未执行需要外部服务的集成测试。未修改前端,未运行前端或浏览器检查。 +- 本批只删除无调用代码,复用现有包测试和编译检查,没有添加仅断言函数不存在的测试。 + ## 后续优先级 | 优先级 | 证据 | 建议边界 | 验证重点 | @@ -100,24 +111,24 @@ ### 后端重复的具体判断 -- Qdrant 与 Weaviate 的 `tokenizeQuery` 函数体相同,但只有 Qdrant 的关键字检索调用它;Weaviate 使用 `WithQuery(params.Query)`。后续应验证并删除 Weaviate 的失效函数,而非为了它增加共享分词层。 +- Qdrant 与 Weaviate 的 `tokenizeQuery` 函数体相同,但只有 Qdrant 的关键字检索调用它;Weaviate 使用 `WithQuery(params.Query)`。第五批已删除 Weaviate 的失效函数,保留 Qdrant 实现。 - IMA、语雀的 `sanitizeFileName` 函数体相同,可以共享;RSS 的近似实现另外处理首尾空白和换行,合并时必须保留这些差异。 - Milvus/Qdrant/Weaviate 的结果映射体相似,但接收不同后端类型。当前各约 17 行,为此引入通用接口或泛型转换层未必减少维护成本。 - 终端和桌面 WebSocket ticket handler 重复检查会话、用户、租户和 token,但签发方式不同。后续可共享前置身份校验,保留 ticket 和连接生命周期各自的契约。 - 资源权限已经有 `internal/application/access`;继续沿已有授权边界收敛,避免新建与其并行的权限工具层。 -### 未引用 Go 私有自由函数候选 +### 第五批已核实并删除的 Go 私有自由函数 -扫描发现 16 个,第一批暂未改动后端。后续逐包确认调用、构建入口和测试后处理: +初次扫描的 16 个候选和 2 个仅被候选调用的辅助函数已清理: | 文件(相对 `internal`) | 函数 | | --- | --- | | `application/repository/retriever/weaviate/repository.go` | `tokenizeQuery` | | `im/wechat/crypto.go` | `encryptAES128ECB` | -| `application/repository/retriever/milvus/filter.go` | `formatValue` | +| `application/repository/retriever/milvus/filter.go` | `formatValue`、`escapeDoubleQuotes` | | `agent/tools/todo_write.go` | `getStringArrayField`、`getStringField` | | `agent/prompts.go` | `formatFileSize` | -| `utils/inject.go` | `extractTableAliasMap` | +| `utils/inject.go` | `extractTableAliasMap`、`collectTableAliases` | | `application/service/wiki_ingest_dedup.go` | `countEntityConceptPages` | | `agent/tools/sandbox_ls.go` | `relativeSkillFileFromImagePath` | | `infrastructure/docparser/json_converter.go` | `indentJSON` | @@ -140,4 +151,4 @@ - 构建仍报告大 chunk 提示;本轮未以 bundle 体积为目标,也没有测量构建前后的体积差异。 - 未修改后端,未运行后端测试;未做浏览器视觉验收。 -下一批优先逐包核实并清理未引用 Go 私有函数候选,再处理后端具体重复逻辑。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。 +下一批优先处理 IMA、语雀等外部来源的重复文件名清理逻辑,核实调用与差异后再决定共享边界。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。 diff --git a/internal/agent/prompts.go b/internal/agent/prompts.go index 4eabc623..34ee9f89 100644 --- a/internal/agent/prompts.go +++ b/internal/agent/prompts.go @@ -11,24 +11,6 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) -// formatFileSize formats file size in human-readable format -func formatFileSize(size int64) string { - const ( - KB = 1024 - MB = 1024 * KB - GB = 1024 * MB - ) - - if size < KB { - return fmt.Sprintf("%d B", size) - } else if size < MB { - return fmt.Sprintf("%.2f KB", float64(size)/KB) - } else if size < GB { - return fmt.Sprintf("%.2f MB", float64(size)/MB) - } - return fmt.Sprintf("%.2f GB", float64(size)/GB) -} - // formatDocSummary cleans and truncates document summaries for table display func formatDocSummary(summary string, maxLen int) string { cleaned := strings.TrimSpace(summary) diff --git a/internal/agent/tools/sandbox_ls.go b/internal/agent/tools/sandbox_ls.go index 1d9f0589..6c9c542c 100644 --- a/internal/agent/tools/sandbox_ls.go +++ b/internal/agent/tools/sandbox_ls.go @@ -281,18 +281,6 @@ func inspectablePathError(requested string) string { return base + ". Use read_file with a listed skill:// resource for skill packages." } -func relativeSkillFileFromImagePath(clean, skillName string) string { - dir, err := sandbox.SkillDirFor(skillName) - if err != nil || clean == dir { - return "" - } - prefix := dir + "/" - if strings.HasPrefix(clean, prefix) { - return strings.TrimPrefix(clean, prefix) - } - return "" -} - // matchingInspectableRoot returns the allowlisted root that contains // clean, or ("", false) when the path sits outside every root. func matchingInspectableRoot(clean string) (string, bool) { diff --git a/internal/agent/tools/todo_write.go b/internal/agent/tools/todo_write.go index 570cd402..0e12d6fd 100644 --- a/internal/agent/tools/todo_write.go +++ b/internal/agent/tools/todo_write.go @@ -199,32 +199,6 @@ func (t *TodoWriteTool) Execute(ctx context.Context, args json.RawMessage) (*typ }, nil } -// Helper function to safely get string field from map -func getStringField(m map[string]interface{}, key string) string { - if val, ok := m[key].(string); ok { - return val - } - return "" -} - -// Helper function to safely get string array field from map -func getStringArrayField(m map[string]interface{}, key string) []string { - if val, ok := m[key].([]interface{}); ok { - result := make([]string, 0, len(val)) - for _, item := range val { - if str, ok := item.(string); ok { - result = append(result, str) - } - } - return result - } - // Handle legacy string format for backward compatibility - if val, ok := m[key].(string); ok && val != "" { - return []string{val} - } - return []string{} -} - // generatePlanOutput generates a formatted plan output func generatePlanOutput(task string, steps []PlanStep) string { output := "Plan created\n\n" diff --git a/internal/application/repository/retriever/milvus/filter.go b/internal/application/repository/retriever/milvus/filter.go index 5d270c1c..e11bf802 100644 --- a/internal/application/repository/retriever/milvus/filter.go +++ b/internal/application/repository/retriever/milvus/filter.go @@ -6,7 +6,6 @@ import ( "maps" "reflect" "strings" - "time" ) const ( @@ -204,33 +203,6 @@ func (c *filter) convertBetweenCondition( }, nil } -func formatValue(value any) string { - switch v := value.(type) { - case string: - return fmt.Sprintf("\"%s\"", escapeDoubleQuotes(v)) - case int, int8, int16, int32, int64: - return fmt.Sprintf("%d", v) - case uint, uint8, uint16, uint32, uint64: - return fmt.Sprintf("%d", v) - case float32, float64: - return fmt.Sprintf("%v", v) - case bool: - if v { - return "true" - } - return "false" - case time.Time: - return fmt.Sprintf("%d", v.Unix()) - default: - return fmt.Sprintf("\"%v\"", value) - } -} - -// escapeDoubleQuotes escapes double quotes in a string for use in Milvus expressions. -func escapeDoubleQuotes(s string) string { - return strings.ReplaceAll(s, "\"", "\\\"") -} - // convertParamName converts field name to a valid Milvus template parameter name. // Milvus template parameters don't support '.' character, so we replace it with '_'. func (c *filter) convertParamName(field string, counter *int) string { diff --git a/internal/application/repository/retriever/weaviate/repository.go b/internal/application/repository/retriever/weaviate/repository.go index 9f1e23d0..f3dcc162 100644 --- a/internal/application/repository/retriever/weaviate/repository.go +++ b/internal/application/repository/retriever/weaviate/repository.go @@ -6,7 +6,6 @@ import ( "maps" "slices" "strings" - "unicode/utf8" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" @@ -1035,30 +1034,3 @@ func fromWeaviateVectorEmbedding(id string, MatchType: matchType, } } - -// tokenizeQuery splits a query string into tokens for OR-based full-text search. -// It uses jieba for professional Chinese word segmentation. -func tokenizeQuery(query string) []string { - query = strings.TrimSpace(query) - if query == "" { - return nil - } - - // Use jieba for segmentation (search mode for better recall) - words := types.Jieba.CutForSearch(query, true) - - // Filter and deduplicate - seen := make(map[string]bool) - result := make([]string, 0, len(words)) - for _, word := range words { - word = strings.TrimSpace(strings.ToLower(word)) - // Skip empty, single-char, and already seen words - if utf8.RuneCountInString(word) < 2 || seen[word] { - continue - } - seen[word] = true - result = append(result, word) - } - - return result -} diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 7c028f00..00cc845c 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -250,14 +250,8 @@ func markKnowledgeProcessing(knowledge *types.Knowledge, now time.Time) { knowledge.UpdatedAt = now } -// buildSplitterConfig creates a SplitterConfig with fallbacks from a KnowledgeBase. -// Defaults mirror chunker.DefaultChunkSize / DefaultChunkOverlap so behavior is -// identical whether callers come through this path or invoke the chunker -// directly with a zero-value config. -func buildSplitterConfig(kb *types.KnowledgeBase) chunker.SplitterConfig { - return buildSplitterConfigFromChunking(kb.ChunkingConfig) -} - +// buildSplitterConfigFromChunking normalizes effective chunking settings with +// the shared chunker defaults. func buildSplitterConfigFromChunking(cc types.ChunkingConfig) chunker.SplitterConfig { return chunker.NormalizeSplitterConfig(chunker.SplitterConfig{ ChunkSize: cc.ChunkSize, diff --git a/internal/application/service/tenant_skill_source.go b/internal/application/service/tenant_skill_source.go index f159f8e6..f1b998d9 100644 --- a/internal/application/service/tenant_skill_source.go +++ b/internal/application/service/tenant_skill_source.go @@ -853,11 +853,6 @@ func getSkillURL( return body, resp.Header.Get("Content-Type"), nil } -func normalizeFetchedSkillArchive(body []byte, contentType, subdir string) ([]byte, error) { - _, archive, err := normalizeFetchedSkill(body, contentType, subdir) - return archive, err -} - func normalizeFetchedSkill(body []byte, contentType, subdir string) (*SkillBundle, []byte, error) { if looksLikeSkillMarkdown(body) { files := map[string][]byte{"SKILL.md": body} diff --git a/internal/application/service/wiki_ingest_dedup.go b/internal/application/service/wiki_ingest_dedup.go index 899f2570..73063ef5 100644 --- a/internal/application/service/wiki_ingest_dedup.go +++ b/internal/application/service/wiki_ingest_dedup.go @@ -67,22 +67,6 @@ type dedupSurface struct { nameGramSets []map[string]struct{} } -// countEntityConceptPages returns how many of the given pages are -// entity- or concept-typed. Used only for logging the prefilter's -// reduction ratio. -func countEntityConceptPages(pages []*types.WikiPage) int { - n := 0 - for _, p := range pages { - if p == nil { - continue - } - if p.PageType == types.WikiPageTypeEntity || p.PageType == types.WikiPageTypeConcept { - n++ - } - } - return n -} - // selectDedupCandidatePages returns the subset of allPages plausibly // related to at least one of newItems. Non-entity/concept pages are // dropped unconditionally. The returned slice preserves the input order diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 05820223..672215b1 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -1512,12 +1512,6 @@ func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) { } } -// mimeTypeByExt returns the MIME type for a given file extension. -func mimeTypeByExt(filename string) string { - ct, _ := secutils.SafeContentTypeByFilename(filename) - return ct -} - // PreviewKnowledgeFile godoc // @Summary 预览知识文件 // @Description 返回知识条目关联的原始文件,Content-Type 根据文件类型设置,用于浏览器内嵌预览 diff --git a/internal/handler/session/helpers.go b/internal/handler/session/helpers.go index 4cb04a7b..c33b29db 100644 --- a/internal/handler/session/helpers.go +++ b/internal/handler/session/helpers.go @@ -440,11 +440,6 @@ func (h *Handler) writeAgentQueryEvent( } } -// getRequestID gets the request ID from gin context -func getRequestID(c *gin.Context) string { - return c.GetString(types.RequestIDContextKey.String()) -} - // Helper function for type assertion with default value func getString(m map[string]interface{}, key string) string { if val, ok := m[key].(string); ok { diff --git a/internal/im/service.go b/internal/im/service.go index 3e68d547..777696f8 100644 --- a/internal/im/service.go +++ b/internal/im/service.go @@ -193,10 +193,6 @@ func cleanIMContent(ctx context.Context, content string, tenant *types.Tenant, d return content } -func imLocalStorageBaseDir() string { - return storageurl.LocalStorageBaseDir() -} - // newIMFileServiceResolver builds a per-message storage backend resolver. The // cache lives for one cleanIMContent / outbound message so a long answer does // not re-create an SDK client for every reference. diff --git a/internal/im/wechat/crypto.go b/internal/im/wechat/crypto.go index 839ec7a2..13d319ca 100644 --- a/internal/im/wechat/crypto.go +++ b/internal/im/wechat/crypto.go @@ -44,29 +44,3 @@ func decryptAES128ECB(ciphertext, key []byte) ([]byte, error) { return plaintext, nil } - -// encryptAES128ECB encrypts data with AES-128-ECB and PKCS#7 padding. -// Used for uploading media files to the iLink CDN. -func encryptAES128ECB(plaintext, key []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, fmt.Errorf("new aes cipher: %w", err) - } - - bs := block.BlockSize() - - // PKCS#7 padding - padLen := bs - len(plaintext)%bs - padded := make([]byte, len(plaintext)+padLen) - copy(padded, plaintext) - for i := len(plaintext); i < len(padded); i++ { - padded[i] = byte(padLen) - } - - ciphertext := make([]byte, len(padded)) - for i := 0; i < len(padded); i += bs { - block.Encrypt(ciphertext[i:i+bs], padded[i:i+bs]) - } - - return ciphertext, nil -} diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index bf170f46..ff415eed 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -74,10 +74,9 @@ type SplitterConfig struct { } // Default chunk sizing constants. Single source of truth for the entire -// chunker package and (via knowledge.go::buildSplitterConfig) the -// knowledge service. The frontend KnowledgeBaseEditorModal mirrors these -// numbers in its initial form state — keep them in sync if you change -// either value here. +// chunker package and the knowledge service via NormalizeSplitterConfig. +// The frontend KnowledgeBaseEditorModal mirrors these numbers in its initial +// form state — keep them in sync if you change either value here. // // DefaultChunkSize = 512 chars: ~100–130 English tokens / ~300 Chinese // tokens. Validated as a strong baseline by the Vecta Feb-2026 benchmark diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index 053bf395..4766462a 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -314,8 +314,8 @@ func runTier(tier StrategyTier, text string, cfg SplitterConfig, profile *DocPro } // ensureDefaults fills in zero-value config fields with sane defaults. -// Mirrors buildSplitterConfig in internal/application/service/knowledge.go -// so direct callers of this package get the same numbers. +// NormalizeSplitterConfig also uses these defaults so service callers and +// direct callers of this package get the same numbers. // // When cfg.TokenLimit is set, ChunkSize is clamped to the character budget // that fits within that token limit (with a 10% safety factor). This makes diff --git a/internal/infrastructure/docparser/json_converter.go b/internal/infrastructure/docparser/json_converter.go index 473d732f..6b776e4e 100644 --- a/internal/infrastructure/docparser/json_converter.go +++ b/internal/infrastructure/docparser/json_converter.go @@ -1,7 +1,6 @@ package docparser import ( - "bytes" "encoding/json" "fmt" "sort" @@ -205,15 +204,6 @@ func formatValue(v interface{}) string { return string(b) } -// indentJSON formats raw JSON bytes with indentation. -func indentJSON(data []byte) (string, error) { - var buf bytes.Buffer - if err := json.Indent(&buf, data, "", " "); err != nil { - return string(data), err - } - return buf.String(), nil -} - // wrapCodeBlock wraps content in a fenced JSON code block. func wrapCodeBlock(content string) string { return "```json\n" + content + "\n```" diff --git a/internal/sandbox/gateway_transport.go b/internal/sandbox/gateway_transport.go index 1898d789..47a27902 100644 --- a/internal/sandbox/gateway_transport.go +++ b/internal/sandbox/gateway_transport.go @@ -154,13 +154,8 @@ func (p *SandboxGatewayTransportPool) dataTransport(target string) http.RoundTri return actual.(http.RoundTripper) } -// newGatewayDataTransport dials target regardless of the request's authority, -// mirroring the SDK's proxy rewrite while adding the outbound address guard -// the SDKs have no notion of. -func newGatewayDataTransport(target string) *http.Transport { - return newGatewayDataTransportWithPolicy(target, DefaultOutboundURLPolicy()) -} - +// newGatewayDataTransportWithPolicy dials target regardless of the request's +// authority, preserving the gateway rewrite and the configured outbound guard. func newGatewayDataTransportWithPolicy(target string, policy OutboundURLPolicy) *http.Transport { dialer := &net.Dialer{ Timeout: 10 * time.Second, diff --git a/internal/utils/inject.go b/internal/utils/inject.go index ddb50fd6..6e67b5fb 100644 --- a/internal/utils/inject.go +++ b/internal/utils/inject.go @@ -976,45 +976,6 @@ func ValidateAndSecureSQL(sql string, opts ...SQLValidationOption) (string, *SQL return securedSQL, validationResult, nil } -// extractTableAliasMap walks the parse tree to build a table_name→alias map. -// When a table has an alias (e.g., "knowledge_bases kb"), the map entry is -// {"knowledge_bases": "kb"}. Without an alias, both key and value are the table name. -func extractTableAliasMap(parseResult *pg_query.ParseResult) map[string]string { - m := make(map[string]string) - if len(parseResult.Stmts) == 0 || parseResult.Stmts[0].Stmt == nil { - return m - } - selectStmt := parseResult.Stmts[0].Stmt.GetSelectStmt() - if selectStmt == nil { - return m - } - for _, fromItem := range selectStmt.FromClause { - collectTableAliases(fromItem, m) - } - return m -} - -// collectTableAliases recursively collects table→alias mappings from FROM clause nodes. -func collectTableAliases(node *pg_query.Node, m map[string]string) { - if node == nil { - return - } - if rv := node.GetRangeVar(); rv != nil { - tableName := strings.ToLower(rv.Relname) - alias := tableName - if rv.Alias != nil && rv.Alias.Aliasname != "" { - alias = strings.ToLower(rv.Alias.Aliasname) - } - m[tableName] = alias - return - } - if je := node.GetJoinExpr(); je != nil { - collectTableAliases(je.Larg, m) - collectTableAliases(je.Rarg, m) - return - } -} - // walkSQLMessages visits children before their parent so rewriting a SELECT // does not revisit trusted predicates added by the server. func walkSQLMessages(message protoreflect.Message, visit func(protoreflect.Message) error) error { diff --git a/website-docs/06-development/03-extension-points.md b/website-docs/06-development/03-extension-points.md index d68c9153..a677baea 100644 --- a/website-docs/06-development/03-extension-points.md +++ b/website-docs/06-development/03-extension-points.md @@ -192,7 +192,7 @@ var splitByHeuristics = func(text string, cfg SplitterConfig, _ *DocProfile) []C - 增加策略常量(如 `StrategyMine = "mine"`)与新的 `StrategyTier`; - 在 `resolveChain`/`resolveChainWithProfile` 的 switch 中为新策略返回 tier 链(建议以 `TierLegacy` 兜底); - 在 `runTier()` 中新增 case; -3. 调用方无需改动:知识库的 `chunking_config.strategy`(JSONB)经 `internal/application/service/knowledge.go` 的 `buildSplitterConfig` 传入; +3. 调用方无需改动:知识库的 `chunking_config.strategy`(JSONB)经 `internal/application/service/knowledge_process.go` 的 `buildSplitterConfigFromChunking` 传入; 4. 用 `SplitWithDiagnostics` 写单测验证 tier 选择与 `ValidateChunks` 验收行为。 ---