From 8dd997f0f406c19ce4ec9acd622456d24eac2130 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 31 Jul 2026 13:44:04 +0800 Subject: [PATCH] refactor(knowledge): make the supported import extension set the single gate The xlsx URL-import fix left the accepted extension set reachable through two indirections and widened ApplyKnowledgeProcessOverrides to gate file types for callers that are not file imports (web URL fetch, connector sync, reparse). Collapse the set into one map that both isValidFileType and the extension check read, move the shared validation next to the other file-type predicates, and return ApplyKnowledgeProcessOverrides to its override-only scope so resolveFileImportProcessConfig is the only import gate. Also drop the unreachable branch in createKnowledgeFromFileURL: getFileType never yields an empty string, so the resolvedFileType fallback and its else arm could not run. Normalize the resolved type before it reaches the record and the task payload, and add an invariant test asserting direct upload and URL import accept the same extensions so #2447 cannot drift back. --- internal/application/service/extract.go | 8 +- .../application/service/knowledge_create.go | 44 +++++------ .../application/service/knowledge_process.go | 2 +- .../service/knowledge_process_config.go | 60 +++++---------- .../service/knowledge_process_config_test.go | 56 +++++++------- .../application/service/knowledge_util.go | 75 +++++++++++++++---- .../service/knowledge_util_filetype_test.go | 35 ++++++++- 7 files changed, 163 insertions(+), 117 deletions(-) diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go index f952ecbe..f9a9d59c 100644 --- a/internal/application/service/extract.go +++ b/internal/application/service/extract.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "os" - "slices" "strings" "time" @@ -157,7 +156,8 @@ func NewDataTableSummaryTask( return nil } -// enqueueDataTableSummaryIfNeeded enqueues table summary work for spreadsheet imports. +// enqueueDataTableSummaryIfNeeded enqueues table summary work for spreadsheet +// imports. fileName is a fallback for older records whose FileType is empty. func enqueueDataTableSummaryIfNeeded( ctx context.Context, client interfaces.TaskEnqueuer, @@ -165,11 +165,11 @@ func enqueueDataTableSummaryIfNeeded( knowledgeID string, fileName, fileType, summaryModelID, embeddingModelID string, ) { - ft := strings.ToLower(strings.TrimPrefix(fileType, ".")) + ft := normalizeFileExtension(fileType) if ft == "" && fileName != "" { ft = getFileType(fileName) } - if !slices.Contains([]string{"csv", "xlsx", "xls"}, ft) { + if !isDataTableFileType(ft) { return } if err := NewDataTableSummaryTask(ctx, client, tenantID, knowledgeID, summaryModelID, embeddingModelID); err != nil { diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index 1b23f0db..b7473c01 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -59,7 +59,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, return nil, err } - // Validate file type + // Early reject before the whole-file hash below. resolveFileImportProcessConfig + // gates the same extension set, but this path must keep returning + // ErrInvalidFileType rather than the shared gate's localized message. logger.Infof(ctx, "Checking file type: %s", fileName) if !isValidFileType(fileName) { logger.Error(ctx, "Invalid file type") @@ -123,7 +125,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, return nil, werrors.NewValidationError("文件名包含非法字符") } - eff, err := resolveKnowledgeFileImportConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel) + eff, err := resolveFileImportProcessConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel) if err != nil { return nil, err } @@ -263,7 +265,7 @@ func isFileURL(rawURL, fileName, fileType string) bool { u, err := url.Parse(rawURL) if err == nil { ext := strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), ".")) - if ext != "" && isAllowedFileURLExtension(ext) { + if ext != "" && isSupportedImportExtension(ext) { return true } } @@ -529,15 +531,13 @@ func (s *knowledgeService) createKnowledgeFromFileURL( fileName = safeFilename } - // Resolve fileType: user-provided > inferred from fileName - if fileType == "" && fileName != "" { + // Resolve fileType: user-provided > inferred from fileName (which already + // falls back to the URL path above). getFileType never returns empty, so an + // undeterminable type surfaces as "unknown" and is rejected below. + if fileType == "" { fileType = getFileType(fileName) } - - resolvedFileType := fileType - if resolvedFileType == "" { - resolvedFileType = getFileType(extractFileNameFromURL(fileURL)) - } + fileType = normalizeFileExtension(fileType) // Use title as display name if fileName is still empty displayName := fileName @@ -604,23 +604,13 @@ func (s *knowledgeService) createKnowledgeFromFileURL( knowledge.Title = displayName } - var eff types.EffectiveProcessConfig - if resolvedFileType != "" { - eff, err = resolveKnowledgeFileImportConfig(ctx, kb, resolvedFileType, processOverrides, enableMultimodel) - if err != nil { - return nil, err - } - if processOverrides != nil { - if err := knowledge.SetProcessOverrides(processOverrides); err != nil { - logger.Errorf(ctx, "Failed to set process overrides: %v", err) - return nil, err - } - } - } else { - eff, err = ApplyKnowledgeProcessOverrides( - ctx, kb, knowledge, processOverrides, nil, enableMultimodel, - ) - if err != nil { + eff, err := resolveFileImportProcessConfig(ctx, kb, fileType, processOverrides, enableMultimodel) + if err != nil { + return nil, err + } + if processOverrides != nil { + if err := knowledge.SetProcessOverrides(processOverrides); err != nil { + logger.Errorf(ctx, "Failed to set process overrides: %v", err) return nil, err } } diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index c26e02a1..6fc9f298 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3184,7 +3184,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return fmt.Errorf("failed to download file from URL: %w", err) } - if resolvedFileType != "" && !isAllowedFileURLExtension(resolvedFileType) { + if resolvedFileType != "" && !isSupportedImportExtension(resolvedFileType) { logger.Errorf(ctx, "Unsupported file type resolved from file URL: %s", resolvedFileType) knowledge.ParseStatus = "failed" knowledge.ErrorMessage = fmt.Sprintf("unsupported file type: %s", resolvedFileType) diff --git a/internal/application/service/knowledge_process_config.go b/internal/application/service/knowledge_process_config.go index 78fc08bb..c65dab33 100644 --- a/internal/application/service/knowledge_process_config.go +++ b/internal/application/service/knowledge_process_config.go @@ -2,7 +2,6 @@ package service import ( "context" - "fmt" "strconv" "strings" @@ -94,29 +93,16 @@ func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgePro return eff } -// validateImportFileType checks file extension constraints shared by direct upload and file-URL import. -func validateImportFileType(fileType string) error { - fileType = strings.ToLower(strings.TrimPrefix(fileType, ".")) - if fileType == "" || fileType == "unknown" { - return werrors.NewBadRequestError("无法确定文件类型") - } - if IsVideoType(fileType) { - return werrors.NewBadRequestError("暂不支持上传视频文件") - } - if !isAllowedFileURLExtension(fileType) { - return werrors.NewBadRequestError(fmt.Sprintf("不支持的文件类型: %s", fileType)) - } - return nil -} - -// validateDefaultFileImportRequirements enforces VLM/ASR prerequisites when no per-import overrides are provided. +// validateDefaultFileImportRequirements enforces the VLM/ASR prerequisites that +// ValidateProcessOverrides would otherwise cover, for imports that ship no +// per-import overrides and therefore fall back to the KB defaults. func validateDefaultFileImportRequirements( ctx context.Context, kb *types.KnowledgeBase, eff types.EffectiveProcessConfig, fileType string, ) error { - fileType = strings.ToLower(strings.TrimPrefix(fileType, ".")) + fileType = normalizeFileExtension(fileType) if IsImageType(fileType) && !eff.VLMConfig.IsEnabled() { logger.Error(ctx, "VLM model is not configured") return werrors.NewBadRequestError("上传图片文件需要设置VLM模型") @@ -128,8 +114,12 @@ func validateDefaultFileImportRequirements( return nil } -// resolveKnowledgeFileImportConfig validates import prerequisites and resolves effective processing config. -func resolveKnowledgeFileImportConfig( +// resolveFileImportProcessConfig is the single gate every file import passes +// through: it rejects unsupported extensions, enforces the VLM/ASR +// prerequisites for the resolved type, and returns the effective processing +// config for task enqueue. Persisting overrides onto the knowledge record stays +// with the caller, which owns the record's lifecycle. +func resolveFileImportProcessConfig( ctx context.Context, kb *types.KnowledgeBase, fileType string, @@ -207,32 +197,18 @@ func ApplyKnowledgeProcessOverrides( fileTypes []string, enableMultimodel *bool, ) (types.EffectiveProcessConfig, error) { - fileType := "" - if len(fileTypes) > 0 { - fileType = fileTypes[0] + eff := ResolveProcessConfig(kb, processOverrides) + if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) { + eff.EnableMultimodel = *enableMultimodel } - - var eff types.EffectiveProcessConfig - var err error - if fileType != "" { - eff, err = resolveKnowledgeFileImportConfig(ctx, kb, fileType, processOverrides, enableMultimodel) - } else { - eff = ResolveProcessConfig(kb, processOverrides) - if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) { - eff.EnableMultimodel = *enableMultimodel - } - if processOverrides != nil { - err = ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes) - } + if processOverrides == nil { + return eff, nil } - if err != nil { + if err := ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes); err != nil { return eff, err } - - if processOverrides != nil && knowledge != nil { - if err := knowledge.SetProcessOverrides(processOverrides); err != nil { - return eff, err - } + if err := knowledge.SetProcessOverrides(processOverrides); err != nil { + return eff, err } return eff, nil } diff --git a/internal/application/service/knowledge_process_config_test.go b/internal/application/service/knowledge_process_config_test.go index ccf7e73e..b72330d2 100644 --- a/internal/application/service/knowledge_process_config_test.go +++ b/internal/application/service/knowledge_process_config_test.go @@ -412,39 +412,55 @@ func TestValidateProcessOverrides_ImageAllowsStorageFallback(t *testing.T) { require.NoError(t, err) } -func TestValidateImportFileType(t *testing.T) { - t.Parallel() - - require.NoError(t, validateImportFileType("xlsx")) - require.NoError(t, validateImportFileType(".XLSX")) - require.Error(t, validateImportFileType("exe")) - require.Error(t, validateImportFileType("mp4")) - require.Error(t, validateImportFileType("")) -} - -func TestResolveKnowledgeFileImportConfig_ImageRequiresVLM(t *testing.T) { +func TestResolveFileImportProcessConfig_ImageRequiresVLM(t *testing.T) { t.Parallel() kb := &types.KnowledgeBase{ VLMConfig: types.VLMConfig{Enabled: false}, } - _, err := resolveKnowledgeFileImportConfig(context.Background(), kb, "png", nil, nil) + _, err := resolveFileImportProcessConfig(context.Background(), kb, "png", nil, nil) require.Error(t, err) var badReq *werrors.AppError require.ErrorAs(t, err, &badReq) } -func TestResolveKnowledgeFileImportConfig_AudioRequiresASR(t *testing.T) { +func TestResolveFileImportProcessConfig_AudioRequiresASR(t *testing.T) { t.Parallel() kb := &types.KnowledgeBase{ ASRConfig: types.ASRConfig{Enabled: false}, } - _, err := resolveKnowledgeFileImportConfig(context.Background(), kb, "mp3", nil, nil) + _, err := resolveFileImportProcessConfig(context.Background(), kb, "mp3", nil, nil) require.Error(t, err) } -func TestApplyKnowledgeProcessOverrides_DefaultImageValidation(t *testing.T) { +// The regression behind #2447: spreadsheets must clear the shared import gate +// even when the caller sends no per-import overrides. +func TestResolveFileImportProcessConfig_SpreadsheetAllowedWithoutOverrides(t *testing.T) { + t.Parallel() + + kb := &types.KnowledgeBase{ChunkingConfig: types.ChunkingConfig{ChunkSize: 512}} + for _, ext := range []string{"xlsx", "xls", "csv"} { + eff, err := resolveFileImportProcessConfig(context.Background(), kb, ext, nil, nil) + require.NoErrorf(t, err, "ext=%s", ext) + require.Equal(t, 512, eff.ChunkingConfig.ChunkSize) + } +} + +func TestResolveFileImportProcessConfig_RejectsUnsupportedAndUndeterminable(t *testing.T) { + t.Parallel() + + kb := &types.KnowledgeBase{} + for _, ext := range []string{"exe", "mp4", "", unknownFileType} { + _, err := resolveFileImportProcessConfig(context.Background(), kb, ext, nil, nil) + require.Errorf(t, err, "ext=%s should be rejected", ext) + } +} + +// ApplyKnowledgeProcessOverrides stays scoped to overrides: import-time file +// type gating belongs to resolveFileImportProcessConfig, so callers that pass +// no overrides (reparse, connector sync) keep their existing behaviour. +func TestApplyKnowledgeProcessOverrides_NoOverridesSkipsImportGate(t *testing.T) { t.Parallel() kb := &types.KnowledgeBase{ @@ -452,17 +468,7 @@ func TestApplyKnowledgeProcessOverrides_DefaultImageValidation(t *testing.T) { } knowledge := &types.Knowledge{} _, err := ApplyKnowledgeProcessOverrides(context.Background(), kb, knowledge, nil, []string{"png"}, nil) - require.Error(t, err) -} - -func TestApplyKnowledgeProcessOverrides_XLSXAllowed(t *testing.T) { - t.Parallel() - - kb := &types.KnowledgeBase{} - knowledge := &types.Knowledge{} - eff, err := ApplyKnowledgeProcessOverrides(context.Background(), kb, knowledge, nil, []string{"xlsx"}, nil) require.NoError(t, err) - require.NotNil(t, eff) } func TestMergeParserEngineOverrides(t *testing.T) { diff --git a/internal/application/service/knowledge_util.go b/internal/application/service/knowledge_util.go index bf84526e..7cd6673b 100644 --- a/internal/application/service/knowledge_util.go +++ b/internal/application/service/knowledge_util.go @@ -13,37 +13,82 @@ import ( "time" filesvc "github.com/Tencent/WeKnora/internal/application/service/file" + werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" ) -// isValidFileType checks if a file type is supported -func isValidFileType(filename string) bool { - switch strings.ToLower(getFileType(filename)) { - case "pdf", "txt", "docx", "doc", "epub", "html", "htm", "mhtml", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json", - "mp3", "wav", "m4a", "flac", "ogg": - return true - default: - return false - } +// unknownFileType is returned by getFileType when a name carries no extension. +const unknownFileType = "unknown" + +// supportedImportFileExtensions is the single source of truth for extensions +// accepted by every knowledge import path: direct upload, file-URL download, +// and the worker's post-download re-check. Keeping one set avoids the drift +// that let direct upload accept xlsx while URL import rejected it (#2447). +var supportedImportFileExtensions = map[string]struct{}{ + "pdf": {}, "txt": {}, "docx": {}, "doc": {}, "epub": {}, + "html": {}, "htm": {}, "mhtml": {}, "md": {}, "markdown": {}, + "png": {}, "jpg": {}, "jpeg": {}, "gif": {}, + "csv": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {}, "json": {}, + "mp3": {}, "wav": {}, "m4a": {}, "flac": {}, "ogg": {}, } -// isAllowedFileURLExtension checks whether a file extension is supported for URL-based file import. -func isAllowedFileURLExtension(ext string) bool { - ext = strings.ToLower(strings.TrimPrefix(ext, ".")) - if ext == "" { +// dataTableFileExtensions are the spreadsheet formats that get an extra +// table-summary task after their document-process task. +var dataTableFileExtensions = map[string]struct{}{ + "csv": {}, "xlsx": {}, "xls": {}, +} + +// normalizeFileExtension lowercases an extension and strips a leading dot so +// callers can pass either "xlsx", ".XLSX", or a raw user-supplied file_type. +func normalizeFileExtension(ext string) string { + return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(ext), ".")) +} + +// isSupportedImportExtension reports whether a bare extension can be imported. +func isSupportedImportExtension(ext string) bool { + ext = normalizeFileExtension(ext) + if ext == "" || ext == unknownFileType { return false } - return isValidFileType("file." + ext) + _, ok := supportedImportFileExtensions[ext] + return ok +} + +// isValidFileType checks if a filename's extension is supported for import. +func isValidFileType(filename string) bool { + return isSupportedImportExtension(getFileType(filename)) +} + +// isDataTableFileType reports whether an extension is a spreadsheet format. +func isDataTableFileType(ext string) bool { + _, ok := dataTableFileExtensions[normalizeFileExtension(ext)] + return ok +} + +// validateImportFileType applies the extension constraints shared by every +// file import path and reports a user-facing reason when one is violated. +func validateImportFileType(fileType string) error { + fileType = normalizeFileExtension(fileType) + if fileType == "" || fileType == unknownFileType { + return werrors.NewBadRequestError("无法确定文件类型") + } + if IsVideoType(fileType) { + return werrors.NewBadRequestError("暂不支持上传视频文件") + } + if !isSupportedImportExtension(fileType) { + return werrors.NewBadRequestError(fmt.Sprintf("不支持的文件类型: %s", fileType)) + } + return nil } // getFileType extracts the file extension from a filename func getFileType(filename string) string { ext := strings.Split(filename, ".") if len(ext) < 2 { - return "unknown" + return unknownFileType } return ext[len(ext)-1] } diff --git a/internal/application/service/knowledge_util_filetype_test.go b/internal/application/service/knowledge_util_filetype_test.go index 7771f53a..a902409f 100644 --- a/internal/application/service/knowledge_util_filetype_test.go +++ b/internal/application/service/knowledge_util_filetype_test.go @@ -23,7 +23,7 @@ func TestIsValidFileTypeHTML(t *testing.T) { } } -func TestIsAllowedFileURLExtension(t *testing.T) { +func TestIsSupportedImportExtension(t *testing.T) { tests := []struct { name string ext string @@ -34,16 +34,45 @@ func TestIsAllowedFileURLExtension(t *testing.T) { {name: "csv", ext: "csv", want: true}, {name: "dot prefix", ext: ".xlsx", want: true}, {name: "uppercase", ext: "XLSX", want: true}, + {name: "surrounding space", ext: " xlsx ", want: true}, {name: "pdf", ext: "pdf", want: true}, {name: "unsupported", ext: "exe", want: false}, + {name: "video", ext: "mp4", want: false}, {name: "empty", ext: "", want: false}, + {name: "unknown sentinel", ext: unknownFileType, want: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := isAllowedFileURLExtension(tt.ext); got != tt.want { - t.Fatalf("isAllowedFileURLExtension(%q) = %v, want %v", tt.ext, got, tt.want) + if got := isSupportedImportExtension(tt.ext); got != tt.want { + t.Fatalf("isSupportedImportExtension(%q) = %v, want %v", tt.ext, got, tt.want) } }) } } + +// Direct upload and URL import must agree on the accepted extension set, +// otherwise #2447 (xlsx accepted on upload, rejected on URL import) regresses. +func TestImportExtensionSetIsSharedAcrossPaths(t *testing.T) { + for ext := range supportedImportFileExtensions { + if !isValidFileType("file." + ext) { + t.Errorf("isValidFileType rejects supported extension %q", ext) + } + if err := validateImportFileType(ext); err != nil { + t.Errorf("validateImportFileType(%q) = %v, want nil", ext, err) + } + } +} + +func TestIsDataTableFileType(t *testing.T) { + for _, ext := range []string{"csv", "xlsx", "xls", ".XLSX"} { + if !isDataTableFileType(ext) { + t.Errorf("isDataTableFileType(%q) = false, want true", ext) + } + } + for _, ext := range []string{"pdf", "png", "", unknownFileType} { + if isDataTableFileType(ext) { + t.Errorf("isDataTableFileType(%q) = true, want false", ext) + } + } +}