diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go index f952ecbe..f9a9d59c 100644 --- a/internal/application/service/extract.go +++ b/internal/application/service/extract.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "os" - "slices" "strings" "time" @@ -157,7 +156,8 @@ func NewDataTableSummaryTask( return nil } -// enqueueDataTableSummaryIfNeeded enqueues table summary work for spreadsheet imports. +// enqueueDataTableSummaryIfNeeded enqueues table summary work for spreadsheet +// imports. fileName is a fallback for older records whose FileType is empty. func enqueueDataTableSummaryIfNeeded( ctx context.Context, client interfaces.TaskEnqueuer, @@ -165,11 +165,11 @@ func enqueueDataTableSummaryIfNeeded( knowledgeID string, fileName, fileType, summaryModelID, embeddingModelID string, ) { - ft := strings.ToLower(strings.TrimPrefix(fileType, ".")) + ft := normalizeFileExtension(fileType) if ft == "" && fileName != "" { ft = getFileType(fileName) } - if !slices.Contains([]string{"csv", "xlsx", "xls"}, ft) { + if !isDataTableFileType(ft) { return } if err := NewDataTableSummaryTask(ctx, client, tenantID, knowledgeID, summaryModelID, embeddingModelID); err != nil { diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index 1b23f0db..b7473c01 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -59,7 +59,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, return nil, err } - // Validate file type + // Early reject before the whole-file hash below. resolveFileImportProcessConfig + // gates the same extension set, but this path must keep returning + // ErrInvalidFileType rather than the shared gate's localized message. logger.Infof(ctx, "Checking file type: %s", fileName) if !isValidFileType(fileName) { logger.Error(ctx, "Invalid file type") @@ -123,7 +125,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, return nil, werrors.NewValidationError("文件名包含非法字符") } - eff, err := resolveKnowledgeFileImportConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel) + eff, err := resolveFileImportProcessConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel) if err != nil { return nil, err } @@ -263,7 +265,7 @@ func isFileURL(rawURL, fileName, fileType string) bool { u, err := url.Parse(rawURL) if err == nil { ext := strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), ".")) - if ext != "" && isAllowedFileURLExtension(ext) { + if ext != "" && isSupportedImportExtension(ext) { return true } } @@ -529,15 +531,13 @@ func (s *knowledgeService) createKnowledgeFromFileURL( fileName = safeFilename } - // Resolve fileType: user-provided > inferred from fileName - if fileType == "" && fileName != "" { + // Resolve fileType: user-provided > inferred from fileName (which already + // falls back to the URL path above). getFileType never returns empty, so an + // undeterminable type surfaces as "unknown" and is rejected below. + if fileType == "" { fileType = getFileType(fileName) } - - resolvedFileType := fileType - if resolvedFileType == "" { - resolvedFileType = getFileType(extractFileNameFromURL(fileURL)) - } + fileType = normalizeFileExtension(fileType) // Use title as display name if fileName is still empty displayName := fileName @@ -604,23 +604,13 @@ func (s *knowledgeService) createKnowledgeFromFileURL( knowledge.Title = displayName } - var eff types.EffectiveProcessConfig - if resolvedFileType != "" { - eff, err = resolveKnowledgeFileImportConfig(ctx, kb, resolvedFileType, processOverrides, enableMultimodel) - if err != nil { - return nil, err - } - if processOverrides != nil { - if err := knowledge.SetProcessOverrides(processOverrides); err != nil { - logger.Errorf(ctx, "Failed to set process overrides: %v", err) - return nil, err - } - } - } else { - eff, err = ApplyKnowledgeProcessOverrides( - ctx, kb, knowledge, processOverrides, nil, enableMultimodel, - ) - if err != nil { + eff, err := resolveFileImportProcessConfig(ctx, kb, fileType, processOverrides, enableMultimodel) + if err != nil { + return nil, err + } + if processOverrides != nil { + if err := knowledge.SetProcessOverrides(processOverrides); err != nil { + logger.Errorf(ctx, "Failed to set process overrides: %v", err) return nil, err } } diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index c26e02a1..6fc9f298 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3184,7 +3184,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return fmt.Errorf("failed to download file from URL: %w", err) } - if resolvedFileType != "" && !isAllowedFileURLExtension(resolvedFileType) { + if resolvedFileType != "" && !isSupportedImportExtension(resolvedFileType) { logger.Errorf(ctx, "Unsupported file type resolved from file URL: %s", resolvedFileType) knowledge.ParseStatus = "failed" knowledge.ErrorMessage = fmt.Sprintf("unsupported file type: %s", resolvedFileType) diff --git a/internal/application/service/knowledge_process_config.go b/internal/application/service/knowledge_process_config.go index 78fc08bb..c65dab33 100644 --- a/internal/application/service/knowledge_process_config.go +++ b/internal/application/service/knowledge_process_config.go @@ -2,7 +2,6 @@ package service import ( "context" - "fmt" "strconv" "strings" @@ -94,29 +93,16 @@ func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgePro return eff } -// validateImportFileType checks file extension constraints shared by direct upload and file-URL import. -func validateImportFileType(fileType string) error { - fileType = strings.ToLower(strings.TrimPrefix(fileType, ".")) - if fileType == "" || fileType == "unknown" { - return werrors.NewBadRequestError("无法确定文件类型") - } - if IsVideoType(fileType) { - return werrors.NewBadRequestError("暂不支持上传视频文件") - } - if !isAllowedFileURLExtension(fileType) { - return werrors.NewBadRequestError(fmt.Sprintf("不支持的文件类型: %s", fileType)) - } - return nil -} - -// validateDefaultFileImportRequirements enforces VLM/ASR prerequisites when no per-import overrides are provided. +// validateDefaultFileImportRequirements enforces the VLM/ASR prerequisites that +// ValidateProcessOverrides would otherwise cover, for imports that ship no +// per-import overrides and therefore fall back to the KB defaults. func validateDefaultFileImportRequirements( ctx context.Context, kb *types.KnowledgeBase, eff types.EffectiveProcessConfig, fileType string, ) error { - fileType = strings.ToLower(strings.TrimPrefix(fileType, ".")) + fileType = normalizeFileExtension(fileType) if IsImageType(fileType) && !eff.VLMConfig.IsEnabled() { logger.Error(ctx, "VLM model is not configured") return werrors.NewBadRequestError("上传图片文件需要设置VLM模型") @@ -128,8 +114,12 @@ func validateDefaultFileImportRequirements( return nil } -// resolveKnowledgeFileImportConfig validates import prerequisites and resolves effective processing config. -func resolveKnowledgeFileImportConfig( +// resolveFileImportProcessConfig is the single gate every file import passes +// through: it rejects unsupported extensions, enforces the VLM/ASR +// prerequisites for the resolved type, and returns the effective processing +// config for task enqueue. Persisting overrides onto the knowledge record stays +// with the caller, which owns the record's lifecycle. +func resolveFileImportProcessConfig( ctx context.Context, kb *types.KnowledgeBase, fileType string, @@ -207,32 +197,18 @@ func ApplyKnowledgeProcessOverrides( fileTypes []string, enableMultimodel *bool, ) (types.EffectiveProcessConfig, error) { - fileType := "" - if len(fileTypes) > 0 { - fileType = fileTypes[0] + eff := ResolveProcessConfig(kb, processOverrides) + if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) { + eff.EnableMultimodel = *enableMultimodel } - - var eff types.EffectiveProcessConfig - var err error - if fileType != "" { - eff, err = resolveKnowledgeFileImportConfig(ctx, kb, fileType, processOverrides, enableMultimodel) - } else { - eff = ResolveProcessConfig(kb, processOverrides) - if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) { - eff.EnableMultimodel = *enableMultimodel - } - if processOverrides != nil { - err = ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes) - } + if processOverrides == nil { + return eff, nil } - if err != nil { + if err := ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes); err != nil { return eff, err } - - if processOverrides != nil && knowledge != nil { - if err := knowledge.SetProcessOverrides(processOverrides); err != nil { - return eff, err - } + if err := knowledge.SetProcessOverrides(processOverrides); err != nil { + return eff, err } return eff, nil } diff --git a/internal/application/service/knowledge_process_config_test.go b/internal/application/service/knowledge_process_config_test.go index ccf7e73e..b72330d2 100644 --- a/internal/application/service/knowledge_process_config_test.go +++ b/internal/application/service/knowledge_process_config_test.go @@ -412,39 +412,55 @@ func TestValidateProcessOverrides_ImageAllowsStorageFallback(t *testing.T) { require.NoError(t, err) } -func TestValidateImportFileType(t *testing.T) { - t.Parallel() - - require.NoError(t, validateImportFileType("xlsx")) - require.NoError(t, validateImportFileType(".XLSX")) - require.Error(t, validateImportFileType("exe")) - require.Error(t, validateImportFileType("mp4")) - require.Error(t, validateImportFileType("")) -} - -func TestResolveKnowledgeFileImportConfig_ImageRequiresVLM(t *testing.T) { +func TestResolveFileImportProcessConfig_ImageRequiresVLM(t *testing.T) { t.Parallel() kb := &types.KnowledgeBase{ VLMConfig: types.VLMConfig{Enabled: false}, } - _, err := resolveKnowledgeFileImportConfig(context.Background(), kb, "png", nil, nil) + _, err := resolveFileImportProcessConfig(context.Background(), kb, "png", nil, nil) require.Error(t, err) var badReq *werrors.AppError require.ErrorAs(t, err, &badReq) } -func TestResolveKnowledgeFileImportConfig_AudioRequiresASR(t *testing.T) { +func TestResolveFileImportProcessConfig_AudioRequiresASR(t *testing.T) { t.Parallel() kb := &types.KnowledgeBase{ ASRConfig: types.ASRConfig{Enabled: false}, } - _, err := resolveKnowledgeFileImportConfig(context.Background(), kb, "mp3", nil, nil) + _, err := resolveFileImportProcessConfig(context.Background(), kb, "mp3", nil, nil) require.Error(t, err) } -func TestApplyKnowledgeProcessOverrides_DefaultImageValidation(t *testing.T) { +// The regression behind #2447: spreadsheets must clear the shared import gate +// even when the caller sends no per-import overrides. +func TestResolveFileImportProcessConfig_SpreadsheetAllowedWithoutOverrides(t *testing.T) { + t.Parallel() + + kb := &types.KnowledgeBase{ChunkingConfig: types.ChunkingConfig{ChunkSize: 512}} + for _, ext := range []string{"xlsx", "xls", "csv"} { + eff, err := resolveFileImportProcessConfig(context.Background(), kb, ext, nil, nil) + require.NoErrorf(t, err, "ext=%s", ext) + require.Equal(t, 512, eff.ChunkingConfig.ChunkSize) + } +} + +func TestResolveFileImportProcessConfig_RejectsUnsupportedAndUndeterminable(t *testing.T) { + t.Parallel() + + kb := &types.KnowledgeBase{} + for _, ext := range []string{"exe", "mp4", "", unknownFileType} { + _, err := resolveFileImportProcessConfig(context.Background(), kb, ext, nil, nil) + require.Errorf(t, err, "ext=%s should be rejected", ext) + } +} + +// ApplyKnowledgeProcessOverrides stays scoped to overrides: import-time file +// type gating belongs to resolveFileImportProcessConfig, so callers that pass +// no overrides (reparse, connector sync) keep their existing behaviour. +func TestApplyKnowledgeProcessOverrides_NoOverridesSkipsImportGate(t *testing.T) { t.Parallel() kb := &types.KnowledgeBase{ @@ -452,17 +468,7 @@ func TestApplyKnowledgeProcessOverrides_DefaultImageValidation(t *testing.T) { } knowledge := &types.Knowledge{} _, err := ApplyKnowledgeProcessOverrides(context.Background(), kb, knowledge, nil, []string{"png"}, nil) - require.Error(t, err) -} - -func TestApplyKnowledgeProcessOverrides_XLSXAllowed(t *testing.T) { - t.Parallel() - - kb := &types.KnowledgeBase{} - knowledge := &types.Knowledge{} - eff, err := ApplyKnowledgeProcessOverrides(context.Background(), kb, knowledge, nil, []string{"xlsx"}, nil) require.NoError(t, err) - require.NotNil(t, eff) } func TestMergeParserEngineOverrides(t *testing.T) { diff --git a/internal/application/service/knowledge_util.go b/internal/application/service/knowledge_util.go index bf84526e..7cd6673b 100644 --- a/internal/application/service/knowledge_util.go +++ b/internal/application/service/knowledge_util.go @@ -13,37 +13,82 @@ import ( "time" filesvc "github.com/Tencent/WeKnora/internal/application/service/file" + werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" ) -// isValidFileType checks if a file type is supported -func isValidFileType(filename string) bool { - switch strings.ToLower(getFileType(filename)) { - case "pdf", "txt", "docx", "doc", "epub", "html", "htm", "mhtml", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json", - "mp3", "wav", "m4a", "flac", "ogg": - return true - default: - return false - } +// unknownFileType is returned by getFileType when a name carries no extension. +const unknownFileType = "unknown" + +// supportedImportFileExtensions is the single source of truth for extensions +// accepted by every knowledge import path: direct upload, file-URL download, +// and the worker's post-download re-check. Keeping one set avoids the drift +// that let direct upload accept xlsx while URL import rejected it (#2447). +var supportedImportFileExtensions = map[string]struct{}{ + "pdf": {}, "txt": {}, "docx": {}, "doc": {}, "epub": {}, + "html": {}, "htm": {}, "mhtml": {}, "md": {}, "markdown": {}, + "png": {}, "jpg": {}, "jpeg": {}, "gif": {}, + "csv": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {}, "json": {}, + "mp3": {}, "wav": {}, "m4a": {}, "flac": {}, "ogg": {}, } -// isAllowedFileURLExtension checks whether a file extension is supported for URL-based file import. -func isAllowedFileURLExtension(ext string) bool { - ext = strings.ToLower(strings.TrimPrefix(ext, ".")) - if ext == "" { +// dataTableFileExtensions are the spreadsheet formats that get an extra +// table-summary task after their document-process task. +var dataTableFileExtensions = map[string]struct{}{ + "csv": {}, "xlsx": {}, "xls": {}, +} + +// normalizeFileExtension lowercases an extension and strips a leading dot so +// callers can pass either "xlsx", ".XLSX", or a raw user-supplied file_type. +func normalizeFileExtension(ext string) string { + return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(ext), ".")) +} + +// isSupportedImportExtension reports whether a bare extension can be imported. +func isSupportedImportExtension(ext string) bool { + ext = normalizeFileExtension(ext) + if ext == "" || ext == unknownFileType { return false } - return isValidFileType("file." + ext) + _, ok := supportedImportFileExtensions[ext] + return ok +} + +// isValidFileType checks if a filename's extension is supported for import. +func isValidFileType(filename string) bool { + return isSupportedImportExtension(getFileType(filename)) +} + +// isDataTableFileType reports whether an extension is a spreadsheet format. +func isDataTableFileType(ext string) bool { + _, ok := dataTableFileExtensions[normalizeFileExtension(ext)] + return ok +} + +// validateImportFileType applies the extension constraints shared by every +// file import path and reports a user-facing reason when one is violated. +func validateImportFileType(fileType string) error { + fileType = normalizeFileExtension(fileType) + if fileType == "" || fileType == unknownFileType { + return werrors.NewBadRequestError("无法确定文件类型") + } + if IsVideoType(fileType) { + return werrors.NewBadRequestError("暂不支持上传视频文件") + } + if !isSupportedImportExtension(fileType) { + return werrors.NewBadRequestError(fmt.Sprintf("不支持的文件类型: %s", fileType)) + } + return nil } // getFileType extracts the file extension from a filename func getFileType(filename string) string { ext := strings.Split(filename, ".") if len(ext) < 2 { - return "unknown" + return unknownFileType } return ext[len(ext)-1] } diff --git a/internal/application/service/knowledge_util_filetype_test.go b/internal/application/service/knowledge_util_filetype_test.go index 7771f53a..a902409f 100644 --- a/internal/application/service/knowledge_util_filetype_test.go +++ b/internal/application/service/knowledge_util_filetype_test.go @@ -23,7 +23,7 @@ func TestIsValidFileTypeHTML(t *testing.T) { } } -func TestIsAllowedFileURLExtension(t *testing.T) { +func TestIsSupportedImportExtension(t *testing.T) { tests := []struct { name string ext string @@ -34,16 +34,45 @@ func TestIsAllowedFileURLExtension(t *testing.T) { {name: "csv", ext: "csv", want: true}, {name: "dot prefix", ext: ".xlsx", want: true}, {name: "uppercase", ext: "XLSX", want: true}, + {name: "surrounding space", ext: " xlsx ", want: true}, {name: "pdf", ext: "pdf", want: true}, {name: "unsupported", ext: "exe", want: false}, + {name: "video", ext: "mp4", want: false}, {name: "empty", ext: "", want: false}, + {name: "unknown sentinel", ext: unknownFileType, want: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := isAllowedFileURLExtension(tt.ext); got != tt.want { - t.Fatalf("isAllowedFileURLExtension(%q) = %v, want %v", tt.ext, got, tt.want) + if got := isSupportedImportExtension(tt.ext); got != tt.want { + t.Fatalf("isSupportedImportExtension(%q) = %v, want %v", tt.ext, got, tt.want) } }) } } + +// Direct upload and URL import must agree on the accepted extension set, +// otherwise #2447 (xlsx accepted on upload, rejected on URL import) regresses. +func TestImportExtensionSetIsSharedAcrossPaths(t *testing.T) { + for ext := range supportedImportFileExtensions { + if !isValidFileType("file." + ext) { + t.Errorf("isValidFileType rejects supported extension %q", ext) + } + if err := validateImportFileType(ext); err != nil { + t.Errorf("validateImportFileType(%q) = %v, want nil", ext, err) + } + } +} + +func TestIsDataTableFileType(t *testing.T) { + for _, ext := range []string{"csv", "xlsx", "xls", ".XLSX"} { + if !isDataTableFileType(ext) { + t.Errorf("isDataTableFileType(%q) = false, want true", ext) + } + } + for _, ext := range []string{"pdf", "png", "", unknownFileType} { + if isDataTableFileType(ext) { + t.Errorf("isDataTableFileType(%q) = true, want false", ext) + } + } +}