refactor(knowledge): make the supported import extension set the single gate

The xlsx URL-import fix left the accepted extension set reachable through two
indirections and widened ApplyKnowledgeProcessOverrides to gate file types for
callers that are not file imports (web URL fetch, connector sync, reparse).

Collapse the set into one map that both isValidFileType and the extension check
read, move the shared validation next to the other file-type predicates, and
return ApplyKnowledgeProcessOverrides to its override-only scope so
resolveFileImportProcessConfig is the only import gate.

Also drop the unreachable branch in createKnowledgeFromFileURL: getFileType
never yields an empty string, so the resolvedFileType fallback and its else arm
could not run. Normalize the resolved type before it reaches the record and the
task payload, and add an invariant test asserting direct upload and URL import
accept the same extensions so #2447 cannot drift back.
main
wizardchen 2026-07-31 13:44:04 +08:00 committed by lyingbug
parent 43afa4c1f8
commit 8dd997f0f4
7 changed files with 163 additions and 117 deletions

View File

@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"os"
"slices"
"strings"
"time"
@ -157,7 +156,8 @@ func NewDataTableSummaryTask(
return nil
}
// enqueueDataTableSummaryIfNeeded enqueues table summary work for spreadsheet imports.
// enqueueDataTableSummaryIfNeeded enqueues table summary work for spreadsheet
// imports. fileName is a fallback for older records whose FileType is empty.
func enqueueDataTableSummaryIfNeeded(
ctx context.Context,
client interfaces.TaskEnqueuer,
@ -165,11 +165,11 @@ func enqueueDataTableSummaryIfNeeded(
knowledgeID string,
fileName, fileType, summaryModelID, embeddingModelID string,
) {
ft := strings.ToLower(strings.TrimPrefix(fileType, "."))
ft := normalizeFileExtension(fileType)
if ft == "" && fileName != "" {
ft = getFileType(fileName)
}
if !slices.Contains([]string{"csv", "xlsx", "xls"}, ft) {
if !isDataTableFileType(ft) {
return
}
if err := NewDataTableSummaryTask(ctx, client, tenantID, knowledgeID, summaryModelID, embeddingModelID); err != nil {

View File

@ -59,7 +59,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return nil, err
}
// Validate file type
// Early reject before the whole-file hash below. resolveFileImportProcessConfig
// gates the same extension set, but this path must keep returning
// ErrInvalidFileType rather than the shared gate's localized message.
logger.Infof(ctx, "Checking file type: %s", fileName)
if !isValidFileType(fileName) {
logger.Error(ctx, "Invalid file type")
@ -123,7 +125,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return nil, werrors.NewValidationError("文件名包含非法字符")
}
eff, err := resolveKnowledgeFileImportConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel)
eff, err := resolveFileImportProcessConfig(ctx, kb, getFileType(safeFilename), processOverrides, enableMultimodel)
if err != nil {
return nil, err
}
@ -263,7 +265,7 @@ func isFileURL(rawURL, fileName, fileType string) bool {
u, err := url.Parse(rawURL)
if err == nil {
ext := strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), "."))
if ext != "" && isAllowedFileURLExtension(ext) {
if ext != "" && isSupportedImportExtension(ext) {
return true
}
}
@ -529,15 +531,13 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
fileName = safeFilename
}
// Resolve fileType: user-provided > inferred from fileName
if fileType == "" && fileName != "" {
// Resolve fileType: user-provided > inferred from fileName (which already
// falls back to the URL path above). getFileType never returns empty, so an
// undeterminable type surfaces as "unknown" and is rejected below.
if fileType == "" {
fileType = getFileType(fileName)
}
resolvedFileType := fileType
if resolvedFileType == "" {
resolvedFileType = getFileType(extractFileNameFromURL(fileURL))
}
fileType = normalizeFileExtension(fileType)
// Use title as display name if fileName is still empty
displayName := fileName
@ -604,23 +604,13 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
knowledge.Title = displayName
}
var eff types.EffectiveProcessConfig
if resolvedFileType != "" {
eff, err = resolveKnowledgeFileImportConfig(ctx, kb, resolvedFileType, processOverrides, enableMultimodel)
if err != nil {
return nil, err
}
if processOverrides != nil {
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
logger.Errorf(ctx, "Failed to set process overrides: %v", err)
return nil, err
}
}
} else {
eff, err = ApplyKnowledgeProcessOverrides(
ctx, kb, knowledge, processOverrides, nil, enableMultimodel,
)
if err != nil {
eff, err := resolveFileImportProcessConfig(ctx, kb, fileType, processOverrides, enableMultimodel)
if err != nil {
return nil, err
}
if processOverrides != nil {
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
logger.Errorf(ctx, "Failed to set process overrides: %v", err)
return nil, err
}
}

View File

@ -3184,7 +3184,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return fmt.Errorf("failed to download file from URL: %w", err)
}
if resolvedFileType != "" && !isAllowedFileURLExtension(resolvedFileType) {
if resolvedFileType != "" && !isSupportedImportExtension(resolvedFileType) {
logger.Errorf(ctx, "Unsupported file type resolved from file URL: %s", resolvedFileType)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("unsupported file type: %s", resolvedFileType)

View File

@ -2,7 +2,6 @@ package service
import (
"context"
"fmt"
"strconv"
"strings"
@ -94,29 +93,16 @@ func ResolveProcessConfig(kb *types.KnowledgeBase, overrides *types.KnowledgePro
return eff
}
// validateImportFileType checks file extension constraints shared by direct upload and file-URL import.
func validateImportFileType(fileType string) error {
fileType = strings.ToLower(strings.TrimPrefix(fileType, "."))
if fileType == "" || fileType == "unknown" {
return werrors.NewBadRequestError("无法确定文件类型")
}
if IsVideoType(fileType) {
return werrors.NewBadRequestError("暂不支持上传视频文件")
}
if !isAllowedFileURLExtension(fileType) {
return werrors.NewBadRequestError(fmt.Sprintf("不支持的文件类型: %s", fileType))
}
return nil
}
// validateDefaultFileImportRequirements enforces VLM/ASR prerequisites when no per-import overrides are provided.
// validateDefaultFileImportRequirements enforces the VLM/ASR prerequisites that
// ValidateProcessOverrides would otherwise cover, for imports that ship no
// per-import overrides and therefore fall back to the KB defaults.
func validateDefaultFileImportRequirements(
ctx context.Context,
kb *types.KnowledgeBase,
eff types.EffectiveProcessConfig,
fileType string,
) error {
fileType = strings.ToLower(strings.TrimPrefix(fileType, "."))
fileType = normalizeFileExtension(fileType)
if IsImageType(fileType) && !eff.VLMConfig.IsEnabled() {
logger.Error(ctx, "VLM model is not configured")
return werrors.NewBadRequestError("上传图片文件需要设置VLM模型")
@ -128,8 +114,12 @@ func validateDefaultFileImportRequirements(
return nil
}
// resolveKnowledgeFileImportConfig validates import prerequisites and resolves effective processing config.
func resolveKnowledgeFileImportConfig(
// resolveFileImportProcessConfig is the single gate every file import passes
// through: it rejects unsupported extensions, enforces the VLM/ASR
// prerequisites for the resolved type, and returns the effective processing
// config for task enqueue. Persisting overrides onto the knowledge record stays
// with the caller, which owns the record's lifecycle.
func resolveFileImportProcessConfig(
ctx context.Context,
kb *types.KnowledgeBase,
fileType string,
@ -207,32 +197,18 @@ func ApplyKnowledgeProcessOverrides(
fileTypes []string,
enableMultimodel *bool,
) (types.EffectiveProcessConfig, error) {
fileType := ""
if len(fileTypes) > 0 {
fileType = fileTypes[0]
eff := ResolveProcessConfig(kb, processOverrides)
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
eff.EnableMultimodel = *enableMultimodel
}
var eff types.EffectiveProcessConfig
var err error
if fileType != "" {
eff, err = resolveKnowledgeFileImportConfig(ctx, kb, fileType, processOverrides, enableMultimodel)
} else {
eff = ResolveProcessConfig(kb, processOverrides)
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
eff.EnableMultimodel = *enableMultimodel
}
if processOverrides != nil {
err = ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes)
}
if processOverrides == nil {
return eff, nil
}
if err != nil {
if err := ValidateProcessOverrides(ctx, kb, processOverrides, fileTypes); err != nil {
return eff, err
}
if processOverrides != nil && knowledge != nil {
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
return eff, err
}
if err := knowledge.SetProcessOverrides(processOverrides); err != nil {
return eff, err
}
return eff, nil
}

View File

@ -412,39 +412,55 @@ func TestValidateProcessOverrides_ImageAllowsStorageFallback(t *testing.T) {
require.NoError(t, err)
}
func TestValidateImportFileType(t *testing.T) {
t.Parallel()
require.NoError(t, validateImportFileType("xlsx"))
require.NoError(t, validateImportFileType(".XLSX"))
require.Error(t, validateImportFileType("exe"))
require.Error(t, validateImportFileType("mp4"))
require.Error(t, validateImportFileType(""))
}
func TestResolveKnowledgeFileImportConfig_ImageRequiresVLM(t *testing.T) {
func TestResolveFileImportProcessConfig_ImageRequiresVLM(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
VLMConfig: types.VLMConfig{Enabled: false},
}
_, err := resolveKnowledgeFileImportConfig(context.Background(), kb, "png", nil, nil)
_, err := resolveFileImportProcessConfig(context.Background(), kb, "png", nil, nil)
require.Error(t, err)
var badReq *werrors.AppError
require.ErrorAs(t, err, &badReq)
}
func TestResolveKnowledgeFileImportConfig_AudioRequiresASR(t *testing.T) {
func TestResolveFileImportProcessConfig_AudioRequiresASR(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
ASRConfig: types.ASRConfig{Enabled: false},
}
_, err := resolveKnowledgeFileImportConfig(context.Background(), kb, "mp3", nil, nil)
_, err := resolveFileImportProcessConfig(context.Background(), kb, "mp3", nil, nil)
require.Error(t, err)
}
func TestApplyKnowledgeProcessOverrides_DefaultImageValidation(t *testing.T) {
// The regression behind #2447: spreadsheets must clear the shared import gate
// even when the caller sends no per-import overrides.
func TestResolveFileImportProcessConfig_SpreadsheetAllowedWithoutOverrides(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{ChunkingConfig: types.ChunkingConfig{ChunkSize: 512}}
for _, ext := range []string{"xlsx", "xls", "csv"} {
eff, err := resolveFileImportProcessConfig(context.Background(), kb, ext, nil, nil)
require.NoErrorf(t, err, "ext=%s", ext)
require.Equal(t, 512, eff.ChunkingConfig.ChunkSize)
}
}
func TestResolveFileImportProcessConfig_RejectsUnsupportedAndUndeterminable(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{}
for _, ext := range []string{"exe", "mp4", "", unknownFileType} {
_, err := resolveFileImportProcessConfig(context.Background(), kb, ext, nil, nil)
require.Errorf(t, err, "ext=%s should be rejected", ext)
}
}
// ApplyKnowledgeProcessOverrides stays scoped to overrides: import-time file
// type gating belongs to resolveFileImportProcessConfig, so callers that pass
// no overrides (reparse, connector sync) keep their existing behaviour.
func TestApplyKnowledgeProcessOverrides_NoOverridesSkipsImportGate(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{
@ -452,17 +468,7 @@ func TestApplyKnowledgeProcessOverrides_DefaultImageValidation(t *testing.T) {
}
knowledge := &types.Knowledge{}
_, err := ApplyKnowledgeProcessOverrides(context.Background(), kb, knowledge, nil, []string{"png"}, nil)
require.Error(t, err)
}
func TestApplyKnowledgeProcessOverrides_XLSXAllowed(t *testing.T) {
t.Parallel()
kb := &types.KnowledgeBase{}
knowledge := &types.Knowledge{}
eff, err := ApplyKnowledgeProcessOverrides(context.Background(), kb, knowledge, nil, []string{"xlsx"}, nil)
require.NoError(t, err)
require.NotNil(t, eff)
}
func TestMergeParserEngineOverrides(t *testing.T) {

View File

@ -13,37 +13,82 @@ import (
"time"
filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// isValidFileType checks if a file type is supported
func isValidFileType(filename string) bool {
switch strings.ToLower(getFileType(filename)) {
case "pdf", "txt", "docx", "doc", "epub", "html", "htm", "mhtml", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json",
"mp3", "wav", "m4a", "flac", "ogg":
return true
default:
return false
}
// unknownFileType is returned by getFileType when a name carries no extension.
const unknownFileType = "unknown"
// supportedImportFileExtensions is the single source of truth for extensions
// accepted by every knowledge import path: direct upload, file-URL download,
// and the worker's post-download re-check. Keeping one set avoids the drift
// that let direct upload accept xlsx while URL import rejected it (#2447).
var supportedImportFileExtensions = map[string]struct{}{
"pdf": {}, "txt": {}, "docx": {}, "doc": {}, "epub": {},
"html": {}, "htm": {}, "mhtml": {}, "md": {}, "markdown": {},
"png": {}, "jpg": {}, "jpeg": {}, "gif": {},
"csv": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {}, "json": {},
"mp3": {}, "wav": {}, "m4a": {}, "flac": {}, "ogg": {},
}
// isAllowedFileURLExtension checks whether a file extension is supported for URL-based file import.
func isAllowedFileURLExtension(ext string) bool {
ext = strings.ToLower(strings.TrimPrefix(ext, "."))
if ext == "" {
// dataTableFileExtensions are the spreadsheet formats that get an extra
// table-summary task after their document-process task.
var dataTableFileExtensions = map[string]struct{}{
"csv": {}, "xlsx": {}, "xls": {},
}
// normalizeFileExtension lowercases an extension and strips a leading dot so
// callers can pass either "xlsx", ".XLSX", or a raw user-supplied file_type.
func normalizeFileExtension(ext string) string {
return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(ext), "."))
}
// isSupportedImportExtension reports whether a bare extension can be imported.
func isSupportedImportExtension(ext string) bool {
ext = normalizeFileExtension(ext)
if ext == "" || ext == unknownFileType {
return false
}
return isValidFileType("file." + ext)
_, ok := supportedImportFileExtensions[ext]
return ok
}
// isValidFileType checks if a filename's extension is supported for import.
func isValidFileType(filename string) bool {
return isSupportedImportExtension(getFileType(filename))
}
// isDataTableFileType reports whether an extension is a spreadsheet format.
func isDataTableFileType(ext string) bool {
_, ok := dataTableFileExtensions[normalizeFileExtension(ext)]
return ok
}
// validateImportFileType applies the extension constraints shared by every
// file import path and reports a user-facing reason when one is violated.
func validateImportFileType(fileType string) error {
fileType = normalizeFileExtension(fileType)
if fileType == "" || fileType == unknownFileType {
return werrors.NewBadRequestError("无法确定文件类型")
}
if IsVideoType(fileType) {
return werrors.NewBadRequestError("暂不支持上传视频文件")
}
if !isSupportedImportExtension(fileType) {
return werrors.NewBadRequestError(fmt.Sprintf("不支持的文件类型: %s", fileType))
}
return nil
}
// getFileType extracts the file extension from a filename
func getFileType(filename string) string {
ext := strings.Split(filename, ".")
if len(ext) < 2 {
return "unknown"
return unknownFileType
}
return ext[len(ext)-1]
}

View File

@ -23,7 +23,7 @@ func TestIsValidFileTypeHTML(t *testing.T) {
}
}
func TestIsAllowedFileURLExtension(t *testing.T) {
func TestIsSupportedImportExtension(t *testing.T) {
tests := []struct {
name string
ext string
@ -34,16 +34,45 @@ func TestIsAllowedFileURLExtension(t *testing.T) {
{name: "csv", ext: "csv", want: true},
{name: "dot prefix", ext: ".xlsx", want: true},
{name: "uppercase", ext: "XLSX", want: true},
{name: "surrounding space", ext: " xlsx ", want: true},
{name: "pdf", ext: "pdf", want: true},
{name: "unsupported", ext: "exe", want: false},
{name: "video", ext: "mp4", want: false},
{name: "empty", ext: "", want: false},
{name: "unknown sentinel", ext: unknownFileType, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isAllowedFileURLExtension(tt.ext); got != tt.want {
t.Fatalf("isAllowedFileURLExtension(%q) = %v, want %v", tt.ext, got, tt.want)
if got := isSupportedImportExtension(tt.ext); got != tt.want {
t.Fatalf("isSupportedImportExtension(%q) = %v, want %v", tt.ext, got, tt.want)
}
})
}
}
// Direct upload and URL import must agree on the accepted extension set,
// otherwise #2447 (xlsx accepted on upload, rejected on URL import) regresses.
func TestImportExtensionSetIsSharedAcrossPaths(t *testing.T) {
for ext := range supportedImportFileExtensions {
if !isValidFileType("file." + ext) {
t.Errorf("isValidFileType rejects supported extension %q", ext)
}
if err := validateImportFileType(ext); err != nil {
t.Errorf("validateImportFileType(%q) = %v, want nil", ext, err)
}
}
}
func TestIsDataTableFileType(t *testing.T) {
for _, ext := range []string{"csv", "xlsx", "xls", ".XLSX"} {
if !isDataTableFileType(ext) {
t.Errorf("isDataTableFileType(%q) = false, want true", ext)
}
}
for _, ext := range []string{"pdf", "png", "", unknownFileType} {
if isDataTableFileType(ext) {
t.Errorf("isDataTableFileType(%q) = true, want false", ext)
}
}
}