feat: standardize storage configuration and enhance backward compatibility

- Updated the `KnowledgeBase` struct to rename `cos_config` to `storage_config` for consistency across the application.
- Implemented a custom `UnmarshalJSON` method to maintain backward compatibility with legacy responses that still use `cos_config`.
- Modified various frontend components to reflect the new `storage_config` naming, ensuring seamless integration with the updated backend structure.
- Enhanced Dockerfiles and service configurations to support the new storage engine setup.

This update improves the clarity of storage configurations and ensures compatibility with existing data structures.
main
wizardchen 2026-02-28 17:39:59 +08:00 committed by lyingbug
parent 31755726a0
commit 393ebf9f92
13 changed files with 168 additions and 25 deletions

View File

@ -6,6 +6,7 @@ package client
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
@ -25,7 +26,7 @@ type KnowledgeBase struct {
EmbeddingModelID string `json:"embedding_model_id"`
SummaryModelID string `json:"summary_model_id"`
VLMConfig VLMConfig `json:"vlm_config"`
StorageConfig StorageConfig `json:"cos_config"`
StorageConfig StorageConfig `json:"storage_config"`
ExtractConfig *ExtractConfig `json:"extract_config"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
@ -99,6 +100,25 @@ type GraphRelation struct {
Type string `json:"type"`
}
// UnmarshalJSON keeps backward compatibility for legacy responses that still
// use `cos_config` instead of `storage_config`.
func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error {
type alias KnowledgeBase
aux := struct {
*alias
LegacyStorageConfig *StorageConfig `json:"cos_config"`
}{
alias: (*alias)(kb),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
if aux.LegacyStorageConfig != nil && kb.StorageConfig == (StorageConfig{}) {
kb.StorageConfig = *aux.LegacyStorageConfig
}
return nil
}
// KnowledgeBaseResponse knowledge base response
type KnowledgeBaseResponse struct {
Success bool `json:"success"`

View File

@ -19,7 +19,7 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \
sed -i "s@deb.debian.org@${APK_MIRROR_ARG}@g" /etc/apt/sources.list.d/debian.sources; \
fi && \
apt-get update && \
apt-get install -y git build-essential
apt-get install -y git build-essential libsqlite3-dev
# Install migrate tool
RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
@ -63,6 +63,7 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \
apt-get update && \
apt-get install -y --no-install-recommends \
build-essential postgresql-client default-mysql-client ca-certificates tzdata sed curl bash vim wget \
libsqlite3-0 \
python3 python3-pip python3-dev libffi-dev libssl-dev \
nodejs npm && \
python3 -m pip install --break-system-packages --upgrade pip setuptools wheel && \

View File

@ -1,5 +1,5 @@
# 构建阶段
FROM node:24-alpine as build-stage
FROM node:24-alpine AS build-stage
WORKDIR /app
@ -22,7 +22,7 @@ COPY . .
RUN pnpm run build
# 生产阶段
FROM nginx:stable-alpine as production-stage
FROM nginx:stable-alpine AS production-stage
# 复制构建产物到nginx服务目录
COPY --from=build-stage /app/dist /usr/share/nginx/html

View File

@ -19,7 +19,7 @@ export function createKnowledgeBase(data: {
enabled: boolean;
model_id?: string;
};
cos_config?: any;
storage_config?: any;
extract_config?: any;
faq_config?: { index_mode: string; question_index_mode?: string };
}) {

View File

@ -131,6 +131,7 @@
<div v-if="!isFAQ && formData" v-show="currentSection === 'storage'" class="section">
<KBStorageSettings
:storage-provider="formData.storageProvider"
:has-files="mode === 'edit' && hasFiles"
@update:storage-provider="handleStorageProviderUpdate"
/>
</div>
@ -374,7 +375,7 @@ const loadKBData = async () => {
separators: kb.chunking_config?.separators || ['\n\n', '\n', '。', '!', '?', ';', ';'],
parserEngineRules: kb.chunking_config?.parser_engine_rules || undefined
},
storageProvider: (kb.cos_config?.provider || 'local') as string,
storageProvider: (kb.storage_config?.provider || 'local') as string,
multimodalConfig: {
enabled: !!kb.vlm_config?.enabled,
vllmModelId: kb.vlm_config?.model_id || ''
@ -519,7 +520,7 @@ const buildSubmitData = () => {
}
// 存储引擎:仅传 provider,参数从全局设置读取
data.cos_config = {
data.storage_config = {
provider: formData.value.storageProvider || 'local'
}
@ -607,7 +608,7 @@ const handleSubmit = async () => {
multimodal: {
enabled: !!data.vlm_config?.enabled
},
storageProvider: data.cos_config?.provider || 'local',
storageProvider: data.storage_config?.provider || 'local',
nodeExtract: {
enabled: data.extract_config?.enabled || false,
text: data.extract_config?.text || '',

View File

@ -143,7 +143,7 @@
<t-icon name="relation" size="14px" />
</div>
</t-tooltip>
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<t-tooltip v-if="kb.vlm_config?.enabled" :content="$t('knowledgeList.features.multimodal')" placement="top">
<div class="feature-badge multimodal">
<t-icon name="image" size="14px" />
</div>
@ -211,7 +211,7 @@
<t-icon name="relation" size="14px" />
</div>
</t-tooltip>
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.storage_config?.provider && kb.storage_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<div class="feature-badge multimodal">
<t-icon name="image" size="14px" />
</div>
@ -308,7 +308,7 @@
<t-icon name="relation" size="14px" />
</div>
</t-tooltip>
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.cos_config?.provider && kb.cos_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<t-tooltip v-if="kb.vlm_config?.enabled || (kb.storage_config?.provider && kb.storage_config?.bucket_name)" :content="$t('knowledgeList.features.multimodal')" placement="top">
<div class="feature-badge multimodal">
<t-icon name="image" size="14px" />
</div>
@ -616,7 +616,7 @@ interface KB {
showMore?: boolean;
vlm_config?: { enabled?: boolean; model_id?: string };
extract_config?: { enabled?: boolean };
cos_config?: { provider?: string; bucket_name?: string };
storage_config?: { provider?: string; bucket_name?: string };
question_generation_config?: { enabled?: boolean; question_count?: number };
knowledge_count?: number;
chunk_count?: number;

View File

@ -24,6 +24,7 @@
size="medium"
placeholder="请选择存储引擎"
style="width: 100%; min-width: 220px;"
:disabled="props.hasFiles"
@change="handleChange"
>
<t-option
@ -40,7 +41,8 @@
</span>
</t-option>
</t-select>
<p v-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
<p v-if="props.hasFiles" class="option-hint locked-hint">知识库中已有文件,无法切换存储引擎。如需更换,请先清空知识库中的所有文件。</p>
<p v-else-if="selectedOption?.desc" class="option-hint">{{ selectedOption.desc }}</p>
<a v-if="showGoSettings" href="javascript:void(0)" class="go-settings" @click.prevent="goToStorageSettings">去全局设置中配置</a>
</div>
</div>
@ -55,6 +57,7 @@ import { useUIStore } from '@/stores/ui'
const props = defineProps<{
storageProvider: string
hasFiles?: boolean
}>()
const emit = defineEmits<{
@ -236,6 +239,10 @@ onMounted(load)
color: #999;
margin: 0;
line-height: 1.4;
&.locked-hint {
color: #e6a23c;
}
}
.go-settings {

View File

@ -7,6 +7,7 @@ import (
"mime/multipart"
"os"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
@ -101,7 +102,8 @@ func (s *localFileService) SaveFile(ctx context.Context,
func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
logger.Infof(ctx, "Getting file: %s", filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, filePath)
candidate := s.normalizePathForBase(filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate)
if err != nil {
logger.Errorf(ctx, "Path traversal denied for GetFile: %v", err)
return nil, fmt.Errorf("invalid file path: %w", err)
@ -123,7 +125,8 @@ func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.Rea
func (s *localFileService) DeleteFile(ctx context.Context, filePath string) error {
logger.Infof(ctx, "Deleting file: %s", filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, filePath)
candidate := s.normalizePathForBase(filePath)
resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate)
if err != nil {
logger.Errorf(ctx, "Path traversal denied for DeleteFile: %v", err)
return fmt.Errorf("invalid file path: %w", err)
@ -180,3 +183,26 @@ func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (str
// Local storage doesn't support URLs, return the path
return filePath, nil
}
// normalizePathForBase keeps backward compatibility for legacy file paths:
// - absolute path: "/data/files/tenant/.."
// - path under base dir: "tenant/.."
// - legacy relative with base prefix: "data/files/tenant/.."
func (s *localFileService) normalizePathForBase(filePath string) string {
clean := filepath.Clean(strings.TrimSpace(filePath))
if clean == "." || clean == "" {
return clean
}
if filepath.IsAbs(clean) {
return clean
}
// Strip duplicated base prefix in legacy relative paths, e.g. "data/files/..."
baseClean := filepath.Clean(s.baseDir)
baseNoSlash := strings.Trim(baseClean, string(filepath.Separator))
cleanNoDot := strings.TrimPrefix(clean, "."+string(filepath.Separator))
if strings.HasPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator)) {
cleanNoDot = strings.TrimPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator))
}
return filepath.Join(baseClean, cleanNoDot)
}

View File

@ -92,7 +92,13 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
logger.Infof(ctx, "[ImageMultimodal] Image downloaded from URL, len=%d", len(imgBytes))
} else {
logger.Infof(ctx, "[ImageMultimodal] Image bytes read from %s, len=%d", payload.ImageLocalPath, len(imgBytes))
defer os.Remove(payload.ImageLocalPath)
// Only clean up the local file when images have been re-uploaded to
// external object storage (COS/TOS) — the serving URL will be an
// absolute http(s) URL in that case. For local and MinIO storage the
// local file IS the serving file and must be kept.
if strings.HasPrefix(payload.ImageURL, "http://") || strings.HasPrefix(payload.ImageURL, "https://") {
defer os.Remove(payload.ImageLocalPath)
}
}
imageInfo := types.ImageInfo{

View File

@ -6756,6 +6756,8 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
}
if hasKBFull {
logger.Infof(ctx, "[storage] buildStorageConfig use kb config: kb=%s provider=%s bucket=%s path_prefix=%s",
kb.ID, provider, sc.BucketName, sc.PathPrefix)
return &types.DocParserStorageConfig{
Provider: strings.ToUpper(provider),
Region: sc.Region,
@ -6809,6 +6811,8 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
}
}
logger.Infof(ctx, "[storage] buildStorageConfig use merged tenant/global config: kb=%s provider=%s bucket=%s path_prefix=%s endpoint=%s",
kb.ID, strings.ToLower(out.Provider), out.BucketName, out.PathPrefix, out.Endpoint)
return &out
}
@ -6817,6 +6821,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno
// Falls back to the global fileSvc when no tenant-level storage config is found.
func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.KnowledgeBase) interfaces.FileService {
if kb == nil {
logger.Infof(ctx, "[storage] resolveFileService fallback default: kb=nil")
return s.fileSvc
}
@ -6828,6 +6833,8 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
}
if provider == "" || tenant == nil || tenant.StorageEngineConfig == nil {
logger.Infof(ctx, "[storage] resolveFileService fallback default: kb=%s provider=%q tenant_cfg=%v",
kb.ID, provider, tenant != nil && tenant.StorageEngineConfig != nil)
return s.fileSvc
}
@ -6837,10 +6844,28 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
switch provider {
case "local":
baseDir := "/data/files"
if sec.Local != nil && sec.Local.PathPrefix != "" {
baseDir = sec.Local.PathPrefix
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
if baseDir == "" {
baseDir = "/data/files"
}
if sec.Local != nil && sec.Local.PathPrefix != "" {
// path_prefix is a subdirectory under local storage base dir, not a replacement.
// Validate it cannot escape baseDir (e.g. path traversal via "..").
rawPrefix := strings.TrimSpace(sec.Local.PathPrefix)
prefix := strings.Trim(rawPrefix, "/\\")
if prefix != "" {
candidate := filepath.Join(baseDir, prefix)
safeBaseDir, pathErr := secutils.SafePathUnderBase(baseDir, candidate)
if pathErr != nil {
logger.Warnf(ctx,
"[storage] invalid local path_prefix ignored: kb=%s raw_prefix=%q err=%v",
kb.ID, rawPrefix, pathErr)
} else {
baseDir = safeBaseDir
}
}
}
logger.Infof(ctx, "[storage] resolveFileService local: kb=%s base_dir=%s", kb.ID, baseDir)
return filesvc.NewLocalFileService(baseDir)
case "minio":
@ -6886,6 +6911,7 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno
logger.Errorf(ctx, "Failed to create %s file service from tenant config: %v, falling back to default", provider, err)
return s.fileSvc
}
logger.Infof(ctx, "[storage] resolveFileService selected: kb=%s provider=%s", kb.ID, provider)
return svc
}
@ -7275,13 +7301,27 @@ func (s *knowledgeService) convert(
return nil, nil
}
imageStorage := s.buildImageStorageConfig(ctx, kb)
if imageStorage == nil {
logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s image_storage=nil(use local temp/files)",
kb.ID, strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)))
} else {
logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s bucket=%s endpoint=%s path_prefix=%s",
kb.ID,
imageStorage["provider"],
imageStorage["bucket_name"],
imageStorage["endpoint"],
imageStorage["path_prefix"],
)
}
req := &types.ReadRequest{
URL: payload.URL,
Title: knowledge.Title,
ParserEngine: parserEngine,
RequestID: payload.RequestId,
ParserEngineOverrides: overrides,
ImageStorage: s.buildImageStorageConfig(ctx, kb),
ImageStorage: imageStorage,
}
if !isURL {

View File

@ -302,6 +302,19 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
if provider == "" {
provider = "local"
}
oldProvider := strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider))
if oldProvider == "" {
oldProvider = "local"
}
if oldProvider != provider {
knowledgeList, err := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx,
kbIdStr, &types.Pagination{Page: 1, PageSize: 1}, "", "", "")
if err == nil && knowledgeList != nil && knowledgeList.Total > 0 {
logger.Error(ctx, "Cannot change storage engine when files exist")
c.Error(errors.NewBadRequestError("知识库中已有文件,无法切换存储引擎"))
return
}
}
kb.StorageConfig = types.StorageConfig{Provider: provider}
// 更新知识图谱配置

View File

@ -619,18 +619,28 @@ func serveLocalFiles(r *gin.Engine) {
logger.Infof(context.Background(), "[Router] Serving local storage files from %s at /files/", absDir)
r.GET("/files/*filepath", func(c *gin.Context) {
reqPath := c.Param("filepath")
fullPath := filepath.Join(absDir, filepath.Clean(reqPath))
if !strings.HasPrefix(fullPath, absDir) {
reqPath := strings.TrimPrefix(c.Param("filepath"), "/")
cleanPath := filepath.Clean(reqPath)
if cleanPath == "." || strings.HasPrefix(cleanPath, "..") {
logger.Warnf(context.Background(), "[Router] Reject invalid /files path: raw=%q clean=%q", c.Param("filepath"), cleanPath)
c.Status(http.StatusForbidden)
return
}
fullPath := filepath.Join(absDir, cleanPath)
if !strings.HasPrefix(fullPath, absDir) {
logger.Warnf(context.Background(), "[Router] Reject escaped /files path: raw=%q full=%q base=%q", c.Param("filepath"), fullPath, absDir)
c.Status(http.StatusForbidden)
return
}
info, err := os.Stat(fullPath)
if err != nil || info.IsDir() {
logger.Warnf(context.Background(), "[Router] /files not found: raw=%q full=%q err=%v", c.Param("filepath"), fullPath, err)
c.Status(http.StatusNotFound)
return
}
logger.Infof(context.Background(), "[Router] /files hit: raw=%q full=%q", c.Param("filepath"), fullPath)
c.File(fullPath)
})
}

View File

@ -60,7 +60,7 @@ type KnowledgeBase struct {
// VLM config
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
// Storage config
StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"`
StorageConfig StorageConfig `yaml:"storage_config" json:"storage_config" gorm:"column:cos_config;type:json"`
// Extract config
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
// FAQConfig stores FAQ specific configuration such as indexing strategy
@ -163,6 +163,25 @@ func (c *StorageConfig) Scan(value interface{}) error {
return json.Unmarshal(b, c)
}
// UnmarshalJSON keeps backward compatibility for legacy clients that still send
// `cos_config`, while standardizing the API field as `storage_config`.
func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error {
type alias KnowledgeBase
aux := struct {
*alias
LegacyStorageConfig *StorageConfig `json:"cos_config"`
}{
alias: (*alias)(kb),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
if aux.LegacyStorageConfig != nil && kb.StorageConfig == (StorageConfig{}) {
kb.StorageConfig = *aux.LegacyStorageConfig
}
return nil
}
// ImageProcessingConfig represents the image processing configuration
type ImageProcessingConfig struct {
// Model ID