WeKnora/internal/config/config.go

285 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

package config
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/go-viper/mapstructure/v2"
"github.com/spf13/viper"
"gopkg.in/yaml.v3"
)
// Config 应用程序总配置
type Config struct {
Conversation *ConversationConfig `yaml:"conversation" json:"conversation"`
Server *ServerConfig `yaml:"server" json:"server"`
KnowledgeBase *KnowledgeBaseConfig `yaml:"knowledge_base" json:"knowledge_base"`
Tenant *TenantConfig `yaml:"tenant" json:"tenant"`
Models []ModelConfig `yaml:"models" json:"models"`
VectorDatabase *VectorDatabaseConfig `yaml:"vector_database" json:"vector_database"`
DocReader *DocReaderConfig `yaml:"docreader" json:"docreader"`
StreamManager *StreamManagerConfig `yaml:"stream_manager" json:"stream_manager"`
ExtractManager *ExtractManagerConfig `yaml:"extract" json:"extract"`
WebSearch *WebSearchConfig `yaml:"web_search" json:"web_search"`
PromptTemplates *PromptTemplatesConfig `yaml:"prompt_templates" json:"prompt_templates"`
}
type DocReaderConfig struct {
Addr string `yaml:"addr" json:"addr"`
}
type VectorDatabaseConfig struct {
Driver string `yaml:"driver" json:"driver"`
}
// ConversationConfig 对话服务配置
type ConversationConfig struct {
MaxRounds int `yaml:"max_rounds" json:"max_rounds"`
KeywordThreshold float64 `yaml:"keyword_threshold" json:"keyword_threshold"`
EmbeddingTopK int `yaml:"embedding_top_k" json:"embedding_top_k"`
VectorThreshold float64 `yaml:"vector_threshold" json:"vector_threshold"`
RerankTopK int `yaml:"rerank_top_k" json:"rerank_top_k"`
RerankThreshold float64 `yaml:"rerank_threshold" json:"rerank_threshold"`
FallbackStrategy string `yaml:"fallback_strategy" json:"fallback_strategy"`
FallbackResponse string `yaml:"fallback_response" json:"fallback_response"`
FallbackPrompt string `yaml:"fallback_prompt" json:"fallback_prompt"`
EnableRewrite bool `yaml:"enable_rewrite" json:"enable_rewrite"`
EnableQueryExpansion bool `yaml:"enable_query_expansion" json:"enable_query_expansion"`
EnableRerank bool `yaml:"enable_rerank" json:"enable_rerank"`
Summary *SummaryConfig `yaml:"summary" json:"summary"`
GenerateSessionTitlePrompt string `yaml:"generate_session_title_prompt" json:"generate_session_title_prompt"`
GenerateSummaryPrompt string `yaml:"generate_summary_prompt" json:"generate_summary_prompt"`
RewritePromptSystem string `yaml:"rewrite_prompt_system" json:"rewrite_prompt_system"`
RewritePromptUser string `yaml:"rewrite_prompt_user" json:"rewrite_prompt_user"`
SimplifyQueryPrompt string `yaml:"simplify_query_prompt" json:"simplify_query_prompt"`
SimplifyQueryPromptUser string `yaml:"simplify_query_prompt_user" json:"simplify_query_prompt_user"`
ExtractEntitiesPrompt string `yaml:"extract_entities_prompt" json:"extract_entities_prompt"`
ExtractRelationshipsPrompt string `yaml:"extract_relationships_prompt" json:"extract_relationships_prompt"`
// GenerateQuestionsPrompt is used to generate questions for document chunks to improve recall
GenerateQuestionsPrompt string `yaml:"generate_questions_prompt" json:"generate_questions_prompt"`
}
// SummaryConfig 摘要配置
type SummaryConfig struct {
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
RepeatPenalty float64 `yaml:"repeat_penalty" json:"repeat_penalty"`
TopK int `yaml:"top_k" json:"top_k"`
TopP float64 `yaml:"top_p" json:"top_p"`
FrequencyPenalty float64 `yaml:"frequency_penalty" json:"frequency_penalty"`
PresencePenalty float64 `yaml:"presence_penalty" json:"presence_penalty"`
Prompt string `yaml:"prompt" json:"prompt"`
ContextTemplate string `yaml:"context_template" json:"context_template"`
Temperature float64 `yaml:"temperature" json:"temperature"`
Seed int `yaml:"seed" json:"seed"`
MaxCompletionTokens int `yaml:"max_completion_tokens" json:"max_completion_tokens"`
NoMatchPrefix string `yaml:"no_match_prefix" json:"no_match_prefix"`
Thinking *bool `yaml:"thinking" json:"thinking"`
}
// ServerConfig 服务器配置
type ServerConfig struct {
Port int `yaml:"port" json:"port"`
Host string `yaml:"host" json:"host"`
LogPath string `yaml:"log_path" json:"log_path"`
ShutdownTimeout time.Duration `yaml:"shutdown_timeout" json:"shutdown_timeout" default:"30s"`
}
// KnowledgeBaseConfig 知识库配置
type KnowledgeBaseConfig struct {
ChunkSize int `yaml:"chunk_size" json:"chunk_size"`
ChunkOverlap int `yaml:"chunk_overlap" json:"chunk_overlap"`
SplitMarkers []string `yaml:"split_markers" json:"split_markers"`
KeepSeparator bool `yaml:"keep_separator" json:"keep_separator"`
ImageProcessing *ImageProcessingConfig `yaml:"image_processing" json:"image_processing"`
}
// ImageProcessingConfig 图像处理配置
type ImageProcessingConfig struct {
EnableMultimodal bool `yaml:"enable_multimodal" json:"enable_multimodal"`
}
// TenantConfig 租户配置
type TenantConfig struct {
DefaultSessionName string `yaml:"default_session_name" json:"default_session_name"`
DefaultSessionTitle string `yaml:"default_session_title" json:"default_session_title"`
DefaultSessionDescription string `yaml:"default_session_description" json:"default_session_description"`
// EnableCrossTenantAccess enables cross-tenant access for users with permission
EnableCrossTenantAccess bool `yaml:"enable_cross_tenant_access" json:"enable_cross_tenant_access"`
}
// PromptTemplate 提示词模板
type PromptTemplate struct {
ID string `yaml:"id" json:"id"`
Name string `yaml:"name" json:"name"`
Description string `yaml:"description" json:"description"`
Content string `yaml:"content" json:"content"`
HasKnowledgeBase bool `yaml:"has_knowledge_base" json:"has_knowledge_base,omitempty"`
HasWebSearch bool `yaml:"has_web_search" json:"has_web_search,omitempty"`
}
// PromptTemplatesConfig 提示词模板配置
type PromptTemplatesConfig struct {
SystemPrompt []PromptTemplate `yaml:"system_prompt" json:"system_prompt"`
ContextTemplate []PromptTemplate `yaml:"context_template" json:"context_template"`
RewriteSystem []PromptTemplate `yaml:"rewrite_system" json:"rewrite_system"`
RewriteUser []PromptTemplate `yaml:"rewrite_user" json:"rewrite_user"`
Fallback []PromptTemplate `yaml:"fallback" json:"fallback"`
}
// ModelConfig 模型配置
type ModelConfig struct {
Type string `yaml:"type" json:"type"`
Source string `yaml:"source" json:"source"`
ModelName string `yaml:"model_name" json:"model_name"`
Parameters map[string]interface{} `yaml:"parameters" json:"parameters"`
}
// StreamManagerConfig 流管理器配置
type StreamManagerConfig struct {
Type string `yaml:"type" json:"type"` // 类型: "memory" 或 "redis"
Redis RedisConfig `yaml:"redis" json:"redis"` // Redis配置
CleanupTimeout time.Duration `yaml:"cleanup_timeout" json:"cleanup_timeout"` // 清理超时,单位秒
}
// RedisConfig Redis配置
type RedisConfig struct {
Address string `yaml:"address" json:"address"` // Redis地址
Username string `yaml:"username" json:"username"` // Redis用户名
Password string `yaml:"password" json:"password"` // Redis密码
DB int `yaml:"db" json:"db"` // Redis数据库
Prefix string `yaml:"prefix" json:"prefix"` // 键前缀
TTL time.Duration `yaml:"ttl" json:"ttl"` // 过期时间(小时)
}
// ExtractManagerConfig 抽取管理器配置
type ExtractManagerConfig struct {
ExtractGraph *types.PromptTemplateStructured `yaml:"extract_graph" json:"extract_graph"`
ExtractEntity *types.PromptTemplateStructured `yaml:"extract_entity" json:"extract_entity"`
FabriText *FebriText `yaml:"fabri_text" json:"fabri_text"`
}
type FebriText struct {
WithTag string `yaml:"with_tag" json:"with_tag"`
WithNoTag string `yaml:"with_no_tag" json:"with_no_tag"`
}
// LoadConfig 从配置文件加载配置
func LoadConfig() (*Config, error) {
// 设置配置文件名和路径
viper.SetConfigName("config") // 配置文件名称(不带扩展名)
viper.SetConfigType("yaml") // 配置文件类型
viper.AddConfigPath(".") // 当前目录
viper.AddConfigPath("./config") // config子目录
viper.AddConfigPath("$HOME/.appname") // 用户目录
viper.AddConfigPath("/etc/appname/") // etc目录
// 启用环境变量替换
viper.AutomaticEnv()
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
// 读取配置文件
if err := viper.ReadInConfig(); err != nil {
return nil, fmt.Errorf("error reading config file: %w", err)
}
// 替换配置中的环境变量引用
configFileContent, err := os.ReadFile(viper.ConfigFileUsed())
if err != nil {
return nil, fmt.Errorf("error reading config file content: %w", err)
}
// 替换${ENV_VAR}格式的环境变量引用
re := regexp.MustCompile(`\${([^}]+)}`)
result := re.ReplaceAllStringFunc(string(configFileContent), func(match string) string {
// 提取环境变量名称(去掉${}部分)
envVar := match[2 : len(match)-1]
// 获取环境变量值,如果不存在则保持原样
if value := os.Getenv(envVar); value != "" {
return value
}
return match
})
// 使用处理后的配置内容
viper.ReadConfig(strings.NewReader(result))
// 解析配置到结构体
var cfg Config
if err := viper.Unmarshal(&cfg, func(dc *mapstructure.DecoderConfig) {
dc.TagName = "yaml"
}); err != nil {
return nil, fmt.Errorf("unable to decode config into struct: %w", err)
}
fmt.Printf("Using configuration file: %s\n", viper.ConfigFileUsed())
// 加载提示词模板(从目录或配置文件)
configDir := filepath.Dir(viper.ConfigFileUsed())
promptTemplates, err := loadPromptTemplates(configDir)
if err != nil {
fmt.Printf("Warning: failed to load prompt templates from directory: %v\n", err)
// 如果目录加载失败,使用配置文件中的模板(如果有)
} else if promptTemplates != nil {
cfg.PromptTemplates = promptTemplates
}
return &cfg, nil
}
// promptTemplateFile 用于解析模板文件
type promptTemplateFile struct {
Templates []PromptTemplate `yaml:"templates"`
}
// loadPromptTemplates 从目录加载提示词模板
func loadPromptTemplates(configDir string) (*PromptTemplatesConfig, error) {
templatesDir := filepath.Join(configDir, "prompt_templates")
// 检查目录是否存在
if _, err := os.Stat(templatesDir); os.IsNotExist(err) {
return nil, nil // 目录不存在,返回nil让调用者使用配置文件中的模板
}
config := &PromptTemplatesConfig{}
// 定义模板文件映射
templateFiles := map[string]*[]PromptTemplate{
"system_prompt.yaml": &config.SystemPrompt,
"context_template.yaml": &config.ContextTemplate,
"rewrite_system.yaml": &config.RewriteSystem,
"rewrite_user.yaml": &config.RewriteUser,
"fallback.yaml": &config.Fallback,
}
// 加载每个模板文件
for filename, target := range templateFiles {
filePath := filepath.Join(templatesDir, filename)
if _, err := os.Stat(filePath); os.IsNotExist(err) {
continue // 文件不存在,跳过
}
data, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %w", filename, err)
}
var file promptTemplateFile
if err := yaml.Unmarshal(data, &file); err != nil {
return nil, fmt.Errorf("failed to parse %s: %w", filename, err)
}
*target = file.Templates
}
return config, nil
}
// WebSearchConfig represents the web search configuration
type WebSearchConfig struct {
Timeout int `yaml:"timeout" json:"timeout"` // 超时时间(秒)
}