feat(m8): P2 pgvector + Ollama 向量化与语义检索
- PostgreSQL 切换 pgvector/pgvector:pg17 镜像;迁移 000024 建 vector 扩展、 knowledge_chunks.embedding vector(1024) + HNSW 余弦索引,retrieval_mode 放宽三态 - OllamaEmbedder 本地 bge-m3 批量嵌入,404 惰性 pull 重试,维度/超时校验,可整体关闭 - SemanticRetriever/HybridRetriever + NewRetriever 按 retrieval_mode 分发,缺 embedder 回退 FTS - 文档入库同步批量向量化;Ollama 故障降级入库 + embedding_failed 事件 - 修复 pgx CopyFrom 对 vector 列二进制编码误读:COPY 基础列后同事务 unnest 批量回填 - 修复降级路径 embeddings=nil 索引越界 panic(Add 与 Reprocess) - 知识库列表 vectorized_chunk_count + 前端三态检索模式选择与向量化覆盖率 - 单测 embedder/retrievers + 集成 TestKnowledgeVectorLifecycle 全绿 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -22,8 +22,9 @@ type Config struct {
|
||||
Audit Audit
|
||||
Outbox Outbox
|
||||
RuntimeData RuntimeData
|
||||
Shadow Shadow
|
||||
Shadow Shadow
|
||||
ObjectStorage ObjectStorage
|
||||
Embeddings Embeddings
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
@@ -125,6 +126,17 @@ type ObjectStorage struct {
|
||||
MaxFileBytes int64
|
||||
}
|
||||
|
||||
// Embeddings 配置本地 Ollama 向量化(默认 bge-m3)。Enabled 为 false 时网关不构造
|
||||
// OllamaEmbedder,知识库退回纯 FTS 检索,AddKnowledgeDocument 不生成 embedding。
|
||||
type Embeddings struct {
|
||||
Enabled bool
|
||||
BaseURL string
|
||||
Model string
|
||||
Dim int
|
||||
BatchSize int
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
cfg := Config{
|
||||
Environment: env("APP_ENV", "local"),
|
||||
@@ -202,6 +214,14 @@ func Load() (Config, error) {
|
||||
UseSSL: boolValue("S3_USE_SSL", false),
|
||||
MaxFileBytes: int64Value("S3_MAX_FILE_BYTES", 128<<20),
|
||||
},
|
||||
Embeddings: Embeddings{
|
||||
Enabled: boolValue("EMBEDDINGS_ENABLED", true),
|
||||
BaseURL: strings.TrimRight(env("OLLAMA_BASE_URL", "http://ollama:11434"), "/"),
|
||||
Model: env("EMBEDDING_MODEL", "bge-m3"),
|
||||
Dim: intValue("EMBEDDING_DIM", 1024),
|
||||
BatchSize: intValue("EMBEDDING_BATCH_SIZE", 64),
|
||||
Timeout: duration("EMBEDDING_TIMEOUT", 120*time.Second),
|
||||
},
|
||||
}
|
||||
|
||||
return cfg, cfg.Validate()
|
||||
@@ -271,6 +291,27 @@ func (c Config) Validate() error {
|
||||
if c.ObjectStorage.MaxFileBytes < 1<<20 || c.ObjectStorage.MaxFileBytes > 512<<20 {
|
||||
errs = append(errs, errors.New("S3_MAX_FILE_BYTES must be between 1 MiB and 512 MiB"))
|
||||
}
|
||||
if c.Embeddings.Enabled {
|
||||
if err := validateHTTPURL(c.Embeddings.BaseURL); err != nil {
|
||||
errs = append(errs, fmt.Errorf("OLLAMA_BASE_URL: %w", err))
|
||||
}
|
||||
if c.Embeddings.Model == "" {
|
||||
errs = append(errs, errors.New("EMBEDDING_MODEL is required when embeddings are enabled"))
|
||||
}
|
||||
if c.Embeddings.Dim < 128 || c.Embeddings.Dim > 8192 {
|
||||
errs = append(errs, errors.New("EMBEDDING_DIM must be between 128 and 8192"))
|
||||
}
|
||||
if c.Embeddings.Dim != 1024 {
|
||||
// 知识库 embedding 列固定为 vector(1024);维度不符会让入库向量报错。
|
||||
errs = append(errs, errors.New("EMBEDDING_DIM must be 1024 to match the vector(1024) column"))
|
||||
}
|
||||
if c.Embeddings.BatchSize < 1 || c.Embeddings.BatchSize > 512 {
|
||||
errs = append(errs, errors.New("EMBEDDING_BATCH_SIZE must be between 1 and 512"))
|
||||
}
|
||||
if c.Embeddings.Timeout < time.Second || c.Embeddings.Timeout > 30*time.Minute {
|
||||
errs = append(errs, errors.New("EMBEDDING_TIMEOUT must be between 1s and 30m"))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user