feat(m8): P2 pgvector + Ollama 向量化与语义检索

- PostgreSQL 切换 pgvector/pgvector:pg17 镜像;迁移 000024 建 vector 扩展、
  knowledge_chunks.embedding vector(1024) + HNSW 余弦索引,retrieval_mode 放宽三态
- OllamaEmbedder 本地 bge-m3 批量嵌入,404 惰性 pull 重试,维度/超时校验,可整体关闭
- SemanticRetriever/HybridRetriever + NewRetriever 按 retrieval_mode 分发,缺 embedder 回退 FTS
- 文档入库同步批量向量化;Ollama 故障降级入库 + embedding_failed 事件
- 修复 pgx CopyFrom 对 vector 列二进制编码误读:COPY 基础列后同事务 unnest 批量回填
- 修复降级路径 embeddings=nil 索引越界 panic(Add 与 Reprocess)
- 知识库列表 vectorized_chunk_count + 前端三态检索模式选择与向量化覆盖率
- 单测 embedder/retrievers + 集成 TestKnowledgeVectorLifecycle 全绿

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
ben
2026-08-12 15:16:32 +08:00
parent 6708c226a5
commit b536672000
26 changed files with 966 additions and 60 deletions
+16 -2
View File
@@ -202,6 +202,20 @@ func main() {
pricingHandler := pricing.NewAdminHTTPHandler(pricingService, identityService)
factCheckHandler := factcheck.NewAdminHTTPHandler(factcheck.NewService(db), identityService)
workbenchService := workbench.NewService(db)
// M8 P2:本地 Ollama 向量化。EMBEDDINGS_ENABLED=false 时不构造 embedder,
// 知识库检索自动回退纯 FTS;Ollama 挂时入库降级(embedding 置 NULL)。
if cfg.Embeddings.Enabled {
workbenchService.SetEmbedder(workbench.NewOllamaEmbedder(workbench.OllamaEmbedderConfig{
BaseURL: cfg.Embeddings.BaseURL,
Model: cfg.Embeddings.Model,
Dim: cfg.Embeddings.Dim,
BatchSize: cfg.Embeddings.BatchSize,
Timeout: cfg.Embeddings.Timeout,
}))
logger.Info("knowledge embeddings enabled", "model", cfg.Embeddings.Model, "base_url", cfg.Embeddings.BaseURL)
} else {
logger.Info("knowledge embeddings disabled, knowledge retrieval uses postgres_fts only")
}
toolCipher, err := cryptox.NewKeyring(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring, "tool-request-headers",
)
@@ -263,7 +277,7 @@ func main() {
}
shadowMiddleware := shadow.New(cfg.Shadow, logger)
governedGateway := shadowMiddleware.Wrap(proxy)
workbenchRuntime := workbench.NewRuntimeHTTPHandler(workbenchService, toolService, workbench.NewPostgreSQLRetriever(workbenchService), apiKeyAuthenticator, governedGateway, workbench.MarketplaceDeps{
workbenchRuntime := workbench.NewRuntimeHTTPHandler(workbenchService, toolService, workbench.NewRetriever(workbenchService, workbenchService.Embedder()), apiKeyAuthenticator, governedGateway, workbench.MarketplaceDeps{
MCPServers: mcpServerService,
Skills: skillService,
Employees: digitalEmployeeService,
@@ -273,7 +287,7 @@ func main() {
workbenchRuntime.SetLogger(logger)
// Wire the fact-check engine: the admin fact-check settings/policies UI now
// actually governs application answers instead of being inert configuration.
factCheckEngine := factcheck.NewEngine(db, workbench.NewFactCheckRetriever(workbench.NewPostgreSQLRetriever(workbenchService)), logger)
factCheckEngine := factcheck.NewEngine(db, workbench.NewFactCheckRetriever(workbench.NewRetriever(workbenchService, workbenchService.Embedder())), logger)
workbenchRuntime.SetFactCheckEngine(factCheckEngine)
portalService := portal.NewService(db, workbenchService, toolService, identityService)
portalService.SetApplicationRuntime(portal.NewRuntimeCredentials(db, apiKeyRepository, applicationKeyCipher), workbenchRuntime)