Files
ai-gateway-go/cmd/gateway-api/main.go
T
superidou b536672000 feat(m8): P2 pgvector + Ollama 向量化与语义检索
- PostgreSQL 切换 pgvector/pgvector:pg17 镜像;迁移 000024 建 vector 扩展、
  knowledge_chunks.embedding vector(1024) + HNSW 余弦索引,retrieval_mode 放宽三态
- OllamaEmbedder 本地 bge-m3 批量嵌入,404 惰性 pull 重试,维度/超时校验,可整体关闭
- SemanticRetriever/HybridRetriever + NewRetriever 按 retrieval_mode 分发,缺 embedder 回退 FTS
- 文档入库同步批量向量化;Ollama 故障降级入库 + embedding_failed 事件
- 修复 pgx CopyFrom 对 vector 列二进制编码误读:COPY 基础列后同事务 unnest 批量回填
- 修复降级路径 embeddings=nil 索引越界 panic(Add 与 Reprocess)
- 知识库列表 vectorized_chunk_count + 前端三态检索模式选择与向量化覆盖率
- 单测 embedder/retrievers + 集成 TestKnowledgeVectorLifecycle 全绿

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-12 15:16:32 +08:00

425 lines
20 KiB
Go

package main
import (
"context"
"errors"
"log/slog"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"aigateway.local/core/internal/apikey"
"aigateway.local/core/internal/audit"
"aigateway.local/core/internal/contentpolicy"
"aigateway.local/core/internal/factcheck"
"aigateway.local/core/internal/gateway"
"aigateway.local/core/internal/identity"
"aigateway.local/core/internal/operations"
"aigateway.local/core/internal/outbox"
"aigateway.local/core/internal/platform/cache"
"aigateway.local/core/internal/platform/config"
"aigateway.local/core/internal/platform/cryptox"
"aigateway.local/core/internal/platform/database"
"aigateway.local/core/internal/platform/health"
"aigateway.local/core/internal/platform/storage"
"aigateway.local/core/internal/platform/httpserver"
"aigateway.local/core/internal/portal"
"aigateway.local/core/internal/pricing"
"aigateway.local/core/internal/provider"
providercontrolplane "aigateway.local/core/internal/provider/controlplane"
provideropenai "aigateway.local/core/internal/provider/openai"
providerruntime "aigateway.local/core/internal/provider/runtime"
"aigateway.local/core/internal/shadow"
"aigateway.local/core/internal/workbench"
)
var version = "dev"
func main() {
logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
cfg, err := config.Load()
if err != nil {
logger.Error("invalid configuration", "error", err)
os.Exit(1)
}
if err := cfg.ValidateRuntime(); err != nil {
logger.Error("invalid runtime configuration", "error", err)
os.Exit(1)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
db, err := database.Open(ctx, cfg.Database)
if err != nil {
logger.Error("database initialization failed", "error", err)
os.Exit(1)
}
if db != nil {
defer db.Close()
}
criticalRedis, err := cache.Open(cfg.Redis.CriticalURL)
if err != nil {
logger.Error("critical redis initialization failed", "error", err)
os.Exit(1)
}
if criticalRedis != nil {
defer criticalRedis.Close()
}
cacheRedis, err := cache.Open(cfg.Redis.CacheURL)
if err != nil {
logger.Error("cache redis initialization failed", "error", err)
os.Exit(1)
}
if cacheRedis != nil {
defer cacheRedis.Close()
}
checker := health.Checker{Timeout: 1500 * time.Millisecond}
checker.Dependencies = append(checker.Dependencies,
health.Dependency{Name: "postgres", Required: true, Probe: func(probeCtx context.Context) error {
if db == nil {
return errors.New("not configured")
}
return db.Ping(probeCtx)
}},
health.Dependency{Name: "redis_critical", Required: true, Probe: func(probeCtx context.Context) error {
if criticalRedis == nil {
return errors.New("not configured")
}
return criticalRedis.Ping(probeCtx).Err()
}},
health.Dependency{Name: "redis_cache", Required: false, Probe: func(probeCtx context.Context) error {
if cacheRedis == nil {
return errors.New("not configured")
}
return cacheRedis.Ping(probeCtx).Err()
}},
)
adapter, err := provideropenai.New(cfg.Upstream.BaseURL, cfg.Upstream.APIKey)
if err != nil {
logger.Error("provider initialization failed", "error", err)
os.Exit(1)
}
var fallbackAdapter provider.Adapter
if cfg.Upstream.FallbackEnabled {
fallbackAdapter = adapter
}
credentialCipher, err := provider.NewCredentialCipher(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring,
)
if err != nil {
logger.Error("credential encryption initialization failed", "error", err)
os.Exit(1)
}
providerRepository := provider.NewRepository(db)
providerResolver := providerruntime.NewResolver(
providerRepository, credentialCipher, fallbackAdapter, cfg.Credentials.ProviderRefreshInterval, logger,
)
providerResolver.SetNotificationClient(criticalRedis)
go providerResolver.Run(ctx)
apiKeyRepository := apikey.NewRepository(db)
bootstrapAPIKey := ""
if cfg.Security.BootstrapAPIKeyEnabled {
bootstrapAPIKey = cfg.Security.BootstrapAPIKey
logger.Warn("bootstrap API key compatibility is enabled; monitor usage and disable after migration")
}
apiKeyAuthenticator := apikey.NewAuthenticator(apiKeyRepository, criticalRedis, bootstrapAPIKey)
apiKeyAuthenticator.SetLogger(logger)
proxy := gateway.NewDynamicProxy(providerResolver, apiKeyAuthenticator, cfg.Server.MaxBodyBytes, logger)
proxy.SetAdmissionController(gateway.NewRedisAdmissionController(criticalRedis))
proxy.SetTokenQuotaController(gateway.NewRedisTokenQuotaController(criticalRedis))
proxy.SetResiliencePolicy(gateway.ResiliencePolicy{
ResponseHeaderTimeout: cfg.Upstream.ResponseHeaderTimeout, MaxRetries: cfg.Upstream.MaxRetries,
RetryBackoff: cfg.Upstream.RetryBackoff, CircuitThreshold: cfg.Upstream.CircuitThreshold,
CircuitOpenDuration: cfg.Upstream.CircuitOpenDuration,
})
auditRecorder := audit.NewRecorder(db, logger, cfg.Audit.QueueSize, cfg.Audit.BatchSize, cfg.Audit.FlushInterval)
auditContext, stopAudit := context.WithCancel(context.Background())
auditStopped := make(chan struct{})
go func() {
auditRecorder.Run(auditContext)
close(auditStopped)
}()
proxy.SetAuditRecorder(auditRecorder)
contentPolicyEngine := contentpolicy.NewEngine(db, cfg.RuntimeData.ContentPolicyRefreshInterval, logger)
pricingService := pricing.NewService(db, cfg.RuntimeData.PricingRefreshInterval, logger)
if err := contentPolicyEngine.Reload(ctx); err != nil {
logger.Error("content policy initialization failed", "error", err)
os.Exit(1)
}
if err := pricingService.Reload(ctx); err != nil {
logger.Error("model pricing initialization failed", "error", err)
os.Exit(1)
}
go contentPolicyEngine.Run(ctx)
go pricingService.Run(ctx)
proxy.SetContentPolicyEngine(contentPolicyEngine)
proxy.SetPricingService(pricingService)
identityRepository := identity.NewRepository(db)
sessionStore := identity.NewSessionStore(criticalRedis, cfg.Auth.SessionTTL)
loginLimiter := identity.NewLoginLimiter(criticalRedis, cfg.Auth.LoginRateLimitMax, cfg.Auth.LoginRateLimitWindow)
totpCipher, err := cryptox.NewKeyring(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring, "totp-secret",
)
if err != nil {
logger.Error("TOTP encryption initialization failed", "error", err)
os.Exit(1)
}
identityService := identity.NewService(identityRepository, sessionStore, loginLimiter, cfg.Auth, totpCipher)
idpCipher, err := cryptox.NewKeyring(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring, "identity-provider-credentials",
)
if err != nil {
logger.Error("identity provider encryption initialization failed", "error", err)
os.Exit(1)
}
identityService.SetIdentityProviderCipher(idpCipher, cfg.Credentials.AllowPrivateProviderURL)
identityHandler := identity.NewHTTPHandler(identityService)
identityManagementHandler := identity.NewManagementHTTPHandler(identityService)
providerHandler := provider.NewAdminHTTPHandler(
providerRepository, credentialCipher, identityService, cfg.Credentials.AllowPrivateProviderURL,
)
providerOperations := providercontrolplane.NewService(
providerRepository, credentialCipher, cfg.Credentials.AllowPrivateProviderURL,
)
providerHandler.SetOperations(providerOperations)
providerHandler.SetChangeHook(func(changeCtx context.Context) error {
reloadErr := providerResolver.Reload(changeCtx)
notifyErr := providerResolver.Notify(changeCtx)
if reloadErr != nil || notifyErr != nil {
logger.Warn("provider change propagation was incomplete", "reload_error", reloadErr, "notify_error", notifyErr)
}
return errors.Join(reloadErr, notifyErr)
})
apiKeyHandler := apikey.NewAdminHTTPHandler(apiKeyRepository, apiKeyAuthenticator, identityService)
apiKeyHandler.SetUsageStore(apikey.NewUsageStore(criticalRedis))
auditHandler := audit.NewAdminHTTPHandler(audit.NewQueryService(db), identityService)
outboxHandler := outbox.NewAdminHTTPHandler(outbox.NewStore(db), identityService)
contentPolicyHandler := contentpolicy.NewAdminHTTPHandler(contentpolicy.NewStore(db), contentPolicyEngine, identityService)
pricingHandler := pricing.NewAdminHTTPHandler(pricingService, identityService)
factCheckHandler := factcheck.NewAdminHTTPHandler(factcheck.NewService(db), identityService)
workbenchService := workbench.NewService(db)
// M8 P2:本地 Ollama 向量化。EMBEDDINGS_ENABLED=false 时不构造 embedder,
// 知识库检索自动回退纯 FTS;Ollama 挂时入库降级(embedding 置 NULL)。
if cfg.Embeddings.Enabled {
workbenchService.SetEmbedder(workbench.NewOllamaEmbedder(workbench.OllamaEmbedderConfig{
BaseURL: cfg.Embeddings.BaseURL,
Model: cfg.Embeddings.Model,
Dim: cfg.Embeddings.Dim,
BatchSize: cfg.Embeddings.BatchSize,
Timeout: cfg.Embeddings.Timeout,
}))
logger.Info("knowledge embeddings enabled", "model", cfg.Embeddings.Model, "base_url", cfg.Embeddings.BaseURL)
} else {
logger.Info("knowledge embeddings disabled, knowledge retrieval uses postgres_fts only")
}
toolCipher, err := cryptox.NewKeyring(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring, "tool-request-headers",
)
if err != nil {
logger.Error("tool credential encryption initialization failed", "error", err)
os.Exit(1)
}
notificationCipher, err := cryptox.NewKeyring(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring, "notification-signing-secret",
)
if err != nil {
logger.Error("notification encryption initialization failed", "error", err)
os.Exit(1)
}
toolService := workbench.NewToolService(workbenchService, toolCipher, cfg.Credentials.AllowPrivateToolURL)
notificationService := workbench.NewNotificationService(workbenchService, notificationCipher, cfg.Credentials.AllowPrivateWebhookURL)
workbenchHandler := workbench.NewAdminHTTPHandler(workbenchService, toolService, notificationService, identityService)
mcpServerCipher, err := cryptox.NewKeyring(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring, "mcp-server-headers",
)
if err != nil {
logger.Error("MCP server credential encryption initialization failed", "error", err)
os.Exit(1)
}
mcpServerService := workbench.NewMCPServerService(workbenchService, mcpServerCipher, cfg.Credentials.AllowPrivateToolURL)
skillService := workbench.NewSkillService(workbenchService)
digitalEmployeeService := workbench.NewDigitalEmployeeService(workbenchService, skillService, toolService, mcpServerService)
marketplaceService := workbench.NewMarketplaceService(workbenchService, mcpServerService, skillService, digitalEmployeeService)
mcpClient := workbench.NewMCPClient(cfg.Credentials.AllowPrivateToolURL, 60*time.Second)
marketplaceHandler := workbench.NewMarketplaceAdminHTTPHandler(marketplaceService, mcpServerService, skillService, digitalEmployeeService, mcpClient, identityService)
// M8: 对象存储(MinIO)文件管理。文件体在 MinIO,元数据在 PostgreSQL。MinIO
// 不暴露主机端口,上传/下载全部经网关代理,凭据只留在 API 容器内。
objectStore, err := storage.NewClient(storage.Config{
Endpoint: cfg.ObjectStorage.Endpoint,
AccessKeyID: cfg.ObjectStorage.AccessKeyID,
SecretAccessKey: cfg.ObjectStorage.SecretAccessKey,
Bucket: cfg.ObjectStorage.Bucket,
Region: cfg.ObjectStorage.Region,
UseSSL: cfg.ObjectStorage.UseSSL,
MaxFileBytes: cfg.ObjectStorage.MaxFileBytes,
})
if err != nil {
logger.Error("object storage client initialization failed", "error", err)
os.Exit(1)
}
if err := objectStore.EnsureBucket(ctx); err != nil {
// 桶未就绪不致命:MinIO 起来后会自动建桶,当前上传请求会得到明确报错。
logger.Warn("object storage bucket is not ready; uploads fail until MinIO is reachable", "error", err)
}
fileService := workbench.NewFileService(workbenchService, objectStore)
filesAdminHandler := workbench.NewFilesAdminHTTPHandler(fileService, identityService)
filesPortalHandler := workbench.NewFilesPortalHTTPHandler(fileService, identityService)
applicationKeyCipher, err := cryptox.NewKeyring(
cfg.Credentials.MasterKey, cfg.Credentials.KEKVersion, cfg.Credentials.KEKKeyring, "application-runtime-key",
)
if err != nil {
logger.Error("application runtime credential initialization failed", "error", err)
os.Exit(1)
}
shadowMiddleware := shadow.New(cfg.Shadow, logger)
governedGateway := shadowMiddleware.Wrap(proxy)
workbenchRuntime := workbench.NewRuntimeHTTPHandler(workbenchService, toolService, workbench.NewRetriever(workbenchService, workbenchService.Embedder()), apiKeyAuthenticator, governedGateway, workbench.MarketplaceDeps{
MCPServers: mcpServerService,
Skills: skillService,
Employees: digitalEmployeeService,
Market: marketplaceService,
MCPClient: mcpClient,
})
workbenchRuntime.SetLogger(logger)
// Wire the fact-check engine: the admin fact-check settings/policies UI now
// actually governs application answers instead of being inert configuration.
factCheckEngine := factcheck.NewEngine(db, workbench.NewFactCheckRetriever(workbench.NewRetriever(workbenchService, workbenchService.Embedder())), logger)
workbenchRuntime.SetFactCheckEngine(factCheckEngine)
portalService := portal.NewService(db, workbenchService, toolService, identityService)
portalService.SetApplicationRuntime(portal.NewRuntimeCredentials(db, apiKeyRepository, applicationKeyCipher), workbenchRuntime)
portalService.SetMarketplace(marketplaceService)
portalHandler := portal.NewHTTPHandler(portalService, identityService)
portalAdminHandler := portal.NewAdminHTTPHandler(portalService, identityService)
startedAt := time.Now()
operationsHandler := operations.NewAdminHTTPHandler(db, identityService, version, startedAt, func(reloadCtx context.Context) error {
return errors.Join(providerResolver.Reload(reloadCtx), contentPolicyEngine.Reload(reloadCtx), pricingService.Reload(reloadCtx))
})
controlMux := http.NewServeMux()
controlMux.Handle("/api/v1/admin/providers", providerHandler)
controlMux.Handle("/api/v1/admin/providers/", providerHandler)
controlMux.Handle("/api/v1/admin/model-routes", providerHandler)
controlMux.Handle("/api/v1/admin/model-routes/", providerHandler)
controlMux.Handle("/api/v1/admin/api-keys", apiKeyHandler)
controlMux.Handle("/api/v1/admin/api-keys/", apiKeyHandler)
controlMux.Handle("/api/v1/admin/audit-events", auditHandler)
controlMux.Handle("/api/v1/admin/usage/", auditHandler)
controlMux.Handle("/api/v1/admin/outbox-events", outboxHandler)
controlMux.Handle("/api/v1/admin/outbox-events/", outboxHandler)
controlMux.Handle("/api/v1/admin/content-policies", contentPolicyHandler)
controlMux.Handle("/api/v1/admin/content-policies/", contentPolicyHandler)
controlMux.Handle("/api/v1/admin/model-prices", pricingHandler)
controlMux.Handle("/api/v1/admin/model-prices/", pricingHandler)
controlMux.Handle("/api/v1/admin/fact-check/", factCheckHandler)
controlMux.Handle("/api/v1/admin/prompt-categories", workbenchHandler)
controlMux.Handle("/api/v1/admin/prompt-categories/", workbenchHandler)
controlMux.Handle("/api/v1/admin/prompts", workbenchHandler)
controlMux.Handle("/api/v1/admin/prompts/", workbenchHandler)
controlMux.Handle("/api/v1/admin/knowledge-bases", workbenchHandler)
controlMux.Handle("/api/v1/admin/knowledge-bases/", workbenchHandler)
controlMux.Handle("/api/v1/admin/tools", workbenchHandler)
controlMux.Handle("/api/v1/admin/tools/", workbenchHandler)
controlMux.Handle("/api/v1/admin/applications", workbenchHandler)
controlMux.Handle("/api/v1/admin/applications/", workbenchHandler)
controlMux.Handle("/api/v1/admin/marketplace-categories", marketplaceHandler)
controlMux.Handle("/api/v1/admin/marketplace-categories/", marketplaceHandler)
controlMux.Handle("/api/v1/admin/mcp-servers", marketplaceHandler)
controlMux.Handle("/api/v1/admin/mcp-servers/", marketplaceHandler)
controlMux.Handle("/api/v1/admin/skills", marketplaceHandler)
controlMux.Handle("/api/v1/admin/skills/", marketplaceHandler)
controlMux.Handle("/api/v1/admin/digital-employees", marketplaceHandler)
controlMux.Handle("/api/v1/admin/digital-employees/", marketplaceHandler)
controlMux.Handle("/api/v1/admin/marketplace/", marketplaceHandler)
controlMux.Handle("/api/v1/portal/marketplace", portalHandler)
controlMux.Handle("/api/v1/portal/marketplace/", portalHandler)
controlMux.Handle("/api/v1/admin/notification-channels", workbenchHandler)
controlMux.Handle("/api/v1/admin/notification-channels/", workbenchHandler)
controlMux.Handle("/api/v1/admin/notification-deliveries", workbenchHandler)
controlMux.Handle("/api/v1/admin/notification-deliveries/", workbenchHandler)
controlMux.Handle("/api/v1/admin/models", portalAdminHandler)
controlMux.Handle("/api/v1/admin/model-requests", portalAdminHandler)
controlMux.Handle("/api/v1/admin/model-requests/", portalAdminHandler)
controlMux.Handle("/api/v1/admin/system-info", operationsHandler)
controlMux.Handle("/api/v1/admin/monitoring/overview", operationsHandler)
controlMux.Handle("/api/v1/admin/files", filesAdminHandler)
controlMux.Handle("/api/v1/admin/files/", filesAdminHandler)
controlMux.Handle("/api/v1/portal/files", filesPortalHandler)
controlMux.Handle("/api/v1/portal/files/", filesPortalHandler)
controlMux.Handle("/api/v1/admin/reload", operationsHandler)
controlMux.Handle("/api/v1/admin/identities/", identityManagementHandler)
controlMux.Handle("/api/v1/admin/departments", identityManagementHandler)
controlMux.Handle("/api/v1/admin/departments/", identityManagementHandler)
controlMux.Handle("/api/v1/admin/identity-providers", identityManagementHandler)
controlMux.Handle("/api/v1/admin/identity-providers/", identityManagementHandler)
controlMux.Handle("/api/v1/admin/saml-providers", identityManagementHandler)
controlMux.Handle("/api/v1/admin/saml-providers/", identityManagementHandler)
controlMux.Handle("/api/v1/portal/applications", portalHandler)
controlMux.Handle("/api/v1/portal/apps/", portalHandler)
controlMux.Handle("/api/v1/portal/catalog", portalHandler)
controlMux.Handle("/api/v1/portal/cost", portalHandler)
controlMux.Handle("/api/v1/portal/docs-info", portalHandler)
controlMux.Handle("/api/v1/portal/knowledge", portalHandler)
controlMux.Handle("/api/v1/portal/logs", portalHandler)
controlMux.Handle("/api/v1/portal/logs/", portalHandler)
controlMux.Handle("/api/v1/portal/model-requests", portalHandler)
controlMux.Handle("/api/v1/portal/model-requests/", portalHandler)
controlMux.Handle("/api/v1/portal/password", portalHandler)
controlMux.Handle("/api/v1/portal/prompts", portalHandler)
controlMux.Handle("/api/v1/portal/prompts/", portalHandler)
controlMux.Handle("/api/v1/portal/stats", portalHandler)
controlMux.Handle("/api/v1/portal/tools", portalHandler)
controlMux.Handle("/api/v1/", identityHandler)
publicMux := http.NewServeMux()
publicMux.Handle("/v1/prompts", workbenchRuntime)
publicMux.Handle("/v1/prompts/", workbenchRuntime)
publicMux.Handle("/v1/knowledge/", workbenchRuntime)
publicMux.Handle("/v1/tools", workbenchRuntime)
publicMux.Handle("/v1/tools/", workbenchRuntime)
publicMux.Handle("/v1/applications/", workbenchRuntime)
publicMux.Handle("/v1/skills/", workbenchRuntime)
publicMux.Handle("/v1/mcp-servers", workbenchRuntime)
publicMux.Handle("/v1/mcp-servers/", workbenchRuntime)
publicMux.Handle("/v1/digital-employees/", workbenchRuntime)
publicMux.Handle("/v1/", governedGateway)
server := httpserver.New(httpserver.Dependencies{
Config: cfg, Logger: logger, Checker: checker, Gateway: publicMux, Control: controlMux,
Version: version, StartedAt: startedAt,
BootstrapUses: apiKeyAuthenticator.BootstrapUses,
ExtraMetrics: func() string { return auditRecorder.Prometheus() + shadowMiddleware.Prometheus() },
})
serverErrors := make(chan error, 1)
go func() {
logger.Info("gateway API started", "address", server.Addr, "version", version)
serverErrors <- server.ListenAndServe()
}()
select {
case <-ctx.Done():
logger.Info("shutdown requested")
case err := <-serverErrors:
if !errors.Is(err, http.ErrServerClosed) {
logger.Error("gateway API stopped unexpectedly", "error", err)
os.Exit(1)
}
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Server.ShutdownTimeout)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil {
logger.Error("graceful shutdown failed", "error", err)
_ = server.Close()
os.Exit(1)
}
stopAudit()
select {
case <-auditStopped:
case <-shutdownCtx.Done():
logger.Warn("audit recorder drain timed out")
}
logger.Info("gateway API stopped")
}