6708c226a5
- 迁移 000023 gateway.file_objects(personal/system 归属隔离 + 部分索引) - internal/platform/storage:minio-go 适配(端点 scheme 剥离、流式 PutObject/Open/Delete) - internal/workbench/files.go:FileService(sha256 校验、PutObject-then-insert 回滚、delete 先删行再删对象) - admin /api/v1/admin/files + portal /api/v1/portal/files 处理器(流式上传下载、Content-Disposition) - RBAC file:read/file:manage;菜单加文件管理 + 门户文件仓库 - compose 增 minio 服务(S3_* anchor、不暴露端口);nginx client_max_body_size 32m→256m - 管理端文件管理页 + 门户个人文件仓;集成测试 TestFileObjectLifecycle 连真 MinIO 通过 - healthz object_storage:true;README/PRODUCTION/进展文档同步 Co-Authored-By: Claude <noreply@anthropic.com>
384 lines
15 KiB
Go
384 lines
15 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type Config struct {
|
|
Environment string
|
|
Server Server
|
|
Database Database
|
|
Redis Redis
|
|
Security Security
|
|
Auth Auth
|
|
Credentials Credentials
|
|
Upstream Upstream
|
|
Audit Audit
|
|
Outbox Outbox
|
|
RuntimeData RuntimeData
|
|
Shadow Shadow
|
|
ObjectStorage ObjectStorage
|
|
}
|
|
|
|
type Server struct {
|
|
Address string
|
|
ReadHeaderTimeout time.Duration
|
|
IdleTimeout time.Duration
|
|
ShutdownTimeout time.Duration
|
|
MaxBodyBytes int64
|
|
}
|
|
|
|
type Database struct {
|
|
URL string
|
|
MaxConns int32
|
|
MinConns int32
|
|
}
|
|
|
|
type Redis struct {
|
|
CriticalURL string
|
|
CacheURL string
|
|
}
|
|
|
|
type Security struct {
|
|
BootstrapAPIKey string
|
|
BootstrapAPIKeyEnabled bool
|
|
}
|
|
|
|
type Auth struct {
|
|
SessionTTL time.Duration
|
|
TOTPChallengeTTL time.Duration
|
|
MaxFailures int
|
|
LockDuration time.Duration
|
|
LoginRateLimitMax int // 单 IP 滑动窗口内的最大登录尝试次数
|
|
LoginRateLimitWindow time.Duration // 登录限流滑动窗口
|
|
}
|
|
|
|
type Credentials struct {
|
|
MasterKey string
|
|
KEKVersion int
|
|
KEKKeyring string
|
|
AllowPrivateProviderURL bool
|
|
AllowPrivateToolURL bool
|
|
AllowPrivateWebhookURL bool
|
|
ProviderRefreshInterval time.Duration
|
|
}
|
|
|
|
type Upstream struct {
|
|
BaseURL string
|
|
APIKey string
|
|
FallbackEnabled bool
|
|
ResponseHeaderTimeout time.Duration
|
|
MaxRetries int
|
|
RetryBackoff time.Duration
|
|
CircuitThreshold int
|
|
CircuitOpenDuration time.Duration
|
|
}
|
|
|
|
type Audit struct {
|
|
QueueSize int
|
|
BatchSize int
|
|
FlushInterval time.Duration
|
|
Retention time.Duration
|
|
UsageRetention time.Duration
|
|
PartitionMonthsAhead int
|
|
MaintenanceInterval time.Duration
|
|
}
|
|
|
|
type Outbox struct {
|
|
Stream string
|
|
BatchSize int
|
|
PollInterval time.Duration
|
|
Lease time.Duration
|
|
MaxAttempts int
|
|
MaxBackoff time.Duration
|
|
StreamMaxLength int64
|
|
MarkerTTL time.Duration
|
|
}
|
|
|
|
type RuntimeData struct {
|
|
ContentPolicyRefreshInterval time.Duration
|
|
PricingRefreshInterval time.Duration
|
|
}
|
|
|
|
type Shadow struct {
|
|
BaseURL string
|
|
APIKey string
|
|
SampleRate float64
|
|
Timeout time.Duration
|
|
MaxBodyBytes int64
|
|
MaxConcurrent int
|
|
}
|
|
|
|
type ObjectStorage struct {
|
|
Endpoint string
|
|
AccessKeyID string
|
|
SecretAccessKey string
|
|
Bucket string
|
|
Region string
|
|
UseSSL bool
|
|
MaxFileBytes int64
|
|
}
|
|
|
|
func Load() (Config, error) {
|
|
cfg := Config{
|
|
Environment: env("APP_ENV", "local"),
|
|
Server: Server{
|
|
Address: env("HTTP_ADDR", "127.0.0.1:8080"),
|
|
ReadHeaderTimeout: duration("HTTP_READ_HEADER_TIMEOUT", 5*time.Second),
|
|
IdleTimeout: duration("HTTP_IDLE_TIMEOUT", 120*time.Second),
|
|
ShutdownTimeout: duration("HTTP_SHUTDOWN_TIMEOUT", 20*time.Second),
|
|
MaxBodyBytes: int64Value("HTTP_MAX_BODY_BYTES", 32<<20),
|
|
},
|
|
Database: Database{
|
|
URL: strings.TrimSpace(os.Getenv("DATABASE_URL")),
|
|
MaxConns: int32(intValue("DATABASE_MAX_CONNS", 40)),
|
|
MinConns: int32(intValue("DATABASE_MIN_CONNS", 4)),
|
|
},
|
|
Redis: Redis{
|
|
CriticalURL: strings.TrimSpace(os.Getenv("REDIS_CRITICAL_URL")),
|
|
CacheURL: strings.TrimSpace(os.Getenv("REDIS_CACHE_URL")),
|
|
},
|
|
Security: Security{
|
|
BootstrapAPIKey: strings.TrimSpace(os.Getenv("GATEWAY_BOOTSTRAP_API_KEY")),
|
|
BootstrapAPIKeyEnabled: boolValue("GATEWAY_BOOTSTRAP_API_KEY_ENABLED", false),
|
|
},
|
|
Auth: Auth{
|
|
SessionTTL: duration("AUTH_SESSION_TTL", 12*time.Hour),
|
|
TOTPChallengeTTL: duration("AUTH_TOTP_CHALLENGE_TTL", 5*time.Minute),
|
|
MaxFailures: intValue("LOGIN_MAX_FAILURES", 5),
|
|
LockDuration: duration("LOGIN_LOCK_DURATION", 15*time.Minute),
|
|
LoginRateLimitMax: intValue("LOGIN_RATE_LIMIT_MAX", 30),
|
|
LoginRateLimitWindow: duration("LOGIN_RATE_LIMIT_WINDOW", 5*time.Minute),
|
|
},
|
|
Credentials: Credentials{
|
|
MasterKey: strings.TrimSpace(os.Getenv("CREDENTIAL_MASTER_KEY")),
|
|
KEKVersion: intValue("CREDENTIAL_KEK_VERSION", 1),
|
|
KEKKeyring: strings.TrimSpace(os.Getenv("CREDENTIAL_KEK_KEYRING")),
|
|
AllowPrivateProviderURL: boolValue("ALLOW_PRIVATE_PROVIDER_URLS", false),
|
|
AllowPrivateToolURL: boolValue("ALLOW_PRIVATE_TOOL_URLS", false),
|
|
AllowPrivateWebhookURL: boolValue("ALLOW_PRIVATE_WEBHOOK_URLS", false),
|
|
ProviderRefreshInterval: duration("PROVIDER_REFRESH_INTERVAL", 5*time.Second),
|
|
},
|
|
Upstream: Upstream{
|
|
BaseURL: strings.TrimRight(env("UPSTREAM_BASE_URL", "https://api.openai.com"), "/"), APIKey: strings.TrimSpace(os.Getenv("UPSTREAM_API_KEY")),
|
|
FallbackEnabled: boolValue("UPSTREAM_FALLBACK_ENABLED", true), ResponseHeaderTimeout: duration("UPSTREAM_RESPONSE_HEADER_TIMEOUT", 60*time.Second),
|
|
MaxRetries: intValue("UPSTREAM_MAX_RETRIES", 2), RetryBackoff: duration("UPSTREAM_RETRY_BACKOFF", 50*time.Millisecond),
|
|
CircuitThreshold: intValue("UPSTREAM_CIRCUIT_THRESHOLD", 5), CircuitOpenDuration: duration("UPSTREAM_CIRCUIT_OPEN_DURATION", 30*time.Second),
|
|
},
|
|
Audit: Audit{
|
|
QueueSize: intValue("AUDIT_QUEUE_SIZE", 4096), BatchSize: intValue("AUDIT_BATCH_SIZE", 200),
|
|
FlushInterval: duration("AUDIT_FLUSH_INTERVAL", time.Second), Retention: duration("AUDIT_RETENTION", 90*24*time.Hour),
|
|
UsageRetention: duration("USAGE_RETENTION", 730*24*time.Hour), PartitionMonthsAhead: intValue("AUDIT_PARTITION_MONTHS_AHEAD", 3),
|
|
MaintenanceInterval: duration("AUDIT_MAINTENANCE_INTERVAL", 6*time.Hour),
|
|
},
|
|
Outbox: Outbox{
|
|
Stream: env("OUTBOX_STREAM", "gateway:{outbox}:events"), BatchSize: intValue("OUTBOX_BATCH_SIZE", 100),
|
|
PollInterval: duration("OUTBOX_POLL_INTERVAL", 500*time.Millisecond), Lease: duration("OUTBOX_LEASE", 30*time.Second),
|
|
MaxAttempts: intValue("OUTBOX_MAX_ATTEMPTS", 10), MaxBackoff: duration("OUTBOX_MAX_BACKOFF", 5*time.Minute),
|
|
StreamMaxLength: int64Value("OUTBOX_STREAM_MAX_LENGTH", 100_000), MarkerTTL: duration("OUTBOX_MARKER_TTL", 30*24*time.Hour),
|
|
},
|
|
RuntimeData: RuntimeData{
|
|
ContentPolicyRefreshInterval: duration("CONTENT_POLICY_REFRESH_INTERVAL", 30*time.Second),
|
|
PricingRefreshInterval: duration("PRICING_REFRESH_INTERVAL", 30*time.Second),
|
|
},
|
|
Shadow: Shadow{
|
|
BaseURL: strings.TrimRight(strings.TrimSpace(os.Getenv("SHADOW_BASE_URL")), "/"),
|
|
APIKey: strings.TrimSpace(os.Getenv("SHADOW_API_KEY")), SampleRate: floatValue("SHADOW_SAMPLE_RATE", 0),
|
|
Timeout: duration("SHADOW_TIMEOUT", 20*time.Second), MaxBodyBytes: int64Value("SHADOW_MAX_BODY_BYTES", 2<<20),
|
|
MaxConcurrent: intValue("SHADOW_MAX_CONCURRENT", 16),
|
|
},
|
|
ObjectStorage: ObjectStorage{
|
|
Endpoint: strings.TrimRight(env("S3_ENDPOINT", "http://minio:9000"), "/"),
|
|
AccessKeyID: env("S3_ACCESS_KEY_ID", "gateway"),
|
|
SecretAccessKey: env("S3_SECRET_ACCESS_KEY", "gateway-secret"),
|
|
Bucket: env("S3_BUCKET", "gateway-files"),
|
|
Region: env("S3_REGION", "us-east-1"),
|
|
UseSSL: boolValue("S3_USE_SSL", false),
|
|
MaxFileBytes: int64Value("S3_MAX_FILE_BYTES", 128<<20),
|
|
},
|
|
}
|
|
|
|
return cfg, cfg.Validate()
|
|
}
|
|
|
|
func (c Config) Validate() error {
|
|
var errs []error
|
|
if c.Server.MaxBodyBytes <= 0 {
|
|
errs = append(errs, errors.New("HTTP_MAX_BODY_BYTES must be positive"))
|
|
}
|
|
if c.Database.MinConns < 0 || c.Database.MaxConns < 1 || c.Database.MinConns > c.Database.MaxConns {
|
|
errs = append(errs, errors.New("database pool sizes are invalid"))
|
|
}
|
|
if c.Auth.SessionTTL < 5*time.Minute || c.Auth.TOTPChallengeTTL < time.Minute || c.Auth.TOTPChallengeTTL > 15*time.Minute || c.Auth.MaxFailures < 1 || c.Auth.LockDuration < time.Minute {
|
|
errs = append(errs, errors.New("authentication limits are invalid"))
|
|
}
|
|
if c.Auth.LoginRateLimitMax < 1 || c.Auth.LoginRateLimitWindow < time.Second {
|
|
errs = append(errs, errors.New("login rate limit is invalid"))
|
|
}
|
|
if c.Credentials.KEKVersion < 1 {
|
|
errs = append(errs, errors.New("CREDENTIAL_KEK_VERSION must be positive"))
|
|
}
|
|
if c.Credentials.ProviderRefreshInterval < time.Second || c.Credentials.ProviderRefreshInterval > time.Minute {
|
|
errs = append(errs, errors.New("PROVIDER_REFRESH_INTERVAL must be between 1s and 1m"))
|
|
}
|
|
if err := validateHTTPURL(c.Upstream.BaseURL); err != nil {
|
|
errs = append(errs, fmt.Errorf("UPSTREAM_BASE_URL: %w", err))
|
|
}
|
|
if c.Upstream.ResponseHeaderTimeout < time.Second || c.Upstream.ResponseHeaderTimeout > 10*time.Minute || c.Upstream.MaxRetries < 0 || c.Upstream.MaxRetries > 5 || c.Upstream.RetryBackoff < 0 || c.Upstream.RetryBackoff > 5*time.Second || c.Upstream.CircuitThreshold < 1 || c.Upstream.CircuitThreshold > 100 || c.Upstream.CircuitOpenDuration < time.Second || c.Upstream.CircuitOpenDuration > 10*time.Minute {
|
|
errs = append(errs, errors.New("upstream resilience settings are invalid"))
|
|
}
|
|
if c.Audit.QueueSize < 100 || c.Audit.QueueSize > 1_000_000 || c.Audit.BatchSize < 1 || c.Audit.BatchSize > c.Audit.QueueSize || c.Audit.FlushInterval < 100*time.Millisecond || c.Audit.FlushInterval > time.Minute {
|
|
errs = append(errs, errors.New("audit buffering settings are invalid"))
|
|
}
|
|
if c.Audit.Retention < 24*time.Hour || c.Audit.Retention > 10*365*24*time.Hour || c.Audit.UsageRetention < c.Audit.Retention || c.Audit.UsageRetention > 10*365*24*time.Hour || c.Audit.PartitionMonthsAhead < 1 || c.Audit.PartitionMonthsAhead > 24 || c.Audit.MaintenanceInterval < time.Hour || c.Audit.MaintenanceInterval > 7*24*time.Hour {
|
|
errs = append(errs, errors.New("audit retention settings are invalid"))
|
|
}
|
|
if !strings.Contains(c.Outbox.Stream, "{outbox}") || c.Outbox.BatchSize < 1 || c.Outbox.BatchSize > 1000 || c.Outbox.PollInterval < 50*time.Millisecond || c.Outbox.PollInterval > time.Minute || c.Outbox.Lease < 5*time.Second || c.Outbox.Lease > 10*time.Minute || c.Outbox.MaxAttempts < 1 || c.Outbox.MaxAttempts > 100 || c.Outbox.MaxBackoff < time.Second || c.Outbox.MaxBackoff > time.Hour || c.Outbox.StreamMaxLength < 1000 || c.Outbox.StreamMaxLength > 100_000_000 || c.Outbox.MarkerTTL < 24*time.Hour || c.Outbox.MarkerTTL > 365*24*time.Hour {
|
|
errs = append(errs, errors.New("outbox delivery settings are invalid"))
|
|
}
|
|
if c.RuntimeData.ContentPolicyRefreshInterval < time.Second || c.RuntimeData.ContentPolicyRefreshInterval > 10*time.Minute || c.RuntimeData.PricingRefreshInterval < time.Second || c.RuntimeData.PricingRefreshInterval > 10*time.Minute {
|
|
errs = append(errs, errors.New("runtime data refresh settings must be between 1s and 10m"))
|
|
}
|
|
if c.Shadow.SampleRate < 0 || c.Shadow.SampleRate > 1 || c.Shadow.Timeout < time.Second || c.Shadow.Timeout > 2*time.Minute || c.Shadow.MaxBodyBytes < 1024 || c.Shadow.MaxBodyBytes > 32<<20 || c.Shadow.MaxConcurrent < 1 || c.Shadow.MaxConcurrent > 1000 {
|
|
errs = append(errs, errors.New("shadow traffic settings are invalid"))
|
|
}
|
|
if c.Shadow.BaseURL != "" {
|
|
if err := validateHTTPURL(c.Shadow.BaseURL); err != nil {
|
|
errs = append(errs, fmt.Errorf("SHADOW_BASE_URL: %w", err))
|
|
}
|
|
if c.Shadow.APIKey == "" {
|
|
errs = append(errs, errors.New("SHADOW_API_KEY is required when SHADOW_BASE_URL is set"))
|
|
}
|
|
}
|
|
if err := validateHTTPURL(c.ObjectStorage.Endpoint); err != nil {
|
|
errs = append(errs, fmt.Errorf("S3_ENDPOINT: %w", err))
|
|
} else if endpointURL, parseErr := url.Parse(c.ObjectStorage.Endpoint); parseErr == nil && endpointURL.Path != "" {
|
|
// minio-go 的 Endpoint 不接受带路径的完整 URL(报 "fully qualified paths")。
|
|
errs = append(errs, errors.New("S3_ENDPOINT must not contain a path"))
|
|
}
|
|
if c.ObjectStorage.AccessKeyID == "" || c.ObjectStorage.SecretAccessKey == "" {
|
|
errs = append(errs, errors.New("S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY are required"))
|
|
}
|
|
if c.ObjectStorage.Bucket == "" || len(c.ObjectStorage.Bucket) > 63 {
|
|
errs = append(errs, errors.New("S3_BUCKET must be a non-empty bucket name of at most 63 characters"))
|
|
}
|
|
if c.ObjectStorage.MaxFileBytes < 1<<20 || c.ObjectStorage.MaxFileBytes > 512<<20 {
|
|
errs = append(errs, errors.New("S3_MAX_FILE_BYTES must be between 1 MiB and 512 MiB"))
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
func (c Config) ValidateRuntime() error {
|
|
var errs []error
|
|
if strings.EqualFold(c.Environment, "production") {
|
|
if c.Database.URL == "" {
|
|
errs = append(errs, errors.New("DATABASE_URL is required in production"))
|
|
}
|
|
if c.Redis.CriticalURL == "" {
|
|
errs = append(errs, errors.New("REDIS_CRITICAL_URL is required in production"))
|
|
}
|
|
if c.Security.BootstrapAPIKeyEnabled {
|
|
if len(c.Security.BootstrapAPIKey) < 32 {
|
|
errs = append(errs, errors.New("GATEWAY_BOOTSTRAP_API_KEY must contain at least 32 characters in production"))
|
|
}
|
|
// Reject the documented placeholder and other obviously weak values
|
|
// so an operator cannot accidentally deploy with the example secret.
|
|
switch strings.ToLower(c.Security.BootstrapAPIKey) {
|
|
case "change-me-in-production", "local-development-key", "changeme", "change-me", "password", "secret":
|
|
errs = append(errs, errors.New("GATEWAY_BOOTSTRAP_API_KEY is set to a known weak default; generate a strong random value"))
|
|
}
|
|
}
|
|
if c.Upstream.FallbackEnabled && c.Upstream.APIKey == "" {
|
|
errs = append(errs, errors.New("UPSTREAM_API_KEY is required in production"))
|
|
}
|
|
if c.Credentials.MasterKey == "" {
|
|
errs = append(errs, errors.New("CREDENTIAL_MASTER_KEY is required in production"))
|
|
}
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
func validateHTTPURL(raw string) error {
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil {
|
|
return errors.New("must be an absolute http(s) URL without user info")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func env(key, fallback string) string {
|
|
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
|
return value
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func duration(key string, fallback time.Duration) time.Duration {
|
|
value := strings.TrimSpace(os.Getenv(key))
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
parsed, err := time.ParseDuration(value)
|
|
if err != nil {
|
|
return fallback
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func intValue(key string, fallback int) int {
|
|
value := strings.TrimSpace(os.Getenv(key))
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
parsed, err := strconv.Atoi(value)
|
|
if err != nil {
|
|
return fallback
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func int64Value(key string, fallback int64) int64 {
|
|
value := strings.TrimSpace(os.Getenv(key))
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
parsed, err := strconv.ParseInt(value, 10, 64)
|
|
if err != nil {
|
|
return fallback
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func floatValue(key string, fallback float64) float64 {
|
|
value := strings.TrimSpace(os.Getenv(key))
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
parsed, err := strconv.ParseFloat(value, 64)
|
|
if err != nil {
|
|
return fallback
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func boolValue(key string, fallback bool) bool {
|
|
value := strings.TrimSpace(os.Getenv(key))
|
|
if value == "" {
|
|
return fallback
|
|
}
|
|
parsed, err := strconv.ParseBool(value)
|
|
if err != nil {
|
|
return fallback
|
|
}
|
|
return parsed
|
|
}
|