feat(m8): P1 MinIO 对象存储与文件管理
- 迁移 000023 gateway.file_objects(personal/system 归属隔离 + 部分索引) - internal/platform/storage:minio-go 适配(端点 scheme 剥离、流式 PutObject/Open/Delete) - internal/workbench/files.go:FileService(sha256 校验、PutObject-then-insert 回滚、delete 先删行再删对象) - admin /api/v1/admin/files + portal /api/v1/portal/files 处理器(流式上传下载、Content-Disposition) - RBAC file:read/file:manage;菜单加文件管理 + 门户文件仓库 - compose 增 minio 服务(S3_* anchor、不暴露端口);nginx client_max_body_size 32m→256m - 管理端文件管理页 + 门户个人文件仓;集成测试 TestFileObjectLifecycle 连真 MinIO 通过 - healthz object_storage:true;README/PRODUCTION/进展文档同步 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -11,18 +11,19 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Environment string
|
||||
Server Server
|
||||
Database Database
|
||||
Redis Redis
|
||||
Security Security
|
||||
Auth Auth
|
||||
Credentials Credentials
|
||||
Upstream Upstream
|
||||
Audit Audit
|
||||
Outbox Outbox
|
||||
RuntimeData RuntimeData
|
||||
Shadow Shadow
|
||||
Environment string
|
||||
Server Server
|
||||
Database Database
|
||||
Redis Redis
|
||||
Security Security
|
||||
Auth Auth
|
||||
Credentials Credentials
|
||||
Upstream Upstream
|
||||
Audit Audit
|
||||
Outbox Outbox
|
||||
RuntimeData RuntimeData
|
||||
Shadow Shadow
|
||||
ObjectStorage ObjectStorage
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
@@ -114,6 +115,16 @@ type Shadow struct {
|
||||
MaxConcurrent int
|
||||
}
|
||||
|
||||
type ObjectStorage struct {
|
||||
Endpoint string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Bucket string
|
||||
Region string
|
||||
UseSSL bool
|
||||
MaxFileBytes int64
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
cfg := Config{
|
||||
Environment: env("APP_ENV", "local"),
|
||||
@@ -182,6 +193,15 @@ func Load() (Config, error) {
|
||||
Timeout: duration("SHADOW_TIMEOUT", 20*time.Second), MaxBodyBytes: int64Value("SHADOW_MAX_BODY_BYTES", 2<<20),
|
||||
MaxConcurrent: intValue("SHADOW_MAX_CONCURRENT", 16),
|
||||
},
|
||||
ObjectStorage: ObjectStorage{
|
||||
Endpoint: strings.TrimRight(env("S3_ENDPOINT", "http://minio:9000"), "/"),
|
||||
AccessKeyID: env("S3_ACCESS_KEY_ID", "gateway"),
|
||||
SecretAccessKey: env("S3_SECRET_ACCESS_KEY", "gateway-secret"),
|
||||
Bucket: env("S3_BUCKET", "gateway-files"),
|
||||
Region: env("S3_REGION", "us-east-1"),
|
||||
UseSSL: boolValue("S3_USE_SSL", false),
|
||||
MaxFileBytes: int64Value("S3_MAX_FILE_BYTES", 128<<20),
|
||||
},
|
||||
}
|
||||
|
||||
return cfg, cfg.Validate()
|
||||
@@ -236,6 +256,21 @@ func (c Config) Validate() error {
|
||||
errs = append(errs, errors.New("SHADOW_API_KEY is required when SHADOW_BASE_URL is set"))
|
||||
}
|
||||
}
|
||||
if err := validateHTTPURL(c.ObjectStorage.Endpoint); err != nil {
|
||||
errs = append(errs, fmt.Errorf("S3_ENDPOINT: %w", err))
|
||||
} else if endpointURL, parseErr := url.Parse(c.ObjectStorage.Endpoint); parseErr == nil && endpointURL.Path != "" {
|
||||
// minio-go 的 Endpoint 不接受带路径的完整 URL(报 "fully qualified paths")。
|
||||
errs = append(errs, errors.New("S3_ENDPOINT must not contain a path"))
|
||||
}
|
||||
if c.ObjectStorage.AccessKeyID == "" || c.ObjectStorage.SecretAccessKey == "" {
|
||||
errs = append(errs, errors.New("S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY are required"))
|
||||
}
|
||||
if c.ObjectStorage.Bucket == "" || len(c.ObjectStorage.Bucket) > 63 {
|
||||
errs = append(errs, errors.New("S3_BUCKET must be a non-empty bucket name of at most 63 characters"))
|
||||
}
|
||||
if c.ObjectStorage.MaxFileBytes < 1<<20 || c.ObjectStorage.MaxFileBytes > 512<<20 {
|
||||
errs = append(errs, errors.New("S3_MAX_FILE_BYTES must be between 1 MiB and 512 MiB"))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/url"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// Config holds the S3-compatible (MinIO) connection settings.
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Bucket string
|
||||
Region string
|
||||
UseSSL bool
|
||||
MaxFileBytes int64
|
||||
}
|
||||
|
||||
// Client is the thin object-store adapter. The domain layer imports this
|
||||
// package instead of minio-go directly, keeping the S3 client at the platform
|
||||
// boundary like cache and database.
|
||||
type Client struct {
|
||||
mc *minio.Client
|
||||
bucket string
|
||||
region string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
func NewClient(cfg Config) (*Client, error) {
|
||||
// minio-go 的 Endpoint 参数必须是裸 host[:port],不接受带 scheme 的完整
|
||||
// URL(否则报 "Endpoint url cannot have fully qualified paths.")。这里剥离
|
||||
// scheme,并由 scheme 推导 Secure(https→true),二者都比 S3_USE_SSL 优先。
|
||||
secure := cfg.UseSSL
|
||||
host := cfg.Endpoint
|
||||
if parsed, err := url.Parse(cfg.Endpoint); err == nil && parsed.Scheme != "" {
|
||||
secure = parsed.Scheme == "https"
|
||||
host = parsed.Host
|
||||
}
|
||||
mc, err := minio.New(host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
||||
Secure: secure,
|
||||
Region: cfg.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mc.SetAppInfo("ai-gateway", "0.10.0")
|
||||
return &Client{mc: mc, bucket: cfg.Bucket, region: cfg.Region, maxBytes: cfg.MaxFileBytes}, nil
|
||||
}
|
||||
|
||||
// EnsureBucket creates the configured bucket if it does not exist. Safe to call
|
||||
// repeatedly; it is a no-op once the bucket exists.
|
||||
func (c *Client) EnsureBucket(ctx context.Context) error {
|
||||
exists, err := c.mc.BucketExists(ctx, c.bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
return c.mc.MakeBucket(ctx, c.bucket, minio.MakeBucketOptions{Region: c.region})
|
||||
}
|
||||
|
||||
func (c *Client) MaxBytes() int64 { return c.maxBytes }
|
||||
|
||||
// PutObject streams a reader to the object store. A size of -1 lets the client
|
||||
// use chunked multipart upload so files are not buffered in memory.
|
||||
func (c *Client) PutObject(ctx context.Context, key string, r io.Reader, size int64, contentType string) (int64, error) {
|
||||
info, err := c.mc.PutObject(ctx, c.bucket, key, r, size, minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
// OpenObject returns a streaming reader plus the object size.
|
||||
func (c *Client) OpenObject(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
obj, err := c.mc.GetObject(ctx, c.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
stat, err := obj.Stat()
|
||||
if err != nil {
|
||||
_ = obj.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
return obj, stat.Size, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteObject(ctx context.Context, key string) error {
|
||||
return c.mc.RemoveObject(ctx, c.bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
Reference in New Issue
Block a user