feat(m8): P1 MinIO 对象存储与文件管理
- 迁移 000023 gateway.file_objects(personal/system 归属隔离 + 部分索引) - internal/platform/storage:minio-go 适配(端点 scheme 剥离、流式 PutObject/Open/Delete) - internal/workbench/files.go:FileService(sha256 校验、PutObject-then-insert 回滚、delete 先删行再删对象) - admin /api/v1/admin/files + portal /api/v1/portal/files 处理器(流式上传下载、Content-Disposition) - RBAC file:read/file:manage;菜单加文件管理 + 门户文件仓库 - compose 增 minio 服务(S3_* anchor、不暴露端口);nginx client_max_body_size 32m→256m - 管理端文件管理页 + 门户个人文件仓;集成测试 TestFileObjectLifecycle 连真 MinIO 通过 - healthz object_storage:true;README/PRODUCTION/进展文档同步 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -73,6 +73,8 @@ const (
|
||||
PermissionDigitalEmployeeManage = "digital_employee:manage"
|
||||
PermissionMarketplaceRead = "marketplace:read"
|
||||
PermissionMarketplaceManage = "marketplace:manage"
|
||||
PermissionFileRead = "file:read"
|
||||
PermissionFileManage = "file:manage"
|
||||
)
|
||||
|
||||
var rolePermissions = map[string][]string{
|
||||
@@ -93,8 +95,9 @@ var rolePermissions = map[string][]string{
|
||||
PermissionSkillRead, PermissionSkillManage,
|
||||
PermissionDigitalEmployeeRead, PermissionDigitalEmployeeManage,
|
||||
PermissionMarketplaceRead, PermissionMarketplaceManage,
|
||||
PermissionFileRead, PermissionFileManage,
|
||||
},
|
||||
"auditor": {PermissionProviderRead, PermissionAPIKeyRead, PermissionAuditRead, PermissionUsageRead, PermissionOutboxRead, PermissionContentPolicyRead, PermissionPricingRead, PermissionPromptRead, PermissionKnowledgeRead, PermissionToolRead, PermissionApplicationRead, PermissionNotificationRead, PermissionMCPServerRead, PermissionSkillRead, PermissionDigitalEmployeeRead, PermissionMarketplaceRead},
|
||||
"auditor": {PermissionProviderRead, PermissionAPIKeyRead, PermissionAuditRead, PermissionUsageRead, PermissionOutboxRead, PermissionContentPolicyRead, PermissionPricingRead, PermissionPromptRead, PermissionKnowledgeRead, PermissionToolRead, PermissionApplicationRead, PermissionNotificationRead, PermissionMCPServerRead, PermissionSkillRead, PermissionDigitalEmployeeRead, PermissionMarketplaceRead, PermissionFileRead},
|
||||
"member": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -377,6 +377,9 @@ func adminMenus(account Account) []map[string]any {
|
||||
if HasPermission(account, PermissionApplicationRead) || HasPermission(account, PermissionApplicationManage) {
|
||||
assetsChildren = append(assetsChildren, map[string]any{"name": "Applications", "path": "applications", "component": "/gateway/applications", "meta": map[string]any{"title": "AI 应用"}})
|
||||
}
|
||||
if HasPermission(account, PermissionFileRead) || HasPermission(account, PermissionFileManage) {
|
||||
assetsChildren = append(assetsChildren, map[string]any{"name": "Files", "path": "files", "component": "/gateway/files", "meta": map[string]any{"title": "文件管理"}})
|
||||
}
|
||||
if len(assetsChildren) > 0 {
|
||||
menus = append(menus, map[string]any{"name": "Assets", "path": "/assets", "component": "/index/index", "meta": map[string]any{"title": "AI 资产", "icon": "ri:box-3-line"}, "children": assetsChildren})
|
||||
}
|
||||
@@ -424,6 +427,7 @@ func portalMenus() []map[string]any {
|
||||
{"name": "PortalPrompts", "path": "prompts", "component": "/portal/prompts", "meta": map[string]any{"title": "Prompt 广场"}},
|
||||
{"name": "PortalUsage", "path": "usage", "component": "/portal/usage", "meta": map[string]any{"title": "我的用量"}},
|
||||
{"name": "PortalAccess", "path": "access", "component": "/portal/access", "meta": map[string]any{"title": "模型权限"}},
|
||||
{"name": "PortalFiles", "path": "files", "component": "/portal/files", "meta": map[string]any{"title": "文件仓库"}},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ func (h *AdminHTTPHandler) systemInfo(w http.ResponseWriter, r *http.Request) {
|
||||
apiresponse.Error(w, http.StatusServiceUnavailable, "系统信息查询失败")
|
||||
return
|
||||
}
|
||||
apiresponse.OK(w, map[string]any{"version": h.version, "go_version": runtime.Version(), "uptime_seconds": int64(time.Since(h.startedAt).Seconds()), "database": "postgresql", "object_storage": false, "clickhouse": false, "enabled_providers": providers, "enabled_models": models, "enabled_api_keys": keys})
|
||||
apiresponse.OK(w, map[string]any{"version": h.version, "go_version": runtime.Version(), "uptime_seconds": int64(time.Since(h.startedAt).Seconds()), "database": "postgresql", "object_storage": true, "clickhouse": false, "enabled_providers": providers, "enabled_models": models, "enabled_api_keys": keys})
|
||||
}
|
||||
|
||||
func (h *AdminHTTPHandler) overview(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -11,18 +11,19 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Environment string
|
||||
Server Server
|
||||
Database Database
|
||||
Redis Redis
|
||||
Security Security
|
||||
Auth Auth
|
||||
Credentials Credentials
|
||||
Upstream Upstream
|
||||
Audit Audit
|
||||
Outbox Outbox
|
||||
RuntimeData RuntimeData
|
||||
Shadow Shadow
|
||||
Environment string
|
||||
Server Server
|
||||
Database Database
|
||||
Redis Redis
|
||||
Security Security
|
||||
Auth Auth
|
||||
Credentials Credentials
|
||||
Upstream Upstream
|
||||
Audit Audit
|
||||
Outbox Outbox
|
||||
RuntimeData RuntimeData
|
||||
Shadow Shadow
|
||||
ObjectStorage ObjectStorage
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
@@ -114,6 +115,16 @@ type Shadow struct {
|
||||
MaxConcurrent int
|
||||
}
|
||||
|
||||
type ObjectStorage struct {
|
||||
Endpoint string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Bucket string
|
||||
Region string
|
||||
UseSSL bool
|
||||
MaxFileBytes int64
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
cfg := Config{
|
||||
Environment: env("APP_ENV", "local"),
|
||||
@@ -182,6 +193,15 @@ func Load() (Config, error) {
|
||||
Timeout: duration("SHADOW_TIMEOUT", 20*time.Second), MaxBodyBytes: int64Value("SHADOW_MAX_BODY_BYTES", 2<<20),
|
||||
MaxConcurrent: intValue("SHADOW_MAX_CONCURRENT", 16),
|
||||
},
|
||||
ObjectStorage: ObjectStorage{
|
||||
Endpoint: strings.TrimRight(env("S3_ENDPOINT", "http://minio:9000"), "/"),
|
||||
AccessKeyID: env("S3_ACCESS_KEY_ID", "gateway"),
|
||||
SecretAccessKey: env("S3_SECRET_ACCESS_KEY", "gateway-secret"),
|
||||
Bucket: env("S3_BUCKET", "gateway-files"),
|
||||
Region: env("S3_REGION", "us-east-1"),
|
||||
UseSSL: boolValue("S3_USE_SSL", false),
|
||||
MaxFileBytes: int64Value("S3_MAX_FILE_BYTES", 128<<20),
|
||||
},
|
||||
}
|
||||
|
||||
return cfg, cfg.Validate()
|
||||
@@ -236,6 +256,21 @@ func (c Config) Validate() error {
|
||||
errs = append(errs, errors.New("SHADOW_API_KEY is required when SHADOW_BASE_URL is set"))
|
||||
}
|
||||
}
|
||||
if err := validateHTTPURL(c.ObjectStorage.Endpoint); err != nil {
|
||||
errs = append(errs, fmt.Errorf("S3_ENDPOINT: %w", err))
|
||||
} else if endpointURL, parseErr := url.Parse(c.ObjectStorage.Endpoint); parseErr == nil && endpointURL.Path != "" {
|
||||
// minio-go 的 Endpoint 不接受带路径的完整 URL(报 "fully qualified paths")。
|
||||
errs = append(errs, errors.New("S3_ENDPOINT must not contain a path"))
|
||||
}
|
||||
if c.ObjectStorage.AccessKeyID == "" || c.ObjectStorage.SecretAccessKey == "" {
|
||||
errs = append(errs, errors.New("S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY are required"))
|
||||
}
|
||||
if c.ObjectStorage.Bucket == "" || len(c.ObjectStorage.Bucket) > 63 {
|
||||
errs = append(errs, errors.New("S3_BUCKET must be a non-empty bucket name of at most 63 characters"))
|
||||
}
|
||||
if c.ObjectStorage.MaxFileBytes < 1<<20 || c.ObjectStorage.MaxFileBytes > 512<<20 {
|
||||
errs = append(errs, errors.New("S3_MAX_FILE_BYTES must be between 1 MiB and 512 MiB"))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/url"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// Config holds the S3-compatible (MinIO) connection settings.
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Bucket string
|
||||
Region string
|
||||
UseSSL bool
|
||||
MaxFileBytes int64
|
||||
}
|
||||
|
||||
// Client is the thin object-store adapter. The domain layer imports this
|
||||
// package instead of minio-go directly, keeping the S3 client at the platform
|
||||
// boundary like cache and database.
|
||||
type Client struct {
|
||||
mc *minio.Client
|
||||
bucket string
|
||||
region string
|
||||
maxBytes int64
|
||||
}
|
||||
|
||||
func NewClient(cfg Config) (*Client, error) {
|
||||
// minio-go 的 Endpoint 参数必须是裸 host[:port],不接受带 scheme 的完整
|
||||
// URL(否则报 "Endpoint url cannot have fully qualified paths.")。这里剥离
|
||||
// scheme,并由 scheme 推导 Secure(https→true),二者都比 S3_USE_SSL 优先。
|
||||
secure := cfg.UseSSL
|
||||
host := cfg.Endpoint
|
||||
if parsed, err := url.Parse(cfg.Endpoint); err == nil && parsed.Scheme != "" {
|
||||
secure = parsed.Scheme == "https"
|
||||
host = parsed.Host
|
||||
}
|
||||
mc, err := minio.New(host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
||||
Secure: secure,
|
||||
Region: cfg.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mc.SetAppInfo("ai-gateway", "0.10.0")
|
||||
return &Client{mc: mc, bucket: cfg.Bucket, region: cfg.Region, maxBytes: cfg.MaxFileBytes}, nil
|
||||
}
|
||||
|
||||
// EnsureBucket creates the configured bucket if it does not exist. Safe to call
|
||||
// repeatedly; it is a no-op once the bucket exists.
|
||||
func (c *Client) EnsureBucket(ctx context.Context) error {
|
||||
exists, err := c.mc.BucketExists(ctx, c.bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
return c.mc.MakeBucket(ctx, c.bucket, minio.MakeBucketOptions{Region: c.region})
|
||||
}
|
||||
|
||||
func (c *Client) MaxBytes() int64 { return c.maxBytes }
|
||||
|
||||
// PutObject streams a reader to the object store. A size of -1 lets the client
|
||||
// use chunked multipart upload so files are not buffered in memory.
|
||||
func (c *Client) PutObject(ctx context.Context, key string, r io.Reader, size int64, contentType string) (int64, error) {
|
||||
info, err := c.mc.PutObject(ctx, c.bucket, key, r, size, minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
// OpenObject returns a streaming reader plus the object size.
|
||||
func (c *Client) OpenObject(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
obj, err := c.mc.GetObject(ctx, c.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
stat, err := obj.Stat()
|
||||
if err != nil {
|
||||
_ = obj.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
return obj, stat.Size, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteObject(ctx context.Context, key string) error {
|
||||
return c.mc.RemoveObject(ctx, c.bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package workbench
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"aigateway.local/core/internal/platform/storage"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// FileObject is the metadata row for one object stored in MinIO. The object
|
||||
// body lives in the bucket; this table is the searchable index plus the access
|
||||
// control (personal scope is bound to a portal user, system scope to admins).
|
||||
type FileObject struct {
|
||||
ID string `json:"id"`
|
||||
ObjectKey string `json:"-"`
|
||||
OriginalName string `json:"original_name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ContentSHA256 string `json:"content_sha256"`
|
||||
Scope string `json:"scope"`
|
||||
OwnerUserID *string `json:"owner_user_id,omitempty"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// FileService stores object metadata in PostgreSQL and the body in MinIO. Both
|
||||
// writes are kept consistent: the object is uploaded first and only then is the
|
||||
// metadata row inserted; any failure rolls the object back.
|
||||
type FileService struct {
|
||||
assets *Service
|
||||
store *storage.Client
|
||||
}
|
||||
|
||||
func NewFileService(assets *Service, store *storage.Client) *FileService {
|
||||
return &FileService{assets: assets, store: store}
|
||||
}
|
||||
|
||||
func (s *FileService) MaxBytes() int64 { return s.store.MaxBytes() }
|
||||
|
||||
// countingReader tallies how many bytes flow through it so uploads can enforce
|
||||
// the configured size ceiling without buffering the whole file in memory.
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n *int64
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
*c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Upload streams the body into the object store and records its metadata. The
|
||||
// owner must be set for the personal scope and nil for the system scope.
|
||||
func (s *FileService) Upload(ctx context.Context, scope string, ownerUserID *string, createdBy, originalName, contentType string, body io.Reader) (FileObject, error) {
|
||||
originalName = strings.TrimSpace(originalName)
|
||||
if originalName == "" || len(originalName) > 255 {
|
||||
return FileObject{}, errors.New("文件名无效")
|
||||
}
|
||||
if scope != "personal" && scope != "system" {
|
||||
return FileObject{}, errors.New("无效的文件范围")
|
||||
}
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
objectKey, err := newUUID()
|
||||
if err != nil {
|
||||
return FileObject{}, err
|
||||
}
|
||||
objectKey = scope + "/" + objectKey
|
||||
var size int64
|
||||
hasher := sha256.New()
|
||||
limited := io.LimitReader(body, s.store.MaxBytes()+1)
|
||||
counted := &countingReader{r: limited, n: &size}
|
||||
rollbackObject := func() { _ = s.store.DeleteObject(ctx, objectKey) }
|
||||
if _, err = s.store.PutObject(ctx, objectKey, io.TeeReader(counted, hasher), -1, contentType); err != nil {
|
||||
return FileObject{}, err
|
||||
}
|
||||
if size > s.store.MaxBytes() {
|
||||
rollbackObject()
|
||||
return FileObject{}, errors.New("文件超过大小上限")
|
||||
}
|
||||
id, err := newUUID()
|
||||
if err != nil {
|
||||
rollbackObject()
|
||||
return FileObject{}, err
|
||||
}
|
||||
var owner any
|
||||
if ownerUserID != nil && strings.TrimSpace(*ownerUserID) != "" {
|
||||
owner = *ownerUserID
|
||||
}
|
||||
var obj FileObject
|
||||
err = s.assets.pool.QueryRow(ctx, `INSERT INTO gateway.file_objects(id,object_key,original_name,content_type,size_bytes,content_sha256,scope,owner_user_id,created_by) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING id::text,object_key,original_name,content_type,size_bytes,content_sha256,scope,owner_user_id::text,created_by,created_at,updated_at`, id, objectKey, originalName, contentType, size, hex.EncodeToString(hasher.Sum(nil)), scope, owner, createdBy).Scan(&obj.ID, &obj.ObjectKey, &obj.OriginalName, &obj.ContentType, &obj.SizeBytes, &obj.ContentSHA256, &obj.Scope, &obj.OwnerUserID, &obj.CreatedBy, &obj.CreatedAt, &obj.UpdatedAt)
|
||||
if err != nil {
|
||||
rollbackObject()
|
||||
return FileObject{}, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
const fileObjectSelect = `SELECT id::text,object_key,original_name,content_type,size_bytes,content_sha256,scope,owner_user_id::text,created_by,created_at,updated_at FROM gateway.file_objects`
|
||||
|
||||
func scanFileObject(row pgx.Row) (FileObject, error) {
|
||||
var obj FileObject
|
||||
err := row.Scan(&obj.ID, &obj.ObjectKey, &obj.OriginalName, &obj.ContentType, &obj.SizeBytes, &obj.ContentSHA256, &obj.Scope, &obj.OwnerUserID, &obj.CreatedBy, &obj.CreatedAt, &obj.UpdatedAt)
|
||||
return obj, mapNotFound(err)
|
||||
}
|
||||
|
||||
func (s *FileService) ListPersonal(ctx context.Context, ownerUserID string) ([]FileObject, error) {
|
||||
rows, err := s.assets.pool.Query(ctx, fileObjectSelect+` WHERE scope='personal' AND owner_user_id=$1 ORDER BY created_at DESC`, ownerUserID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []FileObject{}
|
||||
for rows.Next() {
|
||||
obj, err := scanFileObject(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, obj)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *FileService) ListSystem(ctx context.Context) ([]FileObject, error) {
|
||||
rows, err := s.assets.pool.Query(ctx, fileObjectSelect+` WHERE scope='system' ORDER BY created_at DESC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []FileObject{}
|
||||
for rows.Next() {
|
||||
obj, err := scanFileObject(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, obj)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *FileService) GetPersonal(ctx context.Context, ownerUserID, id string) (FileObject, error) {
|
||||
return scanFileObject(s.assets.pool.QueryRow(ctx, fileObjectSelect+` WHERE id=$1 AND scope='personal' AND owner_user_id=$2`, id, ownerUserID))
|
||||
}
|
||||
|
||||
func (s *FileService) GetSystem(ctx context.Context, id string) (FileObject, error) {
|
||||
return scanFileObject(s.assets.pool.QueryRow(ctx, fileObjectSelect+` WHERE id=$1 AND scope='system'`, id))
|
||||
}
|
||||
|
||||
// Open streams the object body for download.
|
||||
func (s *FileService) Open(ctx context.Context, obj FileObject) (io.ReadCloser, int64, error) {
|
||||
return s.store.OpenObject(ctx, obj.ObjectKey)
|
||||
}
|
||||
|
||||
func (s *FileService) DeletePersonal(ctx context.Context, ownerUserID, id string) error {
|
||||
return s.deleteFile(ctx, id, `scope='personal' AND owner_user_id=$2`, ownerUserID)
|
||||
}
|
||||
|
||||
func (s *FileService) DeleteSystem(ctx context.Context, id string) error {
|
||||
return s.deleteFile(ctx, id, `scope='system'`, nil)
|
||||
}
|
||||
|
||||
// deleteFile removes the metadata row and, best-effort, the object body. The
|
||||
// object removal is best-effort because an orphan in the bucket is recoverable
|
||||
// and must never block the delete that the user asked for.
|
||||
func (s *FileService) deleteFile(ctx context.Context, id, where string, owner any) error {
|
||||
// 只传 SQL 中真实出现的占位符参数:system 范围没有 $2,多传 nil 会让
|
||||
// Postgres 报 "bind message supplies 2 parameters" 错误。
|
||||
args := []any{id}
|
||||
if owner != nil {
|
||||
args = append(args, owner)
|
||||
}
|
||||
var key string
|
||||
if err := s.assets.pool.QueryRow(ctx, `SELECT object_key FROM gateway.file_objects WHERE id=$1 AND `+where, args...).Scan(&key); err != nil {
|
||||
return mapNotFound(err)
|
||||
}
|
||||
tag, err := s.assets.pool.Exec(ctx, `DELETE FROM gateway.file_objects WHERE id=$1 AND `+where, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
_ = s.store.DeleteObject(ctx, key)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package workbench
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"aigateway.local/core/internal/identity"
|
||||
"aigateway.local/core/internal/platform/apiresponse"
|
||||
)
|
||||
|
||||
// FilesAdminHTTPHandler exposes the system-scoped file store to the admin
|
||||
// console. Objects are uploaded through the gateway (never exposing MinIO), so
|
||||
// the S3 credentials stay inside the API container.
|
||||
type FilesAdminHTTPHandler struct {
|
||||
files *FileService
|
||||
identity *identity.Service
|
||||
mux *http.ServeMux
|
||||
}
|
||||
|
||||
func NewFilesAdminHTTPHandler(files *FileService, identityService *identity.Service) *FilesAdminHTTPHandler {
|
||||
h := &FilesAdminHTTPHandler{files: files, identity: identityService, mux: http.NewServeMux()}
|
||||
h.mux.HandleFunc("POST /api/v1/admin/files", h.upload)
|
||||
h.mux.HandleFunc("GET /api/v1/admin/files", h.list)
|
||||
h.mux.HandleFunc("GET /api/v1/admin/files/{id}/download", h.download)
|
||||
h.mux.HandleFunc("DELETE /api/v1/admin/files/{id}", h.delete)
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *FilesAdminHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.mux.ServeHTTP(w, r) }
|
||||
|
||||
func (h *FilesAdminHTTPHandler) require(w http.ResponseWriter, r *http.Request, permission string) (identity.Account, bool) {
|
||||
account, err := h.identity.Authenticate(r.Context(), identity.KindAdmin, r.Header.Get("Authorization"))
|
||||
if err != nil {
|
||||
apiresponse.Error(w, http.StatusUnauthorized, "登录状态无效")
|
||||
return identity.Account{}, false
|
||||
}
|
||||
if !identity.HasPermission(account, permission) {
|
||||
apiresponse.Error(w, http.StatusForbidden, "缺少文件管理权限")
|
||||
return identity.Account{}, false
|
||||
}
|
||||
return account, true
|
||||
}
|
||||
|
||||
// firstFilePart pulls the multipart "file" part out of a streaming request so
|
||||
// uploads never need to buffer the whole body in memory.
|
||||
func firstFilePart(r *http.Request) (*multipart.Part, error) {
|
||||
reader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
part, err := reader.NextPart()
|
||||
if err == io.EOF {
|
||||
return nil, errors.New("no file part")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if part.FormName() == "file" {
|
||||
return part, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// upload accepts either multipart/form-data (browser) or a raw body with a
|
||||
// ?filename= query parameter (scripting). The size ceiling is enforced by the
|
||||
// service's LimitReader, not by buffering.
|
||||
func (h *FilesAdminHTTPHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := h.require(w, r, identity.PermissionFileManage)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var (
|
||||
body io.Reader
|
||||
originalName = strings.TrimSpace(r.URL.Query().Get("filename"))
|
||||
contentType = r.Header.Get("Content-Type")
|
||||
)
|
||||
if strings.HasPrefix(contentType, "multipart/form-data") {
|
||||
part, err := firstFilePart(r)
|
||||
if err != nil {
|
||||
apiresponse.Error(w, http.StatusBadRequest, "缺少上传文件")
|
||||
return
|
||||
}
|
||||
defer part.Close()
|
||||
body = part
|
||||
originalName = part.FileName()
|
||||
if ct := part.Header.Get("Content-Type"); ct != "" {
|
||||
contentType = ct
|
||||
}
|
||||
}
|
||||
if originalName == "" {
|
||||
apiresponse.Error(w, http.StatusBadRequest, "缺少文件名")
|
||||
return
|
||||
}
|
||||
obj, err := h.files.Upload(r.Context(), "system", nil, a.ID, originalName, contentType, body)
|
||||
if err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
apiresponse.OK(w, obj)
|
||||
}
|
||||
|
||||
func (h *FilesAdminHTTPHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := h.require(w, r, identity.PermissionFileRead); !ok {
|
||||
return
|
||||
}
|
||||
items, err := h.files.ListSystem(r.Context())
|
||||
if err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
apiresponse.OK(w, items)
|
||||
}
|
||||
|
||||
func (h *FilesAdminHTTPHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := h.require(w, r, identity.PermissionFileRead); !ok {
|
||||
return
|
||||
}
|
||||
obj, err := h.files.GetSystem(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
serveFileContent(w, r, h.files, obj)
|
||||
}
|
||||
|
||||
func (h *FilesAdminHTTPHandler) delete(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := h.require(w, r, identity.PermissionFileManage); !ok {
|
||||
return
|
||||
}
|
||||
if err := h.files.DeleteSystem(r.Context(), r.PathValue("id")); err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
apiresponse.OK(w, map[string]bool{"deleted": true})
|
||||
}
|
||||
|
||||
// serveFileContent streams the object body straight to the client with a
|
||||
// Content-Disposition so the original filename is preserved on download.
|
||||
func serveFileContent(w http.ResponseWriter, r *http.Request, files *FileService, obj FileObject) {
|
||||
reader, size, err := files.Open(r.Context(), obj)
|
||||
if err != nil {
|
||||
apiresponse.Error(w, http.StatusServiceUnavailable, "文件内容暂不可用")
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+url.PathEscape(obj.OriginalName))
|
||||
w.Header().Set("Content-Type", obj.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
||||
_, _ = io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func fileError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case errors.Is(err, ErrNotFound):
|
||||
apiresponse.Error(w, http.StatusNotFound, "文件不存在")
|
||||
default:
|
||||
apiresponse.Error(w, http.StatusBadRequest, err.Error())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package workbench
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"aigateway.local/core/internal/platform/config"
|
||||
"aigateway.local/core/internal/platform/database"
|
||||
"aigateway.local/core/internal/platform/storage"
|
||||
)
|
||||
|
||||
// TestFileObjectLifecycle runs against real PostgreSQL + MinIO. Set
|
||||
// WORKBENCH_TEST_DATABASE_URL and WORKBENCH_TEST_S3_ENDPOINT to enable it; with
|
||||
// the deploy compose up these are reachable from the build container via the
|
||||
// deploy_default network.
|
||||
func TestFileObjectLifecycle(t *testing.T) {
|
||||
databaseURL := os.Getenv("WORKBENCH_TEST_DATABASE_URL")
|
||||
s3Endpoint := os.Getenv("WORKBENCH_TEST_S3_ENDPOINT")
|
||||
if databaseURL == "" || s3Endpoint == "" {
|
||||
t.Skip("WORKBENCH_TEST_DATABASE_URL and WORKBENCH_TEST_S3_ENDPOINT are not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, config.Database{URL: databaseURL, MaxConns: 8, MinConns: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
store, err := storage.NewClient(storage.Config{
|
||||
Endpoint: s3Endpoint,
|
||||
AccessKeyID: os.Getenv("WORKBENCH_TEST_S3_ACCESS_KEY"),
|
||||
SecretAccessKey: os.Getenv("WORKBENCH_TEST_S3_SECRET_KEY"),
|
||||
Bucket: os.Getenv("WORKBENCH_TEST_S3_BUCKET"),
|
||||
Region: "us-east-1",
|
||||
MaxFileBytes: 8 << 20,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.EnsureBucket(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adminID := "22222222-2222-4222-8222-222222222222"
|
||||
portalID := "33333333-3333-4333-8333-333333333333"
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO gateway.admin_accounts(id,username,password_hash,role) VALUES($1,'m8-files-admin','test','superadmin') ON CONFLICT(id) DO NOTHING`, adminID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO gateway.portal_users(id,account,password_hash,name,active) VALUES($1,'m8-files-user','test','m8',true) ON CONFLICT(id) DO NOTHING`, portalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleanup := func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM gateway.file_objects`)
|
||||
}
|
||||
cleanup()
|
||||
defer cleanup()
|
||||
|
||||
files := NewFileService(NewService(pool), store)
|
||||
|
||||
content := []byte("M8 对象存储文件管理集成测试 payload\nline two\n")
|
||||
owner := portalID
|
||||
sysObj, err := files.Upload(ctx, "system", nil, adminID, "报告.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
personalObj, err := files.Upload(ctx, "personal", &owner, portalID, "notes.txt", "text/plain", bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// sha256 往返一致
|
||||
hasher := sha256.Sum256(content)
|
||||
if personalObj.ContentSHA256 != hex.EncodeToString(hasher[:]) || personalObj.SizeBytes != int64(len(content)) {
|
||||
t.Fatalf("integrity mismatch: sha=%s size=%d", personalObj.ContentSHA256, personalObj.SizeBytes)
|
||||
}
|
||||
|
||||
// 流式读回
|
||||
reader, size, err := files.Open(ctx, personalObj)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := io.ReadAll(reader)
|
||||
_ = reader.Close()
|
||||
if err != nil || int64(len(got)) != size || !bytes.Equal(got, content) {
|
||||
t.Fatalf("roundtrip mismatch: len=%d size=%d err=%v", len(got), size, err)
|
||||
}
|
||||
|
||||
// 列表按范围隔离
|
||||
sysList, err := files.ListSystem(ctx)
|
||||
if err != nil || len(sysList) != 1 || sysList[0].ID != sysObj.ID {
|
||||
t.Fatalf("system list mismatch: %#v err=%v", sysList, err)
|
||||
}
|
||||
myList, err := files.ListPersonal(ctx, portalID)
|
||||
if err != nil || len(myList) != 1 || myList[0].ID != personalObj.ID {
|
||||
t.Fatalf("personal list mismatch: %#v err=%v", myList, err)
|
||||
}
|
||||
otherList, err := files.ListPersonal(ctx, "44444444-4444-4444-8444-444444444444")
|
||||
if err != nil || len(otherList) != 0 {
|
||||
t.Fatalf("other personal list should be empty: %#v err=%v", otherList, err)
|
||||
}
|
||||
|
||||
// 归属隔离:A 不能读 B 的文件
|
||||
if _, err := files.GetPersonal(ctx, adminID, personalObj.ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("expected ErrNotFound for cross-owner read, got %v", err)
|
||||
}
|
||||
if _, err := files.GetSystem(ctx, personalObj.ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("expected ErrNotFound for system read of personal file, got %v", err)
|
||||
}
|
||||
|
||||
// 超限上传被拒(8MiB 上限,用无 EOF 占位 reader 凑 9MiB)
|
||||
if _, err := files.Upload(ctx, "system", nil, adminID, "big.bin", "application/octet-stream", io.LimitReader(zeroReader{}, 9<<20)); err == nil {
|
||||
t.Fatal("expected oversized upload to fail")
|
||||
}
|
||||
|
||||
// 删除后元数据与对象都消失
|
||||
if err := files.DeleteSystem(ctx, sysObj.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := files.GetSystem(ctx, sysObj.ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("expected ErrNotFound after delete, got %v", err)
|
||||
}
|
||||
if err := files.DeletePersonal(ctx, portalID, personalObj.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := files.GetPersonal(ctx, portalID, personalObj.ID); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("expected ErrNotFound after personal delete, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type zeroReader struct{}
|
||||
|
||||
func (zeroReader) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = 0
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package workbench
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"aigateway.local/core/internal/identity"
|
||||
"aigateway.local/core/internal/platform/apiresponse"
|
||||
)
|
||||
|
||||
// FilesPortalHTTPHandler serves each portal user's personal file store. Unlike
|
||||
// the admin handler every object is scoped to the authenticated account, so a
|
||||
// user can only ever see and open their own files.
|
||||
type FilesPortalHTTPHandler struct {
|
||||
files *FileService
|
||||
identity *identity.Service
|
||||
mux *http.ServeMux
|
||||
}
|
||||
|
||||
func NewFilesPortalHTTPHandler(files *FileService, identityService *identity.Service) *FilesPortalHTTPHandler {
|
||||
h := &FilesPortalHTTPHandler{files: files, identity: identityService, mux: http.NewServeMux()}
|
||||
h.mux.HandleFunc("POST /api/v1/portal/files", h.upload)
|
||||
h.mux.HandleFunc("GET /api/v1/portal/files", h.list)
|
||||
h.mux.HandleFunc("GET /api/v1/portal/files/{id}/download", h.download)
|
||||
h.mux.HandleFunc("DELETE /api/v1/portal/files/{id}", h.delete)
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *FilesPortalHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.mux.ServeHTTP(w, r) }
|
||||
|
||||
func (h *FilesPortalHTTPHandler) account(w http.ResponseWriter, r *http.Request) (identity.Account, bool) {
|
||||
account, err := h.identity.Authenticate(r.Context(), identity.KindPortal, r.Header.Get("Authorization"))
|
||||
if err != nil {
|
||||
apiresponse.Error(w, http.StatusUnauthorized, "登录状态无效或已过期")
|
||||
return identity.Account{}, false
|
||||
}
|
||||
return account, true
|
||||
}
|
||||
|
||||
func (h *FilesPortalHTTPHandler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := h.account(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var (
|
||||
body io.Reader
|
||||
originalName = strings.TrimSpace(r.URL.Query().Get("filename"))
|
||||
contentType = r.Header.Get("Content-Type")
|
||||
)
|
||||
if strings.HasPrefix(contentType, "multipart/form-data") {
|
||||
part, err := h.firstFilePart(r)
|
||||
if err != nil {
|
||||
apiresponse.Error(w, http.StatusBadRequest, "缺少上传文件")
|
||||
return
|
||||
}
|
||||
defer part.Close()
|
||||
body = part
|
||||
originalName = part.FileName()
|
||||
if ct := part.Header.Get("Content-Type"); ct != "" {
|
||||
contentType = ct
|
||||
}
|
||||
}
|
||||
if originalName == "" {
|
||||
apiresponse.Error(w, http.StatusBadRequest, "缺少文件名")
|
||||
return
|
||||
}
|
||||
obj, err := h.files.Upload(r.Context(), "personal", &a.ID, a.ID, originalName, contentType, body)
|
||||
if err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
apiresponse.OK(w, obj)
|
||||
}
|
||||
|
||||
func (h *FilesPortalHTTPHandler) firstFilePart(r *http.Request) (*multipart.Part, error) {
|
||||
reader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
part, err := reader.NextPart()
|
||||
if err == io.EOF {
|
||||
return nil, errors.New("no file part")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if part.FormName() == "file" {
|
||||
return part, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FilesPortalHTTPHandler) list(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := h.account(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
items, err := h.files.ListPersonal(r.Context(), a.ID)
|
||||
if err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
apiresponse.OK(w, items)
|
||||
}
|
||||
|
||||
func (h *FilesPortalHTTPHandler) download(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := h.account(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
obj, err := h.files.GetPersonal(r.Context(), a.ID, r.PathValue("id"))
|
||||
if err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
serveFileContent(w, r, h.files, obj)
|
||||
}
|
||||
|
||||
func (h *FilesPortalHTTPHandler) delete(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := h.account(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := h.files.DeletePersonal(r.Context(), a.ID, r.PathValue("id")); err != nil {
|
||||
fileError(w, err)
|
||||
return
|
||||
}
|
||||
apiresponse.OK(w, map[string]bool{"deleted": true})
|
||||
}
|
||||
Reference in New Issue
Block a user