AI Gateway Go 0.10.0 源码快照 + 旗舰版需求规划报告

M0-M7 已完成:核心网关(身份/RBAC/TOTP/OIDC/SAML/Provider/配额/路由/内容策略/审计/定价)+ 资源市场(MCP/Skills/数字员工)。
含 22 个 PostgreSQL 迁移、管理端/门户端前端源码、OpenAPI 契约、部署 compose。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
ben
2026-08-12 11:45:54 +08:00
commit 5759c1862e
807 changed files with 114727 additions and 0 deletions
+119
View File
@@ -0,0 +1,119 @@
package contentpolicy
import (
"encoding/json"
"errors"
"net/http"
"aigateway.local/core/internal/identity"
"aigateway.local/core/internal/platform/apiresponse"
)
type AdminHTTPHandler struct {
store *Store
engine *Engine
identity *identity.Service
mux *http.ServeMux
}
func NewAdminHTTPHandler(store *Store, engine *Engine, identityService *identity.Service) *AdminHTTPHandler {
h := &AdminHTTPHandler{store: store, engine: engine, identity: identityService, mux: http.NewServeMux()}
h.mux.HandleFunc("GET /api/v1/admin/content-policies", h.list)
h.mux.HandleFunc("POST /api/v1/admin/content-policies", h.create)
h.mux.HandleFunc("PUT /api/v1/admin/content-policies/{policy_id}", h.update)
h.mux.HandleFunc("DELETE /api/v1/admin/content-policies/{policy_id}", h.delete)
return h
}
func (h *AdminHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.mux.ServeHTTP(w, r) }
func (h *AdminHTTPHandler) list(w http.ResponseWriter, r *http.Request) {
if _, ok := h.require(w, r, identity.PermissionContentPolicyRead); !ok {
return
}
items, err := h.store.List(r.Context())
if err != nil {
apiresponse.Error(w, 503, "内容策略服务暂不可用")
return
}
apiresponse.OK(w, items)
}
func (h *AdminHTTPHandler) create(w http.ResponseWriter, r *http.Request) {
actor, ok := h.require(w, r, identity.PermissionContentPolicyManage)
if !ok {
return
}
p, ok := decodePolicy(w, r)
if !ok {
return
}
saved, err := h.store.Save(r.Context(), p, actor.ID, true)
h.finish(w, r, saved, err)
}
func (h *AdminHTTPHandler) update(w http.ResponseWriter, r *http.Request) {
actor, ok := h.require(w, r, identity.PermissionContentPolicyManage)
if !ok {
return
}
p, ok := decodePolicy(w, r)
if !ok {
return
}
p.ID = r.PathValue("policy_id")
saved, err := h.store.Save(r.Context(), p, actor.ID, false)
h.finish(w, r, saved, err)
}
func (h *AdminHTTPHandler) delete(w http.ResponseWriter, r *http.Request) {
actor, ok := h.require(w, r, identity.PermissionContentPolicyManage)
if !ok {
return
}
err := h.store.Delete(r.Context(), r.PathValue("policy_id"), actor.ID)
if errors.Is(err, ErrNotFound) {
apiresponse.Error(w, 404, "内容策略不存在")
return
}
if err != nil {
apiresponse.Error(w, 503, "内容策略删除失败")
return
}
_ = h.engine.Reload(r.Context())
apiresponse.OK(w, map[string]bool{"deleted": true})
}
func (h *AdminHTTPHandler) finish(w http.ResponseWriter, r *http.Request, p Policy, err error) {
if errors.Is(err, ErrNotFound) {
apiresponse.Error(w, 404, "内容策略不存在")
return
}
if err != nil {
apiresponse.Error(w, 400, err.Error())
return
}
_ = h.engine.Reload(r.Context())
apiresponse.OK(w, p)
}
func decodePolicy(w http.ResponseWriter, r *http.Request) (Policy, bool) {
var p Policy
d := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20))
d.DisallowUnknownFields()
if d.Decode(&p) != nil {
apiresponse.Error(w, 400, "请求格式无效")
return p, false
}
Normalize(&p)
if err := ValidateInput(p); err != nil {
apiresponse.Error(w, 400, err.Error())
return p, false
}
return p, true
}
func (h *AdminHTTPHandler) require(w http.ResponseWriter, r *http.Request, permission string) (identity.Account, bool) {
a, err := h.identity.Authenticate(r.Context(), identity.KindAdmin, r.Header.Get("Authorization"))
if err != nil {
apiresponse.Error(w, 401, "登录状态无效")
return a, false
}
if !identity.HasPermission(a, permission) {
apiresponse.Error(w, 403, "缺少内容策略权限")
return a, false
}
return a, true
}
+272
View File
@@ -0,0 +1,272 @@
package contentpolicy
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"regexp"
"slices"
"strings"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
var ErrBodyTooLarge = errors.New("request body is too large")
type Rule struct {
Name string `json:"name"`
Pattern string `json:"pattern"`
Replacement string `json:"replacement"`
}
type Policy struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Action string `json:"action"`
Priority int `json:"priority"`
Paths []string `json:"paths"`
Models []string `json:"models"`
APIKeyIDs []string `json:"api_key_ids"`
Rules []Rule `json:"rules"`
Enabled bool `json:"enabled"`
Revision int64 `json:"revision"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type compiledRule struct {
name, replacement string
expression *regexp.Regexp
}
type compiledPolicy struct {
Policy
rules []compiledRule
}
type snapshot struct{ policies []compiledPolicy }
type Engine struct {
pool *pgxpool.Pool
logger *slog.Logger
refresh time.Duration
current atomic.Pointer[snapshot]
}
type Match struct {
PolicyID string `json:"policy_id"`
PolicyName string `json:"policy_name"`
Action string `json:"action"`
Rules []string `json:"rules"`
}
type Result struct {
Matches []Match
Blocked, Redacted bool
}
func NewEngine(pool *pgxpool.Pool, refresh time.Duration, logger *slog.Logger) *Engine {
if refresh <= 0 {
refresh = 30 * time.Second
}
e := &Engine{pool: pool, refresh: refresh, logger: logger}
e.current.Store(&snapshot{})
return e
}
func (e *Engine) Run(ctx context.Context) {
_ = e.Reload(ctx)
ticker := time.NewTicker(e.refresh)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err := e.Reload(ctx); err != nil && e.logger != nil {
e.logger.Warn("content policy refresh failed", "error", err)
}
case <-ctx.Done():
return
}
}
}
func (e *Engine) Reload(ctx context.Context) error {
if e == nil || e.pool == nil {
return errors.New("content policy store unavailable")
}
rows, err := e.pool.Query(ctx, `SELECT id::text,name,description,action,priority,paths,models,api_key_ids::text[],rules,enabled,revision,created_at,updated_at FROM gateway.content_policies WHERE enabled ORDER BY priority DESC,name,id`)
if err != nil {
return fmt.Errorf("load content policies: %w", err)
}
defer rows.Close()
loaded := make([]compiledPolicy, 0)
for rows.Next() {
var policy Policy
var raw []byte
if err := rows.Scan(&policy.ID, &policy.Name, &policy.Description, &policy.Action, &policy.Priority, &policy.Paths, &policy.Models, &policy.APIKeyIDs, &raw, &policy.Enabled, &policy.Revision, &policy.CreatedAt, &policy.UpdatedAt); err != nil {
return err
}
if err := json.Unmarshal(raw, &policy.Rules); err != nil {
return fmt.Errorf("decode policy %s: %w", policy.ID, err)
}
compiled, err := compilePolicy(policy)
if err != nil {
return fmt.Errorf("compile policy %s: %w", policy.ID, err)
}
loaded = append(loaded, compiled)
}
if err := rows.Err(); err != nil {
return err
}
e.current.Store(&snapshot{policies: loaded})
return nil
}
func compilePolicy(policy Policy) (compiledPolicy, error) {
if policy.Action != "audit" && policy.Action != "block" && policy.Action != "redact" {
return compiledPolicy{}, errors.New("unsupported action")
}
if len(policy.Rules) == 0 || len(policy.Rules) > 20 {
return compiledPolicy{}, errors.New("rules must contain 1 to 20 entries")
}
result := compiledPolicy{Policy: policy, rules: make([]compiledRule, 0, len(policy.Rules))}
for _, rule := range policy.Rules {
if strings.TrimSpace(rule.Name) == "" || len(rule.Pattern) == 0 || len(rule.Pattern) > 512 {
return compiledPolicy{}, errors.New("invalid rule name or pattern length")
}
expression, err := regexp.Compile(rule.Pattern)
if err != nil {
return compiledPolicy{}, err
}
replacement := rule.Replacement
if replacement == "" {
replacement = "[REDACTED]"
}
result.rules = append(result.rules, compiledRule{name: rule.Name, replacement: replacement, expression: expression})
}
return result, nil
}
func Validate(policy Policy) error { _, err := compilePolicy(policy); return err }
func (e *Engine) Apply(request *http.Request, maxBody int64, apiKeyID string) (Result, error) {
if e == nil || request.Body == nil || request.Body == http.NoBody || request.Method == http.MethodGet {
return Result{}, nil
}
body, err := io.ReadAll(io.LimitReader(request.Body, maxBody+1))
if err != nil {
return Result{}, err
}
_ = request.Body.Close()
if int64(len(body)) > maxBody {
return Result{}, ErrBodyTooLarge
}
restoreBody(request, body)
var document any
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.UseNumber()
if decoder.Decode(&document) != nil {
return Result{}, nil
}
model := findModel(document)
result := Result{}
for _, policy := range e.current.Load().policies {
if !policyApplies(policy, request.URL.Path, model, apiKeyID) {
continue
}
matched := make([]string, 0)
changed := false
walkText(&document, false, func(value string) string {
for _, rule := range policy.rules {
if rule.expression.MatchString(value) {
if !slices.Contains(matched, rule.name) {
matched = append(matched, rule.name)
}
if policy.Action == "redact" {
next := rule.expression.ReplaceAllString(value, rule.replacement)
changed = changed || next != value
value = next
}
}
}
return value
})
if len(matched) == 0 {
continue
}
result.Matches = append(result.Matches, Match{PolicyID: policy.ID, PolicyName: policy.Name, Action: policy.Action, Rules: matched})
if policy.Action == "block" {
result.Blocked = true
break
}
result.Redacted = result.Redacted || changed
}
if result.Redacted && !result.Blocked {
encoded, err := json.Marshal(document)
if err != nil {
return Result{}, err
}
restoreBody(request, encoded)
}
return result, nil
}
func policyApplies(policy compiledPolicy, path, model, apiKeyID string) bool {
return matches(policy.Paths, path) && matches(policy.Models, model) && matches(policy.APIKeyIDs, apiKeyID)
}
func matches(values []string, candidate string) bool {
if len(values) == 0 {
return true
}
for _, value := range values {
if value == "*" || value == candidate {
return true
}
}
return false
}
var textualFields = map[string]bool{"content": true, "text": true, "input": true, "prompt": true, "instructions": true}
func walkText(value *any, selected bool, transform func(string) string) {
switch current := (*value).(type) {
case map[string]any:
for key, child := range current {
local := child
walkText(&local, selected || textualFields[strings.ToLower(key)], transform)
current[key] = local
}
case []any:
for index, child := range current {
local := child
walkText(&local, selected, transform)
current[index] = local
}
case string:
if selected {
*value = transform(current)
}
}
}
func findModel(document any) string {
if object, ok := document.(map[string]any); ok {
if model, ok := object["model"].(string); ok {
return model
}
}
return ""
}
func restoreBody(request *http.Request, body []byte) {
request.Body = io.NopCloser(bytes.NewReader(body))
request.ContentLength = int64(len(body))
request.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }
}
+44
View File
@@ -0,0 +1,44 @@
package contentpolicy
import (
"io"
"net/http"
"strings"
"testing"
)
func testEngine(t *testing.T, action string) *Engine {
t.Helper()
policy := Policy{ID: "p1", Name: "secret", Action: action, Rules: []Rule{{Name: "token", Pattern: `sk-[A-Za-z0-9]{8,}`, Replacement: "[MASKED]"}}, Enabled: true}
compiled, err := compilePolicy(policy)
if err != nil {
t.Fatal(err)
}
engine := &Engine{}
engine.current.Store(&snapshot{policies: []compiledPolicy{compiled}})
return engine
}
func TestApplyRedactsOnlyTextualPromptFields(t *testing.T) {
request, _ := http.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"gpt-5","messages":[{"content":"use sk-12345678"}],"tools":[{"description":"keep sk-abcdefgh"}]}`))
result, err := testEngine(t, "redact").Apply(request, 1<<20, "key")
if err != nil || !result.Redacted || result.Blocked {
t.Fatalf("unexpected result %#v: %v", result, err)
}
body, _ := io.ReadAll(request.Body)
text := string(body)
if strings.Contains(text, "sk-12345678") || !strings.Contains(text, "keep sk-abcdefgh") {
t.Fatalf("unexpected redaction: %s", text)
}
}
func TestApplyBlocksWithoutReturningMatchedSecret(t *testing.T) {
request, _ := http.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{"input":"sk-12345678"}`))
result, err := testEngine(t, "block").Apply(request, 1<<20, "")
if err != nil || !result.Blocked || len(result.Matches) != 1 {
t.Fatalf("unexpected result %#v: %v", result, err)
}
if strings.Contains(result.Matches[0].Rules[0], "12345678") {
t.Fatal("match metadata leaked secret")
}
}
+156
View File
@@ -0,0 +1,156 @@
package contentpolicy
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
platformid "aigateway.local/core/internal/platform/id"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
)
var ErrNotFound = errors.New("content policy not found")
type Store struct{ pool *pgxpool.Pool }
func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
func (s *Store) List(ctx context.Context) ([]Policy, error) {
rows, err := s.pool.Query(ctx, `SELECT id::text,name,description,action,priority,paths,models,api_key_ids::text[],rules,enabled,revision,created_at,updated_at FROM gateway.content_policies ORDER BY priority DESC,name`)
if err != nil {
return nil, err
}
defer rows.Close()
result := []Policy{}
for rows.Next() {
var p Policy
var raw []byte
if err := rows.Scan(&p.ID, &p.Name, &p.Description, &p.Action, &p.Priority, &p.Paths, &p.Models, &p.APIKeyIDs, &raw, &p.Enabled, &p.Revision, &p.CreatedAt, &p.UpdatedAt); err != nil {
return nil, err
}
if err := json.Unmarshal(raw, &p.Rules); err != nil {
return nil, err
}
result = append(result, p)
}
return result, rows.Err()
}
func (s *Store) Save(ctx context.Context, p Policy, actorID string, create bool) (Policy, error) {
if err := Validate(p); err != nil {
return Policy{}, err
}
rules, _ := json.Marshal(p.Rules)
eventID, _ := platformid.NewUUID()
tx, err := s.pool.Begin(ctx)
if err != nil {
return Policy{}, err
}
defer func() { _ = tx.Rollback(ctx) }()
if create {
p.ID, _ = platformid.NewUUID()
err = tx.QueryRow(ctx, `INSERT INTO gateway.content_policies(id,name,description,action,priority,paths,models,api_key_ids,rules,enabled,created_by) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING revision,created_at,updated_at`, p.ID, p.Name, p.Description, p.Action, p.Priority, p.Paths, p.Models, p.APIKeyIDs, rules, p.Enabled, actorID).Scan(&p.Revision, &p.CreatedAt, &p.UpdatedAt)
} else {
err = tx.QueryRow(ctx, `UPDATE gateway.content_policies SET name=$2,description=$3,action=$4,priority=$5,paths=$6,models=$7,api_key_ids=$8,rules=$9,enabled=$10,revision=revision+1,updated_at=clock_timestamp() WHERE id=$1 RETURNING revision,created_at,updated_at`, p.ID, p.Name, p.Description, p.Action, p.Priority, p.Paths, p.Models, p.APIKeyIDs, rules, p.Enabled).Scan(&p.Revision, &p.CreatedAt, &p.UpdatedAt)
}
if errors.Is(err, pgx.ErrNoRows) {
return Policy{}, ErrNotFound
}
if err != nil {
return Policy{}, err
}
eventType := "content_policy.updated"
if create {
eventType = "content_policy.created"
}
payload, _ := json.Marshal(map[string]any{"content_policy_id": p.ID, "actor_id": actorID, "revision": p.Revision})
_, err = tx.Exec(ctx, `INSERT INTO gateway.outbox_events(event_id,event_type,event_version,aggregate_type,aggregate_id,payload) VALUES($1,$2,1,'content_policy',$3,$4)`, eventID, eventType, p.ID, payload)
if err != nil {
return Policy{}, err
}
return p, tx.Commit(ctx)
}
func (s *Store) Delete(ctx context.Context, id, actorID string) error {
eventID, _ := platformid.NewUUID()
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
tag, err := tx.Exec(ctx, `DELETE FROM gateway.content_policies WHERE id=$1`, id)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return ErrNotFound
}
payload, _ := json.Marshal(map[string]string{"content_policy_id": id, "actor_id": actorID})
if _, err = tx.Exec(ctx, `INSERT INTO gateway.outbox_events(event_id,event_type,event_version,aggregate_type,aggregate_id,payload) VALUES($1,'content_policy.deleted',1,'content_policy',$2,$3)`, eventID, id, payload); err != nil {
return err
}
return tx.Commit(ctx)
}
func Normalize(p *Policy) {
p.Name = strings.TrimSpace(p.Name)
p.Description = strings.TrimSpace(p.Description)
p.Action = strings.ToLower(strings.TrimSpace(p.Action))
p.Paths = normalizeStrings(p.Paths)
p.Models = normalizeStrings(p.Models)
p.APIKeyIDs = normalizeStrings(p.APIKeyIDs)
}
func ValidateInput(p Policy) error {
if p.Name == "" || len(p.Name) > 128 {
return fmt.Errorf("name is required and must not exceed 128 characters")
}
if len(p.Description) > 1000 {
return fmt.Errorf("description is too long")
}
if p.Priority < -100000 || p.Priority > 100000 {
return fmt.Errorf("priority 必须在 -100000 到 100000 之间")
}
if len(p.Paths) > 20 || len(p.Models) > 100 || len(p.APIKeyIDs) > 100 {
return fmt.Errorf("策略范围条目过多")
}
allowedPaths := map[string]bool{"*": true, "/v1/chat/completions": true, "/v1/responses": true, "/v1/embeddings": true, "/v1/messages": true}
for _, value := range p.Paths {
if !allowedPaths[value] {
return fmt.Errorf("不支持的端点范围 %s", value)
}
}
for _, value := range p.Models {
if len(value) > 512 {
return fmt.Errorf("模型名过长")
}
}
for _, value := range p.APIKeyIDs {
var id pgtype.UUID
if id.Scan(value) != nil || !id.Valid {
return fmt.Errorf("API Key ID 格式无效")
}
}
for _, rule := range p.Rules {
if len(rule.Name) > 128 || len(rule.Replacement) > 1024 {
return fmt.Errorf("规则名称或替换文本过长")
}
}
return Validate(p)
}
func normalizeStrings(values []string) []string {
result := make([]string, 0, len(values))
seen := map[string]bool{}
for _, value := range values {
value = strings.TrimSpace(value)
if value != "" && !seen[value] {
seen[value] = true
result = append(result, value)
}
}
return result
}