0.10.1: 安全与业务逻辑加固、新品牌与部署加固

三轮审查修复(60+ 项),相对远端 main(b536672)的关键变更: - 安全: 数据面 SSRF 拨号防护(防 DNS rebinding)/上游凭据剥离/登录防枚举与锁定态统一/可信代理(X-Forwarded-For)限流加固/会话版本失效机制/ 撤销即时传播/弱密钥拒绝启动/脱敏字节级重写(保签名契约) - 业务逻辑: 裸 body 上传 panic/bootstrap 审计管线卡死/定价通配符优先级/ 全局工具可见性/调度器停机补跑/TOTP 挑战令牌消费顺序/熔断探针语义/ >4MB 响应 token 计量/管理员重置密码作废会话等 - 前端: 新 logo(语枢 AI 网关主题)/Provider 凭据异常警示/删除入口/ 后端错误消息透传/localStorage 敏感数据收敛 - 部署: CREDENTIAL_MASTER_KEY 持久化与弱值拒绝/Provider DELETE 接口/ nginx 安全头/worker 内存限制 - 新增迁移 000029(key_hash 索引)/000030(usage_daily 币种维度)
2026-08-13 10:50:51 +08:00
parent b536672000
commit 9501751792
136 changed files with 8024 additions and 1476 deletions
@@ -11,6 +11,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gateway-bootstrap
 RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gateway-outbox-worker ./cmd/gateway-outbox-worker
 RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gateway-maintenance ./cmd/gateway-maintenance
 RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gateway-notification-worker ./cmd/gateway-notification-worker
+RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gateway-scheduler ./cmd/gateway-scheduler
 RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gateway-loadtest ./cmd/gateway-loadtest
 RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gateway-legacy-import ./cmd/gateway-legacy-import

@@ -25,6 +26,7 @@ COPY --from=builder /out/gateway-bootstrap /usr/local/bin/gateway-bootstrap
 COPY --from=builder /out/gateway-outbox-worker /usr/local/bin/gateway-outbox-worker
 COPY --from=builder /out/gateway-maintenance /usr/local/bin/gateway-maintenance
 COPY --from=builder /out/gateway-notification-worker /usr/local/bin/gateway-notification-worker
+COPY --from=builder /out/gateway-scheduler /usr/local/bin/gateway-scheduler
 COPY --from=builder /out/gateway-loadtest /usr/local/bin/gateway-loadtest
 COPY --from=builder /out/gateway-legacy-import /usr/local/bin/gateway-legacy-import
 COPY migrations ./migrations
@@ -7,6 +7,8 @@ required. Neither MinIO nor Ollama is a startup dependency: the gateway warns
 and refuses file uploads until the bucket is reachable, and knowledge-base
 documents are still stored (with embedding set to NULL) when Ollama is down,
 with retrieval falling back to full-text search.
+The scheduler is a separate stateless worker backed by PostgreSQL leases; run at
+least one `scheduler-worker` replica whenever scheduled tasks are enabled.

 ## Prerequisites

@@ -65,6 +67,7 @@ Check status and logs:
 ```bash
 docker compose --env-file deploy/production.env -f deploy/docker-compose.production.yml ps
 docker compose --env-file deploy/production.env -f deploy/docker-compose.production.yml logs --tail=200 gateway-api
+docker compose --env-file deploy/production.env -f deploy/docker-compose.production.yml logs --tail=200 scheduler-worker
 curl --fail http://127.0.0.1:8080/readyz
 ```

@@ -77,6 +80,18 @@ The bundled database URLs use `sslmode=disable` only for the private Compose
 network. When using an external PostgreSQL or Redis service, require TLS and use
 `sslmode=verify-full` / `rediss://` as supported by that service.

+### Scheduled tasks
+
+- `scheduler-worker` calls the gateway over `SCHEDULER_GATEWAY_BASE_URL`; keep it
+  on the private application network and do not expose a scheduler port.
+- Task API keys are encrypted with `CREDENTIAL_MASTER_KEY`. Back up the active
+  key and its historical keyring together with PostgreSQL before rotation.
+- `SCHEDULER_MAX_ATTEMPTS` controls both ordinary execution retries and stale
+  lease recovery. Final success/failure is published through outbox and can be
+  delivered to the selected notification channel and the creator's inbox.
+- Multiple replicas are safe because due tasks and runs are claimed with
+  PostgreSQL row locks and `SKIP LOCKED`.
+
 ### Vectorization and object storage

 - The PostgreSQL image is `pgvector/pgvector:pg17` (data-volume compatible with
@@ -88,3 +103,16 @@ network. When using an external PostgreSQL or Redis service, require TLS and use
 - Back up the `minio-data` and `ollama-models` volumes alongside PostgreSQL.
 - If you previously deployed with `postgres:17-alpine`, back up the PostgreSQL
  volume before switching images.
+
+### CREDENTIAL_MASTER_KEY 持久化（重要）
+
+- 所有加密凭据（Provider API Key、TOTP 密钥、Webhook 签名、调度任务 Key、
+  应用运行时 Key）都用 `CREDENTIAL_MASTER_KEY` 加密。**该密钥必须持久化**：
+  每次部署换新 key 会让全部已存凭据无法解密。
+- 本地 compose 首次启动前执行
+  `openssl rand -base64 32 > deploy/.env`（compose 自动读取该文件），
+  之后重启/重建复用同一 key；`.gitignore` 已排除 `deploy/.env`。
+- 需要轮换时使用管理端「供应商 → 凭据轮换」接口（同一 KEK 版本内重加密），
+  并保留历史 keyring；不要直接更换 `CREDENTIAL_MASTER_KEY` 值。
+- 若误换 key：管理端 Provider 列表会降级显示「凭据无法解密」警示（不会
+  让整个页面报错），需重新保存各 Provider 的 API Key 恢复。
@@ -40,6 +40,11 @@ x-gateway-environment: &gateway-environment
  EMBEDDING_DIM: ${EMBEDDING_DIM:-1024}
  EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
  EMBEDDING_TIMEOUT: ${EMBEDDING_TIMEOUT:-120s}
+  SCHEDULER_GATEWAY_BASE_URL: ${SCHEDULER_GATEWAY_BASE_URL:-http://gateway-api:8080}
+  SCHEDULER_POLL_INTERVAL: ${SCHEDULER_POLL_INTERVAL:-5s}
+  SCHEDULER_EXECUTION_TIMEOUT: ${SCHEDULER_EXECUTION_TIMEOUT:-5m}
+  SCHEDULER_BATCH_SIZE: ${SCHEDULER_BATCH_SIZE:-10}
+  SCHEDULER_MAX_ATTEMPTS: ${SCHEDULER_MAX_ATTEMPTS:-3}

 x-backend-service: &backend-service
  image: ai-gateway-go:${GATEWAY_VERSION:-0.10.0}
@@ -191,6 +196,16 @@ services:
        condition: service_healthy
    restart: unless-stopped

+  scheduler-worker:
+    <<: *backend-service
+    entrypoint: ["gateway-scheduler"]
+    depends_on:
+      migrator:
+        condition: service_completed_successfully
+      gateway-api:
+        condition: service_healthy
+    restart: unless-stopped
+
  # M8: 对象存储。MinIO 不暴露主机端口,上传/下载全部经网关代理;
  # 桶由 gateway-api 启动时的 EnsureBucket 兜底创建。stateful 服务不做加固。
  minio:
@@ -50,7 +50,9 @@ services:
      REDIS_CACHE_URL: redis://redis-cache:6379/0
      GATEWAY_BOOTSTRAP_API_KEY: ${GATEWAY_BOOTSTRAP_API_KEY:-}
      GATEWAY_BOOTSTRAP_API_KEY_ENABLED: ${GATEWAY_BOOTSTRAP_API_KEY_ENABLED:-false}
-      CREDENTIAL_MASTER_KEY: ${CREDENTIAL_MASTER_KEY:-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=}
+      # 必须显式设置强随机密钥(openssl rand -base64 32);默认值已被服务
+      # 端 ValidateRuntime 拒绝,缺失或使用占位值时容器会拒绝启动。
+      CREDENTIAL_MASTER_KEY: ${CREDENTIAL_MASTER_KEY:?CREDENTIAL_MASTER_KEY is required (openssl rand -base64 32)}
      CREDENTIAL_KEK_VERSION: 1
      CREDENTIAL_KEK_KEYRING: ${CREDENTIAL_KEK_KEYRING:-}
      ALLOW_PRIVATE_TOOL_URLS: ${ALLOW_PRIVATE_TOOL_URLS:-false}
@@ -74,6 +76,11 @@ services:
      EMBEDDING_DIM: ${EMBEDDING_DIM:-1024}
      EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
      EMBEDDING_TIMEOUT: ${EMBEDDING_TIMEOUT:-120s}
+      SCHEDULER_GATEWAY_BASE_URL: ${SCHEDULER_GATEWAY_BASE_URL:-http://gateway-api:8080}
+      SCHEDULER_POLL_INTERVAL: ${SCHEDULER_POLL_INTERVAL:-5s}
+      SCHEDULER_EXECUTION_TIMEOUT: ${SCHEDULER_EXECUTION_TIMEOUT:-5m}
+      SCHEDULER_BATCH_SIZE: ${SCHEDULER_BATCH_SIZE:-10}
+      SCHEDULER_MAX_ATTEMPTS: ${SCHEDULER_MAX_ATTEMPTS:-3}
    depends_on:
      postgres:
        condition: service_healthy
@@ -86,7 +93,10 @@ services:
        VERSION: ${GATEWAY_VERSION:-0.10.0}
    environment: *gateway-environment
    ports:
-      - "${GATEWAY_PORT:-8080}:8080"
+      # 仅绑定回环地址:控制台与网关请求都经 admin-web/portal-web 的 nginx
+      # 转发,网关端口不需要对外暴露。直接暴露时,攻击者可以绕过 nginx
+      # 伪造 X-Forwarded-For 并直连 /metrics 等端点。
+      - "127.0.0.1:${GATEWAY_PORT:-8080}:8080"
    depends_on:
      migrator:
        condition: service_completed_successfully
@@ -100,6 +110,10 @@ services:
      ollama:
        condition: service_started
    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 1g

  # M8: 对象存储。MinIO 不暴露主机端口,凭据只留在 API 容器内;
  # 上传/下载全部经网关代理。桶由 gateway-api 启动时的 EnsureBucket 兜底创建。
@@ -161,6 +175,10 @@ services:
      redis-critical:
        condition: service_healthy
    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 512m

  maintenance-worker:
    build:
@@ -174,6 +192,10 @@ services:
      migrator:
        condition: service_completed_successfully
    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 512m

  notification-worker:
    build:
@@ -189,6 +211,29 @@ services:
      redis-critical:
        condition: service_healthy
    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 512m
+
+  scheduler-worker:
+    build:
+      context: ..
+      dockerfile: deploy/Dockerfile
+      args:
+        VERSION: ${GATEWAY_VERSION:-0.10.0}
+    entrypoint: ["gateway-scheduler"]
+    environment: *gateway-environment
+    depends_on:
+      migrator:
+        condition: service_completed_successfully
+      gateway-api:
+        condition: service_started
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 512m

 volumes:
  postgres-data:
@@ -1,6 +1,8 @@
 server {
    listen 80;
    server_name _;
+    # 不暴露 nginx 版本号(server_tokens 同时影响错误页与 Server 头)。
+    server_tokens off;

    # Keep Location headers relative (Location: /admin/) instead of letting
    # nginx absolute_redirect rebuild them from $host + the listening port.
@@ -16,6 +18,11 @@ server {
    root /usr/share/nginx/html;
    index index.html;

+    # 基础安全响应头:防 MIME 嗅探、防点击劫持、限制 Referer 泄露来源页面。
+    add_header X-Content-Type-Options nosniff always;
+    add_header X-Frame-Options DENY always;
+    add_header Referrer-Policy same-origin always;
+
    location = / {
        return 302 /__APP__/;
    }
@@ -58,6 +58,13 @@ EMBEDDING_DIM=1024
 EMBEDDING_BATCH_SIZE=64
 EMBEDDING_TIMEOUT=120s

+# M8 P3 PostgreSQL 权威队列调度器。
+SCHEDULER_GATEWAY_BASE_URL=http://gateway-api:8080
+SCHEDULER_POLL_INTERVAL=5s
+SCHEDULER_EXECUTION_TIMEOUT=5m
+SCHEDULER_BATCH_SIZE=10
+SCHEDULER_MAX_ATTEMPTS=3
+
 # Used only for the one-time bootstrap-admin command; remove after use.
 BOOTSTRAP_ADMIN_USERNAME=admin
 BOOTSTRAP_ADMIN_PASSWORD=CHANGE_ME_AT_LEAST_12_CHARACTERS