feat(m8): P2 pgvector + Ollama 向量化与语义检索

- PostgreSQL 切换 pgvector/pgvector:pg17 镜像;迁移 000024 建 vector 扩展、
  knowledge_chunks.embedding vector(1024) + HNSW 余弦索引,retrieval_mode 放宽三态
- OllamaEmbedder 本地 bge-m3 批量嵌入,404 惰性 pull 重试,维度/超时校验,可整体关闭
- SemanticRetriever/HybridRetriever + NewRetriever 按 retrieval_mode 分发,缺 embedder 回退 FTS
- 文档入库同步批量向量化;Ollama 故障降级入库 + embedding_failed 事件
- 修复 pgx CopyFrom 对 vector 列二进制编码误读:COPY 基础列后同事务 unnest 批量回填
- 修复降级路径 embeddings=nil 索引越界 panic(Add 与 Reprocess)
- 知识库列表 vectorized_chunk_count + 前端三态检索模式选择与向量化覆盖率
- 单测 embedder/retrievers + 集成 TestKnowledgeVectorLifecycle 全绿

Co-Authored-By: Claude <noreply@anthropic.com>

deploy/PRODUCTION.md

@@ -1,9 +1,12 @@
 # Production deployment
 
 This bundle builds the Go services and both Art Design Pro applications from
-source. PostgreSQL, two Redis roles and MinIO (object storage, M8) are included;
-ClickHouse is not required. MinIO is not a startup dependency: the gateway only
-warns and refuses file uploads until the bucket is reachable.
+source. PostgreSQL (with pgvector), two Redis roles, MinIO (object storage, M8)
+and a local Ollama container (vectorization, M8) are included; ClickHouse is not
+required. Neither MinIO nor Ollama is a startup dependency: the gateway warns
+and refuses file uploads until the bucket is reachable, and knowledge-base
+documents are still stored (with embedding set to NULL) when Ollama is down,
+with retrieval falling back to full-text search.
 
 ## Prerequisites
 
@@ -73,3 +76,15 @@ it removes persistent data.
 The bundled database URLs use `sslmode=disable` only for the private Compose
 network. When using an external PostgreSQL or Redis service, require TLS and use
 `sslmode=verify-full` / `rediss://` as supported by that service.
+
+### Vectorization and object storage
+
+- The PostgreSQL image is `pgvector/pgvector:pg17` (data-volume compatible with
+  `postgres:17-alpine`); migration `000024` creates the `vector` extension and
+  adds the HNSW embedding column. `EMBEDDING_DIM` must stay at `1024` to match
+  the `vector(1024)` column.
+- Ollama runs locally and lazily pulls `bge-m3` (~1.2 GiB) on first embedding
+  request. Set `EMBEDDINGS_ENABLED=false` to disable vectorization entirely.
+- Back up the `minio-data` and `ollama-models` volumes alongside PostgreSQL.
+- If you previously deployed with `postgres:17-alpine`, back up the PostgreSQL
+  volume before switching images.

deploy/docker-compose.production.yml

@@ -33,6 +33,13 @@ x-gateway-environment: &gateway-environment
   S3_REGION: ${S3_REGION:-us-east-1}
   S3_USE_SSL: ${S3_USE_SSL:-false}
   S3_MAX_FILE_BYTES: ${S3_MAX_FILE_BYTES:-134217728}
+  # M8 P2:本地 Ollama 向量化。生产可整体关闭(EMBEDDINGS_ENABLED=false)走纯 FTS。
+  EMBEDDINGS_ENABLED: ${EMBEDDINGS_ENABLED:-true}
+  OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-http://ollama:11434}
+  EMBEDDING_MODEL: ${EMBEDDING_MODEL:-bge-m3}
+  EMBEDDING_DIM: ${EMBEDDING_DIM:-1024}
+  EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
+  EMBEDDING_TIMEOUT: ${EMBEDDING_TIMEOUT:-120s}
 
 x-backend-service: &backend-service
   image: ai-gateway-go:${GATEWAY_VERSION:-0.10.0}
@@ -47,8 +54,9 @@ x-backend-service: &backend-service
     - no-new-privileges:true
 
 services:
+  # M8 P2:pgvector 镜像提供 vector 扩展;数据卷与 postgres:17-alpine 兼容。
   postgres:
-    image: postgres:17-alpine
+    image: pgvector/pgvector:pg17
     environment:
       POSTGRES_DB: ${POSTGRES_DB:-gateway}
       POSTGRES_USER: ${POSTGRES_USER:-gateway}
@@ -116,6 +124,9 @@ services:
         condition: service_healthy
       minio:
         condition: service_started
+      # 向量化不阻断 API 启动:ollama 未就绪时知识库入库降级。
+      ollama:
+        condition: service_started
     healthcheck:
       test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/readyz"]
       interval: 10s
@@ -192,6 +203,14 @@ services:
       - minio-data:/data
     restart: unless-stopped
 
+  # M8 P2:本地向量化(默认 bge-m3)。stateful,首次 embed 惰性 pull 模型;
+  # 生产不需要时可移除并设 EMBEDDINGS_ENABLED=false 走纯 FTS。
+  ollama:
+    image: ollama/ollama:latest
+    volumes:
+      - ollama-models:/root/.ollama
+    restart: unless-stopped
+
   bootstrap-admin:
     <<: *backend-service
     profiles: ["tools"]
@@ -209,3 +228,4 @@ volumes:
   postgres-data:
   redis-critical-data:
   minio-data:
+  ollama-models:

deploy/docker-compose.yml

@@ -1,6 +1,8 @@
 services:
+  # M8 P2:pgvector 镜像与 postgres:17-alpine 数据卷兼容;CREATE EXTENSION vector
+  # 由迁移 000024 执行(容器内 gateway 用户即 superuser)。
   postgres:
-    image: postgres:17-alpine
+    image: pgvector/pgvector:pg17
     environment:
       POSTGRES_DB: gateway
       POSTGRES_USER: gateway
@@ -65,6 +67,13 @@ services:
       S3_REGION: ${S3_REGION:-us-east-1}
       S3_USE_SSL: ${S3_USE_SSL:-false}
       S3_MAX_FILE_BYTES: ${S3_MAX_FILE_BYTES:-134217728}
+      # M8 P2:本地 Ollama 向量化(bge-m3)。EMBEDDINGS_ENABLED=false 可整体关闭。
+      EMBEDDINGS_ENABLED: ${EMBEDDINGS_ENABLED:-true}
+      OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-http://ollama:11434}
+      EMBEDDING_MODEL: ${EMBEDDING_MODEL:-bge-m3}
+      EMBEDDING_DIM: ${EMBEDDING_DIM:-1024}
+      EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
+      EMBEDDING_TIMEOUT: ${EMBEDDING_TIMEOUT:-120s}
     depends_on:
       postgres:
         condition: service_healthy
@@ -87,6 +96,9 @@ services:
         condition: service_healthy
       minio:
         condition: service_started
+      # 向量化不阻断 API 启动:ollama 未就绪时 knowledge 入库降级(embedding 置 NULL)。
+      ollama:
+        condition: service_started
     restart: unless-stopped
 
   # M8: 对象存储。MinIO 不暴露主机端口,凭据只留在 API 容器内;
@@ -101,6 +113,14 @@ services:
       - minio-data:/data
     restart: unless-stopped
 
+  # M8 P2:本地向量化。模型默认 bge-m3(~1.2GB),首次 embed 惰性 pull;
+  # 无需对外暴露端口,仅网关容器访问。restart=unless-stopped 以便重启后自愈。
+  ollama:
+    image: ollama/ollama:latest
+    volumes:
+      - ollama-models:/root/.ollama
+    restart: unless-stopped
+
   admin-web:
     build:
       context: ..
@@ -174,3 +194,4 @@ volumes:
   postgres-data:
   redis-critical-data:
   minio-data:
+  ollama-models:

deploy/production.env.example

@@ -48,6 +48,16 @@ S3_REGION=us-east-1
 S3_USE_SSL=false
 S3_MAX_FILE_BYTES=134217728
 
+# M8 P2 向量化（本地 Ollama，默认 bge-m3）。生产可整体关闭走纯 FTS：
+# 设 EMBEDDINGS_ENABLED=false 并移除 compose 的 ollama 服务。
+# EMBEDDING_DIM 必须与迁移 000024 的 vector(1024) 列一致。
+EMBEDDINGS_ENABLED=true
+OLLAMA_BASE_URL=http://ollama:11434
+EMBEDDING_MODEL=bge-m3
+EMBEDDING_DIM=1024
+EMBEDDING_BATCH_SIZE=64
+EMBEDDING_TIMEOUT=120s
+
 # Used only for the one-time bootstrap-admin command; remove after use.
 BOOTSTRAP_ADMIN_USERNAME=admin
 BOOTSTRAP_ADMIN_PASSWORD=CHANGE_ME_AT_LEAST_12_CHARACTERS