superidou/ai-gateway-go
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(m8): P2 pgvector + Ollama 向量化与语义检索

Browse Source 
- PostgreSQL 切换 pgvector/pgvector:pg17 镜像;迁移 000024 建 vector 扩展、
  knowledge_chunks.embedding vector(1024) + HNSW 余弦索引,retrieval_mode 放宽三态
- OllamaEmbedder 本地 bge-m3 批量嵌入,404 惰性 pull 重试,维度/超时校验,可整体关闭
- SemanticRetriever/HybridRetriever + NewRetriever 按 retrieval_mode 分发,缺 embedder 回退 FTS
- 文档入库同步批量向量化;Ollama 故障降级入库 + embedding_failed 事件
- 修复 pgx CopyFrom 对 vector 列二进制编码误读:COPY 基础列后同事务 unnest 批量回填
- 修复降级路径 embeddings=nil 索引越界 panic(Add 与 Reprocess)
- 知识库列表 vectorized_chunk_count + 前端三态检索模式选择与向量化覆盖率
- 单测 embedder/retrievers + 集成 TestKnowledgeVectorLifecycle 全绿

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
ben
2026-08-12 15:16:32 +08:00
parent 6708c226a5
commit b536672000
26 changed files with 966 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/apps/admin/src/api/workbench.ts
+1 -1
View File
@@ -16,7 +16,7 @@ export const fetchPromptVersions=(id:string)=>request.get<PromptVersion[]>({url:
export const createPromptVersion=(id:string,params:{content:string;variables:PromptVariable[];change_note:string;activate:boolean})=>request.post<PromptVersion>({url:`/api/v1/admin/prompts/${id}/versions`,params})
export const activatePromptVersion=(id:string,version:number)=>request.post({url:`/api/v1/admin/prompts/${id}/versions/${version}/activate`})
export interface KnowledgeBase { id:string;name:string;description:string;retrieval_mode:string;chunk_size:number;chunk_overlap:number;department_ids:string[];enabled:boolean;revision:number;document_count:number;chunk_count:number }
export interface KnowledgeBase { id:string;name:string;description:string;retrieval_mode:string;chunk_size:number;chunk_overlap:number;department_ids:string[];enabled:boolean;revision:number;document_count:number;chunk_count:number;vectorized_chunk_count:number }
export interface KnowledgeDocument {id:string;knowledge_base_id:string;title:string;source_type:string;source_uri:string;content_sha256:string;status:string;status_message:string;char_count:number;chunk_count:number;created_at:string}
export interface SearchHit {chunk_id:string;document_id:string;document_title:string;chunk_index:number;content:string;score:number}
export type KnowledgeBaseInput=Pick<KnowledgeBase,'name'|'description'|'retrieval_mode'|'chunk_size'|'chunk_overlap'|'department_ids'|'enabled'>
web/apps/admin/src/views/gateway/knowledge/index.vue
+2 -2
View File
@@ -1,5 +1,5 @@
<template><div class="page-content"><div class="mb-5 flex items-start justify-between"><div><h2 class="text-xl font-semibold">知识库</h2><p class="text-g-500 mt-1 text-sm">PostgreSQL 文本分块与全文检索基线，无需 MinIO/S3；Retriever 接口可后续替换为向量检索</p></div><ElButton type="primary" @click="openCreate">新建知识库</ElButton></div><ElAlert class="mb-4" type="info" :closable="false" title="正文限制 2 MiB 并直接保存在 PostgreSQL；调整分块参数只影响后续入库文档。"/><ElTable v-loading="loading" :data="items" row-key="id"><ElTableColumn prop="name" label="名称" min-width="180"/><ElTableColumn prop="retrieval_mode" label="检索器" width="140"/><ElTableColumn label="分块" width="130"><template #default="{row}">{{row.chunk_size}} / {{row.chunk_overlap}}</template></ElTableColumn><ElTableColumn prop="document_count" label="文档" width="80"/><ElTableColumn prop="chunk_count" label="切片" width="80"/><ElTableColumn label="状态" width="90"><template #default="{row}"><ElTag :type="row.enabled?'success':'info'">{{row.enabled?'启用':'停用'}}</ElTag></template></ElTableColumn><ElTableColumn label="操作" width="250" fixed="right"><template #default="{row}"><ElButton link type="primary" @click="openDocuments(row)">文档</ElButton><ElButton link type="primary" @click="openSearch(row)">检索</ElButton><ElButton link type="primary" @click="openEdit(row)">编辑</ElButton><ElButton link type="danger" @click="remove(row)">删除</ElButton></template></ElTableColumn></ElTable>
<ElDialog v-model="visible" :title="editing?'编辑知识库':'新建知识库'" width="680px"><ElForm label-width="110px"><ElFormItem label="名称"><ElInput v-model="form.name"/></ElFormItem><ElFormItem label="描述"><ElInput v-model="form.description" type="textarea"/></ElFormItem><div class="grid grid-cols-2 gap-x-4"><ElFormItem label="切片字符数"><ElInputNumber v-model="form.chunk_size" :min="200" :max="8000"/></ElFormItem><ElFormItem label="重叠字符数"><ElInputNumber v-model="form.chunk_overlap" :min="0" :max="form.chunk_size/2"/></ElFormItem></div><ElFormItem label="部门范围"><ElSelect v-model="form.department_ids" multiple allow-create filterable class="w-full" placeholder="留空表示全部部门"/></ElFormItem><ElFormItem label="启用"><ElSwitch v-model="form.enabled"/></ElFormItem></ElForm><template #footer><ElButton @click="visible=false">取消</ElButton><ElButton type="primary" @click="save">保存</ElButton></template></ElDialog>
<template><div class="page-content"><div class="mb-5 flex items-start justify-between"><div><h2 class="text-xl font-semibold">知识库</h2><p class="text-g-500 mt-1 text-sm">M8:分块入库支持全文检索(FTS)、语义向量(pgvector+Ollama bge-m3)与混合检索;正文上限 2 MiB</p></div><ElButton type="primary" @click="openCreate">新建知识库</ElButton></div><ElAlert class="mb-4" type="info" :closable="false" title="正文限制 2 MiB 并直接保存在 PostgreSQL；调整分块参数只影响后续入库文档。"/><ElTable v-loading="loading" :data="items" row-key="id"><ElTableColumn prop="name" label="名称" min-width="180"/><ElTableColumn prop="retrieval_mode" label="检索器" width="140"/><ElTableColumn label="分块" width="130"><template #default="{row}">{{row.chunk_size}} / {{row.chunk_overlap}}</template></ElTableColumn><ElTableColumn prop="document_count" label="文档" width="80"/><ElTableColumn prop="chunk_count" label="切片" width="80"/><ElTableColumn label="向量化" width="110"><template #default="{row}"><span :class="row.vectorized_chunk_count===row.chunk_count?'text-emerald-600':'text-amber-600'">{{row.vectorized_chunk_count}}/{{row.chunk_count}}</span></template></ElTableColumn><ElTableColumn label="状态" width="90"><template #default="{row}"><ElTag :type="row.enabled?'success':'info'">{{row.enabled?'启用':'停用'}}</ElTag></template></ElTableColumn><ElTableColumn label="操作" width="250" fixed="right"><template #default="{row}"><ElButton link type="primary" @click="openDocuments(row)">文档</ElButton><ElButton link type="primary" @click="openSearch(row)">检索</ElButton><ElButton link type="primary" @click="openEdit(row)">编辑</ElButton><ElButton link type="danger" @click="remove(row)">删除</ElButton></template></ElTableColumn></ElTable>
<ElDialog v-model="visible" :title="editing?'编辑知识库':'新建知识库'" width="680px"><ElForm label-width="110px"><ElFormItem label="名称"><ElInput v-model="form.name"/></ElFormItem><ElFormItem label="描述"><ElInput v-model="form.description" type="textarea"/></ElFormItem><ElFormItem label="检索模式"><ElSelect v-model="form.retrieval_mode" class="w-full"><ElOption label="全文检索 postgres_fts" value="postgres_fts"/><ElOption label="语义向量 vector" value="vector"/><ElOption label="混合检索 hybrid" value="hybrid"/></ElSelect><p class="mt-1 text-xs text-g-500">vector/hybrid 依赖本地 Ollama(bge-m3);未向量化的旧分块需重新处理后才会被语义召回</p></ElFormItem><div class="grid grid-cols-2 gap-x-4"><ElFormItem label="切片字符数"><ElInputNumber v-model="form.chunk_size" :min="200" :max="8000"/></ElFormItem><ElFormItem label="重叠字符数"><ElInputNumber v-model="form.chunk_overlap" :min="0" :max="form.chunk_size/2"/></ElFormItem></div><ElFormItem label="部门范围"><ElSelect v-model="form.department_ids" multiple allow-create filterable class="w-full" placeholder="留空表示全部部门"/></ElFormItem><ElFormItem label="启用"><ElSwitch v-model="form.enabled"/></ElFormItem></ElForm><template #footer><ElButton @click="visible=false">取消</ElButton><ElButton type="primary" @click="save">保存</ElButton></template></ElDialog>
<ElDrawer v-model="documentsVisible" :title="`${selected?.name||''} · 文档`" size="65%"><ElForm label-width="90px"><ElFormItem label="标题"><ElInput v-model="documentForm.title"/></ElFormItem><ElFormItem label="来源 URI"><ElInput v-model="documentForm.source_uri" placeholder="可选，仅作来源标记"/></ElFormItem><ElFormItem label="正文"><ElInput v-model="documentForm.content" type="textarea" :rows="8" show-word-limit maxlength="2097152"/></ElFormItem><ElFormItem><ElButton type="primary" @click="addDocument">入库并切片</ElButton></ElFormItem></ElForm><ElTable :data="documents"><ElTableColumn prop="title" label="标题"/><ElTableColumn prop="char_count" label="字符" width="90"/><ElTableColumn prop="chunk_count" label="切片" width="80"/><ElTableColumn prop="status" label="状态" width="90"/><ElTableColumn label="操作" width="80"><template #default="{row}"><ElButton link type="danger" @click="removeDocument(row)">删除</ElButton></template></ElTableColumn></ElTable></ElDrawer>
<ElDialog v-model="searchVisible" :title="`${selected?.name||''} · 检索测试`" width="760px"><div class="mb-4 flex gap-2"><ElInput v-model="query" placeholder="输入检索词" @keyup.enter="runSearch"/><ElInputNumber v-model="topK" :min="1" :max="20"/><ElButton type="primary" @click="runSearch">检索</ElButton></div><ElCard v-for="hit in hits" :key="hit.chunk_id" class="mb-3"><template #header>{{hit.document_title}} · #{{hit.chunk_index}} · {{hit.score.toFixed(3)}}</template><div class="whitespace-pre-wrap text-sm">{{hit.content}}</div></ElCard><ElEmpty v-if="!hits.length" description="暂无命中"/></ElDialog></div></template>
<script setup lang="ts">import{KnowledgeBase,KnowledgeBaseInput,KnowledgeDocument,SearchHit,addKnowledgeDocument,createKnowledgeBase,deleteKnowledgeBase,deleteKnowledgeDocument,fetchKnowledgeBases,fetchKnowledgeDocuments,searchKnowledge,updateKnowledgeBase}from'@/api/workbench';import{ElMessage,ElMessageBox}from'element-plus';const loading=ref(false),visible=ref(false),documentsVisible=ref(false),searchVisible=ref(false),editing=ref('');const items=ref<KnowledgeBase[]>([]),documents=ref<KnowledgeDocument[]>([]),hits=ref<SearchHit[]>([]),selected=ref<KnowledgeBase>(),query=ref(''),topK=ref(4);const blank=():KnowledgeBaseInput=>({name:'',description:'',retrieval_mode:'postgres_fts',chunk_size:800,chunk_overlap:100,department_ids:[],enabled:true});const form=reactive<KnowledgeBaseInput>(blank()),documentForm=reactive({title:'',source_type:'text',source_uri:'',content:''});async function load(){loading.value=true;try{items.value=await fetchKnowledgeBases()}finally{loading.value=false}}function openCreate(){editing.value='';Object.assign(form,blank());visible.value=true}function openEdit(row:KnowledgeBase){editing.value=row.id;Object.assign(form,{name:row.name,description:row.description,retrieval_mode:row.retrieval_mode,chunk_size:row.chunk_size,chunk_overlap:row.chunk_overlap,department_ids:[...row.department_ids],enabled:row.enabled});visible.value=true}async function save(){editing.value?await updateKnowledgeBase(editing.value,form):await createKnowledgeBase(form);visible.value=false;await load()}async function remove(row:KnowledgeBase){await ElMessageBox.confirm(`删除“${row.name}”及全部文档？`,'确认',{type:'warning'});await deleteKnowledgeBase(row.id);await load()}async function openDocuments(row:KnowledgeBase){selected.value=row;documents.value=await fetchKnowledgeDocuments(row.id);Object.assign(documentForm,{title:'',source_type:'text',source_uri:'',content:''});documentsVisible.value=true}async function addDocument(){if(!selected.value)return;await addKnowledgeDocument(selected.value.id,documentForm);ElMessage.success('文档已入库');await openDocuments(selected.value);await load()}async function removeDocument(row:KnowledgeDocument){if(!selected.value)return;await deleteKnowledgeDocument(selected.value.id,row.id);await openDocuments(selected.value);await load()}function openSearch(row:KnowledgeBase){selected.value=row;hits.value=[];query.value='';searchVisible.value=true}async function runSearch(){if(!selected.value||!query.value.trim())return;hits.value=await searchKnowledge(selected.value.id,{query:query.value,top_k:topK.value})}onMounted(load)</script>