diff --git a/.agents/knowledge/external-deps-and-gotchas.md b/.agents/knowledge/external-deps-and-gotchas.md index f776e236..d6cfb5d0 100644 --- a/.agents/knowledge/external-deps-and-gotchas.md +++ b/.agents/knowledge/external-deps-and-gotchas.md @@ -9,6 +9,8 @@ | RAGFlow(知识运行时) | `http://100.64.0.8`(及 `:9380`) | `/api/v1/datasets`、`/api/v1/retrieval`;健康检查 `/v1/system/healthz` | | 开发 PG / Redis | 见下方凭据来源 | 远端 PG **15**(用户确认可用,不强制 PG16);Redis `100.64.0.8:6379` | +**Dify 状态(2026-06-28 盘点)**:当前 Muse 1.0.0 运行链路没有接入 Dify,infra 主机也未部署 Dify 相关容器/端口。仓内只剩设计映射与示例测试提及 Dify;因此它不是当前生产运行时依赖。若产品后续要求 Dify 工作流能力,应按新集成从契约、配置、失败闭环、live 验收补齐,不能把它当成已存在基础设施。 + **凭据来源(明文不入库,只记位置)**: - 外部验收:`muse-cloud/scripts/dev/p1r-external-acceptance.env`(`set -a; . 该文件; set +a` 加载;含 New-API base/token、`MUSE_AI_NEW_API_DEFAULT_MODEL_KEY=MiniMax-M2.5`、RAGFlow base/key、GraphRAG 开关)。 - 开发基础设施:`~/.config/muse-repo/infra.env`(PG/Redis 连接 + 真实凭据)。 @@ -26,8 +28,9 @@ - **RAGFlow 建数据集**:`POST /api/v1/datasets` 带 `config:{}` 返回 `code=101 Extra inputs are not permitted`;**只发 `name`** 才 `code=0`。adapter 已改为 createDataset 只发 name,config 走独立 `updateDatasetConfig`。 - **RAGFlow 文档状态轮询**:用真实支持的 `?id=`(单个);多 id 在 adapter 内 fail-closed 为 `VALIDATION_ERROR`。 - **RAGFlow 无 copy/clone dataset API**:复制一份 dataset 必须重走 `createDataset` + 逐文档上传 + `startParse` 重新索引,无法共享原 dataset 的索引(market KB 物化 D0-fork 据此设计,见 [`market-install-downstream-materialization.md`](market-install-downstream-materialization.md))。 -- **RAGFlow 检索 metadata 过滤字段名 bug(潜伏)**:现行代码发 `metadata_filter`,而 RAGFlow 官方契约是 `metadata_condition`,字段名不匹配被 RAGFlow **静默忽略**(不报错、过滤不生效)。任何依赖运行时 metadata 过滤做隔离/筛选的设计都会因此失效;market KB 隔离刻意改用物理隔离(公开副本)绕开,故此 bug 当前潜伏不影响隔离,但仍待修。 +- **RAGFlow 检索 metadata 字段名**:RAGFlow 官方契约是 `metadata_condition`。2026-06-28 已将 Muse HTTP 客户端从旧的 `metadata_filter` 修为 `metadata_condition`,并用请求体单测防回退。即便字段已修,跨可见性安全隔离仍不得依赖运行时 metadata 过滤,market KB 继续使用公开副本物理隔离。 - **GraphRAG attribution**:默认 fail-closed(`ATTRIBUTION_NOT_CONFIGURED`);真跑需 `MUSE_KNOWLEDGE_RAGFLOW_GRAPHRAG_ATTRIBUTION_READY=true` + `..._GRAPHRAG_DATASET_ID`。 +- **New-API/RAGFlow Java 客户端代理污染**:2026-06-28 盘点时,未清代理的 Java/Python 探针曾返回 502,但 `curl --noproxy '*'` 与清代理 live IT 均证明 New-API/RAGFlow 可用。运行时代码已在 New-API 文本生成、导入解析、RAGFlow HTTP 客户端上显式使用 direct `ProxySelector`,避免 macOS/环境代理把 tailnet 内网请求转到本地代理导致假故障。 ## 三、前端 / 构建坑速查 - **studio(Vite + React + TS6.0)**:`erasableSyntaxOnly` 禁用 constructor 参数属性;`exactOptionalPropertyTypes` 下可选属性需规避写法;采纳/Diff 用 **Coarse-to-Fine 分级 Diff** 规避 3000+ 字正文上 O(N×M) LCS 性能爆炸;MSW 仅 `import.meta.env.DEV` 启用。 diff --git a/.agents/knowledge/market-install-downstream-materialization.md b/.agents/knowledge/market-install-downstream-materialization.md index 07a975fe..a6361ea1 100644 --- a/.agents/knowledge/market-install-downstream-materialization.md +++ b/.agents/knowledge/market-install-downstream-materialization.md @@ -53,4 +53,4 @@ market 的安装流程原本只写两样东西——`muse_market_installation` - **副本资源回收待做**。资产被删/版本更替后,旧的公开副本 dataset 没有清理路径。 - **下架/撤权的细粒度补偿仍待设计**。E4 已覆盖 recall→installed_ref 停检索的合规底线;delist/revoke 的差异化策略、已运行任务补偿、生产者自助入口仍后置。 -- **metadata_filter 字段 bug 待修(已知、刻意未依赖)**。代码里发的是 `metadata_filter`,而 RAGFlow 官方契约是 `metadata_condition`,字段名不匹配被 RAGFlow 静默忽略。D0-fork 刻意不依赖运行时 metadata 过滤(改用物理隔离),所以这个 bug 目前潜伏不影响隔离;但它仍是一个待修的真实缺陷。详见 [`external-deps-and-gotchas.md`](external-deps-and-gotchas.md) 的 RAGFlow 集成坑。 +- **metadata 字段名 bug 已修,但不作为隔离边界**。2026-06-28 已将 RAGFlow 检索请求从旧 `metadata_filter` 修为官方 `metadata_condition`,并补请求体回归测试。D0-fork 仍刻意不依赖运行时 metadata 过滤(继续使用公开副本物理隔离),避免外部契约漂移再次变成私有泄露风险。详见 [`external-deps-and-gotchas.md`](external-deps-and-gotchas.md) 的 RAGFlow 集成坑。 diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java index ac3ee7e3..93be5ad9 100644 --- a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java @@ -9,6 +9,7 @@ import org.springframework.util.StringUtils; import java.io.IOException; import java.net.ConnectException; +import java.net.ProxySelector; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -33,6 +34,7 @@ public class NewApiMuseAiImportLlmParser implements MuseAiImportLlmParser { private static final String CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; private static final int MAX_LLM_INPUT_CHARS = 180_000; private static final int MAX_LLM_CHAPTERS = 300; + private static final ProxySelector DIRECT_PROXY_SELECTOR = ProxySelector.of(null); private final MuseAiProperties.NewApi properties; private final HttpClient httpClient; @@ -41,6 +43,8 @@ public class NewApiMuseAiImportLlmParser implements MuseAiImportLlmParser { this.properties = museAiProperties == null ? null : museAiProperties.getNewApi(); this.httpClient = HttpClient.newBuilder() .connectTimeout(Duration.ofSeconds(positive(connectTimeoutSeconds(), 5))) + // 导入解析也调用 tailnet 内网 New-API,必须绕开 JVM/系统代理,避免可用服务被误判为 502。 + .proxy(DIRECT_PROXY_SELECTOR) .build(); } diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/facade/RealNewApiMuseAiRuntimeClient.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/facade/RealNewApiMuseAiRuntimeClient.java index 9cbd05f7..655c954a 100644 --- a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/facade/RealNewApiMuseAiRuntimeClient.java +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/facade/RealNewApiMuseAiRuntimeClient.java @@ -7,6 +7,7 @@ import org.springframework.util.StringUtils; import java.io.IOException; import java.net.ConnectException; +import java.net.ProxySelector; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -29,6 +30,7 @@ public class RealNewApiMuseAiRuntimeClient implements MuseAiRuntimeClient { private static final String CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; private static final int OUTPUT_SNIPPET_LIMIT = 60; + private static final ProxySelector DIRECT_PROXY_SELECTOR = ProxySelector.of(null); private final MuseAiProperties.NewApi properties; private final HttpClient httpClient; @@ -37,6 +39,8 @@ public class RealNewApiMuseAiRuntimeClient implements MuseAiRuntimeClient { this.properties = museAiProperties == null ? null : museAiProperties.getNewApi(); this.httpClient = HttpClient.newBuilder() .connectTimeout(Duration.ofSeconds(positive(connectTimeoutSeconds(), 5))) + // New-API 部署在 tailnet 内网,必须显式直连,避免 JVM/系统代理残留把 live 验证误导成 502。 + .proxy(DIRECT_PROXY_SELECTOR) .build(); } diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/HttpRagFlowKnowledgeRuntimeClient.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/HttpRagFlowKnowledgeRuntimeClient.java index d4c966f2..ce9b30e4 100644 --- a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/HttpRagFlowKnowledgeRuntimeClient.java +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/HttpRagFlowKnowledgeRuntimeClient.java @@ -6,6 +6,7 @@ import org.springframework.util.StringUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.net.ProxySelector; import java.net.URLEncoder; import java.net.URI; import java.net.http.HttpClient; @@ -38,6 +39,7 @@ public class HttpRagFlowKnowledgeRuntimeClient implements RagFlowKnowledgeRuntim private final int retryBudget; private final boolean graphRagAttributionReady; private final HttpClient httpClient; + private static final ProxySelector DIRECT_PROXY_SELECTOR = ProxySelector.of(null); public HttpRagFlowKnowledgeRuntimeClient(String baseUrl, String apiKey, Duration timeout, int retryBudget, boolean graphRagAttributionReady) { @@ -46,7 +48,11 @@ public class HttpRagFlowKnowledgeRuntimeClient implements RagFlowKnowledgeRuntim this.timeout = timeout == null ? Duration.ofSeconds(5) : timeout; this.retryBudget = Math.max(retryBudget, 0); this.graphRagAttributionReady = graphRagAttributionReady; - this.httpClient = HttpClient.newBuilder().connectTimeout(this.timeout).build(); + this.httpClient = HttpClient.newBuilder() + .connectTimeout(this.timeout) + // RAGFlow 部署在 tailnet 内网,显式直连可避免系统代理污染 live IT 与运行时调用。 + .proxy(DIRECT_PROXY_SELECTOR) + .build(); } @Override @@ -152,7 +158,10 @@ public class HttpRagFlowKnowledgeRuntimeClient implements RagFlowKnowledgeRuntim body.put("question", command.question()); body.put("top_k", command.topK()); body.put("similarity_threshold", command.threshold()); - body.put("metadata_filter", command.metadataFilter()); + // RAGFlow 官方 HTTP 契约字段名是 metadata_condition;旧字段 metadata_filter 会被静默忽略。 + if (command.metadataCondition() != null) { + body.put("metadata_condition", command.metadataCondition()); + } return executeJson(Operation.RETRIEVE_CHUNKS, command.correlationId(), command.requestHash(), command.attempt(), "POST", "/api/v1/retrieval", body, null, null); } diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClient.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClient.java index 5499c893..ed970ae2 100644 --- a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClient.java +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClient.java @@ -157,7 +157,7 @@ public interface RagFlowKnowledgeRuntimeClient { record RetrieveChunksCommand(Long tenantId, Long ownerUserId, Long kbId, List ragflowDatasetIds, List ragflowDocumentIds, String question, Integer topK, - Double threshold, Map metadataFilter, + Double threshold, Map metadataCondition, String correlationId, String requestHash, int attempt) { } diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeRetrievalApiImplTest.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeRetrievalApiImplTest.java index 29b7b3f2..618efd8d 100644 --- a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeRetrievalApiImplTest.java +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeRetrievalApiImplTest.java @@ -272,12 +272,12 @@ class MuseKnowledgeRetrievalApiImplTest extends BaseMockitoUnitTest { /** * SPIKE-隔离参数缺位证据(必产):抓取实际发往 RAGFlow 的 RetrieveChunksCommand, - * 断言 document_ids 与 metadata_filter 均为 null —— 检索是"按授权 dataset 整库扫描", + * 断言 document_ids 与 metadata condition 均为 null —— 检索是"按授权 dataset 整库扫描", * 当前根本没有把 chunk 限定到"本安装授权子集"的任何手段。这是越权的机制根因,也指出 * 补隔离(document_ids 限定 / 元数据过滤)的落点必须改 RetrieveChunksCommand 构造处。 */ @Test - void spike_should_sendNullDocumentIdsAndMetadataFilter_provingWholeDatasetScan() { + void spike_should_sendNullDocumentIdsAndMetadataCondition_provingWholeDatasetScan() { when(bindingMapper.selectActiveByWorkId(4001L)).thenReturn(List.of(authorizedBinding(5001L))); when(projectionMapper.selectActiveByWorkId(4001L)).thenReturn(List.of(projection(5001L, "active"))); when(ragflowBindingMapper.selectActiveDatasetByKbId(5001L)).thenReturn(dataset("shared-dataset-D")); @@ -293,7 +293,7 @@ class MuseKnowledgeRetrievalApiImplTest extends BaseMockitoUnitTest { // 整库扫描的证据:检索只带 dataset_ids + question,不带任何把 chunk 限定到授权子集的隔离参数。 assertEquals(List.of("shared-dataset-D"), sent.ragflowDatasetIds()); assertNull(sent.ragflowDocumentIds(), "document_ids 为 null → RAGFlow 对该 dataset 整库返回,无文档级隔离"); - assertNull(sent.metadataFilter(), "metadata_filter 为 null → 无元数据级 chunk 过滤"); + assertNull(sent.metadataCondition(), "metadata_condition 为 null → 无元数据级 chunk 过滤"); } // ======================================================================================== diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClientTest.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClientTest.java index 3284a77e..2f21a0a0 100644 --- a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClientTest.java +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/application/muse/facade/RagFlowKnowledgeRuntimeClientTest.java @@ -365,6 +365,27 @@ class RagFlowKnowledgeRuntimeClientTest { assertTrue(requestBody.contains("dataset_ids")); } + @Test + void retrieveChunksShouldSendMetadataConditionWithOfficialRagflowFieldName() { + // 回归:RAGFlow 检索元数据过滤的官方字段是 metadata_condition; + // metadata_filter 会被外部服务静默忽略,不能作为任何隔离或筛选证据。 + CapturingHttpRagFlowKnowledgeRuntimeClient client = new CapturingHttpRagFlowKnowledgeRuntimeClient( + "https://ragflow.example", "configured-api-key", + "{\"code\":0,\"data\":{\"chunks\":[{\"content\":\"c\",\"score\":0.9}]}}"); + + RagFlowKnowledgeRuntimeClient.RuntimeResult result = client.retrieveChunks( + new RagFlowKnowledgeRuntimeClient.RetrieveChunksCommand(100L, 2001L, 4001L, + List.of("dataset-1"), List.of("doc-1"), "q", 5, 0.2, + Map.of("logic", "and", "conditions", List.of(Map.of( + "name", "source", "comparison_operator", "=", "value", "public"))), + "corr-metadata-condition", "hash-metadata-condition", 1)); + + assertEquals(RagFlowKnowledgeRuntimeClient.Status.SUCCEEDED, result.status()); + String requestBody = new String(client.requests.get(0).body(), StandardCharsets.UTF_8); + assertTrue(requestBody.contains("\"metadata_condition\"")); + assertFalse(requestBody.contains("\"metadata_filter\"")); + } + @Test void listChunksShouldSucceedWithoutTopLevelId() { HttpRagFlowKnowledgeRuntimeClient client = new StubHttpRagFlowKnowledgeRuntimeClient( diff --git a/muse-cloud/scripts/run-p1r-verification.sh b/muse-cloud/scripts/run-p1r-verification.sh index a20c5867..896ef801 100644 --- a/muse-cloud/scripts/run-p1r-verification.sh +++ b/muse-cloud/scripts/run-p1r-verification.sh @@ -224,7 +224,7 @@ run_external_live() { -Dtest="${MODULE_LIVE_TESTS}" \ -DfailIfNoSpecifiedTests=false \ -Dsurefire.failIfNoSpecifiedTests=false \ - -DargLine="-Dhttp.proxyHost= -Dhttp.proxyPort= -Dhttps.proxyHost= -Dhttps.proxyPort= -Djava.net.useSystemProxies=false") + -DargLine="$(live_arg_line)") } case "${MODE}" in