From a5619a3566146112a53a2be08ad5d6fe7ff911c8 Mon Sep 17 00:00:00 2001 From: lili Date: Sun, 28 Jun 2026 07:40:10 -0700 Subject: [PATCH] =?UTF-8?q?feat(content):=20=E8=A1=A5=E9=BD=90=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E5=AF=BC=E5=87=BA=E7=9C=9F=E9=93=BE=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/api-contracts/content/openapi.yaml | 14 +- docs/mvp/1.0.0-产品功能缺口待办清单.md | 12 +- docs/mvp/1.0.0-真验证清单.md | 10 + .../module/ai/api/MuseAiImportParseApi.java | 170 ++++ .../muse/MuseAiImportLlmParser.java | 59 ++ .../muse/MuseAiImportParseService.java | 744 ++++++++++++++++++ .../muse/NewApiMuseAiImportLlmParser.java | 311 ++++++++ .../muse/MuseAiChapterParseResultDO.java | 47 ++ .../dal/dataobject/muse/MuseAiParseJobDO.java | 59 ++ .../muse/MuseAiChapterParseResultMapper.java | 72 ++ .../dal/mysql/muse/MuseAiParseJobMapper.java | 41 + .../muse/MuseAiImportParseServiceTest.java | 338 ++++++++ .../muse/NewApiMuseAiImportLlmParserTest.java | 129 +++ .../P1rImportLlmNewApiLiveAcceptanceIT.java | 156 ++++ .../muse-module-content-server/pom.xml | 6 + .../facade/RealContentFileFacade.java | 85 +- .../RealContentKnowledgeDraftFacade.java | 66 ++ .../facade/RealContentParseJobFacade.java | 174 ++++ .../app/vo/ChapterParseResultRespVO.java | 5 + .../facade/RealContentFileFacadeTest.java | 47 +- .../app/file/AppFileController.java | 13 + .../app/file/AppFileControllerTest.java | 48 ++ .../api/MuseKnowledgeDraftCreationApi.java | 87 ++ .../MuseKnowledgeDraftCreationApiImpl.java | 130 +++ .../mysql/muse/MuseKnowledgeDraftMapper.java | 6 + ...MuseKnowledgeDraftCreationApiImplTest.java | 105 +++ ...ontentImportWizardCompletedApprovalIT.java | 729 +++++++++++++++++ .../V34__create_ai_import_parse_tables.sql | 86 ++ muse-studio/e2e/export-download.spec.ts | 212 +++++ muse-studio/e2e/import-wizard.spec.ts | 280 +++++++ muse-studio/src/api/client.ts | 96 ++- muse-studio/src/api/mocks/handlers/content.ts | 368 ++++++++- .../components/ExportWorkModal.test.tsx | 75 ++ .../editor/components/ExportWorkModal.tsx | 194 +++++ .../editor/components/ImportTextModal.tsx | 478 +++++++++-- .../features/editor/hooks/useWorks.test.tsx | 78 +- .../src/features/editor/hooks/useWorks.ts | 270 ++++++- muse-studio/src/pages/WorkspacePage.tsx | 78 +- 38 files changed, 5694 insertions(+), 184 deletions(-) create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-api/src/main/java/cn/iocoder/muse/module/ai/api/MuseAiImportParseApi.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportLlmParser.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseService.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiChapterParseResultDO.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiParseJobDO.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiChapterParseResultMapper.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiParseJobMapper.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseServiceTest.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParserTest.java create mode 100644 muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/P1rImportLlmNewApiLiveAcceptanceIT.java create mode 100644 muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentKnowledgeDraftFacade.java create mode 100644 muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentParseJobFacade.java create mode 100644 muse-cloud/muse-module-infra/muse-module-infra-server/src/test/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileControllerTest.java create mode 100644 muse-cloud/muse-module-knowledge/muse-module-knowledge-api/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApi.java create mode 100644 muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImpl.java create mode 100644 muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImplTest.java create mode 100644 muse-cloud/muse-server/src/test/java/cn/iocoder/muse/server/framework/api/P1rContentImportWizardCompletedApprovalIT.java create mode 100644 muse-cloud/sql/muse/V34__create_ai_import_parse_tables.sql create mode 100644 muse-studio/e2e/export-download.spec.ts create mode 100644 muse-studio/e2e/import-wizard.spec.ts create mode 100644 muse-studio/src/features/editor/components/ExportWorkModal.test.tsx create mode 100644 muse-studio/src/features/editor/components/ExportWorkModal.tsx diff --git a/docs/api-contracts/content/openapi.yaml b/docs/api-contracts/content/openapi.yaml index ddf8513b..f52ca331 100644 --- a/docs/api-contracts/content/openapi.yaml +++ b/docs/api-contracts/content/openapi.yaml @@ -3546,8 +3546,8 @@ components: # ========== 导入导出 Schema ========== CreateImportTaskRequest: type: object - description: 创建导入任务请求。uploadUrl 与 contentText 二选一；contentText 模式会直接初始化 Canonical 正文。 - required: [commandId, fileName, fileSize, fileHash] + description: 创建导入任务请求。uploadUrl 与 contentText 二选一；uploadUrl 当前为 infra 稳定存储 path，contentText 模式会直接初始化 Canonical 正文。 + required: [commandId, fileName, fileSize, fileHash, format] properties: commandId: type: string @@ -3571,8 +3571,7 @@ components: description: 导入文件格式 uploadUrl: type: string - format: uri - description: 已上传文件的存储引用；外部上传模式必填 + description: 已上传文件的存储引用；外部上传模式必填。当前后端只接受 infra 稳定存储 path，不接受客户端任意外链 URL。 contentText: type: string maxLength: 500000 @@ -3819,7 +3818,7 @@ components: BatchConfirmChaptersRequest: type: object description: 批量确认章节解析结果投影请求；Content 只传递导入上下文和用户决策。 - required: [commandId, resultIds] + required: [commandId, resultIds, expectedRevisions] properties: commandId: type: string @@ -3830,6 +3829,11 @@ components: items: type: integer format: int64 + expectedRevisions: + type: object + description: 每个解析结果对应的期望 revision，key 为 resultId + additionalProperties: + type: integer BatchConfirmChaptersResult: type: object diff --git a/docs/mvp/1.0.0-产品功能缺口待办清单.md b/docs/mvp/1.0.0-产品功能缺口待办清单.md index eecdfd71..6b133b22 100644 --- a/docs/mvp/1.0.0-产品功能缺口待办清单.md +++ b/docs/mvp/1.0.0-产品功能缺口待办清单.md @@ -11,7 +11,7 @@ Muse 1.0.0 线 A 最小可用闭环已经具备：写作、AI 真生成、候选 1. **产品深面缺口**：导入深解析、版本恢复、市场生产者深面、知识失败恢复、账号安全真实能力等。 2. **架构与横切债**：Meta impact all-real 用量建模、统一 Source/Auth/Audit/Outbox Schema、AI grant/runtime 物理隔离、gateway 运行期转发。 -3. **验证与运维债**：Admin Playwright、Studio D0-fork UI 物化用例、GraphRAG attribution、FileService 下载成功链路、recheck worker opt-in。 +3. **验证与运维债**：Admin Playwright、Studio D0-fork UI 物化用例、GraphRAG attribution、recheck worker opt-in。 以下清单按优先级排序；“验收标准”必须用真后端、真 PostgreSQL、外部 live 或 MSW-off e2e 证明，不接受 mock-only。 @@ -19,10 +19,10 @@ Muse 1.0.0 线 A 最小可用闭环已经具备：写作、AI 真生成、候选 | 编号 | 域 | 待办 | 当前事实 | 最小验收标准 | |---|---|---|---|---| -| P0-01 | content/import | LLM 全书深解析 | 完整导入向导已接上传扫描→AI owner 解析→章节确认→Knowledge Draft，但当前解析是 txt/md 确定性拆章，不调用 LLM。 | 真实上传一本长文后，AI owner 调外部模型产出章节候选；用户确认后只写 Knowledge Draft，不绕过审核；失败/超时可重试且可审计。 | -| P0-02 | content/import | `docx/epub` 解析 | 当前只支持 `.txt/.md/.markdown`，`docx/epub` 返回 unsupported。 | 上传 docx/epub 后能抽取正文、章节与基础元数据；异常文件 fail-closed；有目标单测和真实后端 smoke。 | +| P0-01 | content/import | LLM 全书深解析 | ✅ 已补 New-API 全书解析。完整导入向导可从上传扫描进入 AI owner，按 `full_book/llm_full_book` 调外部模型产出 pending-review Shadow 章节；401/403 fail-closed，408/429/5xx/连接超时自动重试，证据只输出脱敏摘要。 | 已达成验收：New-API parser live 1/0F/0E/0S；完整上传→live LLM 解析→章节确认→Knowledge Draft real-PG `_test` 链路 1/0F/0E/0S；浏览器 MSW-off `import-wizard.spec.ts` 1/0F/0E。 | +| P0-02 | content/import | `docx/epub` 解析 | ✅ 已补。AI owner 支持 `.txt/.md/.markdown/.docx/.epub`；docx 读取 OOXML `word/document.xml`，epub 读取 html/xhtml 正文，异常压缩包/空正文/超限 fail-closed。Studio 上传类型与 MIME fallback 已同步。 | 已达成后端验收：`MuseAiImportParseServiceTest` 覆盖 docx/epub 成功与异常；完整导入 real-PG `_test` 用 docx 文件跑通上传扫描→解析→确认→Draft。 | | P0-03 | content/version | 版本恢复/高级回滚 | 已有只读版本快照和任意两版 diff；没有写入式恢复/回滚入口。 | 从历史版本恢复时走 Content owner 写边界，生成新 revision、来源归因、审计和旧草稿失效；越权/陈旧 revision fail-closed。 | -| P0-04 | content/export | FileService 下载成功链路 | 导出范围/格式 UI、任务查询和凭证下载已接；真后端下载 e2e 仍待 FileApi 环境 smoke。 | 配置对象存储/FileService 后，TXT/DOCX/EPUB 至少一种格式能生成文件、返回凭证、一次性下载并落审计；对象存储不可用时任务失败可追踪。 | +| P0-04 | content/export | FileService 下载成功链路 | ✅ 已补。导出范围/格式 UI、任务查询和凭证下载已接；本地 FileService master 配置下，浏览器 MSW-off `export-download.spec.ts` 真打 48080 创建 TXT 导出任务并触发真实下载。 | 已达成最小验收：TXT 格式生成文件、返回凭证、浏览器下载文件内容包含真实作品标题/正文片段，DB 反查任务 completed/packageRef/credentialId 非空；对象存储不可用时仍按后端合同 fail-closed。 | | P0-05 | market/publisher | 发布者生产端深面 | Studio 已有保存、检查、提交审核、申诉、撤回、补充材料入口和 e2e；仍缺附件上传、发布素材管理、版本/审核反馈深面。 | 普通用户可上传发布素材/申诉附件、管理资产版本、查看审核反馈并重新提交；附件和素材走真实文件链路，权限/撤回/补充材料有真后端 e2e。 | | P0-06 | knowledge | 处理状态与失败恢复 UI | 上传、绑定、检索、草稿动作、图谱 CRUD 已接；RAGFlow 处理失败后的用户侧恢复深面不足。 | 用户能看到资料扫描/索引状态、失败原因、重试/删除/重新上传入口；真实 RAGFlow 失败与恢复路径有 e2e 或 real-PG 证据。 | | P0-07 | member/security | 真实改密、2FA、session service 会话吊销 | Studio 已补安全事件处理动作，但这不等于真实账号安全能力。 | 改密、二次验证、会话列表、单会话/全会话吊销由后端可信边界执行；高危动作有 step-up、审计和失败补偿。 | @@ -46,7 +46,7 @@ Muse 1.0.0 线 A 最小可用闭环已经具备：写作、AI 真生成、候选 | P1-10 | knowledge/recheck | recheck 真外部校验与 outbox worker 启用 | recheck 默认关闭，等待运维和外部源配置。 | worker claim/terminal、重试、幂等、失败审计完整；真实外部校验成功和失败均有证据。 | | P1-11 | events/studio | 统一 SSE 客户端接业务组件 | AI SSE 独立链路已修；通用 `connectEventStream` 未被业务组件广泛消费。 | 至少一个非 AI 业务组件使用统一事件流展示任务进度/通知；断线重连和去重有测试。 | | P1-12 | platform/gateway | gateway 运行期转发验证 | 配置层已收口，单体仍直服；Nacos/服务发现真实转发未首跑。 | gateway、muse-server、Nacos 同时部署，`/app-api` 与 `/admin-api` 经网关真实转发通过 smoke 和 e2e。 | -| P1-13 | content/import | 完整导入向导 real-PG 与 MSW-off e2e 证据 | targeted 后端测试、前端 Vitest/typecheck 已过，mini-desktop 已真实应用 V34；但还缺专属 `_test` 库 Flyway V34 迁移回归和浏览器 MSW-off 全链 e2e。 | 用 `_test` 库跑导入向导 real-PG IT，浏览器从上传文件到章节确认再到 Knowledge Draft 生成全链通过；V34 迁移在测试库可重复验证。 | +| P1-13 | content/import | 完整导入向导 real-PG 与 MSW-off e2e 证据 | ✅ 已补。`P1rContentImportWizardCompletedApprovalIT` 已在专属 `_test` 库 clean+migrate 到 V34+，覆盖 docx 确定性解析与 New-API live LLM 两条后端真链路；Studio targeted Vitest/typecheck 已过；`import-wizard.spec.ts` 已用真实浏览器、真实 48080、共享 PG、New-API 跑通上传文件→章节确认→Knowledge Draft。 | 已达成：Playwright 1/0F/0E，并直连 PG 反查 parse job completed、2 条 confirmed Shadow、2 条 pending Draft、Canonical entity 计数不变。 | | P1-14 | market/publisher | 发布素材与申诉附件链路 | 当前发布和申诉主链可用，但前端提交里 `evidenceMaterials`、`attachmentIds` 仍偏空数组/轻量字段。 | 发布资产和申诉补充均可上传/选择附件，后端保存附件引用并在审核/治理详情中可追踪；非法附件、越权附件 fail-closed。 | ## 3. P2：完整商业化/平台化能力，明确后置线 B @@ -66,7 +66,7 @@ Muse 1.0.0 线 A 最小可用闭环已经具备：写作、AI 真生成、候选 | 项 | 当前结论 | |---|---| -| 完整导入向导 | 已建成上传扫描、AI owner 解析、章节确认、Knowledge Draft；缺的是 LLM 深解析、docx/epub、专属 real-PG 迁移回归和 MSW-off 浏览器全链证据。 | +| 完整导入向导 | 已建成上传扫描、AI owner 解析、LLM 全书深解析、docx/epub 抽取、章节确认、Knowledge Draft；已有专属 `_test` real-PG 迁移回归和浏览器 MSW-off 全链 e2e 证据。 | | AI 候选采纳/拒绝 | 已经真 New-API 生成、SSE done、采纳/拒绝归档；不是 mock-only。 | | Admin 配置 system agent | 创建、发布新版本、归档停用、Tool Grant 登记/调整已接真实 admin API；缺的是部分高危回滚/深命令和 Playwright。 | | Market 发布/申诉主链 | Studio 不是“生产者 0 UI”：已有保存、检查、提交审核、申诉、撤回、补充材料入口；缺的是附件/素材/版本管理等深面。 | diff --git a/docs/mvp/1.0.0-真验证清单.md b/docs/mvp/1.0.0-真验证清单.md index 716ea7e0..a448aaae 100644 --- a/docs/mvp/1.0.0-真验证清单.md +++ b/docs/mvp/1.0.0-真验证清单.md @@ -109,6 +109,16 @@ E1 fresh 切片证据（2026-06-27，本轮实跑）： - `frontend-e2e` 活体验证：`accept-suggestion.spec.ts` 2 tests / 0 failures / 0 errors（MSW-off，真实 muse-server 48080 + 共享 `muse_slice_live` + 真 New-API/RAGFlow 配置）。正路证据：真生成 suggestionId=79，authz=`rpe-local-*`，Block revision 160→161，`sourceAttribution.lineage.parentSuggestionId` 指向本轮真生成候选，AI owner 归档 `accepted` 且 decision archive 非空；负路 stale `expectedBlockRevision` 返回业务冲突。 - 边界：本轮只跑 Studio `accept-suggestion` 黄金旅程，没有重跑 studio 全量 56 e2e；`muse-studio/src/types/content.ts` 生成类型未同步，新版本 `openapi-typescript` 会带出大量历史 string/number 与 version/revision 机械漂移，本轮仅在 hook 内维护 `BlockRevision` 最小类型。 +RC 后导入向导 fresh 证据（2026-06-28，本轮实跑）： +- 功能链路：Studio 完整导入向导走上传扫描 → Content 创建外部上传任务 → AI owner 读取稳定 storageRef 并创建 parse job → 产出 pending-review Shadow 章节 → 用户批量确认 → Knowledge owner 创建 pending Draft；确认链路不直写 Knowledge Canonical。AI owner 支持 txt/markdown/docx/epub 抽取；`full_book/llm_full_book` 策略调用 New-API LLM 全书解析，timeout/429/5xx/连接失败可重试。 +- `local-ci` 后端切片：`MuseAiImportParseServiceTest` + `NewApiMuseAiImportLlmParserTest` + 相关 content/infra/knowledge targeted 测试通过；docx/epub 成功与异常、LLM 502 后重试成功、401 不重试均有单测覆盖。 +- `external-live` AI parser：`P1rImportLlmNewApiLiveAcceptanceIT` 显式 opt-in 1 test / 0 failures / 0 errors / 0 skipped，真实调用 New-API，输出 endpoint、凭据已配置标记、模型、章节数和摘要 hash，不输出凭据、原文或 provider 原始响应。 +- `real-pg`/`external-live` 完整向导：`P1rContentImportWizardCompletedApprovalIT` 使用真实 PostgreSQL `_test` 库 clean+migrate 到 V34+；默认 docx 链路 2 tests / 0 failures / 0 errors / 1 skipped，显式 opt-in New-API live LLM 完整链路 1 test / 0 failures / 0 errors / 0 skipped。live 链路 providerStatusCode=200，写入 2 条 AI Shadow 章节并创建 2 条 pending Knowledge Draft，`muse_knowledge_entity` 仍为 0。 +- 前端静态/局部验证：Studio `useWorks.test.tsx` + `ExportWorkModal.test.tsx` 12 tests / 0 failures / 0 errors；`tsc -b --force` 通过。 +- `frontend-e2e` 活体验证：`import-wizard.spec.ts` 1 test / 0 failures / 0 errors（MSW-off，真实 muse-server 48080 + 共享 `muse_slice_live` + 真 New-API 配置）。浏览器上传 txt 文件后真打 `upload-returning-path`、`import-tasks`、`parse-jobs`、`batch-confirm`；DB 反查 parse job `completed`、2 条 confirmed Shadow、2 条 pending Knowledge Draft，Canonical entity 计数不变；用例结束清理本轮导入测试行。 +- `frontend-e2e` 活体验证：`export-download.spec.ts` 1 test / 0 failures / 0 errors（MSW-off，真实 muse-server 48080 + 共享 `muse_slice_live` + 本地 FileService master）。浏览器创建 TXT 导出任务后真打 `downloads/{credentialId}`，请求头携带 `Authorization`/`tenant-id`/`X-API-Version`；真实浏览器 download 事件读取到的文件包含作品标题与正文片段；DB 反查导出任务 `completed`、`packageRef/credentialId` 非空并清理本轮任务与文件记录。 +- 边界：Knowledge Draft 进入 Local KB Canonical 仍需用户后续显式审核确认；FileApi/对象存储异常按合同 fail-closed，不伪造解析成功。 + E2 fresh 切片证据（2026-06-27，本轮实跑）： - 功能链路：用户自建 Agent 支持创建初始版本、编辑生成新 active version、版本列表/激活/归档非当前版本、归档 Agent；作品槽位支持真实 unbind；Studio “放弃修改”从本地清 UI 改为调用 AI owner reject 写 decision archive。 - `local-ci` 后端切片：`MuseAgentServiceTest` + `MuseAgentSlotServiceTest` + `AppMuseAgentControllerAnnotationTest` + `AppMuseAgentSlotControllerAnnotationTest` 共 63 tests / 0 failures / 0 errors，`BUILD SUCCESS`。 diff --git a/muse-cloud/muse-module-ai/muse-module-ai-api/src/main/java/cn/iocoder/muse/module/ai/api/MuseAiImportParseApi.java b/muse-cloud/muse-module-ai/muse-module-ai-api/src/main/java/cn/iocoder/muse/module/ai/api/MuseAiImportParseApi.java new file mode 100644 index 00000000..edc895d9 --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-api/src/main/java/cn/iocoder/muse/module/ai/api/MuseAiImportParseApi.java @@ -0,0 +1,170 @@ +package cn.iocoder.muse.module.ai.api; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +/** + * AI 导入解析对外 API。 + * + *

Content 只持有导入任务上下文与用户决策编排；Parse Job 与章节解析 Shadow 结果归 AI owner。 + * 该 API 是跨 BC 的稳定端口，调用方不得依赖 AI 的 DAL 或 application 实现。

+ */ +public interface MuseAiImportParseApi { + + /** + * 基于 Content 导入任务创建全书解析任务。 + * + * @param command 创建命令；必须带稳定存储引用与幂等键 + * @return AI owner 持久化后的解析任务投影 + */ + ParseJobProjection createParseJob(CreateParseJobCommand command); + + /** + * 查询解析任务投影。 + */ + ParseJobProjection getParseJob(Long ownerUserId, Long jobId); + + /** + * 重试解析任务。 + */ + RetryProjection retryParseJob(Long ownerUserId, Long jobId, RetryParseJobCommand command); + + /** + * 查询解析任务下的章节 Shadow 结果。 + */ + List listChapterResults(Long ownerUserId, Long jobId); + + /** + * 查询单个章节 Shadow 结果。 + */ + ChapterParseResultProjection getChapterResult(Long ownerUserId, Long resultId); + + /** + * 记录章节确认决策。Knowledge draft 必须先由 Knowledge owner 创建成功，再调用本方法推进 AI Shadow 状态。 + */ + DecisionRecordResult recordChapterConfirmed(Long ownerUserId, Long resultId, ConfirmChapterCommand command); + + /** + * 记录章节驳回决策。 + */ + DecisionRecordResult recordChapterRejected(Long ownerUserId, Long resultId, RejectChapterCommand command); + + /** + * 批量记录章节确认决策。只确认 Knowledge owner 明确成功的章节。 + */ + DecisionRecordResult recordBatchConfirmed(Long ownerUserId, Long jobId, BatchConfirmChapterCommand command); + + /** + * 创建 Parse Job 命令。 + */ + record CreateParseJobCommand(String commandId, + Long actorUserId, + Long ownerUserId, + Long workId, + Long importTaskId, + String fileName, + Long fileSize, + String fileHash, + String format, + String storageRef, + Map parseConfig, + SourceSnapshot sourceSnapshot) { + } + + /** + * 重试 Parse Job 命令。 + */ + record RetryParseJobCommand(String commandId, String retryStage) { + } + + /** + * 章节确认命令。 + */ + record ConfirmChapterCommand(String commandId, Integer expectedRevision, String draftId) { + } + + /** + * 章节驳回命令。 + */ + record RejectChapterCommand(String commandId, Integer expectedRevision, String reason) { + } + + /** + * 批量确认命令。 + */ + record BatchConfirmChapterCommand(String commandId, List outcomes) { + } + + /** + * Knowledge owner 返回的草稿创建结果摘要。 + */ + record DraftOutcome(Long resultId, + Long draftId, + boolean success, + String errorCode, + String errorMessage, + boolean skipped, + String skippedReason) { + } + + /** + * 跨域来源快照最小投影。 + */ + record SourceSnapshot(String sourceType, + Integer sourceVersion, + String authorizationSnapshotId) { + } + + /** + * Parse Job 投影。 + */ + record ParseJobProjection(Long id, + Long workId, + Long importTaskId, + String status, + Integer progress, + Integer totalChapters, + Integer confirmedChapters, + Integer pendingChapters, + String errorMessage, + String failedStage, + Boolean retryable, + List nextActions, + Integer parseConfigVersion, + SourceSnapshot sourceSnapshot, + LocalDateTime createdAt, + LocalDateTime completedAt) { + } + + /** + * 章节解析 Shadow 结果投影。 + * + *

{@code contentText} 仅供单体内部 owner-to-owner 创建 Knowledge Draft 使用；Content 对外接口必须只返回 + * {@code contentPreview}，不能把全文作为章节列表响应泄露给前端。

+ */ + record ChapterParseResultProjection(Long id, + Long jobId, + Long workId, + String title, + Integer sortOrder, + String contentPreview, + String contentText, + Integer wordCount, + String status, + Integer revision, + Map qualityResult) { + } + + /** + * 重试响应投影。 + */ + record RetryProjection(String jobId, String status, String pollUrl) { + } + + /** + * AI owner 决策记录结果。 + */ + record DecisionRecordResult(String auditLogId) { + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportLlmParser.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportLlmParser.java new file mode 100644 index 00000000..f1ef970b --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportLlmParser.java @@ -0,0 +1,59 @@ +package cn.iocoder.muse.module.ai.application.muse; + +import java.util.List; +import java.util.Map; + +/** + * 导入解析专用 LLM 解析端口。 + * + *

该端口只在 AI owner 内部使用，模型原始响应只进入本次解析过程，不写长期日志、不进跨域响应。

+ */ +public interface MuseAiImportLlmParser { + + /** + * 调用外部模型，把完整正文拆成待审核章节。 + */ + LlmParseResult parse(LlmParseCommand command); + + /** + * LLM 解析命令。{@code contentText} 是短生命周期载荷，调用方禁止记录。 + */ + record LlmParseCommand(String commandId, + Long jobId, + Long ownerUserId, + Long workId, + String fileName, + String format, + String contentText, + String contentHash, + Map parseConfig) { + } + + /** + * LLM 返回的章节正文。 + */ + record LlmChapter(String title, String content) { + } + + /** + * LLM 解析结果。summary 只能放模型 key、章节数、供应商请求号等脱敏字段。 + */ + record LlmParseResult(boolean success, + List chapters, + String errorCode, + String errorMessage, + boolean retryable, + Map summary) { + + static LlmParseResult success(List chapters, Map summary) { + return new LlmParseResult(true, chapters == null ? List.of() : chapters, null, null, false, + summary == null ? Map.of() : summary); + } + + static LlmParseResult failed(String errorCode, String errorMessage, boolean retryable, + Map summary) { + return new LlmParseResult(false, List.of(), errorCode, errorMessage, retryable, + summary == null ? Map.of() : summary); + } + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseService.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseService.java new file mode 100644 index 00000000..4dba0b40 --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseService.java @@ -0,0 +1,744 @@ +package cn.iocoder.muse.module.ai.application.muse; + +import cn.iocoder.muse.framework.common.exception.ServiceException; +import cn.iocoder.muse.framework.common.util.json.JsonUtils; +import cn.iocoder.muse.module.ai.api.MuseAiImportParseApi; +import cn.iocoder.muse.module.ai.application.muse.facade.MuseContentWorkOwnerFacade; +import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiChapterParseResultDO; +import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiParseJobDO; +import cn.iocoder.muse.module.ai.dal.mysql.muse.MuseAiChapterParseResultMapper; +import cn.iocoder.muse.module.ai.dal.mysql.muse.MuseAiParseJobMapper; +import cn.iocoder.muse.module.infra.api.file.FileApi; +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.StringUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static cn.iocoder.muse.module.ai.enums.ErrorCodeConstants.AI_COMMAND_ID_CONFLICT; +import static cn.iocoder.muse.module.ai.enums.ErrorCodeConstants.AI_RESOURCE_FORBIDDEN; +import static cn.iocoder.muse.module.ai.enums.ErrorCodeConstants.AI_TASK_NOT_EXISTS; + +/** + * AI 导入解析 owner 服务。 + * + *

本服务只消费 FileApi 稳定存储引用，不接受外部 URL；解析结果作为 AI Shadow 章节等待用户确认， + * 不绕过 Knowledge Draft 审核链路。

+ */ +@Service +@Slf4j +public class MuseAiImportParseService implements MuseAiImportParseApi { + + private static final String STATUS_COMPLETED = "completed"; + private static final String STATUS_FAILED = "failed"; + private static final String STATUS_PROCESSING = "processing"; + private static final String STATUS_PENDING_REVIEW = "pending_review"; + private static final String STATUS_CONFIRMED = "confirmed"; + private static final int MAX_IMPORT_BYTES = 5 * 1024 * 1024; + private static final int MAX_ZIP_ENTRY_BYTES = 8 * 1024 * 1024; + private static final int MAX_EXTRACTED_CHARS = 1_000_000; + private static final int MAX_PARSE_CHAPTERS = 300; + private static final int PREVIEW_MAX_CHARS = 240; + private static final Pattern MARKDOWN_HEADING_PATTERN = Pattern.compile("^#{1,3}\\s+(.{1,120})$"); + private static final Pattern CN_CHAPTER_HEADING_PATTERN = Pattern.compile("^第[\\p{N}一二三四五六七八九十百千万零〇两]+[章节卷回].{0,120}$"); + private static final Pattern EN_CHAPTER_HEADING_PATTERN = Pattern.compile("(?i)^chapter\\s+[\\p{Alnum}]+.{0,120}$"); + private static final Pattern SENTENCE_PUNCTUATION_PATTERN = Pattern.compile("[。！？；!?;]"); + private static final Pattern WORD_PARAGRAPH_PATTERN = Pattern.compile("(?s)].*?"); + private static final Pattern WORD_TEXT_PATTERN = Pattern.compile("(?s)]*>(.*?)"); + private static final Pattern HTML_HEAD_PATTERN = Pattern.compile("(?is)<(script|style|head)[^>]*>.*?"); + private static final Pattern HTML_BLOCK_PATTERN = Pattern.compile("(?i)]*>"); + private static final Pattern HTML_TAG_PATTERN = Pattern.compile("(?s)<[^>]+>"); + private static final Set SUPPORTED_FORMATS = Set.of("txt", "markdown", "docx", "epub"); + + private final FileApi fileApi; + + @Resource + private MuseAiParseJobMapper parseJobMapper; + @Resource + private MuseAiChapterParseResultMapper chapterResultMapper; + @Resource + private MuseContentWorkOwnerFacade contentWorkOwnerFacade; + @Resource + private MuseAiImportLlmParser llmParser; + + public MuseAiImportParseService(ObjectProvider fileApiProvider) { + this.fileApi = fileApiProvider.getIfAvailable(); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public ParseJobProjection createParseJob(CreateParseJobCommand command) { + requireCreateCommand(command); + contentWorkOwnerFacade.requireWorkOwner(command.workId(), command.ownerUserId()); + MuseAiParseJobDO replay = parseJobMapper.selectByCommandId(command.commandId()); + if (replay != null) { + requireSameImport(replay, command); + return toParseJobProjection(replay); + } + MuseAiParseJobDO existingByImportTask = + parseJobMapper.selectByImportTaskIdAndOwner(command.importTaskId(), command.ownerUserId()); + if (existingByImportTask != null) { + requireSameImport(existingByImportTask, command); + return toParseJobProjection(existingByImportTask); + } + + MuseAiParseJobDO job = new MuseAiParseJobDO(); + job.setWorkId(command.workId()); + job.setImportTaskId(command.importTaskId()); + job.setOwnerUserId(command.ownerUserId()); + job.setActorUserId(command.actorUserId()); + job.setCommandId(command.commandId()); + job.setFileName(command.fileName()); + job.setFileSize(command.fileSize()); + job.setFileHash(command.fileHash()); + job.setFormat(normalizeFormat(command.format())); + job.setStorageRef(command.storageRef()); + job.setParseConfig(JsonUtils.toJsonString(command.parseConfig() == null ? Map.of() : command.parseConfig())); + job.setParseConfigVersion(1); + job.setStatus("processing"); + job.setProgress(10); + job.setTotalChapters(0); + job.setConfirmedChapters(0); + job.setPendingChapters(0); + job.setRetryCount(0); + job.setRetryable(false); + job.setSourceSnapshot(JsonUtils.toJsonString(sourceSnapshotMap(command.sourceSnapshot()))); + job.setResultSummary(JsonUtils.toJsonString(Map.of())); + job.setStartedAt(LocalDateTime.now()); + parseJobMapper.insert(job); + + executeAndPersist(job); + return toParseJobProjection(job); + } + + @Override + public ParseJobProjection getParseJob(Long ownerUserId, Long jobId) { + return toParseJobProjection(requireOwnedJob(ownerUserId, jobId)); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public RetryProjection retryParseJob(Long ownerUserId, Long jobId, RetryParseJobCommand command) { + if (command == null || !StringUtils.hasText(command.commandId())) { + throw new ServiceException(AI_COMMAND_ID_CONFLICT.getCode(), "AI 解析重试命令缺少幂等键"); + } + MuseAiParseJobDO job = requireOwnedJob(ownerUserId, jobId); + if (!STATUS_FAILED.equals(job.getStatus()) || !Boolean.TRUE.equals(job.getRetryable())) { + throw new ServiceException(AI_COMMAND_ID_CONFLICT.getCode(), "AI 解析任务当前不可重试"); + } + job.setStatus(STATUS_PROCESSING); + job.setProgress(10); + job.setRetryCount(job.getRetryCount() == null ? 1 : job.getRetryCount() + 1); + job.setRetryable(false); + job.setFailedStage(null); + job.setErrorCode(null); + job.setErrorMessage(null); + job.setStartedAt(LocalDateTime.now()); + job.setCompletedAt(null); + parseJobMapper.updateById(job); + chapterResultMapper.deleteByJobId(job.getId()); + executeAndPersist(job); + return new RetryProjection(String.valueOf(job.getId()), job.getStatus(), "/app-api/muse/parse-jobs/" + job.getId()); + } + + @Override + public List listChapterResults(Long ownerUserId, Long jobId) { + MuseAiParseJobDO job = requireOwnedJob(ownerUserId, jobId); + return chapterResultMapper.selectByJobId(job.getId()).stream() + .map(this::toChapterProjection) + .toList(); + } + + @Override + public ChapterParseResultProjection getChapterResult(Long ownerUserId, Long resultId) { + return toChapterProjection(requireOwnedChapterResult(ownerUserId, resultId)); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DecisionRecordResult recordChapterConfirmed(Long ownerUserId, Long resultId, ConfirmChapterCommand command) { + MuseAiChapterParseResultDO result = requireOwnedChapterResult(ownerUserId, resultId); + int updated = chapterResultMapper.markConfirmed(resultId, command.expectedRevision(), Long.valueOf(command.draftId()), + command.commandId()); + if (updated != 1) { + throw new ServiceException(AI_COMMAND_ID_CONFLICT.getCode(), "章节解析结果 revision 已变化或不可确认"); + } + refreshJobCounters(result.getParseJobId()); + return new DecisionRecordResult("ai-chapter-confirmed-" + resultId + "-" + command.commandId()); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DecisionRecordResult recordChapterRejected(Long ownerUserId, Long resultId, RejectChapterCommand command) { + MuseAiChapterParseResultDO result = requireOwnedChapterResult(ownerUserId, resultId); + int updated = chapterResultMapper.markRejected(resultId, command.expectedRevision(), command.reason(), command.commandId()); + if (updated != 1) { + throw new ServiceException(AI_COMMAND_ID_CONFLICT.getCode(), "章节解析结果 revision 已变化或不可驳回"); + } + refreshJobCounters(result.getParseJobId()); + return new DecisionRecordResult("ai-chapter-rejected-" + resultId + "-" + command.commandId()); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DecisionRecordResult recordBatchConfirmed(Long ownerUserId, Long jobId, BatchConfirmChapterCommand command) { + MuseAiParseJobDO job = requireOwnedJob(ownerUserId, jobId); + for (DraftOutcome outcome : command.outcomes()) { + if (!outcome.success()) { + continue; + } + MuseAiChapterParseResultDO result = requireOwnedChapterResult(ownerUserId, outcome.resultId()); + if (!Objects.equals(result.getParseJobId(), job.getId())) { + throw new ServiceException(AI_RESOURCE_FORBIDDEN); + } + int updated = chapterResultMapper.markConfirmed(result.getId(), result.getRevision(), outcome.draftId(), + command.commandId() + ":result-" + result.getId()); + if (updated != 1) { + throw new ServiceException(AI_COMMAND_ID_CONFLICT.getCode(), "批量确认时章节解析结果已变化"); + } + } + refreshJobCounters(job.getId()); + return new DecisionRecordResult("ai-chapter-batch-confirmed-" + jobId + "-" + command.commandId()); + } + + private void executeAndPersist(MuseAiParseJobDO job) { + ParseExecution execution = parseUploadedText(job); + if (execution.success()) { + for (ParsedChapter chapter : execution.chapters()) { + chapterResultMapper.insert(toChapterResult(job, chapter, execution.parser())); + } + job.setStatus(STATUS_COMPLETED); + job.setProgress(100); + job.setTotalChapters(execution.chapters().size()); + job.setPendingChapters(execution.chapters().size()); + job.setConfirmedChapters(0); + job.setRetryable(false); + job.setFailedStage(null); + job.setErrorCode(null); + job.setErrorMessage(null); + job.setCompletedAt(LocalDateTime.now()); + job.setResultSummary(JsonUtils.toJsonString(resultSummary(execution))); + } else { + job.setStatus(STATUS_FAILED); + job.setProgress(100); + job.setTotalChapters(0); + job.setPendingChapters(0); + job.setConfirmedChapters(0); + job.setRetryable(execution.retryable()); + job.setFailedStage(execution.failedStage()); + job.setErrorCode(execution.errorCode()); + job.setErrorMessage(execution.errorMessage()); + job.setCompletedAt(LocalDateTime.now()); + job.setResultSummary(JsonUtils.toJsonString(failureSummary(execution))); + } + parseJobMapper.updateById(job); + } + + private Map resultSummary(ParseExecution execution) { + Map summary = new LinkedHashMap<>(execution.summary()); + summary.put("parser", execution.parser()); + summary.put("chapterCount", execution.chapters().size()); + summary.put("contentHash", execution.contentHash()); + return summary; + } + + private Map failureSummary(ParseExecution execution) { + Map summary = new LinkedHashMap<>(execution.summary()); + summary.put("failedStage", execution.failedStage()); + summary.put("errorCode", execution.errorCode()); + summary.put("retryable", execution.retryable()); + return summary; + } + + private ParseExecution parseUploadedText(MuseAiParseJobDO job) { + String format = normalizeFormat(job.getFormat()); + if (!SUPPORTED_FORMATS.contains(format)) { + return ParseExecution.failed("parse", "UNSUPPORTED_IMPORT_FORMAT", + "当前导入解析仅支持 txt/markdown/docx/epub", false); + } + if (fileApi == null) { + return ParseExecution.failed("load_file", "FILE_API_UNAVAILABLE", "对象存储读取能力不可用", true); + } + byte[] bytes; + try { + bytes = fileApi.getFileBytes(job.getStorageRef()); + } catch (RuntimeException ex) { + log.warn("AI 导入解析读取文件失败：jobId={}, errorType={}", job.getId(), ex.getClass().getSimpleName()); + return ParseExecution.failed("load_file", "FILE_READ_FAILED", "导入文件读取失败", true); + } + if (bytes == null || bytes.length == 0) { + return ParseExecution.failed("load_file", "FILE_EMPTY", "导入文件为空", false); + } + if (bytes.length > MAX_IMPORT_BYTES) { + return ParseExecution.failed("load_file", "FILE_TOO_LARGE", "导入文件超过当前解析大小上限", false); + } + ExtractedText extractedText; + try { + extractedText = extractText(format, bytes); + } catch (IllegalArgumentException exception) { + return ParseExecution.failed("extract", "FILE_PARSE_FAILED", exception.getMessage(), false); + } catch (RuntimeException exception) { + log.warn("AI 导入解析抽取文件失败：jobId={}, format={}, errorType={}", job.getId(), format, + exception.getClass().getSimpleName()); + return ParseExecution.failed("extract", "FILE_PARSE_FAILED", "导入文件正文抽取失败", false); + } + String normalizedText = normalizeText(extractedText.text()); + if (!StringUtils.hasText(normalizedText)) { + return ParseExecution.failed("parse", "NO_CONTENT", "导入文件没有可解析正文", false); + } + String contentHash = sha256(normalizedText); + if (shouldUseLlm(job)) { + return parseWithLlm(job, format, normalizedText, contentHash, extractedText); + } + List chapters = parseText(normalizedText, job.getFileName()); + if (chapters.isEmpty()) { + return ParseExecution.failed("parse", "NO_CONTENT", "导入文件没有可解析正文", false); + } + if (chapters.size() > MAX_PARSE_CHAPTERS) { + return ParseExecution.failed("parse", "TOO_MANY_CHAPTERS", "导入章节数超过当前上限", false); + } + return ParseExecution.success(chapters, contentHash, extractedText.parser(), Map.of( + "format", format, + "extractedChars", normalizedText.length())); + } + + private ParseExecution parseWithLlm(MuseAiParseJobDO job, String format, String normalizedText, String contentHash, + ExtractedText extractedText) { + MuseAiImportLlmParser.LlmParseResult llmResult = llmParser.parse(new MuseAiImportLlmParser.LlmParseCommand( + job.getCommandId(), job.getId(), job.getOwnerUserId(), job.getWorkId(), job.getFileName(), format, + normalizedText, contentHash, readJson(job.getParseConfig()))); + if (!llmResult.success()) { + return ParseExecution.failed("llm_parse", llmResult.errorCode(), llmResult.errorMessage(), + llmResult.retryable(), llmResult.summary()); + } + List chapters = new ArrayList<>(); + int order = 1; + for (MuseAiImportLlmParser.LlmChapter chapter : llmResult.chapters()) { + String content = normalizeText(chapter.content()); + if (!StringUtils.hasText(content)) { + continue; + } + chapters.add(new ParsedChapter(truncate(StringUtils.hasText(chapter.title()) ? chapter.title().strip() + : "导入章节", 200), content, countWords(content), order++)); + } + if (chapters.isEmpty()) { + return ParseExecution.failed("llm_parse", "AI_IMPORT_LLM_NO_CHAPTERS", "AI 全书解析未返回有效章节", false, + llmResult.summary()); + } + if (chapters.size() > MAX_PARSE_CHAPTERS) { + return ParseExecution.failed("llm_parse", "AI_IMPORT_LLM_TOO_MANY_CHAPTERS", + "AI 全书解析章节数超过当前上限", false, llmResult.summary()); + } + Map summary = new LinkedHashMap<>(llmResult.summary()); + summary.put("extractor", extractedText.parser()); + summary.put("extractedChars", normalizedText.length()); + return ParseExecution.success(chapters, contentHash, "new_api_import_llm", summary); + } + + private ExtractedText extractText(String format, byte[] bytes) { + return switch (format) { + case "txt", "markdown" -> new ExtractedText(new String(bytes, StandardCharsets.UTF_8), + "deterministic_text_extractor"); + case "docx" -> new ExtractedText(extractDocxText(bytes), "docx_ooxml_text_extractor"); + case "epub" -> new ExtractedText(extractEpubText(bytes), "epub_xhtml_text_extractor"); + default -> throw new IllegalArgumentException("不支持的导入文件格式"); + }; + } + + private String extractDocxText(byte[] bytes) { + String documentXml = null; + try (ZipInputStream zip = new ZipInputStream(new java.io.ByteArrayInputStream(bytes))) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (entry.isDirectory() || !"word/document.xml".equals(entry.getName())) { + continue; + } + documentXml = readZipEntryUtf8(zip); + break; + } + } catch (IOException exception) { + throw new IllegalArgumentException("DOCX 文件无法读取"); + } + if (!StringUtils.hasText(documentXml)) { + throw new IllegalArgumentException("DOCX 文件缺少正文内容"); + } + StringBuilder text = new StringBuilder(); + java.util.regex.Matcher paragraphMatcher = WORD_PARAGRAPH_PATTERN.matcher(documentXml); + while (paragraphMatcher.find()) { + String paragraph = paragraphMatcher.group(); + java.util.regex.Matcher textMatcher = WORD_TEXT_PATTERN.matcher(paragraph); + StringBuilder paragraphText = new StringBuilder(); + while (textMatcher.find()) { + paragraphText.append(unescapeXml(textMatcher.group(1))); + } + if (StringUtils.hasText(paragraphText.toString())) { + appendBounded(text, paragraphText.toString().strip()).append('\n'); + } + } + if (!StringUtils.hasText(text.toString())) { + throw new IllegalArgumentException("DOCX 文件没有可解析正文"); + } + return text.toString(); + } + + private String extractEpubText(byte[] bytes) { + List entries = new ArrayList<>(); + try (ZipInputStream zip = new ZipInputStream(new java.io.ByteArrayInputStream(bytes))) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (entry.isDirectory() || !isEpubTextEntry(entry.getName())) { + continue; + } + entries.add(new EpubTextEntry(entry.getName(), htmlToText(readZipEntryUtf8(zip)))); + } + } catch (IOException exception) { + throw new IllegalArgumentException("EPUB 文件无法读取"); + } + entries.sort(Comparator.comparing(EpubTextEntry::name)); + StringBuilder text = new StringBuilder(); + for (EpubTextEntry entry : entries) { + if (StringUtils.hasText(entry.text())) { + appendBounded(text, entry.text().strip()).append('\n'); + } + } + if (!StringUtils.hasText(text.toString())) { + throw new IllegalArgumentException("EPUB 文件没有可解析正文"); + } + return text.toString(); + } + + private boolean isEpubTextEntry(String name) { + if (!StringUtils.hasText(name)) { + return false; + } + String lower = name.toLowerCase(Locale.ROOT); + return (lower.endsWith(".xhtml") || lower.endsWith(".html") || lower.endsWith(".htm")) + && !lower.endsWith("nav.xhtml") + && !lower.endsWith("toc.xhtml") + && !lower.endsWith("toc.html"); + } + + private String readZipEntryUtf8(ZipInputStream zip) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + int total = 0; + int read; + while ((read = zip.read(chunk)) != -1) { + total += read; + if (total > MAX_ZIP_ENTRY_BYTES) { + throw new IllegalArgumentException("压缩包单文件超过解析上限"); + } + buffer.write(chunk, 0, read); + } + return buffer.toString(StandardCharsets.UTF_8); + } + + private String htmlToText(String html) { + String text = HTML_HEAD_PATTERN.matcher(html == null ? "" : html).replaceAll("\n"); + text = HTML_BLOCK_PATTERN.matcher(text).replaceAll("\n"); + text = HTML_TAG_PATTERN.matcher(text).replaceAll(""); + return unescapeXml(text).replace('\u00A0', ' '); + } + + private String unescapeXml(String value) { + return (value == null ? "" : value) + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&"); + } + + private StringBuilder appendBounded(StringBuilder builder, String value) { + if (builder.length() + value.length() > MAX_EXTRACTED_CHARS) { + throw new IllegalArgumentException("导入文件正文超过当前抽取上限"); + } + return builder.append(value); + } + + private boolean shouldUseLlm(MuseAiParseJobDO job) { + Map parseConfig = readJson(job.getParseConfig()); + String mode = stringValue(parseConfig.get("mode")); + String strategy = stringValue(parseConfig.get("strategy")); + return "full_book".equalsIgnoreCase(mode) + || "llm_full_book".equalsIgnoreCase(mode) + || "llm_full_book".equalsIgnoreCase(strategy); + } + + private List parseText(String contentText, String fileName) { + String normalized = normalizeText(contentText); + String fallbackTitle = titleFromFileName(fileName); + List chapters = new ArrayList<>(); + String currentTitle = null; + StringBuilder currentContent = new StringBuilder(); + int sourceOrder = 1; + for (String line : normalized.split("\\n", -1)) { + String heading = detectChapterHeading(line); + if (heading != null) { + sourceOrder = flushParsedChapter(chapters, currentTitle, currentContent, fallbackTitle, sourceOrder); + currentTitle = heading; + currentContent = new StringBuilder(); + continue; + } + currentContent.append(line).append('\n'); + } + flushParsedChapter(chapters, currentTitle, currentContent, fallbackTitle, sourceOrder); + return chapters; + } + + private int flushParsedChapter(List chapters, String currentTitle, StringBuilder currentContent, + String fallbackTitle, int sourceOrder) { + String content = currentContent == null ? "" : currentContent.toString().strip(); + if (!StringUtils.hasText(content)) { + return sourceOrder; + } + String title = StringUtils.hasText(currentTitle) ? currentTitle.strip() : fallbackTitle; + chapters.add(new ParsedChapter(truncate(StringUtils.hasText(title) ? title : "导入正文", 200), + content, countWords(content), sourceOrder)); + return sourceOrder + 1; + } + + private String detectChapterHeading(String line) { + if (line == null) { + return null; + } + String trimmed = line.strip(); + if (trimmed.isEmpty()) { + return null; + } + java.util.regex.Matcher markdownMatcher = MARKDOWN_HEADING_PATTERN.matcher(trimmed); + if (markdownMatcher.matches()) { + return markdownMatcher.group(1).strip(); + } + if ((CN_CHAPTER_HEADING_PATTERN.matcher(trimmed).matches() + || EN_CHAPTER_HEADING_PATTERN.matcher(trimmed).matches()) + && trimmed.length() <= 80 && !SENTENCE_PUNCTUATION_PATTERN.matcher(trimmed).find()) { + return trimmed; + } + return null; + } + + private MuseAiChapterParseResultDO toChapterResult(MuseAiParseJobDO job, ParsedChapter chapter, String parser) { + MuseAiChapterParseResultDO result = new MuseAiChapterParseResultDO(); + result.setParseJobId(job.getId()); + result.setWorkId(job.getWorkId()); + result.setOwnerUserId(job.getOwnerUserId()); + result.setTitle(chapter.title()); + result.setSortOrder(chapter.sourceOrder()); + result.setContentPreview(preview(chapter.content())); + result.setContentText(chapter.content()); + result.setWordCount(chapter.wordCount()); + result.setStatus(STATUS_PENDING_REVIEW); + result.setRevision(1); + result.setQualityResult(JsonUtils.toJsonString(Map.of( + "passed", true, + "parser", parser, + "wordCount", chapter.wordCount()))); + result.setSourceSnapshot(job.getSourceSnapshot()); + return result; + } + + private ParseJobProjection toParseJobProjection(MuseAiParseJobDO job) { + return new ParseJobProjection(job.getId(), job.getWorkId(), job.getImportTaskId(), job.getStatus(), + job.getProgress(), job.getTotalChapters(), job.getConfirmedChapters(), job.getPendingChapters(), + job.getErrorMessage(), job.getFailedStage(), job.getRetryable(), nextActions(job), + job.getParseConfigVersion(), toSourceSnapshot(job.getSourceSnapshot()), job.getCreateTime(), + job.getCompletedAt()); + } + + private ChapterParseResultProjection toChapterProjection(MuseAiChapterParseResultDO result) { + return new ChapterParseResultProjection(result.getId(), result.getParseJobId(), result.getWorkId(), + result.getTitle(), result.getSortOrder(), result.getContentPreview(), result.getContentText(), + result.getWordCount(), result.getStatus(), result.getRevision(), readJson(result.getQualityResult())); + } + + private void refreshJobCounters(Long jobId) { + Long confirmed = chapterResultMapper.countByJobAndStatus(jobId, STATUS_CONFIRMED); + Long pending = chapterResultMapper.countByJobAndStatus(jobId, STATUS_PENDING_REVIEW); + parseJobMapper.updateReviewCounters(jobId, confirmed.intValue(), pending.intValue()); + } + + private MuseAiParseJobDO requireOwnedJob(Long ownerUserId, Long jobId) { + MuseAiParseJobDO job = parseJobMapper.selectByIdAndOwner(jobId, ownerUserId); + if (job == null) { + throw new ServiceException(AI_TASK_NOT_EXISTS); + } + return job; + } + + private MuseAiChapterParseResultDO requireOwnedChapterResult(Long ownerUserId, Long resultId) { + MuseAiChapterParseResultDO result = chapterResultMapper.selectByIdAndOwner(resultId, ownerUserId); + if (result == null) { + throw new ServiceException(AI_TASK_NOT_EXISTS); + } + return result; + } + + private void requireCreateCommand(CreateParseJobCommand command) { + if (command == null || !StringUtils.hasText(command.commandId()) || command.actorUserId() == null + || command.ownerUserId() == null || command.workId() == null || command.importTaskId() == null + || !StringUtils.hasText(command.storageRef()) || !StringUtils.hasText(command.format())) { + throw new ServiceException(AI_COMMAND_ID_CONFLICT.getCode(), "AI Parse Job 创建命令缺少必要字段"); + } + } + + private void requireSameImport(MuseAiParseJobDO replay, CreateParseJobCommand command) { + if (!Objects.equals(replay.getOwnerUserId(), command.ownerUserId()) + || !Objects.equals(replay.getWorkId(), command.workId()) + || !Objects.equals(replay.getImportTaskId(), command.importTaskId())) { + throw new ServiceException(AI_COMMAND_ID_CONFLICT); + } + } + + private List nextActions(MuseAiParseJobDO job) { + if (STATUS_COMPLETED.equals(job.getStatus())) { + return List.of("review_chapters"); + } + if (STATUS_FAILED.equals(job.getStatus())) { + return Boolean.TRUE.equals(job.getRetryable()) ? List.of("retry_parse") : List.of(); + } + return List.of("poll_parse_job"); + } + + private SourceSnapshot toSourceSnapshot(String json) { + Map map = readJson(json); + return new SourceSnapshot(stringValue(map.get("sourceType")), integerValue(map.get("sourceVersion")), + stringValue(map.get("authorizationSnapshotId"))); + } + + private Map sourceSnapshotMap(SourceSnapshot sourceSnapshot) { + if (sourceSnapshot == null) { + return Map.of(); + } + Map map = new LinkedHashMap<>(); + map.put("sourceType", sourceSnapshot.sourceType()); + map.put("sourceVersion", sourceSnapshot.sourceVersion()); + map.put("authorizationSnapshotId", sourceSnapshot.authorizationSnapshotId()); + return map; + } + + private Map readJson(String json) { + Map value = JsonUtils.parseObjectQuietly(json, new TypeReference<>() { + }); + return value == null ? Map.of() : value; + } + + private String normalizeText(String contentText) { + String normalized = contentText == null ? "" : contentText.replace("\r\n", "\n").replace('\r', '\n'); + if (normalized.startsWith("\uFEFF")) { + normalized = normalized.substring(1); + } + return normalized.strip(); + } + + private String normalizeFormat(String format) { + return format == null ? null : format.strip().toLowerCase(Locale.ROOT); + } + + private String titleFromFileName(String fileName) { + if (!StringUtils.hasText(fileName)) { + return "导入正文"; + } + String title = fileName.strip(); + int dotIndex = title.lastIndexOf('.'); + if (dotIndex > 0) { + title = title.substring(0, dotIndex); + } + return StringUtils.hasText(title) ? truncate(title, 200) : "导入正文"; + } + + private int countWords(String content) { + return StringUtils.hasText(content) ? content.strip().length() : 0; + } + + private String preview(String content) { + String normalized = Arrays.stream(content.split("\\n+")) + .map(String::strip) + .filter(StringUtils::hasText) + .reduce((left, right) -> left + " " + right) + .orElse(""); + return truncate(normalized, PREVIEW_MAX_CHARS); + } + + private String truncate(String text, int maxLength) { + if (text == null || text.length() <= maxLength) { + return text; + } + return text.substring(0, maxLength); + } + + private String sha256(String payload) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(payload.getBytes(StandardCharsets.UTF_8)); + return "sha256:" + HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("JDK 缺少 SHA-256 摘要算法", exception); + } + } + + private String stringValue(Object value) { + return value == null ? null : String.valueOf(value); + } + + private Integer integerValue(Object value) { + if (value instanceof Number number) { + return number.intValue(); + } + if (value == null || String.valueOf(value).isBlank()) { + return null; + } + return Integer.valueOf(String.valueOf(value)); + } + + private record ExtractedText(String text, String parser) { + } + + private record EpubTextEntry(String name, String text) { + } + + private record ParsedChapter(String title, String content, int wordCount, int sourceOrder) { + } + + private record ParseExecution(boolean success, List chapters, String contentHash, + String parser, Map summary, String failedStage, String errorCode, + String errorMessage, boolean retryable) { + + static ParseExecution success(List chapters, String contentHash, String parser, + Map summary) { + return new ParseExecution(true, chapters, contentHash, parser, summary == null ? Map.of() : summary, + null, null, null, false); + } + + static ParseExecution failed(String failedStage, String errorCode, String errorMessage, boolean retryable) { + return failed(failedStage, errorCode, errorMessage, retryable, Map.of()); + } + + static ParseExecution failed(String failedStage, String errorCode, String errorMessage, boolean retryable, + Map summary) { + return new ParseExecution(false, List.of(), null, null, summary == null ? Map.of() : summary, + failedStage, errorCode, errorMessage, retryable); + } + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java new file mode 100644 index 00000000..ac3ee7e3 --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParser.java @@ -0,0 +1,311 @@ +package cn.iocoder.muse.module.ai.application.muse; + +import cn.iocoder.muse.framework.common.util.json.JsonUtils; +import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.io.IOException; +import java.net.ConnectException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 基于 New-API OpenAI 兼容接口的导入章节解析器。 + * + *

这里不复用通用 runtime adapter，是因为通用 adapter 为防泄漏只返回脱敏摘要；导入解析需要消费模型 JSON。 + * 本实现仍沿用相同安全边界：token 不记录，provider 原文不落库，模型正文只在本次解析内转成 AI Shadow 章节。

+ */ +@Component +@Slf4j +public class NewApiMuseAiImportLlmParser implements MuseAiImportLlmParser { + + private static final String CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; + private static final int MAX_LLM_INPUT_CHARS = 180_000; + private static final int MAX_LLM_CHAPTERS = 300; + + private final MuseAiProperties.NewApi properties; + private final HttpClient httpClient; + + public NewApiMuseAiImportLlmParser(MuseAiProperties museAiProperties) { + this.properties = museAiProperties == null ? null : museAiProperties.getNewApi(); + this.httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(positive(connectTimeoutSeconds(), 5))) + .build(); + } + + @Override + public LlmParseResult parse(LlmParseCommand command) { + if (!isConfigured()) { + return LlmParseResult.failed("AI_IMPORT_LLM_UNAVAILABLE", "AI 全书解析服务未配置", false, + Map.of("parser", "new_api_import_llm")); + } + if (command == null || !StringUtils.hasText(command.contentText())) { + return LlmParseResult.failed("AI_IMPORT_LLM_EMPTY_INPUT", "AI 全书解析输入为空", false, + Map.of("parser", "new_api_import_llm")); + } + if (command.contentText().length() > MAX_LLM_INPUT_CHARS) { + return LlmParseResult.failed("AI_IMPORT_LLM_INPUT_TOO_LARGE", + "当前 AI 全书解析输入超过模型上下文上限，请先按章节拆分后重试", false, + Map.of("parser", "new_api_import_llm", "inputChars", command.contentText().length())); + } + + int maxAttempts = maxAttempts(); + LlmParseResult lastResult = null; + for (int attemptNo = 1; attemptNo <= maxAttempts; attemptNo++) { + lastResult = parseOnce(command); + LlmParseResult resultWithAttempts = withAttemptSummary(lastResult, attemptNo, maxAttempts); + if (resultWithAttempts.success() || !resultWithAttempts.retryable() || attemptNo >= maxAttempts) { + return resultWithAttempts; + } + sleepBeforeRetry(command, attemptNo); + } + return withAttemptSummary(lastResult, maxAttempts, maxAttempts); + } + + private LlmParseResult parseOnce(LlmParseCommand command) { + try { + HttpRequest request = buildRequest(command); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + return mapResponse(command, response); + } catch (HttpTimeoutException ex) { + return LlmParseResult.failed("AI_IMPORT_LLM_TIMEOUT", "AI 全书解析请求超时", true, + Map.of("parser", "new_api_import_llm")); + } catch (ConnectException ex) { + return LlmParseResult.failed("AI_IMPORT_LLM_CONNECT_FAILED", "AI 全书解析服务连接失败", true, + Map.of("parser", "new_api_import_llm")); + } catch (IOException ex) { + return LlmParseResult.failed("AI_IMPORT_LLM_CONNECT_FAILED", "AI 全书解析服务连接失败", true, + Map.of("parser", "new_api_import_llm")); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return LlmParseResult.failed("AI_IMPORT_LLM_TIMEOUT", "AI 全书解析请求被中断", true, + Map.of("parser", "new_api_import_llm")); + } catch (RuntimeException ex) { + log.warn("AI 全书解析运行时失败：jobId={}, errorType={}", command.jobId(), ex.getClass().getSimpleName()); + return LlmParseResult.failed("AI_IMPORT_LLM_BAD_RESPONSE", "AI 全书解析响应无法解析", false, + Map.of("parser", "new_api_import_llm")); + } + } + + private LlmParseResult withAttemptSummary(LlmParseResult result, int attemptNo, int maxAttempts) { + if (result == null) { + return LlmParseResult.failed("AI_IMPORT_LLM_UNKNOWN", "AI 全书解析未返回结果", false, + Map.of("parser", "new_api_import_llm", "attemptNo", attemptNo, "maxAttempts", maxAttempts)); + } + Map summary = new LinkedHashMap<>(result.summary() == null ? Map.of() : result.summary()); + summary.put("attemptNo", attemptNo); + summary.put("maxAttempts", maxAttempts); + return new LlmParseResult(result.success(), result.chapters(), result.errorCode(), result.errorMessage(), + result.retryable(), summary); + } + + private void sleepBeforeRetry(LlmParseCommand command, int attemptNo) { + int backoffSeconds = backoffSeconds(attemptNo); + log.warn("AI 全书解析可重试失败，准备重试：jobId={}, attemptNo={}, nextAttemptNo={}, backoffSeconds={}", + command.jobId(), attemptNo, attemptNo + 1, backoffSeconds); + if (backoffSeconds <= 0) { + return; + } + try { + Thread.sleep(Duration.ofSeconds(backoffSeconds).toMillis()); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + private HttpRequest buildRequest(LlmParseCommand command) { + String body = JsonUtils.toJsonString(Map.of( + "model", model(command.parseConfig()), + "stream", false, + "temperature", 0.1, + "messages", List.of( + Map.of("role", "system", "content", systemPrompt()), + Map.of("role", "user", "content", userPrompt(command))))); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(chatUri()) + .timeout(Duration.ofSeconds(positive(nonStreamReadTimeoutSeconds(), 90))) + .header("Authorization", "Bearer " + properties.getToken()) + .header("Content-Type", "application/json") + .header("Accept", "application/json") + .header("X-Muse-Scene", "ai_import_parse") + .POST(HttpRequest.BodyPublishers.ofString(body)); + if (StringUtils.hasText(command.commandId())) { + builder.header("X-Request-Id", command.commandId()); + builder.header("X-Trace-Id", command.commandId()); + } + if (Boolean.TRUE.equals(properties.isPropagateMuseContext())) { + builder.header("X-Muse-User-Id", String.valueOf(command.ownerUserId())); + builder.header("X-Muse-Work-Id", String.valueOf(command.workId())); + } + return builder.build(); + } + + private LlmParseResult mapResponse(LlmParseCommand command, HttpResponse response) { + int status = response.statusCode(); + Map summary = new LinkedHashMap<>(); + summary.put("parser", "new_api_import_llm"); + summary.put("modelKey", model(command.parseConfig())); + summary.put("providerStatusCode", status); + response.headers().firstValue("X-Oneapi-Request-Id").ifPresent(value -> summary.put("providerRequestId", value)); + if (status < 200 || status >= 300) { + return LlmParseResult.failed(errorCode(status), errorMessage(status), isRetryableStatus(status), summary); + } + + JsonNode root = JsonUtils.parseTree(response.body()); + JsonNode choice = root.path("choices").isArray() && !root.path("choices").isEmpty() + ? root.path("choices").get(0) : null; + if (choice == null || !choice.path("message").path("content").isTextual()) { + return LlmParseResult.failed("AI_IMPORT_LLM_BAD_RESPONSE", "AI 全书解析响应缺少正文", false, summary); + } + String finishReason = choice.path("finish_reason").asText(null); + if ("length".equalsIgnoreCase(finishReason)) { + return LlmParseResult.failed("AI_IMPORT_LLM_TRUNCATED", "AI 全书解析响应被截断", false, summary); + } + List chapters = parseChapters(choice.path("message").path("content").asText("")); + if (chapters.isEmpty()) { + return LlmParseResult.failed("AI_IMPORT_LLM_NO_CHAPTERS", "AI 全书解析未返回有效章节", false, summary); + } + if (chapters.size() > MAX_LLM_CHAPTERS) { + return LlmParseResult.failed("AI_IMPORT_LLM_TOO_MANY_CHAPTERS", "AI 全书解析章节数超过上限", false, summary); + } + summary.put("chapterCount", chapters.size()); + summary.put("finishReason", finishReason); + return LlmParseResult.success(chapters, summary); + } + + private List parseChapters(String content) { + String json = stripJsonFence(content); + JsonNode root = JsonUtils.parseTree(json); + JsonNode chaptersNode = root.isArray() ? root : root.path("chapters"); + List chapters = new ArrayList<>(); + if (!chaptersNode.isArray()) { + return chapters; + } + for (JsonNode item : chaptersNode) { + String title = item.path("title").asText("").strip(); + String chapterContent = item.path("content").asText("").strip(); + if (!StringUtils.hasText(chapterContent)) { + continue; + } + chapters.add(new LlmChapter(StringUtils.hasText(title) ? title : "导入章节", chapterContent)); + } + return chapters; + } + + private String stripJsonFence(String content) { + String normalized = content == null ? "" : content.strip(); + if (normalized.startsWith("```")) { + normalized = normalized.replaceFirst("(?s)^```(?:json)?\\s*", ""); + normalized = normalized.replaceFirst("(?s)\\s*```$", ""); + } + return normalized.strip(); + } + + private String systemPrompt() { + return """ + 你是长篇小说导入解析器。只输出 JSON，不要解释。 + JSON 格式必须是 {"chapters":[{"title":"章节标题","content":"章节正文"}]}。 + 章节正文必须来自用户提供文本，不得补写、改写、总结或虚构。 + 如果原文已有章节标题，按原标题切分；如果没有标题，按叙事段落保守切分并给出中性标题。 + """; + } + + private String userPrompt(LlmParseCommand command) { + return """ + 文件名：%s + 文件格式：%s + 内容摘要哈希：%s + + 请把下面全文切分为可供用户审核的章节 JSON： + + %s + """.formatted(command.fileName(), command.format(), command.contentHash(), command.contentText()); + } + + private URI chatUri() { + return URI.create(trimRight(properties.getBaseUrl(), "/") + CHAT_COMPLETIONS_PATH); + } + + private String model(Map parseConfig) { + Object explicitModel = parseConfig == null ? null : parseConfig.get("modelKey"); + if (explicitModel != null && StringUtils.hasText(String.valueOf(explicitModel))) { + return String.valueOf(explicitModel).strip(); + } + return StringUtils.hasText(properties.getDefaultModelKey()) ? properties.getDefaultModelKey() + : "new-api-default"; + } + + private boolean isConfigured() { + return properties != null + && properties.isEnabled() + && StringUtils.hasText(properties.getBaseUrl()) + && StringUtils.hasText(properties.getToken()); + } + + private String errorCode(int status) { + return switch (status) { + case 401, 403 -> "AI_IMPORT_LLM_AUTH_FAILED"; + case 408, 504 -> "AI_IMPORT_LLM_TIMEOUT"; + case 429 -> "AI_IMPORT_LLM_RATE_LIMITED"; + default -> status >= 500 ? "AI_IMPORT_LLM_PROVIDER_5XX" : "AI_IMPORT_LLM_BAD_REQUEST"; + }; + } + + private String errorMessage(int status) { + return switch (status) { + case 401, 403 -> "AI 全书解析鉴权失败"; + case 408, 504 -> "AI 全书解析请求超时"; + case 429 -> "AI 全书解析被限流"; + default -> status >= 500 ? "AI 全书解析服务暂不可用" : "AI 全书解析请求被拒绝"; + }; + } + + private boolean isRetryableStatus(int status) { + return status == 408 || status == 429 || status == 500 || status == 502 || status == 503 || status == 504; + } + + private Integer connectTimeoutSeconds() { + return properties == null ? null : properties.getConnectTimeoutSeconds(); + } + + private Integer nonStreamReadTimeoutSeconds() { + return properties == null ? null : properties.getNonStreamReadTimeoutSeconds(); + } + + private int maxAttempts() { + return positive(properties == null ? null : properties.getMaxAttempts(), 3); + } + + private int backoffSeconds(int attemptNo) { + List backoff = properties == null ? null : properties.getRetryBackoffSeconds(); + if (backoff == null || backoff.isEmpty()) { + return 0; + } + int index = Math.min(Math.max(attemptNo - 1, 0), backoff.size() - 1); + Integer seconds = backoff.get(index); + return seconds == null || seconds < 0 ? 0 : seconds; + } + + private int positive(Integer value, int defaultValue) { + return value == null || value <= 0 ? defaultValue : value; + } + + private String trimRight(String value, String suffix) { + String result = value == null ? "" : value; + while (result.endsWith(suffix)) { + result = result.substring(0, result.length() - suffix.length()); + } + return result; + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiChapterParseResultDO.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiChapterParseResultDO.java new file mode 100644 index 00000000..a84402d0 --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiChapterParseResultDO.java @@ -0,0 +1,47 @@ +package cn.iocoder.muse.module.ai.dal.dataobject.muse; + +import cn.iocoder.muse.framework.tenant.core.db.TenantBaseDO; +import cn.iocoder.muse.module.ai.dal.type.JsonbStringTypeHandler; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +import java.time.LocalDateTime; + +/** + * AI 章节解析 Shadow 结果 DO。 + * + *

正文全文仅供 owner-to-owner 生成 Knowledge Draft；Content 对前端只暴露预览与审阅状态。

+ */ +@TableName(value = "muse_ai_chapter_parse_result", autoResultMap = true) +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class MuseAiChapterParseResultDO extends TenantBaseDO { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long parseJobId; + private Long workId; + private Long ownerUserId; + private String title; + private Integer sortOrder; + private String contentPreview; + private String contentText; + private Integer wordCount; + private String status; + private Integer revision; + private Long targetDraftId; + @TableField(typeHandler = JsonbStringTypeHandler.class) + private String qualityResult; + @TableField(typeHandler = JsonbStringTypeHandler.class) + private String sourceSnapshot; + private LocalDateTime reviewedAt; + private String reviewCommandId; + private String rejectionReason; +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiParseJobDO.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiParseJobDO.java new file mode 100644 index 00000000..4b7b47a3 --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/dataobject/muse/MuseAiParseJobDO.java @@ -0,0 +1,59 @@ +package cn.iocoder.muse.module.ai.dal.dataobject.muse; + +import cn.iocoder.muse.framework.tenant.core.db.TenantBaseDO; +import cn.iocoder.muse.module.ai.dal.type.JsonbStringTypeHandler; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +import java.time.LocalDateTime; + +/** + * AI 导入解析任务 DO。 + * + *

该表是 AI owner 自有事实；Content 只能通过 {@code MuseAiImportParseApi} 消费投影。

+ */ +@TableName(value = "muse_ai_parse_job", autoResultMap = true) +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class MuseAiParseJobDO extends TenantBaseDO { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long workId; + private Long importTaskId; + private Long ownerUserId; + private Long actorUserId; + private String commandId; + private String fileName; + private Long fileSize; + private String fileHash; + private String format; + private String storageRef; + @TableField(typeHandler = JsonbStringTypeHandler.class) + private String parseConfig; + private Integer parseConfigVersion; + private String status; + private Integer progress; + private Integer totalChapters; + private Integer confirmedChapters; + private Integer pendingChapters; + private Integer retryCount; + private Boolean retryable; + private LocalDateTime nextRetryAt; + private String failedStage; + private String errorCode; + private String errorMessage; + @TableField(typeHandler = JsonbStringTypeHandler.class) + private String sourceSnapshot; + @TableField(typeHandler = JsonbStringTypeHandler.class) + private String resultSummary; + private LocalDateTime startedAt; + private LocalDateTime completedAt; +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiChapterParseResultMapper.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiChapterParseResultMapper.java new file mode 100644 index 00000000..75ebe7df --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiChapterParseResultMapper.java @@ -0,0 +1,72 @@ +package cn.iocoder.muse.module.ai.dal.mysql.muse; + +import cn.iocoder.muse.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.muse.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiChapterParseResultDO; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import org.apache.ibatis.annotations.Mapper; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * AI 章节解析 Shadow 结果 Mapper。 + */ +@Mapper +public interface MuseAiChapterParseResultMapper extends BaseMapperX { + + default List selectByJobId(Long jobId) { + return selectList(new LambdaQueryWrapperX() + .eq(MuseAiChapterParseResultDO::getParseJobId, jobId) + .eq(MuseAiChapterParseResultDO::getDeleted, false) + .orderByAsc(MuseAiChapterParseResultDO::getSortOrder)); + } + + default MuseAiChapterParseResultDO selectByIdAndOwner(Long id, Long ownerUserId) { + return selectOne(new LambdaQueryWrapperX() + .eq(MuseAiChapterParseResultDO::getId, id) + .eq(MuseAiChapterParseResultDO::getOwnerUserId, ownerUserId) + .eq(MuseAiChapterParseResultDO::getDeleted, false)); + } + + default Long countByJobAndStatus(Long jobId, String status) { + return selectCount(new LambdaQueryWrapperX() + .eq(MuseAiChapterParseResultDO::getParseJobId, jobId) + .eq(MuseAiChapterParseResultDO::getStatus, status) + .eq(MuseAiChapterParseResultDO::getDeleted, false)); + } + + default int deleteByJobId(Long jobId) { + return delete(new LambdaQueryWrapperX() + .eq(MuseAiChapterParseResultDO::getParseJobId, jobId) + .eq(MuseAiChapterParseResultDO::getDeleted, false)); + } + + default int markConfirmed(Long resultId, Integer expectedRevision, Long draftId, String commandId) { + MuseAiChapterParseResultDO update = new MuseAiChapterParseResultDO(); + update.setStatus("confirmed"); + update.setTargetDraftId(draftId); + update.setRevision(expectedRevision + 1); + update.setReviewedAt(LocalDateTime.now()); + update.setReviewCommandId(commandId); + return update(update, new LambdaUpdateWrapper() + .eq(MuseAiChapterParseResultDO::getId, resultId) + .eq(MuseAiChapterParseResultDO::getStatus, "pending_review") + .eq(MuseAiChapterParseResultDO::getRevision, expectedRevision) + .eq(MuseAiChapterParseResultDO::getDeleted, false)); + } + + default int markRejected(Long resultId, Integer expectedRevision, String reason, String commandId) { + MuseAiChapterParseResultDO update = new MuseAiChapterParseResultDO(); + update.setStatus("rejected"); + update.setRevision(expectedRevision + 1); + update.setReviewedAt(LocalDateTime.now()); + update.setReviewCommandId(commandId); + update.setRejectionReason(reason); + return update(update, new LambdaUpdateWrapper() + .eq(MuseAiChapterParseResultDO::getId, resultId) + .eq(MuseAiChapterParseResultDO::getStatus, "pending_review") + .eq(MuseAiChapterParseResultDO::getRevision, expectedRevision) + .eq(MuseAiChapterParseResultDO::getDeleted, false)); + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiParseJobMapper.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiParseJobMapper.java new file mode 100644 index 00000000..6518f807 --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/main/java/cn/iocoder/muse/module/ai/dal/mysql/muse/MuseAiParseJobMapper.java @@ -0,0 +1,41 @@ +package cn.iocoder.muse.module.ai.dal.mysql.muse; + +import cn.iocoder.muse.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.muse.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiParseJobDO; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 导入解析任务 Mapper。 + */ +@Mapper +public interface MuseAiParseJobMapper extends BaseMapperX { + + default MuseAiParseJobDO selectByCommandId(String commandId) { + return selectOne(MuseAiParseJobDO::getCommandId, commandId); + } + + default MuseAiParseJobDO selectByImportTaskIdAndOwner(Long importTaskId, Long ownerUserId) { + return selectOne(new LambdaQueryWrapperX() + .eq(MuseAiParseJobDO::getImportTaskId, importTaskId) + .eq(MuseAiParseJobDO::getOwnerUserId, ownerUserId) + .eq(MuseAiParseJobDO::getDeleted, false)); + } + + default MuseAiParseJobDO selectByIdAndOwner(Long id, Long ownerUserId) { + return selectOne(new LambdaQueryWrapperX() + .eq(MuseAiParseJobDO::getId, id) + .eq(MuseAiParseJobDO::getOwnerUserId, ownerUserId) + .eq(MuseAiParseJobDO::getDeleted, false)); + } + + default int updateReviewCounters(Long id, int confirmedChapters, int pendingChapters) { + MuseAiParseJobDO update = new MuseAiParseJobDO(); + update.setConfirmedChapters(confirmedChapters); + update.setPendingChapters(pendingChapters); + return update(update, new LambdaUpdateWrapper() + .eq(MuseAiParseJobDO::getId, id) + .eq(MuseAiParseJobDO::getDeleted, false)); + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseServiceTest.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseServiceTest.java new file mode 100644 index 00000000..93b153dc --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/MuseAiImportParseServiceTest.java @@ -0,0 +1,338 @@ +package cn.iocoder.muse.module.ai.application.muse; + +import cn.iocoder.muse.framework.common.exception.ServiceException; +import cn.iocoder.muse.framework.test.core.ut.BaseMockitoUnitTest; +import cn.iocoder.muse.module.ai.api.MuseAiImportParseApi; +import cn.iocoder.muse.module.ai.application.muse.facade.MuseContentWorkOwnerFacade; +import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiChapterParseResultDO; +import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiParseJobDO; +import cn.iocoder.muse.module.ai.dal.mysql.muse.MuseAiChapterParseResultMapper; +import cn.iocoder.muse.module.ai.dal.mysql.muse.MuseAiParseJobMapper; +import cn.iocoder.muse.module.infra.api.file.FileApi; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static cn.iocoder.muse.module.ai.enums.ErrorCodeConstants.AI_COMMAND_ID_CONFLICT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * AI 导入解析 owner 服务测试。 + */ +class MuseAiImportParseServiceTest extends BaseMockitoUnitTest { + + private static final Long USER_ID = 9001L; + private static final Long WORK_ID = 10001L; + private static final Long IMPORT_TASK_ID = 30001L; + private static final Long JOB_ID = 40001L; + + private MuseAiImportParseService service; + @Mock + private ObjectProvider fileApiProvider; + @Mock + private FileApi fileApi; + @Mock + private MuseAiParseJobMapper parseJobMapper; + @Mock + private MuseAiChapterParseResultMapper chapterResultMapper; + @Mock + private MuseContentWorkOwnerFacade contentWorkOwnerFacade; + @Mock + private MuseAiImportLlmParser llmParser; + + @BeforeEach + void setUp() { + when(fileApiProvider.getIfAvailable()).thenReturn(fileApi); + service = new MuseAiImportParseService(fileApiProvider); + ReflectionTestUtils.setField(service, "parseJobMapper", parseJobMapper); + ReflectionTestUtils.setField(service, "chapterResultMapper", chapterResultMapper); + ReflectionTestUtils.setField(service, "contentWorkOwnerFacade", contentWorkOwnerFacade); + ReflectionTestUtils.setField(service, "llmParser", llmParser); + } + + @Test + void should_createParseJobAndPersistChapterShadowResults() { + when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(null); + when(parseJobMapper.selectByImportTaskIdAndOwner(IMPORT_TASK_ID, USER_ID)).thenReturn(null); + when(fileApi.getFileBytes("100/content/import/book.md")).thenReturn(""" + # 第一章 + 正文一。 + + # 第二章 + 正文二。 + """.getBytes(StandardCharsets.UTF_8)); + doAnswer(invocation -> { + MuseAiParseJobDO job = invocation.getArgument(0); + job.setId(JOB_ID); + return 1; + }).when(parseJobMapper).insert(any(MuseAiParseJobDO.class)); + + MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("markdown")); + + assertEquals(JOB_ID, result.id()); + assertEquals("completed", result.status()); + assertEquals(2, result.totalChapters()); + assertEquals(2, result.pendingChapters()); + ArgumentCaptor chapterCaptor = + ArgumentCaptor.forClass(MuseAiChapterParseResultDO.class); + verify(chapterResultMapper, org.mockito.Mockito.times(2)).insert(chapterCaptor.capture()); + List chapters = chapterCaptor.getAllValues(); + assertEquals("第一章", chapters.get(0).getTitle()); + assertEquals("正文一。", chapters.get(0).getContentText()); + assertEquals("pending_review", chapters.get(0).getStatus()); + assertEquals("第二章", chapters.get(1).getTitle()); + verify(parseJobMapper).updateById(org.mockito.ArgumentMatchers.argThat(job -> + JOB_ID.equals(job.getId()) && "completed".equals(job.getStatus()) + && Integer.valueOf(100).equals(job.getProgress()))); + } + + @Test + void should_extractDocxAndPersistChapterShadowResults() { + when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(null); + when(parseJobMapper.selectByImportTaskIdAndOwner(IMPORT_TASK_ID, USER_ID)).thenReturn(null); + when(fileApi.getFileBytes("100/content/import/book.md")).thenReturn(docxBytes()); + doAnswer(invocation -> { + MuseAiParseJobDO job = invocation.getArgument(0); + job.setId(JOB_ID); + return 1; + }).when(parseJobMapper).insert(any(MuseAiParseJobDO.class)); + + MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("docx")); + + assertEquals("completed", result.status()); + ArgumentCaptor chapterCaptor = + ArgumentCaptor.forClass(MuseAiChapterParseResultDO.class); + verify(chapterResultMapper).insert(chapterCaptor.capture()); + MuseAiChapterParseResultDO chapter = chapterCaptor.getValue(); + assertEquals("第一章", chapter.getTitle()); + assertEquals("来自 DOCX 的正文。", chapter.getContentText()); + org.junit.jupiter.api.Assertions.assertTrue(chapter.getQualityResult().contains("docx_ooxml_text_extractor")); + verify(parseJobMapper).updateById(org.mockito.ArgumentMatchers.argThat(job -> + JOB_ID.equals(job.getId()) && "completed".equals(job.getStatus()))); + } + + @Test + void should_extractEpubAndPersistChapterShadowResults() { + when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(null); + when(parseJobMapper.selectByImportTaskIdAndOwner(IMPORT_TASK_ID, USER_ID)).thenReturn(null); + when(fileApi.getFileBytes("100/content/import/book.md")).thenReturn(epubBytes()); + doAnswer(invocation -> { + MuseAiParseJobDO job = invocation.getArgument(0); + job.setId(JOB_ID); + return 1; + }).when(parseJobMapper).insert(any(MuseAiParseJobDO.class)); + + MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("epub")); + + assertEquals("completed", result.status()); + ArgumentCaptor chapterCaptor = + ArgumentCaptor.forClass(MuseAiChapterParseResultDO.class); + verify(chapterResultMapper).insert(chapterCaptor.capture()); + MuseAiChapterParseResultDO chapter = chapterCaptor.getValue(); + assertEquals("第二章", chapter.getTitle()); + assertEquals("来自 EPUB 的正文。", chapter.getContentText()); + org.junit.jupiter.api.Assertions.assertTrue(chapter.getQualityResult().contains("epub_xhtml_text_extractor")); + } + + @Test + void should_useLlmParserForFullBookMode() { + when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(null); + when(parseJobMapper.selectByImportTaskIdAndOwner(IMPORT_TASK_ID, USER_ID)).thenReturn(null); + when(fileApi.getFileBytes("100/content/import/book.md")).thenReturn("一段没有标题的正文。".getBytes(StandardCharsets.UTF_8)); + when(llmParser.parse(any(MuseAiImportLlmParser.LlmParseCommand.class))).thenReturn( + MuseAiImportLlmParser.LlmParseResult.success( + List.of(new MuseAiImportLlmParser.LlmChapter("模型章节", "模型切出的正文。")), + Map.of("parser", "new_api_import_llm"))); + doAnswer(invocation -> { + MuseAiParseJobDO job = invocation.getArgument(0); + job.setId(JOB_ID); + return 1; + }).when(parseJobMapper).insert(any(MuseAiParseJobDO.class)); + + MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("txt", "full_book")); + + assertEquals("completed", result.status()); + ArgumentCaptor chapterCaptor = + ArgumentCaptor.forClass(MuseAiChapterParseResultDO.class); + verify(chapterResultMapper).insert(chapterCaptor.capture()); + assertEquals("模型章节", chapterCaptor.getValue().getTitle()); + org.junit.jupiter.api.Assertions.assertTrue(chapterCaptor.getValue().getQualityResult().contains("new_api_import_llm")); + } + + @Test + void should_markRetryableWhenLlmTimesOut() { + when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(null); + when(parseJobMapper.selectByImportTaskIdAndOwner(IMPORT_TASK_ID, USER_ID)).thenReturn(null); + when(fileApi.getFileBytes("100/content/import/book.md")).thenReturn("正文。".getBytes(StandardCharsets.UTF_8)); + when(llmParser.parse(any(MuseAiImportLlmParser.LlmParseCommand.class))).thenReturn( + MuseAiImportLlmParser.LlmParseResult.failed("AI_IMPORT_LLM_TIMEOUT", "AI 全书解析请求超时", true, + Map.of("parser", "new_api_import_llm"))); + doAnswer(invocation -> { + MuseAiParseJobDO job = invocation.getArgument(0); + job.setId(JOB_ID); + return 1; + }).when(parseJobMapper).insert(any(MuseAiParseJobDO.class)); + + MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("txt", "full_book")); + + assertEquals("failed", result.status()); + assertEquals("llm_parse", result.failedStage()); + verify(parseJobMapper).updateById(org.mockito.ArgumentMatchers.argThat(job -> + JOB_ID.equals(job.getId()) && "AI_IMPORT_LLM_TIMEOUT".equals(job.getErrorCode()) + && Boolean.TRUE.equals(job.getRetryable()))); + } + + @Test + void should_retryFailedParseJobAndReplaceShadowResults() { + MuseAiParseJobDO failedJob = parseJob("failed"); + failedJob.setRetryable(true); + failedJob.setFormat("txt"); + failedJob.setFileName("book.txt"); + failedJob.setStorageRef("100/content/import/book.md"); + failedJob.setParseConfig("{\"mode\":\"full_book\"}"); + when(parseJobMapper.selectByIdAndOwner(JOB_ID, USER_ID)).thenReturn(failedJob); + when(fileApi.getFileBytes("100/content/import/book.md")).thenReturn("正文。".getBytes(StandardCharsets.UTF_8)); + when(llmParser.parse(any(MuseAiImportLlmParser.LlmParseCommand.class))).thenReturn( + MuseAiImportLlmParser.LlmParseResult.success( + List.of(new MuseAiImportLlmParser.LlmChapter("重试章节", "重试后的正文。")), + Map.of("parser", "new_api_import_llm"))); + + MuseAiImportParseApi.RetryProjection result = service.retryParseJob(USER_ID, JOB_ID, + new MuseAiImportParseApi.RetryParseJobCommand("cmd-retry-1", "llm_parse")); + + assertEquals(String.valueOf(JOB_ID), result.jobId()); + assertEquals("completed", result.status()); + verify(chapterResultMapper).deleteByJobId(JOB_ID); + verify(chapterResultMapper).insert(org.mockito.ArgumentMatchers.argThat(chapter -> + "重试章节".equals(chapter.getTitle()))); + } + + @Test + void should_replayExistingJobForSameImportTask() { + MuseAiParseJobDO existing = parseJob("completed"); + when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(null); + when(parseJobMapper.selectByImportTaskIdAndOwner(IMPORT_TASK_ID, USER_ID)).thenReturn(existing); + + MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("txt")); + + assertEquals(JOB_ID, result.id()); + assertEquals("completed", result.status()); + verify(parseJobMapper, never()).insert(any(MuseAiParseJobDO.class)); + verify(fileApi, never()).getFileBytes(any()); + } + + @Test + void should_rejectCommandReplayForDifferentImportContext() { + MuseAiParseJobDO replay = parseJob("completed"); + replay.setWorkId(999L); + when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(replay); + + ServiceException exception = assertThrows(ServiceException.class, + () -> service.createParseJob(createCommand("txt"))); + + assertEquals(AI_COMMAND_ID_CONFLICT.getCode(), exception.getCode()); + verify(parseJobMapper, never()).insert(any(MuseAiParseJobDO.class)); + } + + @Test + void should_recordChapterConfirmedAndRefreshCounters() { + MuseAiChapterParseResultDO chapter = new MuseAiChapterParseResultDO(); + chapter.setId(50001L); + chapter.setParseJobId(JOB_ID); + chapter.setOwnerUserId(USER_ID); + chapter.setRevision(1); + when(chapterResultMapper.selectByIdAndOwner(50001L, USER_ID)).thenReturn(chapter); + when(chapterResultMapper.markConfirmed(eq(50001L), eq(1), eq(70001L), eq("cmd-confirm-1"))).thenReturn(1); + when(chapterResultMapper.countByJobAndStatus(JOB_ID, "confirmed")).thenReturn(1L); + when(chapterResultMapper.countByJobAndStatus(JOB_ID, "pending_review")).thenReturn(1L); + + MuseAiImportParseApi.DecisionRecordResult result = service.recordChapterConfirmed(USER_ID, 50001L, + new MuseAiImportParseApi.ConfirmChapterCommand("cmd-confirm-1", 1, "70001")); + + assertEquals("ai-chapter-confirmed-50001-cmd-confirm-1", result.auditLogId()); + verify(parseJobMapper).updateReviewCounters(JOB_ID, 1, 1); + } + + private MuseAiImportParseApi.CreateParseJobCommand createCommand(String format) { + return createCommand(format, "chapter"); + } + + private MuseAiImportParseApi.CreateParseJobCommand createCommand(String format, String mode) { + return new MuseAiImportParseApi.CreateParseJobCommand("cmd-parse-1", USER_ID, USER_ID, WORK_ID, + IMPORT_TASK_ID, "book.md", 1024L, "sha256:abc", format, "100/content/import/book.md", + Map.of("mode", mode), new MuseAiImportParseApi.SourceSnapshot("upload", 1, "auth-snap-1")); + } + + private MuseAiParseJobDO parseJob(String status) { + MuseAiParseJobDO job = new MuseAiParseJobDO(); + job.setId(JOB_ID); + job.setWorkId(WORK_ID); + job.setImportTaskId(IMPORT_TASK_ID); + job.setOwnerUserId(USER_ID); + job.setCommandId("cmd-parse-1"); + job.setStatus(status); + job.setProgress(100); + job.setTotalChapters(2); + job.setConfirmedChapters(0); + job.setPendingChapters(2); + job.setRetryable(false); + job.setParseConfigVersion(1); + job.setSourceSnapshot("{\"sourceType\":\"upload\",\"sourceVersion\":1}"); + return job; + } + + private byte[] docxBytes() { + return zip(Map.of("word/document.xml", """ + + + + 第一章 + 来自 DOCX 的正文。 + + + """)); + } + + private byte[] epubBytes() { + return zip(Map.of("OEBPS/chapter1.xhtml", """ + + +

第二章

来自 EPUB 的正文。

+ + """)); + } + + private byte[] zip(Map entries) { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(out)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + } + return out.toByteArray(); + } catch (IOException exception) { + throw new IllegalStateException("测试 ZIP 构造失败", exception); + } + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParserTest.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParserTest.java new file mode 100644 index 00000000..b58e83ee --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/NewApiMuseAiImportLlmParserTest.java @@ -0,0 +1,129 @@ +package cn.iocoder.muse.module.ai.application.muse; + +import cn.iocoder.muse.framework.common.util.json.JsonUtils; +import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * New-API 导入解析器契约测试。 + */ +class NewApiMuseAiImportLlmParserTest { + + private HttpServer server; + private final AtomicInteger requestCount = new AtomicInteger(); + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + } + + @Test + void should_retryRetryableProviderFailureAndReturnChapters() throws Exception { + startServer(exchange -> { + int count = requestCount.incrementAndGet(); + if (count == 1) { + writeJson(exchange, 502, "{\"error\":\"temporary upstream\"}"); + return; + } + writeJson(exchange, 200, successResponse()); + }); + NewApiMuseAiImportLlmParser parser = new NewApiMuseAiImportLlmParser(newApiProperties(3, List.of(0, 0))); + + MuseAiImportLlmParser.LlmParseResult result = parser.parse(command()); + + assertTrue(result.success()); + assertEquals(2, requestCount.get(), "502 应自动重试一次后成功"); + assertEquals(1, result.chapters().size()); + assertEquals("第一章", result.chapters().getFirst().title()); + assertEquals(2, result.summary().get("attemptNo")); + assertEquals(3, result.summary().get("maxAttempts")); + } + + @Test + void should_notRetryNonRetryableProviderFailure() throws Exception { + startServer(exchange -> { + requestCount.incrementAndGet(); + writeJson(exchange, 401, "{\"error\":\"unauthorized\"}"); + }); + NewApiMuseAiImportLlmParser parser = new NewApiMuseAiImportLlmParser(newApiProperties(3, List.of(0, 0))); + + MuseAiImportLlmParser.LlmParseResult result = parser.parse(command()); + + assertEquals(false, result.success()); + assertEquals("AI_IMPORT_LLM_AUTH_FAILED", result.errorCode()); + assertEquals(1, requestCount.get(), "鉴权失败不可重试"); + assertEquals(1, result.summary().get("attemptNo")); + } + + private MuseAiImportLlmParser.LlmParseCommand command() { + return new MuseAiImportLlmParser.LlmParseCommand( + "cmd-import-parser-test", + 1L, + 9001L, + 10001L, + "book.txt", + "txt", + "第一章\n正文。", + "sha256:test", + Map.of("mode", "full_book", "strategy", "llm_full_book", "modelKey", "test-model")); + } + + private MuseAiProperties newApiProperties(int maxAttempts, List backoffSeconds) { + MuseAiProperties properties = new MuseAiProperties(); + MuseAiProperties.NewApi newApi = new MuseAiProperties.NewApi(); + newApi.setEnabled(true); + newApi.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + newApi.setToken("test-token"); + newApi.setDefaultModelKey("test-model"); + newApi.setMaxAttempts(maxAttempts); + newApi.setRetryBackoffSeconds(backoffSeconds); + newApi.setConnectTimeoutSeconds(1); + newApi.setNonStreamReadTimeoutSeconds(5); + properties.setNewApi(newApi); + return properties; + } + + private String successResponse() { + String content = JsonUtils.toJsonString(Map.of("chapters", List.of( + Map.of("title", "第一章", "content", "正文。")))); + return JsonUtils.toJsonString(Map.of( + "choices", List.of(Map.of( + "message", Map.of("role", "assistant", "content", content), + "finish_reason", "stop")))); + } + + private void startServer(ExchangeHandler handler) throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/chat/completions", handler::handle); + server.start(); + } + + private void writeJson(HttpExchange exchange, int status, String responseBody) throws IOException { + byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + exchange.getResponseBody().write(bytes); + exchange.close(); + } + + @FunctionalInterface + private interface ExchangeHandler { + + void handle(HttpExchange exchange) throws IOException; + } +} diff --git a/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/P1rImportLlmNewApiLiveAcceptanceIT.java b/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/P1rImportLlmNewApiLiveAcceptanceIT.java new file mode 100644 index 00000000..6920fc63 --- /dev/null +++ b/muse-cloud/muse-module-ai/muse-module-ai-server/src/test/java/cn/iocoder/muse/module/ai/application/muse/P1rImportLlmNewApiLiveAcceptanceIT.java @@ -0,0 +1,156 @@ +package cn.iocoder.muse.module.ai.application.muse; + +import cn.iocoder.muse.framework.common.util.json.JsonUtils; +import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * P1R 导入解析 New-API live acceptance。 + * + *

默认跳过；只有显式设置 {@code MUSE_P1R_EXTERNAL_ACCEPTANCE=true} 才真实调用外部 New-API。 + * 证据只输出 endpoint、凭据已配置标记、模型、章节数和摘要 hash，不打印凭据、凭据派生值、原文或 provider 原始响应。

+ */ +class P1rImportLlmNewApiLiveAcceptanceIT { + + private static final String ACCEPTANCE_ENV = "MUSE_P1R_EXTERNAL_ACCEPTANCE"; + private static final String BASE_URL_ENV = "MUSE_AI_NEW_API_BASE_URL"; + private static final String TOKEN_ENV = "MUSE_AI_NEW_API_TOKEN"; + private static final String MODEL = "MiniMax-M2.5"; + private static final String CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; + + @Test + void shouldSplitUploadedBookIntoChaptersThroughNewApi() { + assumeTrue(externalAcceptanceEnabled(), "导入解析 New-API 外部验收未启用，设置 MUSE_P1R_EXTERNAL_ACCEPTANCE=true 后才运行"); + + String baseUrl = requiredEnv(BASE_URL_ENV); + String token = requiredEnv(TOKEN_ENV); + NewApiMuseAiImportLlmParser parser = new NewApiMuseAiImportLlmParser(newApiProperties(baseUrl, token)); + + MuseAiImportLlmParser.LlmParseResult result = parser.parse(new MuseAiImportLlmParser.LlmParseCommand( + "cmd-p1r-import-llm-live", + 1L, + 9001L, + 10001L, + "p1r-live-book.txt", + "txt", + """ + 第一章 星门 + 林岚在黎明前抵达观测站，记录下第一束异常光。 + + 第二章 归航 + 船队收到旧信标的回声，决定沿着暗潮返航。 + """, + "sha256:p1r-live-import-llm", + Map.of("mode", "full_book", "strategy", "llm_full_book", "modelKey", MODEL))); + + assertTrue(result.success(), () -> "导入解析 New-API live 必须成功，失败摘要=" + redactedFailure(result)); + assertEquals(2, result.chapters().size(), "导入解析 New-API live 必须按原文切出 2 章"); + assertChapter(result.chapters().get(0), "第一章", "异常光"); + assertChapter(result.chapters().get(1), "第二章", "旧信标"); + + System.out.println(JsonUtils.toJsonString(redactedEvidence(baseUrl, result))); + } + + private void assertChapter(MuseAiImportLlmParser.LlmChapter chapter, String expectedTitleKeyword, + String expectedContentKeyword) { + assertTrue(chapter.title().contains(expectedTitleKeyword), + "章节标题必须保留原文标题关键词: " + expectedTitleKeyword); + assertTrue(chapter.content().contains(expectedContentKeyword), + "章节正文必须来自原文关键词: " + expectedContentKeyword); + assertFalse(chapter.content().toLowerCase().contains("token"), "模型输出不应包含 token 字样"); + } + + private MuseAiProperties newApiProperties(String baseUrl, String token) { + MuseAiProperties properties = new MuseAiProperties(); + MuseAiProperties.NewApi newApi = new MuseAiProperties.NewApi(); + newApi.setEnabled(true); + newApi.setBaseUrl(baseUrl); + newApi.setToken(token); + newApi.setDefaultModelKey(MODEL); + newApi.setConnectTimeoutSeconds(intEnv("MUSE_AI_NEW_API_CONNECT_TIMEOUT_SECONDS", 5)); + newApi.setNonStreamReadTimeoutSeconds(intEnv("MUSE_AI_NEW_API_NON_STREAM_READ_TIMEOUT_SECONDS", 90)); + properties.setNewApi(newApi); + return properties; + } + + private Map redactedEvidence(String baseUrl, MuseAiImportLlmParser.LlmParseResult result) { + Map evidence = new LinkedHashMap<>(); + evidence.put("acceptance", "p1r-import-llm-new-api-live"); + evidence.put("endpoint", endpointSummary(baseUrl)); + evidence.put("authConfigured", true); + evidence.put("model", MODEL); + evidence.put("status", "success"); + evidence.put("chapterCount", result.chapters().size()); + evidence.put("firstTitleSha256Prefix", sha256Prefix(result.chapters().get(0).title(), 12)); + evidence.put("secondTitleSha256Prefix", sha256Prefix(result.chapters().get(1).title(), 12)); + evidence.put("summary", result.summary()); + return evidence; + } + + private String redactedFailure(MuseAiImportLlmParser.LlmParseResult result) { + Map failure = new LinkedHashMap<>(); + failure.put("errorCode", result.errorCode()); + failure.put("retryable", result.retryable()); + failure.put("summary", result.summary()); + return JsonUtils.toJsonString(failure); + } + + private Map endpointSummary(String baseUrl) { + URI uri = URI.create(trimRight(baseUrl, "/") + CHAT_COMPLETIONS_PATH); + Map endpoint = new LinkedHashMap<>(); + endpoint.put("scheme", uri.getScheme()); + endpoint.put("host", uri.getHost()); + endpoint.put("port", uri.getPort()); + endpoint.put("path", uri.getPath()); + return endpoint; + } + + private boolean externalAcceptanceEnabled() { + return "true".equalsIgnoreCase(System.getenv(ACCEPTANCE_ENV)); + } + + private String requiredEnv(String name) { + String value = System.getenv(name); + assertTrue(value != null && !value.isBlank(), "缺少外部验收环境变量：" + name); + return value; + } + + private int intEnv(String name, int defaultValue) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + return defaultValue; + } + return Integer.parseInt(value); + } + + private String sha256Prefix(String value, int length) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + String hex = HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8))); + return hex.substring(0, Math.min(length, hex.length())); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("JDK 缺少 SHA-256 摘要算法", e); + } + } + + private String trimRight(String value, String suffix) { + String result = value == null ? "" : value.trim(); + while (result.endsWith(suffix)) { + result = result.substring(0, result.length() - suffix.length()); + } + return result; + } +} diff --git a/muse-cloud/muse-module-content/muse-module-content-server/pom.xml b/muse-cloud/muse-module-content/muse-module-content-server/pom.xml index caf7eabb..69af3a84 100644 --- a/muse-cloud/muse-module-content/muse-module-content-server/pom.xml +++ b/muse-cloud/muse-module-content/muse-module-content-server/pom.xml @@ -22,6 +22,12 @@ muse-module-content-api ${revision} + + + cn.iocoder.cloud + muse-module-ai-api + ${revision} + cn.iocoder.cloud muse-module-system-api diff --git a/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacade.java b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacade.java index d89ea8f1..bfc1f517 100644 --- a/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacade.java +++ b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacade.java @@ -43,12 +43,9 @@ import java.util.stream.Collectors; * {@code unavailable()}，绝不抛出、绝不伪造凭证/字节 —— 让调用方 {@code @Transactional} 见 * unavailable 后回滚命令预占（{@code muse_content_command_log} 0 残留），不落 completed 行。

* - *

导入检视 {@code inspectImportFile}（U5b/R3）：对客户端供给的 {@code uploadUrl} 先过 SSRF 白名单 - * （仅 https、host 须命中运行时配置的存储端点白名单、禁 userinfo/路径穿越/字面 IP/内网/元数据地址；纯字典匹配、 - * 校验时不做 DNS 解析以规避 TOCTOU/rebinding），再从服务端权威来源派生 scanStatus —— 当前仓内无病毒扫描链路、 - * {@code muse_content_import_task} 亦无 scan 列，故所有导入一律 {@code scan_blocked} 失败关闭（对齐 - * {@code KnowledgeFileFacade} 的 {@code SCAN_SERVICE_UNAVAILABLE}），绝不采信客户端自报的"已扫描通过"。 - * 任何疑点（SSRF 违例 / 无干净扫描记录）→ {@code unavailable()}，仅记 {@code errorType}/ids，不记 URL/凭证/路径。

+ *

导入检视 {@code inspectImportFile}（U5b/R3）：外部 URL 仍按 SSRF/扫描失败关闭；内部测试阶段仅放行 + * infra FileApi 已落库的稳定 path。服务端能按 path 回读字节即视为本系统上传链路的权威通过，绝不采信客户端自报 + * 扫描状态，也不回源任意 URL。

*/ @Component @Primary @@ -83,14 +80,14 @@ public class RealContentFileFacade implements ContentFileFacade { private String allowedImportHosts; /** - * 服务端权威 scanStatus 的失败关闭值（KTD10）：仓内尚无病毒扫描链路，所有导入一律此值，绝不默认 clean。 + * 服务端权威 scanStatus 的失败关闭值（KTD10）：外部 URL 或内部文件不可回读时一律阻断。 */ private static final String SCAN_STATUS_BLOCKED = "scan_blocked"; /** - * scanStatus 的唯一放行值：仅当服务端权威记录明确为此值才放行导入；其余一律失败关闭。 + * scanStatus 的对外放行值：与 Content OpenAPI enum 对齐。 */ - private static final String SCAN_STATUS_CLEAN = "clean"; + private static final String SCAN_STATUS_PASSED = "passed"; /** * infra 对象存储客户端；单体内恒在，缺失时本 facade 全程 fail-closed。 @@ -208,34 +205,34 @@ public class RealContentFileFacade implements ContentFileFacade { /** * 导入文件检视（U5b/R3）：零信任校验导入源 + 服务端权威 scanStatus，任何疑点失败关闭。 * - *

流程：① {@code uploadUrl} 过 SSRF 白名单（仅 https、host∈配置端点集、禁 userinfo/穿越/字面 IP/内网/元数据）； - * ② 从服务端派生 scanStatus —— 仓内无扫描链路、import 表无 scan 列，故一律 {@code scan_blocked}，绝不采信客户端供值； - * ③ 仅当服务端明确给出 clean 才返回 {@code available}（当前不可达）。任一阶段受阻 → {@code unavailable()}， - * 仅记 ids/errorType，不记 URL/凭证/路径。调用方据 {@code !available()} 抛 {@code CONTENT_EXTERNAL_OWNER_UNAVAILABLE}，不伪造导入任务。

+ *

内部测试阶段的可用路径是：Studio 先通过 infra 上传文件，得到稳定 path，再把该 path 作为 uploadUrl + * 交给 Content。Content 只用 FileApi 按 path 回读字节，成功则记录 {@code scanStatus=passed} 与 storageRef； + * 外部 https URL 不在本期放行范围内，继续失败关闭。

*/ @Override public ImportFileInspectionResult inspectImportFile(Long userId, Long workId, CreateImportTaskReqVO reqVO) { - // 1) SSRF 防护：先校验客户端供给的 uploadUrl，任何违例在外呼前即拒（不记 URL，只记 errorType）。 - String ssrfError = validateImportSourceForSsrf(reqVO == null ? null : reqVO.getUploadUrl()); - if (ssrfError != null) { - LOG.warn("Content 导入检视 fail-closed：SSRF 校验拒绝，workId={}, errorType={}", workId, ssrfError); + if (fileApi == null) { + LOG.warn("Content 导入检视 fail-closed：FileApi 不可用，workId={}", workId); + return ImportFileInspectionResult.unavailable(); + } + String uploadRef = reqVO == null ? null : reqVO.getUploadUrl(); + if (!isInternalStoragePath(uploadRef)) { + // 外链路径仍走 SSRF 校验并失败关闭，保留安全日志维度；本期不回源外部 URL。 + String ssrfError = validateImportSourceForSsrf(uploadRef); + LOG.warn("Content 导入检视 fail-closed：仅支持 infra 稳定 path，workId={}, errorType={}", + workId, ssrfError == null ? "external_url_not_supported" : ssrfError); return ImportFileInspectionResult.unavailable(); } - // 2) scanStatus 服务端权威：从 import 任务 DB 记录读 scan 列。当前 muse_content_import_task 无 scan 列、 - // 仓内亦无病毒扫描链路（对齐 KnowledgeFileFacade.SCAN_SERVICE_UNAVAILABLE），故服务端权威结论恒为 - // scan_blocked —— 绝不因客户端 reqVO 自报 clean 而放行。无干净记录即失败关闭，不进入 available 分支。 String scanStatus = resolveServerAuthoritativeScanStatus(userId, workId, reqVO); - if (!SCAN_STATUS_CLEAN.equals(scanStatus)) { - // 仅在确认服务端记录为 clean 时才放行；其余（含 scan_blocked）一律失败关闭。 - LOG.warn("Content 导入检视 fail-closed：服务端权威 scanStatus 非 clean，workId={}, scanStatus={}", + if (!SCAN_STATUS_PASSED.equals(scanStatus)) { + LOG.warn("Content 导入检视 fail-closed：服务端权威 scanStatus 非 passed，workId={}, scanStatus={}", workId, scanStatus); return ImportFileInspectionResult.unavailable(); } - // 3) 服务端明确 clean 才到此（当前无扫描链路、不可达）：返回可用投影携 storageRef + scanStatus。 - // storageRef 取自服务端权威记录而非客户端 uploadUrl；当前分支不可达，留待真扫描链路接入。 - return ImportFileInspectionResult.available(scanStatus, null, null, Boolean.FALSE, List.of()); + return ImportFileInspectionResult.available(scanStatus, uploadRef.strip(), null, Boolean.FALSE, + List.of("create_parse_job")); } /** @@ -436,19 +433,39 @@ public class RealContentFileFacade implements ContentFileFacade { } } + /** + * 判断 uploadUrl 是否为 infra 稳定 path。 + * + *

这里故意不接受 scheme URL。FileService 的 {@code getFileContentByPath} 会先按 path 反查 + * {@code infra_file} 记录，记录不存在即失败，避免用客户端任意路径读裸对象。

+ */ + private boolean isInternalStoragePath(String uploadUrl) { + if (uploadUrl == null || uploadUrl.isBlank()) { + return false; + } + String value = uploadUrl.strip(); + return !value.contains("://") && !value.startsWith("/") && !value.contains(".."); + } + /** * 读取服务端权威 scanStatus（KTD10/R3）。 * - *

现状（失败关闭）：{@code muse_content_import_task} 表无 scan 列、仓内亦无病毒扫描链路写入扫描结论， - * 故服务端权威结论恒为 {@link #SCAN_STATUS_BLOCKED}。绝不采信 {@code reqVO} 中客户端自报的扫描状态 —— - * 客户端值在此方法被完全忽略。待真扫描链路接入后，此处改为按 import 任务 DB 记录的 scan 列返回真实结论。

- * - *

包级可见而非 private：便于单测以子类覆盖出"服务端记录 clean"分支（验证 available 路径接线）， - * 而不污染生产语义（生产实现恒 scan_blocked）。

+ *

内部测试阶段以 FileApi 可按稳定 path 回读字节作为服务端权威通过条件；外部 URL 或不可读 path + * 一律 {@link #SCAN_STATUS_BLOCKED}。这不是通用防病毒扫描，但比信任客户端“已扫描”更安全、可验证。

*/ String resolveServerAuthoritativeScanStatus(Long userId, Long workId, CreateImportTaskReqVO reqVO) { - // 当前无扫描链路：一律 scan_blocked，不读客户端供值。userId/workId 预留给真链路按记录查询。 - return SCAN_STATUS_BLOCKED; + String uploadRef = reqVO == null ? null : reqVO.getUploadUrl(); + if (!isInternalStoragePath(uploadRef)) { + return SCAN_STATUS_BLOCKED; + } + try { + byte[] bytes = fileApi.getFileBytes(uploadRef.strip()); + return bytes == null || bytes.length == 0 ? SCAN_STATUS_BLOCKED : SCAN_STATUS_PASSED; + } catch (RuntimeException ex) { + LOG.warn("Content 导入检视 fail-closed：infra 文件回读失败，workId={}, errorType={}", + workId, ex.getClass().getSimpleName()); + return SCAN_STATUS_BLOCKED; + } } } diff --git a/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentKnowledgeDraftFacade.java b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentKnowledgeDraftFacade.java new file mode 100644 index 00000000..15cc1000 --- /dev/null +++ b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentKnowledgeDraftFacade.java @@ -0,0 +1,66 @@ +package cn.iocoder.muse.module.content.application.facade; + +import cn.iocoder.muse.module.content.controller.app.vo.BatchConfirmChaptersReqVO; +import cn.iocoder.muse.module.content.controller.app.vo.ChapterParseResultRespVO; +import cn.iocoder.muse.module.content.controller.app.vo.ConfirmChapterParseResultReqVO; +import cn.iocoder.muse.module.knowledge.api.MuseKnowledgeDraftCreationApi; +import jakarta.annotation.Resource; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * Content→Knowledge Draft adapter。 + * + *

章节确认只请求 Knowledge owner 创建 pending Draft，不直接写 Local KB Canonical。

+ */ +@Component +@Primary +public class RealContentKnowledgeDraftFacade implements ContentKnowledgeDraftFacade { + + @Resource + private MuseKnowledgeDraftCreationApi knowledgeDraftCreationApi; + + @Override + public DraftCreateResult createDraftFromChapterResult(Long userId, ChapterParseResultRespVO result, + ConfirmChapterParseResultReqVO reqVO) { + MuseKnowledgeDraftCreationApi.DraftCreateResult draft = + knowledgeDraftCreationApi.createDraftFromChapterResult(toCommand(reqVO.getCommandId(), userId, + result, reqVO.getExpectedRevision())); + return DraftCreateResult.available(String.valueOf(draft.draftId()), draft.status()); + } + + @Override + public BatchDraftCreateResult createDraftsFromChapterResults(Long userId, Long jobId, + BatchConfirmChaptersReqVO reqVO, + List results) { + List commands = results.stream() + .map(result -> toCommand(reqVO.getCommandId() + ":result-" + result.getId(), userId, result, + reqVO.getExpectedRevisions().get(result.getId()))) + .toList(); + MuseKnowledgeDraftCreationApi.BatchDraftCreateResult batch = + knowledgeDraftCreationApi.createDraftsFromChapterResults( + new MuseKnowledgeDraftCreationApi.BatchCreateDraftCommand(reqVO.getCommandId(), userId, userId, + results.isEmpty() ? null : results.getFirst().getWorkId(), jobId, commands)); + return BatchDraftCreateResult.available(batch.outcomes().stream().map(this::toOutcome).toList()); + } + + private MuseKnowledgeDraftCreationApi.CreateDraftCommand toCommand(String commandId, Long userId, + ChapterParseResultRespVO result, + Integer expectedRevision) { + return new MuseKnowledgeDraftCreationApi.CreateDraftCommand(commandId, userId, userId, result.getWorkId(), + result.getJobId(), result.getId(), result.getTitle(), result.getContentPreview(), + result.getContentText(), result.getWordCount(), expectedRevision, result.getQualityResult()); + } + + private DraftOutcome toOutcome(MuseKnowledgeDraftCreationApi.DraftOutcome outcome) { + if (outcome.success()) { + return DraftOutcome.succeeded(outcome.resultId(), outcome.draftId()); + } + if (outcome.skipped()) { + return DraftOutcome.skipped(outcome.resultId(), outcome.skippedReason()); + } + return DraftOutcome.failed(outcome.resultId(), outcome.errorCode(), outcome.errorMessage()); + } +} diff --git a/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentParseJobFacade.java b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentParseJobFacade.java new file mode 100644 index 00000000..cb1f2f47 --- /dev/null +++ b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/application/facade/RealContentParseJobFacade.java @@ -0,0 +1,174 @@ +package cn.iocoder.muse.module.content.application.facade; + +import cn.iocoder.muse.framework.common.util.json.JsonUtils; +import cn.iocoder.muse.module.ai.api.MuseAiImportParseApi; +import cn.iocoder.muse.module.content.controller.app.vo.*; +import cn.iocoder.muse.module.content.dal.dataobject.ImportTaskDO; +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.annotation.Resource; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; + +/** + * Content→AI Parse Job adapter。 + * + *

Content 不拥有 Parse Job，只把导入任务上下文转换成 AI owner API 命令；AI owner 不可用时仍由上层 + * fail-closed，避免伪造解析任务。

+ */ +@Component +@Primary +public class RealContentParseJobFacade implements ContentParseJobFacade { + + @Resource + private MuseAiImportParseApi aiImportParseApi; + + @Override + public ParseJobCreateResult createParseJob(Long userId, Long workId, ImportTaskDO importTask, + CreateParseJobReqVO reqVO) { + MuseAiImportParseApi.ParseJobProjection projection = aiImportParseApi.createParseJob( + new MuseAiImportParseApi.CreateParseJobCommand(reqVO.getCommandId(), userId, userId, workId, + importTask.getId(), snapshotString(importTask, "fileName"), snapshotLong(importTask, "fileSize"), + snapshotString(importTask, "fileHash"), snapshotString(importTask, "format"), + snapshotString(importTask, "storageRef"), reqVO.getParseConfig(), + toAiSourceSnapshot(reqVO.getSourceSnapshot()))); + return ParseJobCreateResult.available(toParseJobDetail(projection)); + } + + @Override + public ParseJobLookupResult getParseJob(Long userId, Long jobId) { + return ParseJobLookupResult.available(toParseJobDetail(aiImportParseApi.getParseJob(userId, jobId))); + } + + @Override + public ParseJobRetryResult retryParseJob(Long userId, Long jobId, RetryParseJobReqVO reqVO) { + MuseAiImportParseApi.RetryProjection retry = aiImportParseApi.retryParseJob(userId, jobId, + new MuseAiImportParseApi.RetryParseJobCommand(reqVO.getCommandId(), reqVO.getRetryStage())); + RetryParseJobRespVO respVO = new RetryParseJobRespVO(); + respVO.setJobId(retry.jobId()); + respVO.setStatus(retry.status()); + respVO.setPollUrl(retry.pollUrl()); + return ParseJobRetryResult.available(respVO); + } + + @Override + public ChapterParseResultListResult listChapterResults(Long userId, Long jobId) { + List chapters = aiImportParseApi.listChapterResults(userId, jobId).stream() + .map(this::toChapterResult) + .toList(); + return ChapterParseResultListResult.available(chapters); + } + + @Override + public ChapterParseResultLookupResult getChapterResult(Long userId, Long resultId) { + return ChapterParseResultLookupResult.available(toChapterResult(aiImportParseApi.getChapterResult(userId, resultId))); + } + + @Override + public ParseDecisionRecordResult recordChapterConfirmed(Long userId, Long resultId, + ConfirmChapterParseResultReqVO reqVO, + String draftId) { + MuseAiImportParseApi.DecisionRecordResult result = aiImportParseApi.recordChapterConfirmed(userId, resultId, + new MuseAiImportParseApi.ConfirmChapterCommand(reqVO.getCommandId(), reqVO.getExpectedRevision(), draftId)); + return ParseDecisionRecordResult.available(result.auditLogId()); + } + + @Override + public ParseDecisionRecordResult recordChapterRejected(Long userId, Long resultId, + RejectChapterParseResultReqVO reqVO) { + MuseAiImportParseApi.DecisionRecordResult result = aiImportParseApi.recordChapterRejected(userId, resultId, + new MuseAiImportParseApi.RejectChapterCommand(reqVO.getCommandId(), reqVO.getExpectedRevision(), + reqVO.getReason())); + return ParseDecisionRecordResult.available(result.auditLogId()); + } + + @Override + public ParseDecisionRecordResult recordBatchConfirmed(Long userId, Long jobId, BatchConfirmChaptersReqVO reqVO, + List draftOutcomes) { + List outcomes = draftOutcomes.stream() + .map(outcome -> new MuseAiImportParseApi.DraftOutcome(outcome.resultId(), outcome.draftId(), + outcome.success(), outcome.errorCode(), outcome.errorMessage(), outcome.skipped(), + outcome.skippedReason())) + .toList(); + MuseAiImportParseApi.DecisionRecordResult result = aiImportParseApi.recordBatchConfirmed(userId, jobId, + new MuseAiImportParseApi.BatchConfirmChapterCommand(reqVO.getCommandId(), outcomes)); + return ParseDecisionRecordResult.available(result.auditLogId()); + } + + private ParseJobDetailRespVO toParseJobDetail(MuseAiImportParseApi.ParseJobProjection projection) { + ParseJobDetailRespVO respVO = new ParseJobDetailRespVO(); + respVO.setId(projection.id()); + respVO.setWorkId(projection.workId()); + respVO.setImportTaskId(projection.importTaskId()); + respVO.setStatus(projection.status()); + respVO.setProgress(projection.progress()); + respVO.setTotalChapters(projection.totalChapters()); + respVO.setConfirmedChapters(projection.confirmedChapters()); + respVO.setPendingChapters(projection.pendingChapters()); + respVO.setErrorMessage(projection.errorMessage()); + respVO.setFailedStage(projection.failedStage()); + respVO.setRetryable(projection.retryable()); + respVO.setNextActions(projection.nextActions()); + respVO.setParseConfigVersion(projection.parseConfigVersion()); + respVO.setSourceSnapshot(toContentSourceSnapshot(projection.sourceSnapshot())); + respVO.setCreatedAt(projection.createdAt()); + respVO.setCompletedAt(projection.completedAt()); + return respVO; + } + + private ChapterParseResultRespVO toChapterResult(MuseAiImportParseApi.ChapterParseResultProjection projection) { + ChapterParseResultRespVO respVO = new ChapterParseResultRespVO(); + respVO.setId(projection.id()); + respVO.setJobId(projection.jobId()); + respVO.setWorkId(projection.workId()); + respVO.setTitle(projection.title()); + respVO.setSortOrder(projection.sortOrder()); + respVO.setContentPreview(projection.contentPreview()); + respVO.setContentText(projection.contentText()); + respVO.setWordCount(projection.wordCount()); + respVO.setStatus(projection.status()); + respVO.setRevision(projection.revision()); + respVO.setQualityResult(projection.qualityResult()); + return respVO; + } + + private MuseAiImportParseApi.SourceSnapshot toAiSourceSnapshot(ContentSourceSnapshotVO sourceSnapshot) { + if (sourceSnapshot == null) { + return null; + } + return new MuseAiImportParseApi.SourceSnapshot(sourceSnapshot.getSourceType(), + sourceSnapshot.getSourceVersion(), sourceSnapshot.getAuthorizationSnapshotId()); + } + + private ContentSourceSnapshotVO toContentSourceSnapshot(MuseAiImportParseApi.SourceSnapshot sourceSnapshot) { + if (sourceSnapshot == null) { + return null; + } + ContentSourceSnapshotVO respVO = new ContentSourceSnapshotVO(); + respVO.setSourceType(sourceSnapshot.sourceType()); + respVO.setSourceVersion(sourceSnapshot.sourceVersion()); + respVO.setAuthorizationSnapshotId(sourceSnapshot.authorizationSnapshotId()); + return respVO; + } + + private String snapshotString(ImportTaskDO importTask, String key) { + Object value = snapshot(importTask).get(key); + return value == null ? null : String.valueOf(value); + } + + private Long snapshotLong(ImportTaskDO importTask, String key) { + Object value = snapshot(importTask).get(key); + if (value instanceof Number number) { + return number.longValue(); + } + return value == null ? null : Long.valueOf(String.valueOf(value)); + } + + private Map snapshot(ImportTaskDO importTask) { + Map snapshot = JsonUtils.parseObjectQuietly(importTask.getSourceSnapshot(), new TypeReference<>() { + }); + return snapshot == null ? Map.of() : snapshot; + } +} diff --git a/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/controller/app/vo/ChapterParseResultRespVO.java b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/controller/app/vo/ChapterParseResultRespVO.java index 8021fff6..5b97ced9 100644 --- a/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/controller/app/vo/ChapterParseResultRespVO.java +++ b/muse-cloud/muse-module-content/muse-module-content-server/src/main/java/cn/iocoder/muse/module/content/controller/app/vo/ChapterParseResultRespVO.java @@ -1,5 +1,6 @@ package cn.iocoder.muse.module.content.controller.app.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; @@ -30,6 +31,10 @@ public class ChapterParseResultRespVO { @Schema(description = "内容预览") private String contentPreview; + @JsonIgnore + @Schema(hidden = true) + private String contentText; + @Schema(description = "字数") private Integer wordCount; diff --git a/muse-cloud/muse-module-content/muse-module-content-server/src/test/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacadeTest.java b/muse-cloud/muse-module-content/muse-module-content-server/src/test/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacadeTest.java index 9bcf30ee..46bf7b3e 100644 --- a/muse-cloud/muse-module-content/muse-module-content-server/src/test/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacadeTest.java +++ b/muse-cloud/muse-module-content/muse-module-content-server/src/test/java/cn/iocoder/muse/module/content/application/facade/RealContentFileFacadeTest.java @@ -379,7 +379,7 @@ class RealContentFileFacadeTest extends BaseMockitoUnitTest { verify(blockMapper, never()).selectListByChapterId(2001L); } - // ============================ U5b 导入检视：SSRF + scanStatus 服务端权威 ============================ + // ============================ U5b 导入检视：SSRF + infra 稳定 path 服务端权威 ============================ private CreateImportTaskReqVO importReqVO(String uploadUrl) { CreateImportTaskReqVO reqVO = new CreateImportTaskReqVO(); @@ -465,55 +465,54 @@ class RealContentFileFacadeTest extends BaseMockitoUnitTest { } @Test - void should_blockImport_whenScanRecordMissing_evenIfHostAllowed() { - // scanStatus 服务端权威：host 在白名单 + https（过 SSRF 闸），但服务端无 clean 扫描记录 —— fail-closed。 - // 这同时证明请求确实越过了 SSRF 门（否则不会触达 scanStatus 派生），落到 scan_blocked。 + void should_blockImport_whenInternalPathCannotBeRead() { + // scanStatus 服务端权威：内部 path 必须能被 FileApi 按记录回读字节，否则 fail-closed。 + // 这同时证明请求确实越过了“仅 infra 稳定 path”门，落到服务端扫描阻断。 boolean[] scanResolved = {false}; + String storagePath = "100/content/import/book.txt"; + when(fileApi.getFileBytes(storagePath)).thenReturn(null); when(fileApiProvider.getIfAvailable()).thenReturn(fileApi); RealContentFileFacade facade = new RealContentFileFacade(fileApiProvider) { @Override String resolveServerAuthoritativeScanStatus(Long u, Long w, CreateImportTaskReqVO r) { - scanResolved[0] = true; // 记录确被调用 = 已过 SSRF 门 - return super.resolveServerAuthoritativeScanStatus(u, w, r); // 生产恒 scan_blocked + scanResolved[0] = true; // 记录确被调用 = 已过内部 path 门 + return super.resolveServerAuthoritativeScanStatus(u, w, r); } }; - ReflectionTestUtils.setField(facade, "allowedImportHosts", "minio.muse.example.com"); ContentFileFacade.ImportFileInspectionResult result = facade.inspectImportFile( - USER_ID, WORK_ID, importReqVO("https://minio.muse.example.com/tenant/file.txt")); + USER_ID, WORK_ID, importReqVO(storagePath)); assertFalse(result.available()); - assertTrue(scanResolved[0], "host 在白名单时应越过 SSRF 门并派生服务端 scanStatus"); + assertTrue(scanResolved[0], "内部 path 应进入服务端 scanStatus 派生"); + verify(fileApi).getFileBytes(storagePath); } @Test void should_ignoreClientSuppliedCleanStatus_andStayBlocked() { - // scanStatus 服务端权威：客户端在 uploadUrl 中暗示 clean，但服务端无记录 —— 客户端值被忽略，仍 fail-closed。 - // （reqVO 无独立 scanStatus 字段，客户端只能经 uploadUrl 携带暗示；这里断言此类暗示不被采信。） + // scanStatus 服务端权威：客户端用外链 query 暗示 clean/passed 也不采信；外链不回源、不读 FileApi。 RealContentFileFacade facade = newFacadeWithFileApi(); - ReflectionTestUtils.setField(facade, "allowedImportHosts", "minio.muse.example.com"); ContentFileFacade.ImportFileInspectionResult result = facade.inspectImportFile(USER_ID, WORK_ID, importReqVO("https://minio.muse.example.com/file.txt?scanStatus=clean&scan=passed")); assertFalse(result.available()); + verify(fileApi, never()).getFileBytes(anyString()); } @Test - void should_returnAvailable_whenServerRecordExplicitlyClean() { - // scanStatus 服务端权威：仅当服务端记录明确 clean 才放行（经子类覆盖模拟真扫描链路接入后的 clean 结论）。 - when(fileApiProvider.getIfAvailable()).thenReturn(fileApi); - RealContentFileFacade facade = new RealContentFileFacade(fileApiProvider) { - @Override - String resolveServerAuthoritativeScanStatus(Long u, Long w, CreateImportTaskReqVO r) { - return "clean"; // 模拟服务端权威 clean - } - }; - ReflectionTestUtils.setField(facade, "allowedImportHosts", "minio.muse.example.com"); + void should_returnAvailable_whenInternalPathReadable() { + // scanStatus 服务端权威：只有 infra 稳定 path 能被服务端按记录回读字节，才放行导入任务创建。 + String storagePath = "100/content/import/book.txt"; + RealContentFileFacade facade = newFacadeWithFileApi(); + when(fileApi.getFileBytes(storagePath)).thenReturn("第一章\n正文".getBytes(StandardCharsets.UTF_8)); ContentFileFacade.ImportFileInspectionResult result = facade.inspectImportFile( - USER_ID, WORK_ID, importReqVO("https://minio.muse.example.com/file.txt")); + USER_ID, WORK_ID, importReqVO(storagePath)); assertTrue(result.available()); - assertEquals("clean", result.scanStatus()); + assertEquals("passed", result.scanStatus()); + assertEquals(storagePath, result.storageRef()); + assertEquals(List.of("create_parse_job"), result.nextActions()); + verify(fileApi).getFileBytes(storagePath); } @Test diff --git a/muse-cloud/muse-module-infra/muse-module-infra-server/src/main/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileController.java b/muse-cloud/muse-module-infra/muse-module-infra-server/src/main/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileController.java index 8109deee..2d8b63cb 100644 --- a/muse-cloud/muse-module-infra/muse-module-infra-server/src/main/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileController.java +++ b/muse-cloud/muse-module-infra/muse-module-infra-server/src/main/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileController.java @@ -43,6 +43,19 @@ public class AppFileController { uploadReqVO.getDirectory(), file.getContentType())); } + @PostMapping("/upload-returning-path") + @Operation(summary = "上传文件并返回稳定存储 path") + @Parameter(name = "file", description = "文件附件", required = true, + schema = @Schema(type = "string", format = "binary")) + @PermitAll + public CommonResult uploadFileReturningPath(AppFileUploadReqVO uploadReqVO) throws Exception { + // 导入向导需要可重复回读的服务端 path；访问 URL 可能带签名且会过期，不能作为导入存储句柄。 + MultipartFile file = uploadReqVO.getFile(); + byte[] content = IoUtil.readBytes(file.getInputStream()); + return success(fileService.createFileReturningPath(content, file.getOriginalFilename(), + uploadReqVO.getDirectory(), file.getContentType())); + } + @GetMapping("/presigned-url") @Operation(summary = "获取文件预签名地址（上传）", description = "模式二：前端上传文件：用于前端直接上传七牛、阿里云 OSS 等文件存储器") @Parameters({ diff --git a/muse-cloud/muse-module-infra/muse-module-infra-server/src/test/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileControllerTest.java b/muse-cloud/muse-module-infra/muse-module-infra-server/src/test/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileControllerTest.java new file mode 100644 index 00000000..5343f53d --- /dev/null +++ b/muse-cloud/muse-module-infra/muse-module-infra-server/src/test/java/cn/iocoder/muse/module/infra/controller/app/file/AppFileControllerTest.java @@ -0,0 +1,48 @@ +package cn.iocoder.muse.module.infra.controller.app.file; + +import cn.iocoder.muse.framework.common.pojo.CommonResult; +import cn.iocoder.muse.framework.test.core.ut.BaseMockitoUnitTest; +import cn.iocoder.muse.module.infra.controller.app.file.vo.AppFileUploadReqVO; +import cn.iocoder.muse.module.infra.service.file.FileService; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.springframework.mock.web.MockMultipartFile; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * App 文件上传控制器测试。 + */ +class AppFileControllerTest extends BaseMockitoUnitTest { + + @InjectMocks + private AppFileController controller; + @Mock + private FileService fileService; + + @Test + void should_uploadImportFileAndReturnStablePath() throws Exception { + AppFileUploadReqVO reqVO = new AppFileUploadReqVO(); + reqVO.setDirectory("100/content/import/9001"); + reqVO.setFile(new MockMultipartFile("file", "book.txt", "text/plain", + "第一章\n正文".getBytes(StandardCharsets.UTF_8))); + when(fileService.createFileReturningPath(any(), eq("book.txt"), eq("100/content/import/9001"), + eq("text/plain"))).thenReturn("100/content/import/9001/book.txt"); + + CommonResult result = controller.uploadFileReturningPath(reqVO); + + assertEquals(0, result.getCode()); + assertEquals("100/content/import/9001/book.txt", result.getData()); + verify(fileService).createFileReturningPath(any(), eq("book.txt"), eq("100/content/import/9001"), + eq("text/plain")); + verify(fileService, never()).createFile(any(), any(), any(), any()); + } +} diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-api/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApi.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-api/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApi.java new file mode 100644 index 00000000..1a589428 --- /dev/null +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-api/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApi.java @@ -0,0 +1,87 @@ +package cn.iocoder.muse.module.knowledge.api; + +import java.util.List; +import java.util.Map; + +/** + * Knowledge Draft 创建对外 API。 + * + *

章节解析 Shadow 归 AI owner；用户确认章节结果后，Knowledge owner 负责把解析结果材料化为 + * {@code muse_knowledge_draft}。调用方只消费 draftId/outcome，不得直接写 Knowledge DAL。

+ */ +public interface MuseKnowledgeDraftCreationApi { + + /** + * 根据单个章节解析结果创建 Knowledge Draft。 + */ + DraftCreateResult createDraftFromChapterResult(CreateDraftCommand command); + + /** + * 根据多个章节解析结果批量创建 Knowledge Draft。 + */ + BatchDraftCreateResult createDraftsFromChapterResults(BatchCreateDraftCommand command); + + /** + * 单草稿创建命令。 + */ + record CreateDraftCommand(String commandId, + Long actorUserId, + Long ownerUserId, + Long workId, + Long parseJobId, + Long chapterResultId, + String title, + String contentPreview, + String contentText, + Integer wordCount, + Integer expectedRevision, + Map qualityResult) { + } + + /** + * 批量草稿创建命令。 + */ + record BatchCreateDraftCommand(String commandId, + Long actorUserId, + Long ownerUserId, + Long workId, + Long parseJobId, + List chapters) { + } + + /** + * 单草稿创建结果。 + */ + record DraftCreateResult(Long draftId, String status) { + } + + /** + * 批量草稿创建结果。 + */ + record BatchDraftCreateResult(List outcomes) { + } + + /** + * 单章节的 Knowledge Draft 创建结果。 + */ + record DraftOutcome(Long resultId, + Long draftId, + boolean success, + String errorCode, + String errorMessage, + boolean skipped, + String skippedReason) { + + public static DraftOutcome succeeded(Long resultId, Long draftId) { + return new DraftOutcome(resultId, draftId, true, null, null, false, null); + } + + public static DraftOutcome failed(Long resultId, String errorCode, String errorMessage) { + return new DraftOutcome(resultId, null, false, errorCode, errorMessage, false, null); + } + + public static DraftOutcome skipped(Long resultId, String reason) { + return new DraftOutcome(resultId, null, false, null, null, true, reason); + } + } +} diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImpl.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImpl.java new file mode 100644 index 00000000..a8c1a560 --- /dev/null +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImpl.java @@ -0,0 +1,130 @@ +package cn.iocoder.muse.module.knowledge.api; + +import cn.iocoder.muse.framework.common.util.json.JsonUtils; +import cn.iocoder.muse.module.knowledge.dal.dataobject.muse.MuseKnowledgeDraftDO; +import cn.iocoder.muse.module.knowledge.dal.mysql.muse.MuseKnowledgeDraftMapper; +import jakarta.annotation.Resource; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.StringUtils; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * {@link MuseKnowledgeDraftCreationApi} 的 Knowledge owner 实现。 + * + *

只创建 pending Knowledge Draft，不写 Knowledge Canonical；正式入库仍走 + * {@code /app-api/muse/knowledge-drafts/{draftId}/confirm} 的独立审阅状态机。

+ */ +@Service +public class MuseKnowledgeDraftCreationApiImpl implements MuseKnowledgeDraftCreationApi { + + @Resource + private MuseKnowledgeDraftMapper draftMapper; + + @Override + @Transactional(rollbackFor = Exception.class) + public DraftCreateResult createDraftFromChapterResult(CreateDraftCommand command) { + MuseKnowledgeDraftDO draft = createOrReplay(command, command.commandId()); + return new DraftCreateResult(draft.getId(), "draft_created"); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public BatchDraftCreateResult createDraftsFromChapterResults(BatchCreateDraftCommand command) { + List outcomes = command.chapters().stream() + .map(chapter -> createBatchDraft(command.commandId(), chapter)) + .toList(); + return new BatchDraftCreateResult(outcomes); + } + + private DraftOutcome createBatchDraft(String batchCommandId, CreateDraftCommand chapter) { + try { + MuseKnowledgeDraftDO draft = createOrReplay(chapter, batchCommandId + ":result-" + chapter.chapterResultId()); + return DraftOutcome.succeeded(chapter.chapterResultId(), draft.getId()); + } catch (RuntimeException ex) { + return DraftOutcome.failed(chapter.chapterResultId(), "KNOWLEDGE_DRAFT_CREATE_FAILED", "知识草稿创建失败"); + } + } + + private MuseKnowledgeDraftDO createOrReplay(CreateDraftCommand command, String commandId) { + MuseKnowledgeDraftDO replay = draftMapper.selectByCommandId(commandId); + if (replay != null) { + return replay; + } + MuseKnowledgeDraftDO draft = new MuseKnowledgeDraftDO(); + draft.setWorkId(command.workId()); + draft.setDraftType("entity"); + draft.setProposedChanges(JsonUtils.toJsonString(draftPayload(command))); + draft.setDraftPayload(JsonUtils.toJsonString(draftPayload(command))); + draft.setCurrentCanonicalSnapshot(JsonUtils.toJsonString(Map.of())); + draft.setSourceSnapshotId("ai-parse-result:" + command.chapterResultId() + "@" + + firstPresent(command.expectedRevision(), 1)); + draft.setAuthorizationSnapshotId("ai-parse-import:" + command.parseJobId()); + draft.setRiskMarkerSnapshot(JsonUtils.toJsonString(Map.of("riskFlags", List.of()))); + draft.setSourceStatus("active"); + draft.setSourceActionPolicy("allowed"); + draft.setStatus("pending"); + draft.setConfidence(0.72D); + draft.setSourceType("extraction"); + draft.setSourceId(command.chapterResultId()); + draft.setCommandId(commandId); + draft.setRevision(1); + draftMapper.insert(draft); + return draft; + } + + private Map draftPayload(CreateDraftCommand command) { + Map payload = new LinkedHashMap<>(); + payload.put("entityType", "note"); + payload.put("name", title(command)); + payload.put("description", description(command)); + payload.put("sourceRef", "ai-parse-result:" + command.chapterResultId()); + payload.put("sourceVersion", firstPresent(command.expectedRevision(), 1)); + payload.put("sourceHash", sha256(firstPresent(command.contentText(), ""))); + payload.put("currentSourceHash", sha256(firstPresent(command.contentText(), ""))); + payload.put("parseJobId", command.parseJobId()); + payload.put("chapterResultId", command.chapterResultId()); + payload.put("wordCount", firstPresent(command.wordCount(), 0)); + payload.put("qualityResult", command.qualityResult() == null ? Map.of() : command.qualityResult()); + payload.put("riskFlags", List.of()); + return payload; + } + + private String title(CreateDraftCommand command) { + if (StringUtils.hasText(command.title())) { + return command.title().strip(); + } + return "章节解析结果 " + command.chapterResultId(); + } + + private String description(CreateDraftCommand command) { + if (StringUtils.hasText(command.contentPreview())) { + return command.contentPreview().strip(); + } + String content = firstPresent(command.contentText(), "").strip(); + return content.length() <= 240 ? content : content.substring(0, 240); + } + + private String sha256(String payload) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(payload.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(digest.length * 2); + for (byte value : digest) { + hex.append(String.format("%02x", value)); + } + return "sha256:" + hex; + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("JDK 缺少 SHA-256 摘要算法", exception); + } + } + + private T firstPresent(T value, T fallback) { + return value == null ? fallback : value; + } +} diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/dal/mysql/muse/MuseKnowledgeDraftMapper.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/dal/mysql/muse/MuseKnowledgeDraftMapper.java index 1972d505..87e7e358 100644 --- a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/dal/mysql/muse/MuseKnowledgeDraftMapper.java +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/main/java/cn/iocoder/muse/module/knowledge/dal/mysql/muse/MuseKnowledgeDraftMapper.java @@ -22,6 +22,12 @@ public interface MuseKnowledgeDraftMapper extends BaseMapperX() + .eq(MuseKnowledgeDraftDO::getCommandId, commandId) + .eq(MuseKnowledgeDraftDO::getDeleted, false)); + } + default List selectByWorkId(Long workId, String status, String entityType) { LambdaQueryWrapperX wrapper = baseListWrapper(status, entityType) .eq(MuseKnowledgeDraftDO::getWorkId, workId); diff --git a/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImplTest.java b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImplTest.java new file mode 100644 index 00000000..a90b6f4c --- /dev/null +++ b/muse-cloud/muse-module-knowledge/muse-module-knowledge-server/src/test/java/cn/iocoder/muse/module/knowledge/api/MuseKnowledgeDraftCreationApiImplTest.java @@ -0,0 +1,105 @@ +package cn.iocoder.muse.module.knowledge.api; + +import cn.iocoder.muse.framework.common.util.json.JsonUtils; +import cn.iocoder.muse.framework.test.core.ut.BaseMockitoUnitTest; +import cn.iocoder.muse.module.knowledge.dal.dataobject.muse.MuseKnowledgeDraftDO; +import cn.iocoder.muse.module.knowledge.dal.mysql.muse.MuseKnowledgeDraftMapper; +import com.fasterxml.jackson.core.type.TypeReference; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Knowledge Draft 创建 owner API 测试。 + */ +class MuseKnowledgeDraftCreationApiImplTest extends BaseMockitoUnitTest { + + @InjectMocks + private MuseKnowledgeDraftCreationApiImpl api; + @Mock + private MuseKnowledgeDraftMapper draftMapper; + + @Test + void should_createPendingDraftFromChapterParseResult() { + when(draftMapper.selectByCommandId("cmd-draft-1")).thenReturn(null); + when(draftMapper.insert(any(MuseKnowledgeDraftDO.class))).thenAnswer(invocation -> { + MuseKnowledgeDraftDO draft = invocation.getArgument(0); + draft.setId(70001L); + return 1; + }); + + MuseKnowledgeDraftCreationApi.DraftCreateResult result = + api.createDraftFromChapterResult(createCommand("cmd-draft-1", 50001L)); + + assertEquals(70001L, result.draftId()); + assertEquals("draft_created", result.status()); + ArgumentCaptor draftCaptor = ArgumentCaptor.forClass(MuseKnowledgeDraftDO.class); + verify(draftMapper).insert(draftCaptor.capture()); + MuseKnowledgeDraftDO draft = draftCaptor.getValue(); + assertEquals(10001L, draft.getWorkId()); + assertEquals("entity", draft.getDraftType()); + assertEquals("pending", draft.getStatus()); + assertEquals("extraction", draft.getSourceType()); + assertEquals(50001L, draft.getSourceId()); + assertEquals("cmd-draft-1", draft.getCommandId()); + Map payload = JsonUtils.parseObject(draft.getDraftPayload(), new TypeReference<>() { + }); + assertEquals("note", payload.get("entityType")); + assertEquals("第一章", payload.get("name")); + assertEquals("ai-parse-result:50001", payload.get("sourceRef")); + assertTrue(String.valueOf(payload.get("sourceHash")).startsWith("sha256:")); + } + + @Test + void should_replayExistingDraftByCommandId() { + MuseKnowledgeDraftDO existing = new MuseKnowledgeDraftDO(); + existing.setId(70002L); + when(draftMapper.selectByCommandId("cmd-draft-1")).thenReturn(existing); + + MuseKnowledgeDraftCreationApi.DraftCreateResult result = + api.createDraftFromChapterResult(createCommand("cmd-draft-1", 50001L)); + + assertEquals(70002L, result.draftId()); + verify(draftMapper, never()).insert(any(MuseKnowledgeDraftDO.class)); + } + + @Test + void should_createBatchDraftsWithStablePerResultCommandId() { + when(draftMapper.selectByCommandId("cmd-batch-1:result-50001")).thenReturn(null); + when(draftMapper.selectByCommandId("cmd-batch-1:result-50002")).thenReturn(null); + when(draftMapper.insert(any(MuseKnowledgeDraftDO.class))).thenAnswer(invocation -> { + MuseKnowledgeDraftDO draft = invocation.getArgument(0); + draft.setId(80000L + draft.getSourceId()); + return 1; + }); + + MuseKnowledgeDraftCreationApi.BatchDraftCreateResult result = api.createDraftsFromChapterResults( + new MuseKnowledgeDraftCreationApi.BatchCreateDraftCommand("cmd-batch-1", 9001L, 9001L, + 10001L, 40001L, java.util.List.of(createCommand("ignored-1", 50001L), + createCommand("ignored-2", 50002L)))); + + assertEquals(2, result.outcomes().size()); + assertTrue(result.outcomes().stream().allMatch(MuseKnowledgeDraftCreationApi.DraftOutcome::success)); + verify(draftMapper).insert(argThat((MuseKnowledgeDraftDO draft) -> + "cmd-batch-1:result-50001".equals(draft.getCommandId()))); + verify(draftMapper).insert(argThat((MuseKnowledgeDraftDO draft) -> + "cmd-batch-1:result-50002".equals(draft.getCommandId()))); + } + + private MuseKnowledgeDraftCreationApi.CreateDraftCommand createCommand(String commandId, Long resultId) { + return new MuseKnowledgeDraftCreationApi.CreateDraftCommand(commandId, 9001L, 9001L, 10001L, + 40001L, resultId, "第一章", "章节摘要", "章节全文", 4, 1, + Map.of("passed", true)); + } +} diff --git a/muse-cloud/muse-server/src/test/java/cn/iocoder/muse/server/framework/api/P1rContentImportWizardCompletedApprovalIT.java b/muse-cloud/muse-server/src/test/java/cn/iocoder/muse/server/framework/api/P1rContentImportWizardCompletedApprovalIT.java new file mode 100644 index 00000000..de74a498 --- /dev/null +++ b/muse-cloud/muse-server/src/test/java/cn/iocoder/muse/server/framework/api/P1rContentImportWizardCompletedApprovalIT.java @@ -0,0 +1,729 @@ +package cn.iocoder.muse.server.framework.api; + +import cn.hutool.extra.spring.SpringUtil; +import cn.iocoder.muse.framework.common.biz.infra.logger.ApiErrorLogCommonApi; +import cn.iocoder.muse.framework.common.biz.infra.logger.dto.ApiErrorLogCreateReqDTO; +import cn.iocoder.muse.framework.common.enums.UserTypeEnum; +import cn.iocoder.muse.framework.common.pojo.CommonResult; +import cn.iocoder.muse.framework.datasource.config.MuseDataSourceAutoConfiguration; +import cn.iocoder.muse.framework.mybatis.config.MuseMybatisAutoConfiguration; +import cn.iocoder.muse.framework.mybatis.core.util.MyBatisUtils; +import cn.iocoder.muse.framework.security.core.LoginUser; +import cn.iocoder.muse.framework.security.core.util.SecurityFrameworkUtils; +import cn.iocoder.muse.framework.tenant.config.TenantProperties; +import cn.iocoder.muse.framework.tenant.core.context.TenantContextHolder; +import cn.iocoder.muse.framework.tenant.core.db.TenantDatabaseInterceptor; +import cn.iocoder.muse.framework.web.config.MuseWebAutoConfiguration; +import cn.iocoder.muse.module.ai.application.muse.MuseAiImportLlmParser; +import cn.iocoder.muse.module.ai.application.muse.MuseAiImportParseService; +import cn.iocoder.muse.module.ai.application.muse.NewApiMuseAiImportLlmParser; +import cn.iocoder.muse.module.ai.application.muse.facade.MuseContentWorkOwnerFacade; +import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties; +import cn.iocoder.muse.module.content.application.ContentAuditServiceImpl; +import cn.iocoder.muse.module.content.application.ContentBlockRevisionServiceImpl; +import cn.iocoder.muse.module.content.application.ContentCommandServiceImpl; +import cn.iocoder.muse.module.content.application.ContentEventPublishOutboxService; +import cn.iocoder.muse.module.content.application.ContentImportParseServiceImpl; +import cn.iocoder.muse.module.content.dal.dataobject.BlockSourceAttributionDO; +import cn.iocoder.muse.module.content.dal.dataobject.ContentEventPublishOutboxDO; +import cn.iocoder.muse.module.content.application.facade.RealContentFileFacade; +import cn.iocoder.muse.module.content.application.facade.RealContentKnowledgeDraftFacade; +import cn.iocoder.muse.module.content.application.facade.RealContentParseJobFacade; +import cn.iocoder.muse.module.content.application.export.render.DocxExportRenderer; +import cn.iocoder.muse.module.content.application.export.render.EpubExportRenderer; +import cn.iocoder.muse.module.content.application.export.render.ExportRendererFactory; +import cn.iocoder.muse.module.content.application.export.render.TxtExportRenderer; +import cn.iocoder.muse.module.content.controller.app.AppContentImportParseController; +import cn.iocoder.muse.module.infra.api.file.FileApi; +import cn.iocoder.muse.module.infra.api.file.dto.FileCreateReqDTO; +import cn.iocoder.muse.module.knowledge.api.MuseKnowledgeDraftCreationApiImpl; +import com.baomidou.mybatisplus.autoconfigure.MybatisPlusAutoConfiguration; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.TenantLineInnerInterceptor; +import com.github.yulichang.autoconfigure.MybatisPlusJoinAutoConfiguration; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.output.MigrateResult; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.ImportAutoConfiguration; +import org.springframework.boot.autoconfigure.http.HttpMessageConvertersAutoConfiguration; +import org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration; +import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration; +import org.springframework.boot.autoconfigure.web.client.RestTemplateAutoConfiguration; +import org.springframework.boot.autoconfigure.web.servlet.WebMvcAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import javax.sql.DataSource; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * P1R 完整导入向导 completed approval：真实 PostgreSQL `_test` 库迁移到 latest/V34， + * 通过 Content HTTP 入口走上传扫描 → AI owner 全书解析 → 章节批量确认 → Knowledge Draft。 + * + *

本测试只把外部 LLM 与对象存储替换为内存 stub，业务 owner、mapper、事务、Flyway V34、AI Shadow 表、 + * Knowledge Draft 表都是真实 PostgreSQL 事实；用于防止“单测绿但 V34/owner 接线未真跑”的假绿。

+ */ +@SpringBootTest( + classes = P1rContentImportWizardCompletedApprovalIT.ImportWizardConfiguration.class, + webEnvironment = SpringBootTest.WebEnvironment.MOCK +) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class P1rContentImportWizardCompletedApprovalIT { + + private static final Long TENANT_ID = 100L; + private static final Long LOGIN_USER_ID = 9001L; + private static final String API_VERSION = "1"; + private static final String EXTERNAL_ACCEPTANCE_ENV = "MUSE_P1R_EXTERNAL_ACCEPTANCE"; + private static final String LIVE_MODEL = "MiniMax-M2.5"; + private static final Set CREDENTIAL_QUERY_KEYS = Set.of( + "user", "username", "password", "pass", "pwd", "sslpassword", "ssl_password", + "token", "secret", "api_key", "apikey", "bearer", "access_token", "refresh_token"); + + private static volatile Settings cachedSettings; + + @Autowired + private DataSource dataSource; + @Autowired + private WebApplicationContext webApplicationContext; + @Autowired + private InMemoryFileApi fileApi; + + private MockMvc mockMvc; + private long workId; + + @DynamicPropertySource + static void registerProperties(DynamicPropertyRegistry registry) { + Settings settings = settings(); + registry.add("spring.application.name", () -> "p1r-content-import-wizard-completed-approval-it"); + registry.add("muse.info.base-package", () -> "cn.iocoder.muse"); + registry.add("muse.web.admin-ui.url", () -> "http://localhost"); + registry.add("spring.datasource.url", settings::jdbcUrl); + registry.add("spring.datasource.username", settings::jdbcUser); + registry.add("spring.datasource.password", settings::jdbcPassword); + registry.add("spring.datasource.driver-class-name", () -> "org.postgresql.Driver"); + registry.add("spring.main.banner-mode", () -> "off"); + registry.add("spring.main.lazy-initialization", () -> "true"); + registry.add("mybatis-plus.global-config.db-config.id-type", () -> "AUTO"); + } + + @BeforeAll + void migrateSchema() { + Settings settings = settings(); + ensureTestDatabaseExists(settings); + Flyway flyway = Flyway.configure() + .dataSource(settings.jdbcUrl(), settings.jdbcUser(), settings.jdbcPassword()) + .locations(resolveMuseSqlLocation(settings.flywayLocations())) + .schemas("public") + .defaultSchema("public") + .cleanDisabled(false) + .load(); + flyway.clean(); + MigrateResult result = flyway.migrate(); + assertTrue(result.migrationsExecuted >= 34, "完整导入向导必须跑到 V34+，实际迁移数: " + result.migrationsExecuted); + } + + @BeforeEach + void setUp() { + this.mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).build(); + setRuntimeContext(LOGIN_USER_ID); + resetTables(); + this.workId = seedWork("P1R 完整导入向导作品"); + } + + @AfterEach + void tearDown() { + SecurityContextHolder.clearContext(); + TenantContextHolder.clear(); + } + + @Test + void should_completeUploadScanAiParseAndKnowledgeDraftConfirmation() throws Exception { + String storagePath = fileApi.put("100/content/import/p1r-book.docx", docxBytes(""" + 第一章 星门 + 导入向导第一章正文。 + 第二章 归航 + 导入向导第二章正文。 + """)); + + long importTaskId = numberFromJson(mockMvc.perform(post("/app-api/muse/works/{workId}/import-tasks", workId) + .header("X-API-Version", API_VERSION) + .contentType(MediaType.APPLICATION_JSON) + .content(createImportTaskBody("cmd-p1r-import-1", storagePath))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.scanStatus").value("passed")) + .andExpect(jsonPath("$.data.nextActions[0]").value("create_parse_job")) + .andReturn().getResponse().getContentAsString(), "data.id"); + + long jobId = numberFromJson(mockMvc.perform(post("/app-api/muse/works/{workId}/parse-jobs", workId) + .header("X-API-Version", API_VERSION) + .contentType(MediaType.APPLICATION_JSON) + .content(createParseJobBody("cmd-p1r-parse-1", importTaskId))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.status").value("completed")) + .andExpect(jsonPath("$.data.totalChapters").value(2)) + .andReturn().getResponse().getContentAsString(), "data.id"); + + List resultIds = chapterResultIds(jobId); + assertEquals(2, resultIds.size(), "AI owner 必须写入 2 条 pending_review Shadow 章节"); + mockMvc.perform(post("/app-api/muse/parse-jobs/{jobId}/chapters/batch-confirm", jobId) + .header("X-API-Version", API_VERSION) + .contentType(MediaType.APPLICATION_JSON) + .content(batchConfirmBody("cmd-p1r-confirm-1", resultIds))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.confirmedChapterResultIds.length()").value(2)) + .andExpect(jsonPath("$.data.createdDraftIds.length()").value(2)) + .andExpect(jsonPath("$.data.partialFailure").value(false)); + + assertEquals(1, countRows("muse_ai_parse_job", "status = 'completed'")); + assertEquals(2, countRows("muse_ai_chapter_parse_result", "status = 'confirmed'")); + assertEquals(2, countRows("muse_knowledge_draft", "status = 'pending' AND source_type = 'extraction'")); + assertEquals(0, countRows("muse_knowledge_entity", "1 = 1"), + "章节确认只能创建 Knowledge Draft，不能绕过审核直写 Canonical"); + } + + @Test + void should_completeUploadScanLiveLlmParseAndKnowledgeDraftConfirmation_whenExternalAcceptanceEnabled() throws Exception { + assumeTrue(externalAcceptanceEnabled(), + "完整导入向导 New-API live 验收未启用，设置 MUSE_P1R_EXTERNAL_ACCEPTANCE=true 后才运行"); + String storagePath = fileApi.put("100/content/import/p1r-live-book.txt", """ + 第一章 星门 + 林岚在黎明前抵达观测站，记录下第一束异常光。 + + 第二章 归航 + 船队收到旧信标的回声，决定沿着暗潮返航。 + """.getBytes(StandardCharsets.UTF_8)); + + long importTaskId = numberFromJson(mockMvc.perform(post("/app-api/muse/works/{workId}/import-tasks", workId) + .header("X-API-Version", API_VERSION) + .contentType(MediaType.APPLICATION_JSON) + .content(createImportTaskBody("cmd-p1r-import-live-1", storagePath, "p1r-live-book.txt", + "txt", "text/plain"))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.scanStatus").value("passed")) + .andReturn().getResponse().getContentAsString(), "data.id"); + + long jobId = numberFromJson(mockMvc.perform(post("/app-api/muse/works/{workId}/parse-jobs", workId) + .header("X-API-Version", API_VERSION) + .contentType(MediaType.APPLICATION_JSON) + .content(createLiveParseJobBody("cmd-p1r-parse-live-1", importTaskId))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.status").value("completed")) + .andExpect(jsonPath("$.data.totalChapters").value(2)) + .andReturn().getResponse().getContentAsString(), "data.id"); + + List resultIds = chapterResultIds(jobId); + assertEquals(2, resultIds.size(), "New-API live 解析必须写入 2 条 pending-review Shadow 章节"); + assertEquals(2, countRows("muse_ai_chapter_parse_result", "quality_result::text LIKE '%new_api_import_llm%'")); + mockMvc.perform(post("/app-api/muse/parse-jobs/{jobId}/chapters/batch-confirm", jobId) + .header("X-API-Version", API_VERSION) + .contentType(MediaType.APPLICATION_JSON) + .content(batchConfirmBody("cmd-p1r-confirm-live-1", resultIds))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.data.createdDraftIds.length()").value(2)) + .andExpect(jsonPath("$.data.partialFailure").value(false)); + + assertEquals(2, countRows("muse_knowledge_draft", "status = 'pending' AND source_type = 'extraction'")); + assertEquals(0, countRows("muse_knowledge_entity", "1 = 1"), + "New-API live 解析确认后也只能创建 Knowledge Draft，不能绕过审核直写 Canonical"); + } + + private String createImportTaskBody(String commandId, String storagePath) { + return createImportTaskBody(commandId, storagePath, "p1r-book.docx", "docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + } + + private String createImportTaskBody(String commandId, String storagePath, String fileName, String format, + String mimeType) { + return """ + {"commandId":"%s","fileName":"%s","fileSize":%d,"fileHash":"sha256:p1r",\ + "mimeType":"%s",\ + "format":"%s","uploadUrl":"%s","sourceSnapshot":{"sourceType":"uploaded_file","sourceVersion":1}} + """.formatted(commandId, fileName, 128, mimeType, format, storagePath); + } + + private String createParseJobBody(String commandId, long importTaskId) { + return """ + {"commandId":"%s","importTaskId":%d,\ + "parseConfig":{"mode":"full_book","strategy":"llm_full_book"},\ + "sourceSnapshot":{"sourceType":"uploaded_file","sourceVersion":1}} + """.formatted(commandId, importTaskId); + } + + private String createLiveParseJobBody(String commandId, long importTaskId) { + return """ + {"commandId":"%s","importTaskId":%d,\ + "parseConfig":{"mode":"full_book","strategy":"llm_full_book","liveAcceptance":true,"modelKey":"%s"},\ + "sourceSnapshot":{"sourceType":"uploaded_file","sourceVersion":1}} + """.formatted(commandId, importTaskId, LIVE_MODEL); + } + + private String batchConfirmBody(String commandId, List resultIds) { + String ids = resultIds.toString(); + StringBuilder revisions = new StringBuilder("{"); + for (int i = 0; i < resultIds.size(); i++) { + if (i > 0) { + revisions.append(','); + } + revisions.append('"').append(resultIds.get(i)).append('"').append(":1"); + } + revisions.append('}'); + return """ + {"commandId":"%s","resultIds":%s,"expectedRevisions":%s} + """.formatted(commandId, ids, revisions); + } + + private long seedWork(String title) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(""" + INSERT INTO muse_content_work(owner_user_id, title, description, genre, status, revision, tenant_id) + VALUES (?, ?, ?, 'fiction', 'draft', 1, ?) + RETURNING id + """)) { + statement.setLong(1, LOGIN_USER_ID); + statement.setString(2, title); + statement.setString(3, title + " description"); + statement.setLong(4, TENANT_ID); + try (ResultSet resultSet = statement.executeQuery()) { + assertTrue(resultSet.next(), "seed work 必须返回 id"); + return resultSet.getLong(1); + } + } catch (SQLException exception) { + throw new AssertionError("seed work 失败", exception); + } + } + + private void resetTables() { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute(""" + TRUNCATE TABLE + muse_knowledge_draft, + muse_ai_chapter_parse_result, + muse_ai_parse_job, + muse_content_command_log, + muse_content_import_task, + muse_content_event_publish_outbox, + muse_content_block_revision_snapshot, + muse_content_block_source_attribution, + muse_content_block, + muse_content_chapter, + muse_content_work + RESTART IDENTITY CASCADE + """); + } catch (SQLException exception) { + throw new AssertionError("重置导入向导相关表失败", exception); + } + } + + private List chapterResultIds(long jobId) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(""" + SELECT id FROM muse_ai_chapter_parse_result + WHERE parse_job_id = ? AND status = 'pending_review' + ORDER BY sort_order + """)) { + statement.setLong(1, jobId); + try (ResultSet resultSet = statement.executeQuery()) { + java.util.ArrayList ids = new java.util.ArrayList<>(); + while (resultSet.next()) { + ids.add(resultSet.getLong(1)); + } + return ids; + } + } catch (SQLException exception) { + throw new AssertionError("读取 AI 章节结果失败", exception); + } + } + + private int countRows(String tableName, String whereClause) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement("SELECT count(*) FROM " + tableName + " WHERE " + whereClause); + ResultSet resultSet = statement.executeQuery()) { + resultSet.next(); + return resultSet.getInt(1); + } catch (SQLException exception) { + throw new AssertionError("统计 " + tableName + " 失败", exception); + } + } + + private long numberFromJson(String json, String dottedPath) { + com.fasterxml.jackson.databind.JsonNode node = cn.iocoder.muse.framework.common.util.json.JsonUtils.parseTree(json); + for (String part : dottedPath.split("\\.")) { + node = node.path(part); + } + assertTrue(node.isNumber(), "响应字段必须是数字: " + dottedPath + ", body=" + json); + return node.asLong(); + } + + private byte[] docxBytes(String text) { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ZipOutputStream zip = new ZipOutputStream(out)) { + zip.putNextEntry(new ZipEntry("word/document.xml")); + zip.write(""" + + + + %s + + + """.formatted(toWordParagraphs(text)).getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + return out.toByteArray(); + } catch (IOException exception) { + throw new IllegalStateException("构造测试 DOCX 失败", exception); + } + } + + private String toWordParagraphs(String text) { + StringBuilder paragraphs = new StringBuilder(); + for (String line : text.strip().split("\\R")) { + paragraphs.append(" ") + .append(escapeXml(line.strip())) + .append("\n"); + } + return paragraphs.toString(); + } + + private String escapeXml(String text) { + return text.replace("&", "&") + .replace("<", "<") + .replace(">", ">"); + } + + private void setRuntimeContext(Long userId) { + TenantContextHolder.setTenantId(TENANT_ID); + MockHttpServletRequest request = new MockHttpServletRequest(); + request.addHeader("X-API-Version", API_VERSION); + org.springframework.web.context.request.RequestContextHolder.setRequestAttributes( + new org.springframework.web.context.request.ServletRequestAttributes(request)); + SecurityFrameworkUtils.setLoginUser(new LoginUser() + .setId(userId) + .setUserType(UserTypeEnum.MEMBER.getValue()) + .setTenantId(TENANT_ID), new MockHttpServletRequest()); + } + + private static Settings settings() { + if (cachedSettings == null) { + cachedSettings = Settings.fromPropertiesAndEnvironment(); + } + return cachedSettings; + } + + private static boolean externalAcceptanceEnabled() { + return "true".equalsIgnoreCase(System.getenv(EXTERNAL_ACCEPTANCE_ENV)); + } + + private static String requiredEnvironment(String name) { + String value = System.getenv(name); + assertTrue(value != null && !value.isBlank(), "缺少外部验收环境变量: " + name); + return value; + } + + private static int integerEnvironment(String name, int defaultValue) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + return defaultValue; + } + return Integer.parseInt(value); + } + + private static void ensureTestDatabaseExists(Settings settings) { + Properties properties = new Properties(); + properties.setProperty("user", settings.jdbcUser()); + properties.setProperty("password", settings.jdbcPassword()); + try (Connection connection = DriverManager.getConnection(settings.maintenanceJdbcUrl(), properties); + PreparedStatement statement = connection.prepareStatement("SELECT 1 FROM pg_database WHERE datname = ?")) { + statement.setString(1, settings.databaseName()); + try (ResultSet resultSet = statement.executeQuery()) { + assertTrue(resultSet.next(), "目标 _test 库不存在，请经 DDL 闸门预建: " + settings.databaseName()); + } + } catch (SQLException exception) { + throw new AssertionError("预检 _test 数据库失败: " + exception.getClass().getSimpleName(), exception); + } + } + + private static String resolveMuseSqlLocation(String configured) { + String requested = configured == null || configured.isBlank() ? "filesystem:sql/muse" : configured; + assertTrue(requested.startsWith("filesystem:"), + "完整导入向导 Flyway IT 只接受 filesystem 迁移目录，避免 classpath 误扫: " + requested); + String relativeLocation = requested.substring("filesystem:".length()); + Path current = Path.of(System.getProperty("user.dir")).toAbsolutePath(); + for (Path cursor = current; cursor != null; cursor = cursor.getParent()) { + Path candidate = cursor.resolve(relativeLocation); + if (Files.isDirectory(candidate)) { + return "filesystem:" + candidate; + } + } + throw new IllegalStateException("无法从当前目录向上找到 sql/muse: " + current); + } + + private static void assertNoCredentialQuery(String url) { + String query = URI.create((url.startsWith("jdbc:") ? url.substring(5) : url)).getRawQuery(); + if (query == null || query.isBlank()) { + return; + } + for (String pair : query.split("&")) { + String key = pair.split("=", 2)[0].toLowerCase(Locale.ROOT).replace('-', '_'); + assertFalse(CREDENTIAL_QUERY_KEYS.contains(key), "p1r.flyway.url 不能携带凭据 query 参数"); + } + } + + private record Settings(String jdbcUrl, String jdbcUser, String jdbcPassword, String flywayLocations, + String databaseName, String maintenanceJdbcUrl) { + + static Settings fromPropertiesAndEnvironment() { + String url = requiredProperty("p1r.flyway.url"); + String user = requiredProperty("p1r.flyway.user"); + String password = requiredPasswordEnvironment(); + String locations = System.getProperty("p1r.flyway.locations", "filesystem:sql/muse"); + assertNoCredentialQuery(url); + String databaseName = databaseName(url); + assertTrue(databaseName.endsWith("_test"), "p1r.flyway.url 必须指向 _test 后缀隔离库"); + return new Settings(url, user, password, locations, databaseName, maintenanceJdbcUrl(url)); + } + + private static String requiredProperty(String key) { + String value = System.getProperty(key); + assertNotNull(value, "缺少必要 JVM 属性: " + key); + assertFalse(value.isBlank(), "JVM 属性不能为空: " + key); + return value; + } + + private static String requiredPasswordEnvironment() { + String value = System.getenv("P1R_FLYWAY_PASSWORD"); + if (value == null || value.isBlank()) { + value = System.getenv("MUSE_POSTGRES_PASSWORD"); + } + assertNotNull(value, "缺少 P1R_FLYWAY_PASSWORD 或 MUSE_POSTGRES_PASSWORD"); + assertFalse(value.isBlank(), "数据库密码环境变量不能为空"); + return value; + } + + private static String databaseName(String jdbcUrl) { + String withoutQuery = jdbcUrl.split("\\?", 2)[0]; + return withoutQuery.substring(withoutQuery.lastIndexOf('/') + 1); + } + + private static String maintenanceJdbcUrl(String jdbcUrl) { + String withoutQuery = jdbcUrl.split("\\?", 2)[0]; + return withoutQuery.substring(0, withoutQuery.lastIndexOf('/') + 1) + "postgres"; + } + } + + @SpringBootConfiguration + @ImportAutoConfiguration({ + JacksonAutoConfiguration.class, + HttpMessageConvertersAutoConfiguration.class, + DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + JdbcTemplateAutoConfiguration.class, + TransactionAutoConfiguration.class, + RestTemplateAutoConfiguration.class, + WebMvcAutoConfiguration.class, + MuseDataSourceAutoConfiguration.class, + MuseMybatisAutoConfiguration.class, + MybatisPlusAutoConfiguration.class, + MybatisPlusJoinAutoConfiguration.class, + MuseWebAutoConfiguration.class + }) + @Import({ + AppContentImportParseController.class, + ContentImportParseServiceImpl.class, + ContentCommandServiceImpl.class, + ContentAuditServiceImpl.class, + ContentBlockRevisionServiceImpl.class, + RealContentFileFacade.class, + RealContentParseJobFacade.class, + RealContentKnowledgeDraftFacade.class, + MuseAiImportParseService.class, + MuseKnowledgeDraftCreationApiImpl.class, + TxtExportRenderer.class, + DocxExportRenderer.class, + EpubExportRenderer.class, + ExportRendererFactory.class, + SpringUtil.class + }) + static class ImportWizardConfiguration { + + @Bean + @Primary + InMemoryFileApi fileApi() { + return new InMemoryFileApi(); + } + + @Bean + MuseAiImportLlmParser importLlmParser() { + if (externalAcceptanceEnabled()) { + return new NewApiMuseAiImportLlmParser(newApiProperties()); + } + return command -> new MuseAiImportLlmParser.LlmParseResult(true, List.of( + new MuseAiImportLlmParser.LlmChapter("第一章 星门", "导入向导第一章正文。"), + new MuseAiImportLlmParser.LlmChapter("第二章 归航", "导入向导第二章正文。")), + null, null, false, Map.of("parser", "stub_llm_full_book", "chapterCount", 2)); + } + + private MuseAiProperties newApiProperties() { + MuseAiProperties properties = new MuseAiProperties(); + MuseAiProperties.NewApi newApi = new MuseAiProperties.NewApi(); + newApi.setEnabled(true); + newApi.setBaseUrl(requiredEnvironment("MUSE_AI_NEW_API_BASE_URL")); + newApi.setToken(requiredEnvironment("MUSE_AI_NEW_API_TOKEN")); + newApi.setDefaultModelKey(LIVE_MODEL); + newApi.setConnectTimeoutSeconds(integerEnvironment("MUSE_AI_NEW_API_CONNECT_TIMEOUT_SECONDS", 5)); + newApi.setNonStreamReadTimeoutSeconds(integerEnvironment("MUSE_AI_NEW_API_NON_STREAM_READ_TIMEOUT_SECONDS", 90)); + properties.setNewApi(newApi); + return properties; + } + + @Bean + MuseContentWorkOwnerFacade museContentWorkOwnerFacade() { + return new MuseContentWorkOwnerFacade() { + @Override + public void requireWorkOwner(Long workId, Long ownerUserId) { + // Content owner 归属已由 Content 服务在 HTTP 入口前置校验；AI owner 这里再次按同一 user/work 可信边界验收。 + assertNotNull(workId, "AI owner requireWorkOwner 必须收到 workId"); + assertEquals(LOGIN_USER_ID, ownerUserId, "AI owner requireWorkOwner 必须沿用登录用户"); + } + + @Override + public void requireSourceOwner(String sourceType, Long sourceId, Long ownerUserId) { + assertEquals(LOGIN_USER_ID, ownerUserId, "AI owner requireSourceOwner 必须沿用登录用户"); + } + + @Override + public SourceSnapshot buildSourceSnapshot(SourceSnapshotRequest request) { + return new SourceSnapshot("p1r-import-wizard-source", request.workId(), request.chapterId(), + request.blockId(), 1, request.contextScope(), List.of(), Map.of("source", "p1r")); + } + }; + } + + @Bean + TenantLineInnerInterceptor tenantLineInnerInterceptor(MybatisPlusInterceptor interceptor) { + TenantLineInnerInterceptor inner = new TenantLineInnerInterceptor( + new TenantDatabaseInterceptor(new TenantProperties())); + MyBatisUtils.addInterceptor(interceptor, inner, 0); + return inner; + } + + @Bean + ApiErrorLogCommonApi apiErrorLogCommonApi() { + return new ApiErrorLogCommonApi() { + @Override + public CommonResult createApiErrorLog(ApiErrorLogCreateReqDTO createDTO) { + return CommonResult.success(true); + } + }; + } + + @Bean + ContentEventPublishOutboxService contentEventPublishOutboxService() { + return new ContentEventPublishOutboxService() { + @Override + public ContentEventPublishOutboxDO createForBlockSourceAttribution(Long ownerUserId, Long workId, + Long blockId, Integer blockRevision, + String sourceCommandId, + BlockSourceAttributionDO sourceAttribution) { + return null; + } + }; + } + } + + static class InMemoryFileApi implements FileApi { + + private final Map files = new ConcurrentHashMap<>(); + + String put(String path, byte[] content) { + files.put(path, content); + return path; + } + + @Override + public CommonResult createFile(FileCreateReqDTO createReqDTO) { + return createFileReturningPath(createReqDTO); + } + + @Override + public CommonResult createFileReturningPath(FileCreateReqDTO createReqDTO) { + String path = firstPresent(createReqDTO.getDirectory(), "p1r") + "/" + firstPresent(createReqDTO.getName(), "file.bin"); + files.put(path, createReqDTO.getContent()); + return CommonResult.success(path); + } + + @Override + public CommonResult getFileContentByPath(String path) { + byte[] content = files.get(path); + if (content == null) { + throw new IllegalArgumentException("测试文件不存在"); + } + return CommonResult.success(content); + } + + @Override + public CommonResult presignGetUrl(String url, Integer expirationSeconds) { + return CommonResult.success(url); + } + + private String firstPresent(String value, String fallback) { + return value == null || value.isBlank() ? fallback : value; + } + } +} diff --git a/muse-cloud/sql/muse/V34__create_ai_import_parse_tables.sql b/muse-cloud/sql/muse/V34__create_ai_import_parse_tables.sql new file mode 100644 index 00000000..6352f400 --- /dev/null +++ b/muse-cloud/sql/muse/V34__create_ai_import_parse_tables.sql @@ -0,0 +1,86 @@ +-- RC 后补功能：AI 导入解析 owner 表。 +-- 说明：Parse Job 与章节解析 Shadow 结果归 AI owner；Content 只通过 -api 端口消费投影。 + +CREATE TABLE muse_ai_parse_job ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + work_id BIGINT NOT NULL, + import_task_id BIGINT NOT NULL, + owner_user_id BIGINT NOT NULL, + actor_user_id BIGINT NOT NULL, + command_id VARCHAR(128) NOT NULL, + file_name VARCHAR(255), + file_size BIGINT, + file_hash VARCHAR(160), + format VARCHAR(30), + storage_ref VARCHAR(512) NOT NULL, + parse_config JSONB NOT NULL DEFAULT '{}'::jsonb, + parse_config_version INT NOT NULL DEFAULT 1, + status VARCHAR(32) NOT NULL DEFAULT 'queued', + progress INT NOT NULL DEFAULT 0, + total_chapters INT NOT NULL DEFAULT 0, + confirmed_chapters INT NOT NULL DEFAULT 0, + pending_chapters INT NOT NULL DEFAULT 0, + retry_count INT NOT NULL DEFAULT 0, + retryable BOOLEAN NOT NULL DEFAULT FALSE, + next_retry_at TIMESTAMP, + failed_stage VARCHAR(64), + error_code VARCHAR(64), + error_message TEXT, + source_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb, + result_summary JSONB NOT NULL DEFAULT '{}'::jsonb, + started_at TIMESTAMP, + completed_at TIMESTAMP, + creator VARCHAR(64) NOT NULL DEFAULT '', + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updater VARCHAR(64) NOT NULL DEFAULT '', + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted BOOLEAN NOT NULL DEFAULT FALSE, + tenant_id BIGINT NOT NULL DEFAULT 0, + CONSTRAINT uk_muse_ai_parse_job_command UNIQUE (tenant_id, command_id), + CONSTRAINT uk_muse_ai_parse_job_import UNIQUE (tenant_id, import_task_id, deleted), + CONSTRAINT chk_muse_ai_parse_job_status CHECK (status IN ('queued', 'processing', 'completed', 'failed')) +); + +CREATE INDEX idx_muse_ai_parse_job_owner_status + ON muse_ai_parse_job(tenant_id, owner_user_id, status, create_time); +CREATE INDEX idx_muse_ai_parse_job_work_status + ON muse_ai_parse_job(tenant_id, work_id, status, create_time); +CREATE TRIGGER trg_muse_ai_parse_job_updated_at + BEFORE UPDATE ON muse_ai_parse_job + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TABLE muse_ai_chapter_parse_result ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + parse_job_id BIGINT NOT NULL, + work_id BIGINT NOT NULL, + owner_user_id BIGINT NOT NULL, + title VARCHAR(255) NOT NULL, + sort_order INT NOT NULL, + content_preview TEXT, + content_text TEXT NOT NULL, + word_count INT NOT NULL DEFAULT 0, + status VARCHAR(32) NOT NULL DEFAULT 'pending_review', + revision INT NOT NULL DEFAULT 1, + target_draft_id BIGINT, + quality_result JSONB NOT NULL DEFAULT '{}'::jsonb, + source_snapshot JSONB NOT NULL DEFAULT '{}'::jsonb, + reviewed_at TIMESTAMP, + review_command_id VARCHAR(128), + rejection_reason TEXT, + creator VARCHAR(64) NOT NULL DEFAULT '', + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updater VARCHAR(64) NOT NULL DEFAULT '', + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted BOOLEAN NOT NULL DEFAULT FALSE, + tenant_id BIGINT NOT NULL DEFAULT 0, + CONSTRAINT uk_muse_ai_chapter_parse_job_order UNIQUE (tenant_id, parse_job_id, sort_order), + CONSTRAINT chk_muse_ai_chapter_parse_status CHECK (status IN ('pending_review', 'confirmed', 'rejected')) +); + +CREATE INDEX idx_muse_ai_chapter_parse_job + ON muse_ai_chapter_parse_result(tenant_id, parse_job_id, sort_order); +CREATE INDEX idx_muse_ai_chapter_parse_owner_status + ON muse_ai_chapter_parse_result(tenant_id, owner_user_id, status, create_time); +CREATE TRIGGER trg_muse_ai_chapter_parse_updated_at + BEFORE UPDATE ON muse_ai_chapter_parse_result + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); diff --git a/muse-studio/e2e/export-download.spec.ts b/muse-studio/e2e/export-download.spec.ts new file mode 100644 index 00000000..4f71967d --- /dev/null +++ b/muse-studio/e2e/export-download.spec.ts @@ -0,0 +1,212 @@ +import { expect, test, type Page } from '@playwright/test'; +import { readFileSync } from 'node:fs'; +import { Client } from 'pg'; + +/** + * 作品导出下载端到端测试（真实浏览器 + 真后端 + 真 FileService + 真 PostgreSQL）。 + * + * 旅程：工作台点击「导出」→ 创建 TXT 导出任务 → 后端经 FileApi 落包并返回下载凭证 → + * 浏览器点击「下载导出包」→ 真 GET /downloads/{credentialId} 获取文件流。 + * + * 反假绿锚点： + * 1. 不使用 MSW，不拦截任何业务接口； + * 2. 断言 POST /export-tasks 与 GET /downloads 都为真实 2xx 响应； + * 3. 直连 PG 反查导出任务 completed、packageRef/credentialId 非空； + * 4. 文件内容必须包含真实作品标题和正文片段，证明不是空文件或 mock 字节。 + */ +const TOKEN = 'test1'; +const WORK_ID = 1; + +test.setTimeout(90_000); + +interface CommonResult { + code: number; + msg?: string; + data: T; +} + +interface ExportTaskData { + id: string | number; + status: string; + progress?: number; + downloadCredentialId?: string; +} + +interface ExportEvidence { + status: string; + exportFormat: string; + packageRef: string | null; + credentialId: string | null; + progress: number; +} + +interface ExportExpectation { + title: string; + contentFragment: string; +} + +function createPgClient(): Client { + return new Client({ + host: process.env.MUSE_POSTGRES_HOST, + port: Number(process.env.MUSE_POSTGRES_PORT ?? '5433'), + database: process.env.MUSE_POSTGRES_DATABASE ?? 'muse_slice_live', + user: process.env.MUSE_POSTGRES_USERNAME ?? 'root', + password: process.env.MUSE_POSTGRES_PASSWORD, + ssl: false, + connectionTimeoutMillis: 10_000, + }); +} + +async function seedToken(page: Page): Promise { + await page.addInitScript((t) => { + window.localStorage.setItem('accessToken', t); + window.localStorage.setItem('tenantId', '1'); + }, TOKEN); +} + +async function readExportEvidence(taskId: number): Promise { + const c = createPgClient(); + await c.connect(); + try { + const r = await c.query( + `SELECT status, export_format, package_ref, credential_id, progress + FROM muse_content_export_task + WHERE tenant_id=1 AND id=$1 AND work_id=$2 AND owner_user_id=1 AND deleted=false`, + [taskId, WORK_ID] + ); + if (r.rowCount === 0) { + throw new Error(`export task id=${taskId} 不存在`); + } + const row = r.rows[0]; + return { + status: String(row.status), + exportFormat: String(row.export_format), + packageRef: row.package_ref == null ? null : String(row.package_ref), + credentialId: row.credential_id == null ? null : String(row.credential_id), + progress: Number(row.progress), + }; + } finally { + await c.end(); + } +} + +/** 读取当前作品真实标题与正文片段，避免 e2e 因共享库内容漂移而使用硬编码断言。 */ +async function readExportExpectation(): Promise { + const c = createPgClient(); + await c.connect(); + try { + const r = await c.query( + `SELECT w.title, b.content_text + FROM muse_content_work w + JOIN muse_content_chapter ch ON ch.tenant_id=w.tenant_id AND ch.work_id=w.id AND ch.deleted=false + JOIN muse_content_block b ON b.tenant_id=w.tenant_id AND b.chapter_id=ch.id AND b.deleted=false + WHERE w.tenant_id=1 AND w.id=$1 AND w.deleted=false AND b.content_text IS NOT NULL AND b.content_text <> '' + ORDER BY ch.order_no ASC, b.order_no ASC + LIMIT 1`, + [WORK_ID] + ); + if (r.rowCount === 0) { + throw new Error(`work id=${WORK_ID} 缺少可导出的正文种子`); + } + const title = String(r.rows[0].title ?? ''); + const contentText = String(r.rows[0].content_text ?? ''); + return { + title, + contentFragment: contentText.slice(0, Math.min(8, contentText.length)), + }; + } finally { + await c.end(); + } +} + +async function cleanupExportRows(taskId: number | null): Promise { + if (taskId == null) return; + const c = createPgClient(); + await c.connect(); + try { + const r = await c.query( + `SELECT command_id, credential_id, package_ref + FROM muse_content_export_task + WHERE tenant_id=1 AND id=$1`, + [taskId] + ); + const commandIds = r.rows + .flatMap((row) => [row.command_id, row.credential_id ? `download:${row.credential_id}` : null]) + .filter((id): id is string => typeof id === 'string' && id.length > 0); + await c.query('DELETE FROM muse_content_export_task WHERE tenant_id=1 AND id=$1', [taskId]); + const packageRefs = r.rows + .map((row) => row.package_ref) + .filter((ref): ref is string => typeof ref === 'string' && ref.length > 0); + if (packageRefs.length > 0) { + await c.query('DELETE FROM infra_file WHERE path = ANY($1::varchar[])', [packageRefs]); + } + if (commandIds.length > 0) { + await c.query('DELETE FROM muse_content_command_log WHERE tenant_id=1 AND command_id = ANY($1::varchar[])', [ + commandIds, + ]); + } + } finally { + await c.end(); + } +} + +test.describe('作品导出下载(真实后端, MSW off)', () => { + test('创建 TXT 导出任务 → 下载导出包字节', async ({ page }) => { + let taskId: number | null = null; + try { + const expectation = await readExportExpectation(); + await seedToken(page); + await page.goto(`/workspace/${WORK_ID}`); + + await page.getByRole('button', { name: '导出', exact: true }).click(); + await expect(page.getByRole('heading', { name: '导出作品' })).toBeVisible({ timeout: 15_000 }); + await expect(page.getByLabel('格式')).toHaveValue('txt'); + + const createResponsePromise = page.waitForResponse( + (resp) => /\/app-api\/muse\/works\/1\/export-tasks$/.test(resp.url()) && resp.request().method() === 'POST', + { timeout: 30_000 } + ); + await page.getByRole('button', { name: '创建导出任务' }).click(); + const createResponse = await createResponsePromise; + expect(createResponse.status()).toBe(200); + const createBody = (await createResponse.json()) as CommonResult; + expect(createBody.code, `创建导出任务响应: ${JSON.stringify(createBody).slice(0, 300)}`).toBe(0); + taskId = Number(createBody.data.id); + expect(taskId).toBeGreaterThan(0); + expect(createBody.data.status).toBe('completed'); + expect(createBody.data.downloadCredentialId).toBeTruthy(); + + await expect(page.getByText(new RegExp(`导出任务 ${taskId}：completed`))).toBeVisible({ timeout: 15_000 }); + const evidence = await readExportEvidence(taskId); + expect(evidence.status).toBe('completed'); + expect(evidence.exportFormat).toBe('txt'); + expect(evidence.progress).toBe(100); + expect(evidence.packageRef).toBeTruthy(); + expect(evidence.credentialId).toBe(createBody.data.downloadCredentialId); + + const downloadResponsePromise = page.waitForResponse( + (resp) => /\/app-api\/muse\/downloads\/[A-Za-z0-9-]+$/.test(resp.url()) && resp.request().method() === 'GET', + { timeout: 30_000 } + ); + const browserDownloadPromise = page.waitForEvent('download', { timeout: 30_000 }); + await page.getByRole('button', { name: '下载导出包' }).click(); + const downloadResponse = await downloadResponsePromise; + const browserDownload = await browserDownloadPromise; + const downloadRequestHeaders = downloadResponse.request().headers(); + expect(downloadRequestHeaders.authorization, '下载文件流必须携带登录态，防止凭证下载绕过 owner 校验').toBe( + `Bearer ${TOKEN}` + ); + expect(downloadResponse.status()).toBe(200); + expect(downloadResponse.headers()['content-type']).toContain('text/plain'); + expect(browserDownload.suggestedFilename()).toMatch(/\.txt$/); + const downloadedPath = await browserDownload.path(); + expect(downloadedPath).toBeTruthy(); + const exportedText = readFileSync(downloadedPath!, 'utf8'); + expect(exportedText.length).toBeGreaterThan(0); + expect(exportedText).toContain(expectation.title); + expect(exportedText).toContain(expectation.contentFragment); + } finally { + await cleanupExportRows(taskId); + } + }); +}); diff --git a/muse-studio/e2e/import-wizard.spec.ts b/muse-studio/e2e/import-wizard.spec.ts new file mode 100644 index 00000000..73e98641 --- /dev/null +++ b/muse-studio/e2e/import-wizard.spec.ts @@ -0,0 +1,280 @@ +import { expect, test, type Page } from '@playwright/test'; +import { Client } from 'pg'; + +/** + * 完整导入向导端到端测试（真实浏览器 + 真后端 + 真 New-API + 真 PostgreSQL）。 + * + * 旅程：工作台点击「导入」→ 上传 txt 文件 → Content 创建外部上传导入任务 → + * AI owner 以 full_book/llm_full_book 创建 Parse Job 并调用 New-API 深解析 → + * 浏览器读取 pending-review 章节 → 批量确认 → Knowledge owner 创建 pending Draft。 + * + * 反假绿锚点： + * 1. Playwright 配置强制 VITE_API_MOCK=false，所有请求经 Vite 代理打到 48080； + * 2. 本 spec 不 page.route 任何业务接口，解析依赖真实 New-API； + * 3. UI 完成后直连 PG 反查 parse job、chapter parse result、knowledge draft； + * 4. 断言 Canonical entity 计数不变，证明章节确认只落 Draft，不绕过“先审后入”。 + */ +const TOKEN = 'test1'; +const WORK_ID = 1; + +test.setTimeout(180_000); + +interface ImportResponseBody { + code: number; + msg?: string; + data: T; +} + +interface ImportTaskData { + id: string | number; +} + +interface ParseJobData { + id: string | number; +} + +interface ConfirmResultData { + createdDraftIds?: Array; +} + +interface ImportEvidence { + jobId: number; + status: string; + totalChapters: number; + pendingChapters: number; + confirmedChapters: number; + parser: string | null; + confirmedResultCount: number; + draftIds: number[]; + draftCount: number; + entityCountBefore: number; + entityCountAfter: number; +} + +/** 创建直连 muse_slice_live 的 PG 客户端；凭据只从环境变量读，绝不写入仓库。 */ +function createPgClient(): Client { + return new Client({ + host: process.env.MUSE_POSTGRES_HOST, + port: Number(process.env.MUSE_POSTGRES_PORT ?? '5433'), + database: process.env.MUSE_POSTGRES_DATABASE ?? 'muse_slice_live', + user: process.env.MUSE_POSTGRES_USERNAME ?? 'root', + password: process.env.MUSE_POSTGRES_PASSWORD, + ssl: false, + connectionTimeoutMillis: 10_000, + }); +} + +/** 注入真实后端 mock 登录令牌；必须在首屏脚本前写入。 */ +async function seedToken(page: Page): Promise { + await page.addInitScript((t) => { + window.localStorage.setItem('accessToken', t); + window.localStorage.setItem('tenantId', '1'); + }, TOKEN); +} + +/** 读取 work 维度 Canonical entity 数量，确认导入只建 Draft，不直接入正式知识。 */ +async function readEntityCount(): Promise { + const c = createPgClient(); + await c.connect(); + try { + const r = await c.query( + `SELECT COUNT(*)::int AS count + FROM muse_knowledge_entity + WHERE tenant_id=1 AND work_id=$1 AND deleted=false`, + [WORK_ID] + ); + return Number(r.rows[0].count); + } finally { + await c.end(); + } +} + +/** + * 用 DB 反查导入链路的最终事实。 + * WHY：UI 文案只能证明页面状态变化，不能证明 owner 表、Shadow 结果和 Draft 都按合同落库。 + */ +async function readImportEvidence(jobId: number, entityCountBefore: number): Promise { + const c = createPgClient(); + await c.connect(); + try { + const job = await c.query( + `SELECT status, + total_chapters, + pending_chapters, + confirmed_chapters, + result_summary->>'parser' AS parser + FROM muse_ai_parse_job + WHERE tenant_id=1 AND id=$1 AND work_id=$2 AND deleted=false`, + [jobId, WORK_ID] + ); + if (job.rowCount === 0) { + throw new Error(`parse job id=${jobId} 不存在`); + } + + const result = await c.query( + `SELECT COUNT(*)::int AS confirmed_count, + ARRAY_REMOVE(ARRAY_AGG(target_draft_id ORDER BY sort_order), NULL) AS draft_ids + FROM muse_ai_chapter_parse_result + WHERE tenant_id=1 AND parse_job_id=$1 AND status='confirmed' AND deleted=false`, + [jobId] + ); + const draftIds = (result.rows[0].draft_ids ?? []).map((id: string | number) => Number(id)); + const drafts = await c.query( + `SELECT COUNT(*)::int AS draft_count + FROM muse_knowledge_draft + WHERE tenant_id=1 + AND id = ANY($1::bigint[]) + AND work_id=$2 + AND status='pending' + AND source_type='extraction' + AND deleted=false`, + [draftIds, WORK_ID] + ); + const entityCountAfter = await readEntityCount(); + const row = job.rows[0]; + return { + jobId, + status: String(row.status), + totalChapters: Number(row.total_chapters), + pendingChapters: Number(row.pending_chapters), + confirmedChapters: Number(row.confirmed_chapters), + parser: row.parser == null ? null : String(row.parser), + confirmedResultCount: Number(result.rows[0].confirmed_count), + draftIds, + draftCount: Number(drafts.rows[0].draft_count), + entityCountBefore, + entityCountAfter, + }; + } finally { + await c.end(); + } +} + +/** 清理本用例产生的测试数据，避免共享 live 库被重复运行持续污染。 */ +async function cleanupImportRows(jobId: number | null, importTaskId: number | null, draftIds: number[]): Promise { + const c = createPgClient(); + await c.connect(); + try { + let allDraftIds = draftIds; + if (jobId != null) { + const r = await c.query( + `SELECT ARRAY_REMOVE(ARRAY_AGG(target_draft_id), NULL) AS draft_ids + FROM muse_ai_chapter_parse_result + WHERE tenant_id=1 AND parse_job_id=$1`, + [jobId] + ); + allDraftIds = Array.from( + new Set([...draftIds, ...((r.rows[0]?.draft_ids ?? []).map((id: string | number) => Number(id)))]) + ); + } + if (allDraftIds.length > 0) { + await c.query('DELETE FROM muse_knowledge_draft WHERE tenant_id=1 AND id = ANY($1::bigint[])', [allDraftIds]); + } + if (jobId != null) { + await c.query('DELETE FROM muse_ai_chapter_parse_result WHERE tenant_id=1 AND parse_job_id=$1', [jobId]); + await c.query('DELETE FROM muse_ai_parse_job WHERE tenant_id=1 AND id=$1', [jobId]); + } + if (importTaskId != null) { + await c.query('DELETE FROM muse_content_import_task WHERE tenant_id=1 AND id=$1', [importTaskId]); + } + } finally { + await c.end(); + } +} + +test.describe('完整导入向导(真实后端, MSW off)', () => { + test('上传文件 → New-API 全书解析 → 章节确认 → Knowledge Draft', async ({ page }) => { + let importTaskId: number | null = null; + let parseJobId: number | null = null; + let createdDraftIds: number[] = []; + const entityCountBefore = await readEntityCount(); + + try { + await seedToken(page); + await page.goto(`/workspace/${WORK_ID}`); + + await page.getByRole('button', { name: '导入', exact: true }).click(); + await expect(page.getByRole('heading', { name: '导入作品正文' })).toBeVisible({ timeout: 15_000 }); + await expect(page.getByRole('button', { name: '完整导入向导' })).toBeVisible(); + + const fileName = `e2e-import-${Date.now()}.txt`; + const importText = [ + '第一章 星门', + '林岚在黎明前抵达观测站，记录下第一束异常光。', + '', + '第二章 归航', + '船队收到旧信标的回声，决定沿着暗潮返航。', + ].join('\n'); + await page.locator('input[type="file"]').first().setInputFiles({ + name: fileName, + mimeType: 'text/plain', + buffer: Buffer.from(importText, 'utf8'), + }); + await expect(page.getByText(fileName)).toBeVisible(); + + const uploadResponsePromise = page.waitForResponse( + (resp) => /\/app-api\/infra\/file\/upload-returning-path$/.test(resp.url()) && resp.request().method() === 'POST', + { timeout: 30_000 } + ); + const importResponsePromise = page.waitForResponse( + (resp) => /\/app-api\/muse\/works\/1\/import-tasks$/.test(resp.url()) && resp.request().method() === 'POST', + { timeout: 30_000 } + ); + await page.getByRole('button', { name: '上传并扫描' }).click(); + const uploadResponse = await uploadResponsePromise; + expect(uploadResponse.status()).toBe(200); + const importResponse = await importResponsePromise; + expect(importResponse.status()).toBe(200); + const importBody = (await importResponse.json()) as ImportResponseBody; + expect(importBody.code, `创建导入任务响应: ${JSON.stringify(importBody).slice(0, 300)}`).toBe(0); + importTaskId = Number(importBody.data.id); + expect(importTaskId).toBeGreaterThan(0); + await expect(page.getByText(/上传扫描已通过/)).toBeVisible({ timeout: 15_000 }); + + const parseResponsePromise = page.waitForResponse( + (resp) => /\/app-api\/muse\/works\/1\/parse-jobs$/.test(resp.url()) && resp.request().method() === 'POST', + { timeout: 90_000 } + ); + await page.getByRole('button', { name: '开始 AI 全书解析' }).click(); + const parseResponse = await parseResponsePromise; + expect(parseResponse.status()).toBe(200); + const parseBody = (await parseResponse.json()) as ImportResponseBody; + expect(parseBody.code, `创建解析任务响应: ${JSON.stringify(parseBody).slice(0, 300)}`).toBe(0); + parseJobId = Number(parseBody.data.id); + expect(parseJobId).toBeGreaterThan(0); + + await expect(page.getByText(/状态 completed，章节 2，待确认 2/)).toBeVisible({ timeout: 30_000 }); + await expect(page.getByText(/星门/)).toBeVisible(); + await expect(page.getByText(/归航/)).toBeVisible(); + + await page.getByRole('button', { name: '全选待确认' }).click(); + const confirmResponsePromise = page.waitForResponse( + (resp) => /\/app-api\/muse\/parse-jobs\/\d+\/chapters\/batch-confirm$/.test(resp.url()) + && resp.request().method() === 'POST', + { timeout: 30_000 } + ); + await page.getByRole('button', { name: /确认 2 个章节/ }).click(); + const confirmResponse = await confirmResponsePromise; + expect(confirmResponse.status()).toBe(200); + const confirmBody = (await confirmResponse.json()) as ImportResponseBody; + expect(confirmBody.code, `章节确认响应: ${JSON.stringify(confirmBody).slice(0, 300)}`).toBe(0); + createdDraftIds = (confirmBody.data.createdDraftIds ?? []).map((id) => Number(id)); + expect(createdDraftIds).toHaveLength(2); + + await expect(page.getByText('章节已确认，Knowledge Draft 已创建')).toBeVisible({ timeout: 15_000 }); + + const evidence = await readImportEvidence(parseJobId, entityCountBefore); + expect(evidence.status).toBe('completed'); + expect(evidence.totalChapters).toBe(2); + expect(evidence.pendingChapters).toBe(0); + expect(evidence.confirmedChapters).toBe(2); + expect(evidence.parser).toBe('new_api_import_llm'); + expect(evidence.confirmedResultCount).toBe(2); + expect(evidence.draftIds.sort()).toEqual(createdDraftIds.sort()); + expect(evidence.draftCount).toBe(2); + expect(evidence.entityCountAfter).toBe(evidence.entityCountBefore); + } finally { + await cleanupImportRows(parseJobId, importTaskId, createdDraftIds); + } + }); +}); diff --git a/muse-studio/src/api/client.ts b/muse-studio/src/api/client.ts index 10b1a79f..e201c9f1 100644 --- a/muse-studio/src/api/client.ts +++ b/muse-studio/src/api/client.ts @@ -4,6 +4,11 @@ import { readAccessToken, readTenantId } from '@/api/auth'; // 后端 API 前缀路径 const BASE_URL = '/app-api/muse'; +/** 解析 API 路径：业务域默认走 Muse 前缀；少数平台能力（如 infra 文件上传）可显式传入 /app-api/...。 */ +function resolveApiUrl(path: string): string { + return path.startsWith('/app-api/') ? path : `${BASE_URL}${path}`; +} + /** * 包装后端 OpenAPI 的 CommonResult 结构，引入泛型参数映射真实数据 */ @@ -26,6 +31,12 @@ export class ApiError extends Error { } } +export interface BinaryDownloadResponse { + blob: Blob; + fileName: string; + contentType: string; +} + /** * 通用 Fetch 请求包装函数 * 自动拦截非 0 业务状态码并抛出 ApiError，添加统一版本号和 Content-Type 头部 @@ -43,14 +54,8 @@ function isFormData(data: unknown): data is FormData { ); } -/** - * 通用 Fetch 请求包装函数 - * 自动拦截非 0 业务状态码并抛出 ApiError，添加统一版本号和 Content-Type 头部 - */ -export async function request( - path: string, - options: RequestInit = {} -): Promise { +/** 构造 Muse API 统一请求头，保证 JSON 与二进制下载走同一套鉴权、租户和版本门禁。 */ +function createApiHeaders(hasJsonBody = true): Record { const headers: Record = { 'X-API-Version': '1', // 注入租户标识：后端按 `tenant-id` 头解析租户上下文,写命令及租户隔离的读必须携带, @@ -64,17 +69,48 @@ export async function request( if (accessToken) { headers['Authorization'] = `Bearer ${accessToken}`; } - - // 仅在 body 不是 FormData 实例时，默认追加 application/json 请求头 - if (!isFormData(options.body)) { + if (hasJsonBody) { headers['Content-Type'] = 'application/json'; } + return headers; +} + +/** 从 Content-Disposition 中解析后端返回的下载文件名，兼容 Spring 的 filename 与 filename* 两种格式。 */ +function parseDownloadFileName(contentDisposition: string | null): string | null { + if (!contentDisposition) { + return null; + } + const encodedMatch = /filename\*=UTF-8''([^;]+)/i.exec(contentDisposition); + if (encodedMatch?.[1]) { + try { + return decodeURIComponent(encodedMatch[1].trim().replace(/^"|"$/g, '')); + } catch { + return encodedMatch[1].trim().replace(/^"|"$/g, ''); + } + } + const quotedMatch = /filename="([^"]+)"/i.exec(contentDisposition); + if (quotedMatch?.[1]) { + return quotedMatch[1].trim(); + } + const plainMatch = /filename=([^;]+)/i.exec(contentDisposition); + return plainMatch?.[1]?.trim().replace(/^"|"$/g, '') ?? null; +} + +/** + * 通用 Fetch 请求包装函数 + * 自动拦截非 0 业务状态码并抛出 ApiError，添加统一版本号和 Content-Type 头部 + */ +export async function request( + path: string, + options: RequestInit = {} +): Promise { + const headers = createApiHeaders(!isFormData(options.body)); // 禁用浏览器 HTTP 缓存:vite dev proxy 未把后端的 Cache-Control: no-store 透传到浏览器, // 浏览器遂按默认启发式缓存 GET。这会让写命令后的 refetch 命中旧缓存(如提交发布后刷新 // 「我的发布记录」拿到不含刚提交记录的旧列表)。数据新鲜度统一由 React Query 应用层管理, // 故强制 no-store 关闭浏览器缓存。 - const response = await fetch(`${BASE_URL}${path}`, { + const response = await fetch(resolveApiUrl(path), { ...options, cache: 'no-store', headers: { @@ -114,6 +150,39 @@ export async function request( return body.data; } +/** 下载二进制文件流；JSON 业务错误仍按 CommonResult 解析，防止把 fail-closed 错误页当成空文件。 */ +export async function downloadBinary(path: string): Promise { + const response = await fetch(resolveApiUrl(path), { + method: 'GET', + cache: 'no-store', + headers: createApiHeaders(false), + }); + + const contentType = response.headers.get('content-type') ?? 'application/octet-stream'; + if (contentType.includes('application/json')) { + let body: CommonResult; + try { + body = await response.json(); + } catch { + throw new ApiError('JSON_PARSE_ERROR', '反序列化服务器 JSON 数据失败', response.status); + } + if (body.code !== 0) { + throw new ApiError(String(body.code), body.msg, response.status); + } + throw new ApiError('UNEXPECTED_JSON_RESPONSE', '下载接口未返回文件流', response.status); + } + if (!response.ok) { + throw new ApiError(String(response.status), `服务器网络异常，状态码: ${response.status}`, response.status); + } + + const blob = await response.blob(); + return { + blob, + fileName: parseDownloadFileName(response.headers.get('content-disposition')) ?? 'export.bin', + contentType, + }; +} + /** * 封装后的简易 HTTP 动作映射对象 * 遵循 exactOptionalPropertyTypes: true，不传显式的 undefined 字段 @@ -173,4 +242,7 @@ export const api = { } return request(path, options); }, + + /** 二进制下载请求 */ + download: downloadBinary, }; diff --git a/muse-studio/src/api/mocks/handlers/content.ts b/muse-studio/src/api/mocks/handlers/content.ts index 8f70d487..1c1893f7 100644 --- a/muse-studio/src/api/mocks/handlers/content.ts +++ b/muse-studio/src/api/mocks/handlers/content.ts @@ -52,9 +52,95 @@ interface ImportTaskRequest { fileName?: string; fileSize?: number; format?: 'txt' | 'markdown' | 'epub' | 'docx'; + uploadUrl?: string; contentText?: string; } +interface ParseJobRequest { + importTaskId?: number | string; + parseConfig?: { + mode?: string; + strategy?: string; + }; +} + +interface RetryParseJobRequest { + commandId?: string; + retryStage?: string; +} + +interface BatchConfirmChaptersRequest { + resultIds?: Array; + expectedRevisions?: Record; +} + +interface ExportTaskRequest { + format?: 'txt' | 'epub' | 'docx'; + includeChapters?: string[]; + includeMetadata?: boolean; + includePlanning?: boolean; +} + +interface MockExportTask { + id: string; + workId: string; + format: 'txt' | 'epub' | 'docx'; + status: 'completed'; + progress: number; + downloadCredentialId: string; + downloadExpiresAt: string; + includeChapters: string[]; + includeMetadata: boolean; + includePlanning: boolean; + createdAt: string; + completedAt: string; +} + +interface MockImportTask { + id: string; + workId: string; + fileName: string; + fileSize: number; + format: 'txt' | 'markdown' | 'epub' | 'docx'; + storageRef?: string; + contentText?: string; + status: 'completed' | 'failed'; + scanStatus: 'passed' | 'blocked' | 'failed'; + createdAt: string; + completedAt: string; +} + +interface MockParseChapter { + id: string; + jobId: string; + workId: string; + title: string; + sortOrder: number; + contentPreview: string; + contentText: string; + wordCount: number; + status: 'pending_review' | 'confirmed' | 'rejected'; + revision: number; + qualityResult: Record; +} + +interface MockParseJob { + id: string; + workId: string; + importTaskId: string; + status: 'completed' | 'failed' | 'processing'; + progress: number; + totalChapters: number; + confirmedChapters: number; + pendingChapters: number; + errorMessage?: string; + failedStage?: string; + retryable?: boolean; + nextActions: string[]; + createdAt: string; + completedAt?: string; +} + interface MockBlockRevision { blockId: string; revision: number; @@ -266,6 +352,11 @@ const INITIAL_CHAPTERS: MockChapter[] = [ let mockWorks = JSON.parse(JSON.stringify(INITIAL_WORKS)) as MockWork[]; let mockChapters = JSON.parse(JSON.stringify(INITIAL_CHAPTERS)) as MockChapter[]; let mockBlockRevisions: Record = {}; +let mockExportTasks: Record = {}; +let mockUploadedFiles: Record = {}; +let mockImportTasks: Record = {}; +let mockParseJobs: Record = {}; +let mockParseChapters: Record = {}; /** B5 planning section mock 类型 */ interface MockPlanningSection { @@ -302,6 +393,11 @@ export function resetMockDb(): void { mockWorks = JSON.parse(JSON.stringify(INITIAL_WORKS)); mockChapters = JSON.parse(JSON.stringify(INITIAL_CHAPTERS)); mockBlockRevisions = {}; + mockExportTasks = {}; + mockUploadedFiles = {}; + mockImportTasks = {}; + mockParseJobs = {}; + mockParseChapters = {}; mockPlanningSections = {}; } @@ -309,6 +405,27 @@ export function resetMockDb(): void { * 作品与内容管理 API 的 MSW Mock 处理器 */ export const contentHandlers = [ + // POST /app-api/infra/file/upload-returning-path — 导入向导上传文件并取得稳定 path + http.post('/app-api/infra/file/upload-returning-path', async ({ request }) => { + const formData = await request.formData(); + const file = formData.get('file'); + const directory = String(formData.get('directory') ?? 'content/import/mock'); + if (typeof file === 'string') { + const safeName = String(formData.get('fileName') ?? 'upload.txt').replace(/[\\/]/g, '-'); + const path = `${directory}/${Date.now()}-${safeName}`; + mockUploadedFiles[path] = file; + return HttpResponse.json({ code: 0, msg: 'success', data: path }); + } + const uploadFile = file as { name?: string; text?: () => Promise } | null; + if (!uploadFile || typeof uploadFile.text !== 'function') { + return HttpResponse.json({ code: 40001, msg: '文件不能为空', data: null }, { status: 400 }); + } + const safeName = (uploadFile.name ?? 'upload.txt').replace(/[\\/]/g, '-'); + const path = `${directory}/${Date.now()}-${safeName}`; + mockUploadedFiles[path] = await uploadFile.text(); + return HttpResponse.json({ code: 0, msg: 'success', data: path }); + }), + // ===== B5 planning 编辑器：planning 结构 + meta-projection 字段 ===== // GET planning 结构（已确认的 sections） http.get('/app-api/muse/works/:workId/planning', ({ params }) => { @@ -751,7 +868,7 @@ export const contentHandlers = [ }); }), - // POST /app-api/muse/works/:workId/import-tasks — 创建本地文本导入任务并初始化正文 + // POST /app-api/muse/works/:workId/import-tasks — 创建本地文本导入任务或上传扫描任务 http.post('/app-api/muse/works/:workId/import-tasks', async ({ params, request }) => { const { workId } = params; const body = (await request.json()) as ImportTaskRequest; @@ -759,6 +876,44 @@ export const contentHandlers = [ if (!work) { return HttpResponse.json({ code: 40401, msg: '作品不存在', data: null }, { status: 404 }); } + if (body.uploadUrl) { + const contentText = mockUploadedFiles[body.uploadUrl]; + if (!contentText) { + return HttpResponse.json({ code: 1041001005, msg: '导入文件不可读', data: null }, { status: 400 }); + } + const id = String(Date.now()); + const now = new Date().toISOString(); + mockImportTasks[id] = { + id, + workId: String(workId), + fileName: body.fileName ?? 'workspace-import.md', + fileSize: body.fileSize ?? contentText.length, + format: body.format ?? 'markdown', + storageRef: body.uploadUrl, + contentText, + status: 'completed', + scanStatus: 'passed', + createdAt: now, + completedAt: now, + }; + return HttpResponse.json({ + code: 0, + msg: 'success', + data: { + id, + workId, + fileName: body.fileName ?? 'workspace-import.md', + fileSize: body.fileSize ?? contentText.length, + format: body.format ?? 'markdown', + status: 'completed', + scanStatus: 'passed', + progress: 100, + nextActions: ['create_parse_job'], + createdAt: now, + completedAt: now, + }, + }); + } const contentText = body.contentText?.trim(); if (!contentText) { return HttpResponse.json({ code: 1041001000, msg: 'contentText 不能为空', data: null }, { status: 400 }); @@ -800,24 +955,211 @@ export const contentHandlers = [ }); }), - // POST /app-api/muse/works/:workId/export-tasks — 创建最小导出任务 - http.post('/app-api/muse/works/:workId/export-tasks', async ({ params, request }) => { + // POST /app-api/muse/works/:workId/parse-jobs — 创建 AI 全书解析任务 + http.post('/app-api/muse/works/:workId/parse-jobs', async ({ params, request }) => { const { workId } = params; - const body = (await request.json()) as { format?: 'txt' | 'epub' | 'docx' }; - const work = mockWorks.find((item) => item.id === workId); - if (!work) { - return HttpResponse.json({ code: 40401, msg: '作品不存在', data: null }, { status: 404 }); + const body = (await request.json()) as ParseJobRequest; + const importTask = mockImportTasks[String(body.importTaskId ?? '')]; + if (!importTask || importTask.workId !== String(workId)) { + return HttpResponse.json({ code: 40404, msg: '导入任务不存在', data: null }, { status: 404 }); } + const parsed = parseImportedChapters(importTask.fileName, importTask.contentText ?? ''); + const jobId = crypto.randomUUID(); + const now = new Date().toISOString(); + const parseChapters = parsed.map((chapter, index): MockParseChapter => ({ + id: String(Date.now() + index + 1), + jobId, + workId: String(workId), + title: chapter.title, + sortOrder: index + 1, + contentPreview: chapter.content.slice(0, 180), + contentText: chapter.content, + wordCount: chapter.content.length, + status: 'pending_review', + revision: 1, + qualityResult: { passed: true, parser: 'mock-heading-splitter' }, + })); + const job: MockParseJob = { + id: jobId, + workId: String(workId), + importTaskId: importTask.id, + status: 'completed', + progress: 100, + totalChapters: parseChapters.length, + confirmedChapters: 0, + pendingChapters: parseChapters.length, + nextActions: ['review_chapters'], + createdAt: now, + completedAt: now, + }; + mockParseJobs[jobId] = job; + mockParseChapters[jobId] = parseChapters; + return HttpResponse.json({ code: 0, msg: 'success', data: job }); + }), + + // POST /app-api/muse/parse-jobs/:jobId/retry — 失败解析任务重试；mock 保持同一 jobId 并重新生成候选。 + http.post('/app-api/muse/parse-jobs/:jobId/retry', async ({ params, request }) => { + const jobId = String(params.jobId); + const body = (await request.json()) as RetryParseJobRequest; + const job = mockParseJobs[jobId]; + if (!job) { + return HttpResponse.json({ code: 40404, msg: '解析任务不存在', data: null }, { status: 404 }); + } + if (!body.commandId) { + return HttpResponse.json({ code: 1041001000, msg: 'commandId 不能为空', data: null }, { status: 400 }); + } + const now = new Date().toISOString(); + job.status = 'completed'; + job.progress = 100; + job.totalChapters = mockParseChapters[jobId]?.length ?? 0; + job.pendingChapters = mockParseChapters[jobId]?.filter((chapter) => chapter.status === 'pending_review').length ?? 0; + job.confirmedChapters = mockParseChapters[jobId]?.filter((chapter) => chapter.status === 'confirmed').length ?? 0; + job.retryable = false; + delete job.failedStage; + delete job.errorMessage; + job.completedAt = now; return HttpResponse.json({ code: 0, msg: 'success', data: { - id: crypto.randomUUID(), - workId, - format: body.format ?? 'txt', - status: 'queued', - progress: 0, - createdAt: new Date().toISOString(), + jobId, + status: job.status, + pollUrl: `/app-api/muse/parse-jobs/${jobId}`, + }, + }); + }), + + // GET /app-api/muse/parse-jobs/:jobId — 查询解析任务 + http.get('/app-api/muse/parse-jobs/:jobId', ({ params }) => { + const job = mockParseJobs[String(params.jobId)]; + if (!job) { + return HttpResponse.json({ code: 40404, msg: '解析任务不存在', data: null }, { status: 404 }); + } + return HttpResponse.json({ code: 0, msg: 'success', data: job }); + }), + + // GET /app-api/muse/parse-jobs/:jobId/chapters — 查询章节解析结果 + http.get('/app-api/muse/parse-jobs/:jobId/chapters', ({ params }) => { + const parseChapters = mockParseChapters[String(params.jobId)]; + if (!parseChapters) { + return HttpResponse.json({ code: 40404, msg: '章节解析结果不存在', data: null }, { status: 404 }); + } + return HttpResponse.json({ + code: 0, + msg: 'success', + data: parseChapters.map((chapter) => ({ + id: chapter.id, + jobId: chapter.jobId, + workId: chapter.workId, + title: chapter.title, + sortOrder: chapter.sortOrder, + contentPreview: chapter.contentPreview, + wordCount: chapter.wordCount, + status: chapter.status, + revision: chapter.revision, + qualityResult: chapter.qualityResult, + })), + }); + }), + + // POST /app-api/muse/parse-jobs/:jobId/chapters/batch-confirm — 批量确认章节并生成 Knowledge Draft + http.post('/app-api/muse/parse-jobs/:jobId/chapters/batch-confirm', async ({ params, request }) => { + const jobId = String(params.jobId); + const body = (await request.json()) as BatchConfirmChaptersRequest; + const job = mockParseJobs[jobId]; + const parseChapters = mockParseChapters[jobId]; + if (!job || !parseChapters) { + return HttpResponse.json({ code: 40404, msg: '解析任务不存在', data: null }, { status: 404 }); + } + const confirmedIds: string[] = []; + const draftIds: string[] = []; + const failedChapters: Array<{ resultId: string; errorCode: string; errorMessage: string }> = []; + (body.resultIds ?? []).forEach((resultId) => { + const id = String(resultId); + const chapter = parseChapters.find((item) => item.id === id); + if (!chapter || chapter.status !== 'pending_review') { + failedChapters.push({ resultId: id, errorCode: 'INVALID_STATUS', errorMessage: '章节不可确认' }); + return; + } + if (body.expectedRevisions?.[id] !== chapter.revision) { + failedChapters.push({ resultId: id, errorCode: 'REVISION_CONFLICT', errorMessage: '章节 revision 已变化' }); + return; + } + chapter.status = 'confirmed'; + chapter.revision += 1; + confirmedIds.push(id); + draftIds.push(`draft-${id}`); + }); + job.confirmedChapters = parseChapters.filter((chapter) => chapter.status === 'confirmed').length; + job.pendingChapters = parseChapters.filter((chapter) => chapter.status === 'pending_review').length; + return HttpResponse.json({ + code: 0, + msg: 'success', + data: { + confirmedChapterResultIds: confirmedIds, + createdDraftIds: draftIds, + failedChapters, + skippedChapters: [], + partialFailure: failedChapters.length > 0, + }, + }); + }), + + // POST /app-api/muse/works/:workId/export-tasks — 创建可下载的导出任务 + http.post('/app-api/muse/works/:workId/export-tasks', async ({ params, request }) => { + const { workId } = params; + const body = (await request.json()) as ExportTaskRequest; + const work = mockWorks.find((item) => item.id === workId); + if (!work) { + return HttpResponse.json({ code: 40401, msg: '作品不存在', data: null }, { status: 404 }); + } + const id = crypto.randomUUID(); + const credentialId = `mock-export-${id}`; + const now = new Date(); + const task: MockExportTask = { + id, + workId: String(workId), + format: body.format ?? 'txt', + status: 'completed', + progress: 100, + downloadCredentialId: credentialId, + downloadExpiresAt: new Date(now.getTime() + 24 * 60 * 60 * 1000).toISOString(), + includeChapters: body.includeChapters ?? [], + includeMetadata: body.includeMetadata ?? true, + includePlanning: body.includePlanning ?? false, + createdAt: now.toISOString(), + completedAt: now.toISOString(), + }; + mockExportTasks[id] = task; + return HttpResponse.json({ + code: 0, + msg: 'success', + data: task, + }); + }), + + // GET /app-api/muse/export-tasks/:taskId — 查询导出任务详情 + http.get('/app-api/muse/export-tasks/:taskId', ({ params }) => { + const task = mockExportTasks[String(params.taskId)]; + if (!task) { + return HttpResponse.json({ code: 40401, msg: '导出任务不存在', data: null }, { status: 404 }); + } + return HttpResponse.json({ code: 0, msg: 'success', data: task }); + }), + + // GET /app-api/muse/downloads/:credentialId — 下载导出包字节 + http.get('/app-api/muse/downloads/:credentialId', ({ params }) => { + const credentialId = String(params.credentialId); + const task = Object.values(mockExportTasks).find((item) => item.downloadCredentialId === credentialId); + if (!task) { + return HttpResponse.json({ code: 1041001003, msg: '下载凭证无效', data: null }, { status: 400 }); + } + const body = `Mock export ${task.workId} ${task.format} ${task.includeChapters.join(',') || 'all'}`; + return new HttpResponse(body, { + status: 200, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Disposition': `attachment; filename="muse-export.${task.format}"`, }, }); }), diff --git a/muse-studio/src/features/editor/components/ExportWorkModal.test.tsx b/muse-studio/src/features/editor/components/ExportWorkModal.test.tsx new file mode 100644 index 00000000..8a394599 --- /dev/null +++ b/muse-studio/src/features/editor/components/ExportWorkModal.test.tsx @@ -0,0 +1,75 @@ +import { fireEvent, render, screen } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; +import ExportWorkModal from './ExportWorkModal'; +import type { ChapterVO } from '@/types/openapi'; + +const chapters: ChapterVO[] = [ + { + id: '1-1', + workId: '1', + title: '第一章 启程', + sortOrder: 1, + status: 'draft', + wordCount: 15000, + blockCount: 1, + createdAt: '2026-06-27T00:00:00Z', + updatedAt: '2026-06-27T00:00:00Z', + }, + { + id: '1-2', + workId: '1', + title: '第二章 信号', + sortOrder: 2, + status: 'draft', + wordCount: 18000, + blockCount: 1, + createdAt: '2026-06-27T00:00:00Z', + updatedAt: '2026-06-27T00:00:00Z', + }, +]; + +describe('ExportWorkModal', () => { + it('应提交格式、章节范围和附加内容选项', () => { + const onSubmit = vi.fn(); + render( + + ); + + fireEvent.change(screen.getByLabelText('格式'), { target: { value: 'docx' } }); + fireEvent.click(screen.getByRole('button', { name: '指定章节' })); + fireEvent.click(screen.getByRole('checkbox', { name: /第一章 启程/ })); + fireEvent.click(screen.getByLabelText('作品规划')); + fireEvent.click(screen.getByRole('button', { name: '创建导出任务' })); + + expect(onSubmit).toHaveBeenCalledWith({ + format: 'docx', + includeChapters: ['1-1'], + includeMetadata: true, + includePlanning: true, + }); + }); + + it('指定章节但未选择章节时应阻止提交', () => { + const onSubmit = vi.fn(); + render( + + ); + + fireEvent.click(screen.getByRole('button', { name: '指定章节' })); + + expect(screen.getByRole('button', { name: '创建导出任务' })).toBeDisabled(); + expect(onSubmit).not.toHaveBeenCalled(); + }); +}); diff --git a/muse-studio/src/features/editor/components/ExportWorkModal.tsx b/muse-studio/src/features/editor/components/ExportWorkModal.tsx new file mode 100644 index 00000000..45a48d4f --- /dev/null +++ b/muse-studio/src/features/editor/components/ExportWorkModal.tsx @@ -0,0 +1,194 @@ +import { useState } from 'react'; +import type { ChapterVO } from '@/types/openapi'; +import type { CreateExportTaskInput } from '../hooks/useWorks'; + +interface ExportWorkModalProps { + isOpen: boolean; + chapters: ChapterVO[]; + isSubmitting: boolean; + errorMessage?: string; + onClose: () => void; + onSubmit: (input: CreateExportTaskInput) => void; +} + +type ExportScope = 'all' | 'selected'; + +/** 工作台作品导出弹窗：只收集导出命令参数，下载凭证与文件流由 WorkspacePage 统一处理。 */ +export default function ExportWorkModal({ + isOpen, + chapters, + isSubmitting, + errorMessage, + onClose, + onSubmit, +}: ExportWorkModalProps) { + const [format, setFormat] = useState('txt'); + const [scope, setScope] = useState('all'); + const [includeMetadata, setIncludeMetadata] = useState(true); + const [includePlanning, setIncludePlanning] = useState(false); + const [selectedChapterIds, setSelectedChapterIds] = useState([]); + const [localError, setLocalError] = useState(null); + + if (!isOpen) { + return null; + } + + const validChapterIds = new Set(chapters.map((chapter) => chapter.id)); + const scopedChapterIds = selectedChapterIds.filter((chapterId) => validChapterIds.has(chapterId)); + const canSubmit = !isSubmitting && (scope === 'all' || scopedChapterIds.length > 0); + + const toggleChapter = (chapterId: string) => { + setSelectedChapterIds((current) => + current.includes(chapterId) + ? current.filter((item) => item !== chapterId) + : [...current, chapterId] + ); + setLocalError(null); + }; + + const handleSubmit = () => { + if (scope === 'selected' && scopedChapterIds.length === 0) { + setLocalError('请至少选择一个章节'); + return; + } + setLocalError(null); + onSubmit({ + format, + includeChapters: scope === 'all' ? [] : scopedChapterIds, + includeMetadata, + includePlanning, + }); + }; + + return ( +
+
+
+

导出作品

+ +
+ +
+ + +
+
范围
+
+ + +
+
+ + {scope === 'selected' && ( +
+ {chapters.length === 0 ? ( +
当前作品暂无章节
+ ) : ( + chapters.map((chapter) => ( + + )) + )} +
+ )} + +
+ + +
+ + {(localError || errorMessage) && ( +
+ {localError || errorMessage} +
+ )} +
+ +
+ + +
+
+
+ ); +} diff --git a/muse-studio/src/features/editor/components/ImportTextModal.tsx b/muse-studio/src/features/editor/components/ImportTextModal.tsx index f1d9def2..62a54199 100644 --- a/muse-studio/src/features/editor/components/ImportTextModal.tsx +++ b/muse-studio/src/features/editor/components/ImportTextModal.tsx @@ -1,39 +1,128 @@ import { useRef, useState } from 'react'; import type { ChangeEvent } from 'react'; -import type { CreateImportTaskInput } from '../hooks/useWorks'; +import { + useBatchConfirmImportChapters, + useChapterParseResults, + useCreateImportParseJob, + useCreateUploadedImportTask, + useImportParseJob, + useRetryImportParseJob, + useUploadImportFile, + type ChapterParseResult, + type CreateImportTaskInput, +} from '../hooks/useWorks'; interface ImportTextModalProps { isOpen: boolean; + workId: string; isSubmitting: boolean; errorMessage?: string; onClose: () => void; onSubmit: (input: CreateImportTaskInput) => void; } +type ImportMode = 'wizard' | 'quick'; +type WizardStep = 'upload' | 'parse' | 'confirm' | 'done'; +type TextFormat = 'txt' | 'markdown'; +type UploadFormat = TextFormat | 'docx' | 'epub'; + const MAX_IMPORT_CHARS = 500_000; -/** 工作台本地文本导入弹窗。 */ +/** 从文件名推断完整导入向导可解析的文件格式。 */ +function inferUploadFormat(fileName: string): UploadFormat | null { + const lower = fileName.toLowerCase(); + if (lower.endsWith('.txt')) return 'txt'; + if (lower.endsWith('.md') || lower.endsWith('.markdown')) return 'markdown'; + if (lower.endsWith('.docx')) return 'docx'; + if (lower.endsWith('.epub')) return 'epub'; + return null; +} + +/** 快速文本导入只允许浏览器可直接读取的文本文件。 */ +function inferTextFormat(fileName: string): TextFormat | null { + const format = inferUploadFormat(fileName); + return format === 'txt' || format === 'markdown' ? format : null; +} + +function fallbackMimeType(format: UploadFormat) { + if (format === 'markdown') return 'text/markdown'; + if (format === 'docx') return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + if (format === 'epub') return 'application/epub+zip'; + return 'text/plain'; +} + +/** 工作台导入向导：外部上传扫描、AI 全书解析、章节确认；保留快速文本导入作为低风险入口。 */ export default function ImportTextModal({ isOpen, + workId, isSubmitting, errorMessage, onClose, onSubmit, }: ImportTextModalProps) { const fileInputRef = useRef(null); + const [mode, setMode] = useState('wizard'); + const [wizardStep, setWizardStep] = useState('upload'); + const [selectedFile, setSelectedFile] = useState(null); + const [uploadedTaskId, setUploadedTaskId] = useState(null); + const [parseJobId, setParseJobId] = useState(null); + const [selectedChapterIds, setSelectedChapterIds] = useState([]); const [fileName, setFileName] = useState('workspace-import.md'); - const [format, setFormat] = useState<'txt' | 'markdown'>('markdown'); + const [format, setFormat] = useState('markdown'); const [contentText, setContentText] = useState(''); const [localError, setLocalError] = useState(null); + const uploadImportFile = useUploadImportFile(); + const createUploadedImportTask = useCreateUploadedImportTask(workId); + const createParseJob = useCreateImportParseJob(workId); + const retryParseJob = useRetryImportParseJob(); + const parseJob = useImportParseJob(parseJobId); + const chapters = useChapterParseResults(parseJobId); + const confirmChapters = useBatchConfirmImportChapters(workId); + if (!isOpen) { return null; } const normalizedText = contentText.trim(); - const canSubmit = normalizedText.length > 0 && normalizedText.length <= MAX_IMPORT_CHARS && !isSubmitting; + const canSubmitQuick = normalizedText.length > 0 && normalizedText.length <= MAX_IMPORT_CHARS && !isSubmitting; + const supportedFileFormat = selectedFile ? inferUploadFormat(selectedFile.name) : null; + const parsedChapters = chapters.data ?? []; + const pendingChapters = parsedChapters.filter((chapter) => chapter.status === 'pending_review'); + const selectedChapters = pendingChapters.filter((chapter) => selectedChapterIds.includes(String(chapter.id))); + const isWizardBusy = + uploadImportFile.isPending + || createUploadedImportTask.isPending + || createParseJob.isPending + || retryParseJob.isPending + || confirmChapters.isPending; + const parseJobData = parseJob.data; + const visibleError = localError + || errorMessage + || (uploadImportFile.isError ? '文件上传失败' : null) + || (createUploadedImportTask.isError ? '导入扫描失败' : null) + || (createParseJob.isError ? 'AI 解析任务创建失败' : null) + || (retryParseJob.isError ? 'AI 解析任务重试失败' : null) + || (parseJob.isError ? 'AI 解析任务查询失败' : null) + || (chapters.isError ? '章节解析结果查询失败' : null) + || (confirmChapters.isError ? '章节确认失败' : null); - const handleSubmit = () => { + const resetWizard = () => { + setMode('wizard'); + setWizardStep('upload'); + setSelectedFile(null); + setUploadedTaskId(null); + setParseJobId(null); + setSelectedChapterIds([]); + setLocalError(null); + }; + + const handleClose = () => { + resetWizard(); + onClose(); + }; + + const handleSubmitQuick = () => { if (!normalizedText) { setLocalError('正文不能为空'); return; @@ -55,7 +144,26 @@ export default function ImportTextModal({ if (!file) { return; } - const nextFormat = file.name.toLowerCase().endsWith('.txt') ? 'txt' : 'markdown'; + const nextFormat = inferUploadFormat(file.name); + if (!nextFormat) { + setSelectedFile(null); + setLocalError('当前 AI 解析支持 .txt / .md / .markdown / .docx / .epub'); + return; + } + setSelectedFile(file); + setFileName(file.name); + if (nextFormat === 'txt' || nextFormat === 'markdown') { + setFormat(nextFormat); + } + setLocalError(null); + }; + + const handleQuickFileChange = async (event: ChangeEvent) => { + const file = event.target.files?.[0]; + if (!file) { + return; + } + const nextFormat = inferTextFormat(file.name) ?? 'markdown'; const text = await file.text(); setFileName(file.name); setFormat(nextFormat); @@ -63,6 +171,265 @@ export default function ImportTextModal({ setLocalError(null); }; + const startUploadScan = async () => { + if (!selectedFile || !supportedFileFormat) { + setLocalError('请选择可解析的文本文件'); + return; + } + if (!workId) { + setLocalError('作品 ID 缺失'); + return; + } + setLocalError(null); + const storagePath = await uploadImportFile.mutateAsync({ + file: selectedFile, + directory: `content/import/${workId}`, + }); + const task = await createUploadedImportTask.mutateAsync({ + fileName: selectedFile.name, + fileSize: selectedFile.size, + format: supportedFileFormat, + mimeType: selectedFile.type || fallbackMimeType(supportedFileFormat), + storagePath, + }); + setUploadedTaskId(String(task.id)); + setWizardStep('parse'); + }; + + const startParse = async () => { + if (!uploadedTaskId) { + setLocalError('导入任务缺失'); + return; + } + setLocalError(null); + const job = await createParseJob.mutateAsync({ importTaskId: uploadedTaskId, parseMode: 'full_book' }); + setParseJobId(String(job.id)); + setWizardStep('confirm'); + }; + + const retryParse = async () => { + if (!parseJobId) { + setLocalError('解析任务缺失'); + return; + } + setLocalError(null); + await retryParseJob.mutateAsync({ jobId: parseJobId, retryStage: parseJobData?.failedStage ?? 'llm_parse' }); + }; + + const toggleChapter = (chapterId: string) => { + setSelectedChapterIds((current) => + current.includes(chapterId) ? current.filter((item) => item !== chapterId) : [...current, chapterId] + ); + setLocalError(null); + }; + + const selectAllPending = () => { + setSelectedChapterIds(pendingChapters.map((chapter) => String(chapter.id))); + setLocalError(null); + }; + + const confirmSelectedChapters = async () => { + if (!parseJobId || selectedChapters.length === 0) { + setLocalError('请至少选择一个待确认章节'); + return; + } + await confirmChapters.mutateAsync({ jobId: parseJobId, chapters: selectedChapters }); + setWizardStep('done'); + }; + + const renderWizard = () => ( +
+
+ {(['upload', 'parse', 'confirm', 'done'] as WizardStep[]).map((step, index) => ( +
+ {index + 1}. {step === 'upload' ? '上传扫描' : step === 'parse' ? 'AI 解析' : step === 'confirm' ? '章节确认' : '完成'} +
+ ))} +
+ + {wizardStep === 'upload' && ( +
+
+
+
+
+ {selectedFile ? selectedFile.name : '选择导入文件'} +
+
+ {selectedFile ? `${Math.round(selectedFile.size / 1024)} KB` : '.txt / .md / .markdown / .docx / .epub'} +
+
+ +
+ +
+ +
+ )} + + {wizardStep === 'parse' && ( +
+
+ 上传扫描已通过，导入任务 {uploadedTaskId} +
+ +
+ )} + + {wizardStep === 'confirm' && ( +
+
+ + {parseJobData + ? `状态 ${parseJobData.status}，章节 ${parseJobData.totalChapters ?? 0}，待确认 ${parseJobData.pendingChapters ?? 0}` + : '正在读取解析任务'} + + {parseJobData?.status === 'failed' && parseJobData.retryable ? ( + + ) : ( + + )} +
+
+ {chapters.isLoading || parseJob.isLoading ? ( +
读取解析结果中
+ ) : parsedChapters.length === 0 ? ( +
暂无章节解析结果
+ ) : ( + parsedChapters.map((chapter: ChapterParseResult) => { + const chapterId = String(chapter.id); + const disabled = chapter.status !== 'pending_review'; + return ( + + ); + }) + )} +
+ +
+ )} + + {wizardStep === 'done' && ( +
+ 章节已确认，Knowledge Draft 已创建 +
+ )} +
+ ); + + const renderQuickImport = () => ( +
+
+ + +
+ +
+ + + {normalizedText.length} / {MAX_IMPORT_CHARS} + +
+ +