feat(ai): S8 导入解析切 Dify chat 解析器 + provider 选择
MuseAiImportParseService 从单点注入 NewApiMuseAiImportLlmParser 改为按
muse.ai.import.provider(缺省 dify)从 List<MuseAiImportLlmParser> 选择;来源标识
由选中 parser 的 summary.parser 自报(dify_import_llm / new_api_import_llm)。
新增 DifyMuseAiImportLlmParser:走 Dify「muse-全书解析」chat app(passthrough M3、
克隆自写作 app)的 /chat-messages blocking 调用,把镜像 New-API 的 system+user 解析
prompt 合并成一条 query 发出,从 answer 消费章节 JSON。语义逐条对齐 New-API 版:
401/403 fail-closed 不重试、408/429/5xx 退避重试、输入上限 180k、空/超 300 章节、
脱敏 summary、直连绕代理、key 不落库不打印。截断检测差异:Dify chat 无 finish_reason,
改为对 answer 做 JSON 完整性校验(解析失败→AI_IMPORT_LLM_TRUNCATED 不可重试)。
MuseAiProperties.Dify 补 maxAttempts/retryBackoffSeconds;新增 ImportParse.provider。
规格允许 app/workflow;创始人定 chat app 顶替空 workflow 壳 fed4d25c。p1r 配置
MUSE_AI_DIFY_PARSER_APP_ID/API_KEY 指向新 chat app 17438ceb。
验证(独立重跑核验,不采信子代理自报):
- ai 模块单测 20/0/0/0(DifyImportLlmParser 7、ImportParseService provider 选择 11、
NewApi 2)+ content ImportParseService IT 16/0/0/0,MVN_EXIT=0 BUILD SUCCESS
- Dify 解析 app 直连 /v1/chat-messages 冒烟真出严格章节 JSON {"chapters":[...]}
- 代码 review:确认真 chat 契约(/chat-messages、读 answer、无 workflow 残留)
未做:完整上传链 live 全书解析(import→对象存储→storageRef→parse job→LLM)属 S9
黄金旅程/import-wizard.spec.ts 范畴——inline contentText 走同步 splitter 绕 LLM,
LLM 路径需对象存储上传流,非本次 parser 改动引入。
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
lili
parent
bdc07a4321
commit
0b109a2961
@ -0,0 +1,359 @@
|
||||
package cn.iocoder.muse.module.ai.application.muse;
|
||||
|
||||
import cn.iocoder.muse.framework.common.util.json.JsonUtils;
|
||||
import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.ConnectException;
|
||||
import java.net.ProxySelector;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.net.http.HttpTimeoutException;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 基于 Dify「muse-全书解析」chat app(passthrough,pre_prompt 空)的导入章节解析器。
|
||||
*
|
||||
* <p>语义与 {@link NewApiMuseAiImportLlmParser} 逐条对齐:401/403 fail-closed 不重试、408/429/5xx 退避重试、
|
||||
* token/provider 原文不落库不打印、模型正文只在本次解析内转成 AI Shadow 章节。差异仅在传输载体:
|
||||
* 走 Dify chat 的 blocking 调用,把镜像 New-API 的 system+user prompt 合并成一条 query 发出,
|
||||
* 从 {@code answer} 消费模型产出的章节 JSON 字符串。</p>
|
||||
*
|
||||
* <p>截断检测:Dify chat 没有 New-API 那种 provider 级 {@code finish_reason=length} 信号,只能靠
|
||||
* 对模型产出 JSON 本身做完整性校验——JSON 解析失败即判定输出被截断(见 {@link #mapChatResult}),
|
||||
* 再叠加章节数合理性(空 / 超上限)检查。</p>
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class DifyMuseAiImportLlmParser implements MuseAiImportLlmParser {
|
||||
|
||||
/** summary/落库来源标识,与 New-API 版对称。 */
|
||||
private static final String PARSER_TAG = "dify_import_llm";
|
||||
/** provider 选择键,归一化后与 muse.ai.import.provider 比较。 */
|
||||
private static final String PROVIDER_KEY = "dify";
|
||||
/** S8 全书解析 chat app 固定凭据 ref;真实 API Key 由 env 注入到 muse.ai.dify.credentials,禁止落库/打印。 */
|
||||
private static final String PARSER_CREDENTIAL_REF = "dify-parser-s1";
|
||||
/** Dify chat blocking 端点;baseUrl 已含 /v1,故只拼 /chat-messages(app 由 API Key 确定)。 */
|
||||
private static final String CHAT_MESSAGES_PATH = "/chat-messages";
|
||||
private static final String RESPONSE_MODE_BLOCKING = "blocking";
|
||||
private static final int MAX_LLM_INPUT_CHARS = 180_000;
|
||||
private static final int MAX_LLM_CHAPTERS = 300;
|
||||
private static final ProxySelector DIRECT_PROXY_SELECTOR = ProxySelector.of(null);
|
||||
|
||||
private final MuseAiProperties.Dify properties;
|
||||
private final HttpClient httpClient;
|
||||
|
||||
public DifyMuseAiImportLlmParser(MuseAiProperties museAiProperties) {
|
||||
this.properties = museAiProperties == null ? null : museAiProperties.getDify();
|
||||
this.httpClient = HttpClient.newBuilder()
|
||||
.connectTimeout(Duration.ofSeconds(positive(connectTimeoutSeconds(), 5)))
|
||||
// 全书解析同样调用内网 Dify chat app,必须绕开 JVM/系统代理,避免可用服务被误判为 502。
|
||||
.proxy(DIRECT_PROXY_SELECTOR)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String providerKey() {
|
||||
return PROVIDER_KEY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public LlmParseResult parse(LlmParseCommand command) {
|
||||
// fail-closed:Dify 未启用 / baseUrl 缺失 / dify-parser-s1 凭据缺失,一律不可重试地判为未配置。
|
||||
if (!isConfigured()) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_UNAVAILABLE", "AI 全书解析服务未配置", false,
|
||||
Map.of("parser", PARSER_TAG));
|
||||
}
|
||||
if (command == null || !StringUtils.hasText(command.contentText())) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_EMPTY_INPUT", "AI 全书解析输入为空", false,
|
||||
Map.of("parser", PARSER_TAG));
|
||||
}
|
||||
if (command.contentText().length() > MAX_LLM_INPUT_CHARS) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_INPUT_TOO_LARGE",
|
||||
"当前 AI 全书解析输入超过模型上下文上限，请先按章节拆分后重试", false,
|
||||
Map.of("parser", PARSER_TAG, "inputChars", command.contentText().length()));
|
||||
}
|
||||
|
||||
int maxAttempts = maxAttempts();
|
||||
LlmParseResult lastResult = null;
|
||||
for (int attemptNo = 1; attemptNo <= maxAttempts; attemptNo++) {
|
||||
lastResult = parseOnce(command);
|
||||
LlmParseResult resultWithAttempts = withAttemptSummary(lastResult, attemptNo, maxAttempts);
|
||||
// 成功 / 不可重试(如鉴权失败、坏响应)/ 已到上限:立即返回,不再退避。
|
||||
if (resultWithAttempts.success() || !resultWithAttempts.retryable() || attemptNo >= maxAttempts) {
|
||||
return resultWithAttempts;
|
||||
}
|
||||
sleepBeforeRetry(command, attemptNo);
|
||||
}
|
||||
return withAttemptSummary(lastResult, maxAttempts, maxAttempts);
|
||||
}
|
||||
|
||||
private LlmParseResult parseOnce(LlmParseCommand command) {
|
||||
try {
|
||||
HttpRequest request = buildRequest(command);
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
return mapResponse(command, response);
|
||||
} catch (HttpTimeoutException ex) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_TIMEOUT", "AI 全书解析请求超时", true,
|
||||
Map.of("parser", PARSER_TAG));
|
||||
} catch (ConnectException ex) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_CONNECT_FAILED", "AI 全书解析服务连接失败", true,
|
||||
Map.of("parser", PARSER_TAG));
|
||||
} catch (IOException ex) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_CONNECT_FAILED", "AI 全书解析服务连接失败", true,
|
||||
Map.of("parser", PARSER_TAG));
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_TIMEOUT", "AI 全书解析请求被中断", true,
|
||||
Map.of("parser", PARSER_TAG));
|
||||
} catch (RuntimeException ex) {
|
||||
// 只记脱敏错误类型,绝不打印响应体/凭据;Dify envelope 无法解析走此分支,不可重试。
|
||||
log.warn("AI 全书解析(Dify)运行时失败：jobId={}, errorType={}", command.jobId(), ex.getClass().getSimpleName());
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_BAD_RESPONSE", "AI 全书解析响应无法解析", false,
|
||||
Map.of("parser", PARSER_TAG));
|
||||
}
|
||||
}
|
||||
|
||||
private LlmParseResult withAttemptSummary(LlmParseResult result, int attemptNo, int maxAttempts) {
|
||||
if (result == null) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_UNKNOWN", "AI 全书解析未返回结果", false,
|
||||
Map.of("parser", PARSER_TAG, "attemptNo", attemptNo, "maxAttempts", maxAttempts));
|
||||
}
|
||||
Map<String, Object> summary = new LinkedHashMap<>(result.summary() == null ? Map.of() : result.summary());
|
||||
summary.put("attemptNo", attemptNo);
|
||||
summary.put("maxAttempts", maxAttempts);
|
||||
return new LlmParseResult(result.success(), result.chapters(), result.errorCode(), result.errorMessage(),
|
||||
result.retryable(), summary);
|
||||
}
|
||||
|
||||
private void sleepBeforeRetry(LlmParseCommand command, int attemptNo) {
|
||||
int backoffSeconds = backoffSeconds(attemptNo);
|
||||
log.warn("AI 全书解析(Dify)可重试失败，准备重试：jobId={}, attemptNo={}, nextAttemptNo={}, backoffSeconds={}",
|
||||
command.jobId(), attemptNo, attemptNo + 1, backoffSeconds);
|
||||
if (backoffSeconds <= 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Thread.sleep(Duration.ofSeconds(backoffSeconds).toMillis());
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private HttpRequest buildRequest(LlmParseCommand command) {
|
||||
// chat app 是 passthrough(pre_prompt 空),故把 system 指令与 user 数据合并成一条 query 发出;inputs 留空。
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("inputs", Map.of());
|
||||
payload.put("query", systemPrompt() + "\n\n" + userPrompt(command));
|
||||
payload.put("response_mode", RESPONSE_MODE_BLOCKING);
|
||||
payload.put("conversation_id", "");
|
||||
payload.put("user", user(command));
|
||||
String body = JsonUtils.toJsonString(payload);
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder()
|
||||
.uri(chatMessagesUri())
|
||||
.timeout(Duration.ofSeconds(positive(nonStreamReadTimeoutSeconds(), 90)))
|
||||
// 凭据只在 Authorization 头出现,来自安全配置解析,不进 body/日志/summary。
|
||||
.header("Authorization", "Bearer " + apiKey(PARSER_CREDENTIAL_REF))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Accept", "application/json")
|
||||
.header("X-Muse-Scene", "ai_import_parse")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(body));
|
||||
if (StringUtils.hasText(command.commandId())) {
|
||||
builder.header("X-Request-Id", command.commandId());
|
||||
builder.header("X-Trace-Id", command.commandId());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private LlmParseResult mapResponse(LlmParseCommand command, HttpResponse<String> response) {
|
||||
int status = response.statusCode();
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("parser", PARSER_TAG);
|
||||
summary.put("providerStatusCode", status);
|
||||
response.headers().firstValue("X-Request-Id").ifPresent(value -> summary.put("providerRequestId", value));
|
||||
// HTTP 层错误:错误码/重试策略与 New-API 版逐条一致(401/403 不重试,408/429/5xx 重试)。
|
||||
if (status < 200 || status >= 300) {
|
||||
return LlmParseResult.failed(errorCode(status), errorMessage(status), isRetryableStatus(status), summary);
|
||||
}
|
||||
|
||||
// Dify chat blocking 成功响应:{"answer":"<JSON字符串>","metadata":{...},"message_id":..}。
|
||||
// chat app 无 workflow 的 status/outputs 概念:HTTP 200 且 answer 非空即视为成功,进入 JSON 解析。
|
||||
JsonNode root = JsonUtils.parseTree(response.body());
|
||||
String messageId = textOrNull(root.path("message_id"));
|
||||
if (StringUtils.hasText(messageId)) {
|
||||
summary.put("providerRequestId", messageId);
|
||||
}
|
||||
String answer = textOrNull(root.path("answer"));
|
||||
if (!StringUtils.hasText(answer)) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_BAD_RESPONSE", "AI 全书解析响应缺少正文", false, summary);
|
||||
}
|
||||
return mapChatResult(answer, summary);
|
||||
}
|
||||
|
||||
private LlmParseResult mapChatResult(String answer, Map<String, Object> summary) {
|
||||
String json = stripJsonFence(answer);
|
||||
JsonNode resultRoot;
|
||||
try {
|
||||
resultRoot = JsonUtils.parseTree(json);
|
||||
} catch (RuntimeException ex) {
|
||||
// 截断检测核心:Dify 无 finish_reason 信号,模型产出 JSON 不完整(解析失败)即视为输出被截断。
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_TRUNCATED", "AI 全书解析响应被截断", false, summary);
|
||||
}
|
||||
List<LlmChapter> chapters = extractChapters(resultRoot);
|
||||
if (chapters.isEmpty()) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_NO_CHAPTERS", "AI 全书解析未返回有效章节", false, summary);
|
||||
}
|
||||
if (chapters.size() > MAX_LLM_CHAPTERS) {
|
||||
return LlmParseResult.failed("AI_IMPORT_LLM_TOO_MANY_CHAPTERS", "AI 全书解析章节数超过上限", false, summary);
|
||||
}
|
||||
summary.put("chapterCount", chapters.size());
|
||||
return LlmParseResult.success(chapters, summary);
|
||||
}
|
||||
|
||||
private List<LlmChapter> extractChapters(JsonNode resultRoot) {
|
||||
JsonNode chaptersNode = resultRoot.isArray() ? resultRoot : resultRoot.path("chapters");
|
||||
List<LlmChapter> chapters = new ArrayList<>();
|
||||
if (!chaptersNode.isArray()) {
|
||||
return chapters;
|
||||
}
|
||||
for (JsonNode item : chaptersNode) {
|
||||
String title = item.path("title").asText("").strip();
|
||||
String content = item.path("content").asText("").strip();
|
||||
if (!StringUtils.hasText(content)) {
|
||||
continue;
|
||||
}
|
||||
chapters.add(new LlmChapter(StringUtils.hasText(title) ? title : "导入章节", content));
|
||||
}
|
||||
return chapters;
|
||||
}
|
||||
|
||||
private String stripJsonFence(String content) {
|
||||
String normalized = content == null ? "" : content.strip();
|
||||
if (normalized.startsWith("```")) {
|
||||
normalized = normalized.replaceFirst("(?s)^```(?:json)?\\s*", "");
|
||||
normalized = normalized.replaceFirst("(?s)\\s*```$", "");
|
||||
}
|
||||
return normalized.strip();
|
||||
}
|
||||
|
||||
private String systemPrompt() {
|
||||
// 文案照搬 NewApiMuseAiImportLlmParser.systemPrompt();chat app 为 passthrough,须由 query 携带全部指令。
|
||||
return """
|
||||
你是长篇小说导入解析器。只输出 JSON，不要解释。
|
||||
JSON 格式必须是 {"chapters":[{"title":"章节标题","content":"章节正文"}]}。
|
||||
章节正文必须来自用户提供文本，不得补写、改写、总结或虚构。
|
||||
如果原文已有章节标题，按原标题切分；如果没有标题，按叙事段落保守切分并给出中性标题。
|
||||
""";
|
||||
}
|
||||
|
||||
private String userPrompt(LlmParseCommand command) {
|
||||
// 文案照搬 NewApiMuseAiImportLlmParser.userPrompt(),保证与 New-API 版产出同一份严格章节 JSON。
|
||||
return """
|
||||
文件名：%s
|
||||
文件格式：%s
|
||||
内容摘要哈希：%s
|
||||
|
||||
请把下面全文切分为可供用户审核的章节 JSON：
|
||||
|
||||
%s
|
||||
""".formatted(command.fileName(), command.format(), command.contentHash(), command.contentText());
|
||||
}
|
||||
|
||||
private URI chatMessagesUri() {
|
||||
return URI.create(trimRight(properties.getBaseUrl(), "/") + CHAT_MESSAGES_PATH);
|
||||
}
|
||||
|
||||
private String user(LlmParseCommand command) {
|
||||
Long owner = command == null ? null : command.ownerUserId();
|
||||
return "muse-import-" + (owner == null ? "unknown" : owner);
|
||||
}
|
||||
|
||||
private String apiKey(String credentialRef) {
|
||||
if (properties == null || properties.getCredentials() == null) {
|
||||
return null;
|
||||
}
|
||||
return properties.getCredentials().stream()
|
||||
.filter(credential -> credential != null && credentialRef.equals(credential.getRef()))
|
||||
.map(MuseAiProperties.Dify.Credential::getApiKey)
|
||||
.filter(StringUtils::hasText)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
private boolean isConfigured() {
|
||||
return properties != null
|
||||
&& properties.isEnabled()
|
||||
&& StringUtils.hasText(properties.getBaseUrl())
|
||||
&& StringUtils.hasText(apiKey(PARSER_CREDENTIAL_REF));
|
||||
}
|
||||
|
||||
private String errorCode(int status) {
|
||||
return switch (status) {
|
||||
case 401, 403 -> "AI_IMPORT_LLM_AUTH_FAILED";
|
||||
case 408, 504 -> "AI_IMPORT_LLM_TIMEOUT";
|
||||
case 429 -> "AI_IMPORT_LLM_RATE_LIMITED";
|
||||
default -> status >= 500 ? "AI_IMPORT_LLM_PROVIDER_5XX" : "AI_IMPORT_LLM_BAD_REQUEST";
|
||||
};
|
||||
}
|
||||
|
||||
private String errorMessage(int status) {
|
||||
return switch (status) {
|
||||
case 401, 403 -> "AI 全书解析鉴权失败";
|
||||
case 408, 504 -> "AI 全书解析请求超时";
|
||||
case 429 -> "AI 全书解析被限流";
|
||||
default -> status >= 500 ? "AI 全书解析服务暂不可用" : "AI 全书解析请求被拒绝";
|
||||
};
|
||||
}
|
||||
|
||||
private boolean isRetryableStatus(int status) {
|
||||
return status == 408 || status == 429 || status == 500 || status == 502 || status == 503 || status == 504;
|
||||
}
|
||||
|
||||
private Integer connectTimeoutSeconds() {
|
||||
return properties == null ? null : properties.getConnectTimeoutSeconds();
|
||||
}
|
||||
|
||||
private Integer nonStreamReadTimeoutSeconds() {
|
||||
return properties == null ? null : properties.getNonStreamReadTimeoutSeconds();
|
||||
}
|
||||
|
||||
private int maxAttempts() {
|
||||
return positive(properties == null ? null : properties.getMaxAttempts(), 3);
|
||||
}
|
||||
|
||||
private int backoffSeconds(int attemptNo) {
|
||||
List<Integer> backoff = properties == null ? null : properties.getRetryBackoffSeconds();
|
||||
if (backoff == null || backoff.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
int index = Math.min(Math.max(attemptNo - 1, 0), backoff.size() - 1);
|
||||
Integer seconds = backoff.get(index);
|
||||
return seconds == null || seconds < 0 ? 0 : seconds;
|
||||
}
|
||||
|
||||
private int positive(Integer value, int defaultValue) {
|
||||
return value == null || value <= 0 ? defaultValue : value;
|
||||
}
|
||||
|
||||
private String trimRight(String value, String suffix) {
|
||||
String result = value == null ? "" : value;
|
||||
while (result.endsWith(suffix)) {
|
||||
result = result.substring(0, result.length() - suffix.length());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String textOrNull(JsonNode node) {
|
||||
return node == null || node.isMissingNode() || node.isNull() ? null : node.asText();
|
||||
}
|
||||
}
|
||||
@ -15,6 +15,13 @@ public interface MuseAiImportLlmParser {
|
||||
*/
|
||||
LlmParseResult parse(LlmParseCommand command);
|
||||
|
||||
/**
|
||||
* 该 parser 归属的 runtime provider 标识（如 {@code new-api} / {@code dify}）。
|
||||
*
|
||||
* <p>供 {@code muse.ai.import.provider} 在多实现并存时按 provider 选择;归一化比较由调用方负责。</p>
|
||||
*/
|
||||
String providerKey();
|
||||
|
||||
/**
|
||||
* LLM 解析命令。{@code contentText} 是短生命周期载荷，调用方禁止记录。
|
||||
*/
|
||||
|
||||
@ -4,6 +4,7 @@ import cn.iocoder.muse.framework.common.exception.ServiceException;
|
||||
import cn.iocoder.muse.framework.common.util.json.JsonUtils;
|
||||
import cn.iocoder.muse.module.ai.api.MuseAiImportParseApi;
|
||||
import cn.iocoder.muse.module.ai.application.muse.facade.MuseContentWorkOwnerFacade;
|
||||
import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties;
|
||||
import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiChapterParseResultDO;
|
||||
import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiParseJobDO;
|
||||
import cn.iocoder.muse.module.ai.dal.mysql.muse.MuseAiChapterParseResultMapper;
|
||||
@ -61,6 +62,8 @@ public class MuseAiImportParseService implements MuseAiImportParseApi {
|
||||
private static final int MAX_EXTRACTED_CHARS = 1_000_000;
|
||||
private static final int MAX_PARSE_CHAPTERS = 300;
|
||||
private static final int PREVIEW_MAX_CHARS = 240;
|
||||
// 全书解析 provider 缺省 dify(S8 从 New-API 切到 Dify;New-API 链路保留以便回退)。
|
||||
private static final String DEFAULT_IMPORT_PROVIDER = "dify";
|
||||
private static final Pattern MARKDOWN_HEADING_PATTERN = Pattern.compile("^#{1,3}\\s+(.{1,120})$");
|
||||
private static final Pattern CN_CHAPTER_HEADING_PATTERN = Pattern.compile("^第[\\p{N}一二三四五六七八九十百千万零〇两]+[章节卷回].{0,120}$");
|
||||
private static final Pattern EN_CHAPTER_HEADING_PATTERN = Pattern.compile("(?i)^chapter\\s+[\\p{Alnum}]+.{0,120}$");
|
||||
@ -80,8 +83,14 @@ public class MuseAiImportParseService implements MuseAiImportParseApi {
|
||||
private MuseAiChapterParseResultMapper chapterResultMapper;
|
||||
@Resource
|
||||
private MuseContentWorkOwnerFacade contentWorkOwnerFacade;
|
||||
/**
|
||||
* 所有全书解析 parser 实现(new-api / dify),运行时按 muse.ai.import.provider 选择;
|
||||
* 用 List 注入而非单点,避免两实现并存时的注入歧义。
|
||||
*/
|
||||
@Resource
|
||||
private MuseAiImportLlmParser llmParser;
|
||||
private List<MuseAiImportLlmParser> importLlmParsers;
|
||||
@Resource
|
||||
private MuseAiProperties museAiProperties;
|
||||
|
||||
public MuseAiImportParseService(ObjectProvider<FileApi> fileApiProvider) {
|
||||
this.fileApi = fileApiProvider.getIfAvailable();
|
||||
@ -326,7 +335,12 @@ public class MuseAiImportParseService implements MuseAiImportParseApi {
|
||||
|
||||
private ParseExecution parseWithLlm(MuseAiParseJobDO job, String format, String normalizedText, String contentHash,
|
||||
ExtractedText extractedText) {
|
||||
MuseAiImportLlmParser.LlmParseResult llmResult = llmParser.parse(new MuseAiImportLlmParser.LlmParseCommand(
|
||||
MuseAiImportLlmParser parser = resolveImportLlmParser();
|
||||
if (parser == null) {
|
||||
// 没有任何可用的全书解析 parser 实现:直接 fail-closed,不落任何 shadow 章节。
|
||||
return ParseExecution.failed("llm_parse", "AI_IMPORT_LLM_UNAVAILABLE", "AI 全书解析服务未配置", false);
|
||||
}
|
||||
MuseAiImportLlmParser.LlmParseResult llmResult = parser.parse(new MuseAiImportLlmParser.LlmParseCommand(
|
||||
job.getCommandId(), job.getId(), job.getOwnerUserId(), job.getWorkId(), job.getFileName(), format,
|
||||
normalizedText, contentHash, readJson(job.getParseConfig())));
|
||||
if (!llmResult.success()) {
|
||||
@ -354,7 +368,59 @@ public class MuseAiImportParseService implements MuseAiImportParseApi {
|
||||
Map<String, Object> summary = new LinkedHashMap<>(llmResult.summary());
|
||||
summary.put("extractor", extractedText.parser());
|
||||
summary.put("extractedChars", normalizedText.length());
|
||||
return ParseExecution.success(chapters, contentHash, "new_api_import_llm", summary);
|
||||
// 来源标识由选中 parser 在 summary.parser 自报(new_api_import_llm / dify_import_llm),避免此处硬编码 provider。
|
||||
String parserTag = stringValue(llmResult.summary() == null ? null : llmResult.summary().get("parser"));
|
||||
return ParseExecution.success(chapters, contentHash,
|
||||
StringUtils.hasText(parserTag) ? parserTag : "ai_import_llm", summary);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 {@code muse.ai.import.provider} 选择全书解析 parser(缺省 dify)。
|
||||
*
|
||||
* <p>WHY:S8 把全书解析从 New-API 切到 Dify,但两条链路并存以便回退。单实现场景(仅注册一个 parser,
|
||||
* 或单测注入单个 mock)直接返回,不依赖配置;多实现时按归一化 provider 匹配,匹配不到回退默认再兜底首个。</p>
|
||||
*/
|
||||
private MuseAiImportLlmParser resolveImportLlmParser() {
|
||||
List<MuseAiImportLlmParser> parsers = importLlmParsers;
|
||||
if (parsers == null || parsers.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (parsers.size() == 1) {
|
||||
return parsers.get(0);
|
||||
}
|
||||
String provider = importProvider();
|
||||
MuseAiImportLlmParser matched = matchProvider(parsers, provider);
|
||||
if (matched != null) {
|
||||
return matched;
|
||||
}
|
||||
// 配置的 provider 无对应实现:回退缺省 dify,再兜底首个,保证解析链路不空转。
|
||||
log.warn("AI 全书解析 provider 无匹配实现，回退默认：configuredProvider={}", provider);
|
||||
MuseAiImportLlmParser fallback = matchProvider(parsers, DEFAULT_IMPORT_PROVIDER);
|
||||
return fallback != null ? fallback : parsers.get(0);
|
||||
}
|
||||
|
||||
private MuseAiImportLlmParser matchProvider(List<MuseAiImportLlmParser> parsers, String provider) {
|
||||
for (MuseAiImportLlmParser parser : parsers) {
|
||||
if (provider.equals(normalizeProvider(parser.providerKey()))) {
|
||||
return parser;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String importProvider() {
|
||||
String configured = museAiProperties == null || museAiProperties.getImport() == null
|
||||
? null : museAiProperties.getImport().getProvider();
|
||||
String normalized = normalizeProvider(configured);
|
||||
return StringUtils.hasText(normalized) ? normalized : DEFAULT_IMPORT_PROVIDER;
|
||||
}
|
||||
|
||||
private String normalizeProvider(String provider) {
|
||||
if (!StringUtils.hasText(provider)) {
|
||||
return null;
|
||||
}
|
||||
// 统一小写并把下划线视作连字符,兼容 new_api / new-api / NEW_API 等写法。
|
||||
return provider.trim().toLowerCase(Locale.ROOT).replace('_', '-');
|
||||
}
|
||||
|
||||
private ExtractedText extractText(String format, byte[] bytes) {
|
||||
|
||||
@ -48,6 +48,11 @@ public class NewApiMuseAiImportLlmParser implements MuseAiImportLlmParser {
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String providerKey() {
|
||||
return "new-api";
|
||||
}
|
||||
|
||||
@Override
|
||||
public LlmParseResult parse(LlmParseCommand command) {
|
||||
if (!isConfigured()) {
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
package cn.iocoder.muse.module.ai.framework.ai.config;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import java.util.List;
|
||||
@ -75,6 +77,22 @@ public class MuseAiProperties {
|
||||
*/
|
||||
private Dify dify = new Dify();
|
||||
|
||||
/**
|
||||
* Muse 导入全书解析 provider 选择(S8)。
|
||||
*
|
||||
* <p>字段名不能直接叫 {@code import}(Java 关键字),故手写 {@link #getImport()} 让
|
||||
* {@code muse.ai.import.provider} 走标量 relaxed-binding(如 {@code MUSE_AI_IMPORT_PROVIDER})。</p>
|
||||
*/
|
||||
@Getter(AccessLevel.NONE)
|
||||
private final ImportParse importParse = new ImportParse();
|
||||
|
||||
/**
|
||||
* 绑定 {@code muse.ai.import.*};返回固定实例,provider 缺省 dify。
|
||||
*/
|
||||
public ImportParse getImport() {
|
||||
return importParse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Muse Events 发布配置。
|
||||
*/
|
||||
@ -272,6 +290,16 @@ public class MuseAiProperties {
|
||||
private Integer nonStreamReadTimeoutSeconds = 180;
|
||||
private Integer totalTimeoutSeconds = 180;
|
||||
|
||||
/**
|
||||
* 全书解析(S8)可重试失败的最大尝试次数。408/429/5xx 退避重试,401/403 fail-closed 不重试。
|
||||
*/
|
||||
private Integer maxAttempts = 3;
|
||||
|
||||
/**
|
||||
* 各次重试前的退避秒数;下标对应第几次重试,超出取最后一档。
|
||||
*/
|
||||
private List<Integer> retryBackoffSeconds = List.of(1, 2, 4);
|
||||
|
||||
@Data
|
||||
public static class Credential {
|
||||
|
||||
@ -332,4 +360,16 @@ public class MuseAiProperties {
|
||||
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class ImportParse {
|
||||
|
||||
/**
|
||||
* 导入全书解析 provider,缺省 dify;可用 MUSE_AI_IMPORT_PROVIDER 覆盖为 new-api。
|
||||
*
|
||||
* <p>取值经服务层归一化(小写、下划线视作连字符),故 new_api / new-api 等价。</p>
|
||||
*/
|
||||
private String provider = "dify";
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,217 @@
|
||||
package cn.iocoder.muse.module.ai.application.muse;
|
||||
|
||||
import cn.iocoder.muse.framework.common.util.json.JsonUtils;
|
||||
import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties;
|
||||
import com.sun.net.httpserver.HttpExchange;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Dify 导入解析器契约测试。
|
||||
*
|
||||
* <p>用本地 HttpServer 桩 Dify chat blocking 端点(/chat-messages),覆盖成功多章、鉴权 fail-closed 不重试、
|
||||
* 限流重试后成功、5xx 可重试耗尽、answer 非 JSON 判截断、空章节、未配置 fail-closed。</p>
|
||||
*/
|
||||
class DifyMuseAiImportLlmParserTest {
|
||||
|
||||
private HttpServer server;
|
||||
private final AtomicInteger requestCount = new AtomicInteger();
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (server != null) {
|
||||
server.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_parseMultipleChaptersOnChatSuccess() throws Exception {
|
||||
startServer(exchange -> {
|
||||
requestCount.incrementAndGet();
|
||||
writeJson(exchange, 200, chatAnswerBody(twoChaptersAnswerJson()));
|
||||
});
|
||||
DifyMuseAiImportLlmParser parser = new DifyMuseAiImportLlmParser(difyProperties(3, List.of(0, 0)));
|
||||
|
||||
MuseAiImportLlmParser.LlmParseResult result = parser.parse(command());
|
||||
|
||||
assertTrue(result.success());
|
||||
assertEquals(1, requestCount.get());
|
||||
assertEquals(2, result.chapters().size());
|
||||
assertEquals("第一章", result.chapters().getFirst().title());
|
||||
assertEquals("dify_import_llm", result.summary().get("parser"));
|
||||
assertEquals(2, result.summary().get("chapterCount"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_notRetryAuthFailure() throws Exception {
|
||||
startServer(exchange -> {
|
||||
requestCount.incrementAndGet();
|
||||
writeJson(exchange, 401, "{\"error\":\"unauthorized\"}");
|
||||
});
|
||||
DifyMuseAiImportLlmParser parser = new DifyMuseAiImportLlmParser(difyProperties(3, List.of(0, 0)));
|
||||
|
||||
MuseAiImportLlmParser.LlmParseResult result = parser.parse(command());
|
||||
|
||||
assertFalse(result.success());
|
||||
assertEquals("AI_IMPORT_LLM_AUTH_FAILED", result.errorCode());
|
||||
assertFalse(result.retryable());
|
||||
assertEquals(1, requestCount.get(), "鉴权失败不可重试");
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_retryRateLimitedThenReturnChapters() throws Exception {
|
||||
startServer(exchange -> {
|
||||
int count = requestCount.incrementAndGet();
|
||||
if (count == 1) {
|
||||
writeJson(exchange, 429, "{\"error\":\"rate limited\"}");
|
||||
return;
|
||||
}
|
||||
writeJson(exchange, 200, chatAnswerBody(twoChaptersAnswerJson()));
|
||||
});
|
||||
DifyMuseAiImportLlmParser parser = new DifyMuseAiImportLlmParser(difyProperties(3, List.of(0, 0)));
|
||||
|
||||
MuseAiImportLlmParser.LlmParseResult result = parser.parse(command());
|
||||
|
||||
assertTrue(result.success());
|
||||
assertEquals(2, requestCount.get(), "429 应自动重试一次后成功");
|
||||
assertEquals(2, result.summary().get("attemptNo"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_retryProvider5xxUntilExhausted() throws Exception {
|
||||
// chat app 无 workflow status 概念;HTTP 5xx 归为 provider 5xx 可重试,耗尽后仍以可重试失败返回。
|
||||
startServer(exchange -> {
|
||||
requestCount.incrementAndGet();
|
||||
writeJson(exchange, 502, "{\"error\":\"bad gateway\"}");
|
||||
});
|
||||
DifyMuseAiImportLlmParser parser = new DifyMuseAiImportLlmParser(difyProperties(2, List.of(0, 0)));
|
||||
|
||||
MuseAiImportLlmParser.LlmParseResult result = parser.parse(command());
|
||||
|
||||
assertFalse(result.success());
|
||||
assertEquals("AI_IMPORT_LLM_PROVIDER_5XX", result.errorCode());
|
||||
assertTrue(result.retryable());
|
||||
assertEquals(2, requestCount.get(), "5xx 应重试到 maxAttempts");
|
||||
assertEquals(2, result.summary().get("attemptNo"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_failTruncatedWhenAnswerNotValidJson() throws Exception {
|
||||
startServer(exchange -> {
|
||||
requestCount.incrementAndGet();
|
||||
// answer 是不完整 JSON:Dify chat 无 finish_reason,靠 JSON 完整性判定输出被截断。
|
||||
writeJson(exchange, 200, chatAnswerBody("{\"chapters\":[{\"title\":\"第一章\",\"content\":\"正文"));
|
||||
});
|
||||
DifyMuseAiImportLlmParser parser = new DifyMuseAiImportLlmParser(difyProperties(3, List.of(0, 0)));
|
||||
|
||||
MuseAiImportLlmParser.LlmParseResult result = parser.parse(command());
|
||||
|
||||
assertFalse(result.success());
|
||||
assertEquals("AI_IMPORT_LLM_TRUNCATED", result.errorCode());
|
||||
assertFalse(result.retryable());
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_failNoChaptersWhenChaptersEmpty() throws Exception {
|
||||
startServer(exchange -> {
|
||||
requestCount.incrementAndGet();
|
||||
writeJson(exchange, 200, chatAnswerBody("{\"chapters\":[]}"));
|
||||
});
|
||||
DifyMuseAiImportLlmParser parser = new DifyMuseAiImportLlmParser(difyProperties(3, List.of(0, 0)));
|
||||
|
||||
MuseAiImportLlmParser.LlmParseResult result = parser.parse(command());
|
||||
|
||||
assertFalse(result.success());
|
||||
assertEquals("AI_IMPORT_LLM_NO_CHAPTERS", result.errorCode());
|
||||
assertFalse(result.retryable());
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_failUnavailableWhenNotConfigured() {
|
||||
// Dify 未启用(默认 enabled=false):fail-closed,不发起任何 HTTP。
|
||||
DifyMuseAiImportLlmParser parser = new DifyMuseAiImportLlmParser(new MuseAiProperties());
|
||||
|
||||
MuseAiImportLlmParser.LlmParseResult result = parser.parse(command());
|
||||
|
||||
assertFalse(result.success());
|
||||
assertEquals("AI_IMPORT_LLM_UNAVAILABLE", result.errorCode());
|
||||
assertFalse(result.retryable());
|
||||
assertEquals(0, requestCount.get());
|
||||
}
|
||||
|
||||
private MuseAiImportLlmParser.LlmParseCommand command() {
|
||||
return new MuseAiImportLlmParser.LlmParseCommand(
|
||||
"cmd-dify-import-parser-test",
|
||||
1L,
|
||||
9001L,
|
||||
10001L,
|
||||
"book.txt",
|
||||
"txt",
|
||||
"第一章\n正文。",
|
||||
"sha256:test",
|
||||
Map.of("mode", "full_book", "strategy", "llm_full_book"));
|
||||
}
|
||||
|
||||
private MuseAiProperties difyProperties(int maxAttempts, List<Integer> backoffSeconds) {
|
||||
MuseAiProperties properties = new MuseAiProperties();
|
||||
MuseAiProperties.Dify dify = new MuseAiProperties.Dify();
|
||||
dify.setEnabled(true);
|
||||
dify.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort());
|
||||
MuseAiProperties.Dify.Credential credential = new MuseAiProperties.Dify.Credential();
|
||||
credential.setRef("dify-parser-s1");
|
||||
credential.setApiKey("test-dify-key");
|
||||
dify.setCredentials(List.of(credential));
|
||||
dify.setMaxAttempts(maxAttempts);
|
||||
dify.setRetryBackoffSeconds(backoffSeconds);
|
||||
dify.setConnectTimeoutSeconds(1);
|
||||
dify.setNonStreamReadTimeoutSeconds(5);
|
||||
properties.setDify(dify);
|
||||
return properties;
|
||||
}
|
||||
|
||||
private String twoChaptersAnswerJson() {
|
||||
return JsonUtils.toJsonString(Map.of("chapters", List.of(
|
||||
Map.of("title", "第一章", "content", "正文一。"),
|
||||
Map.of("title", "第二章", "content", "正文二。"))));
|
||||
}
|
||||
|
||||
private String chatAnswerBody(String answerJson) {
|
||||
// Dify chat blocking 响应形状:answer 承载模型产出的章节 JSON 字符串。
|
||||
return JsonUtils.toJsonString(Map.of(
|
||||
"answer", answerJson,
|
||||
"message_id", "msg-dify-test",
|
||||
"metadata", Map.of()));
|
||||
}
|
||||
|
||||
private void startServer(ExchangeHandler handler) throws IOException {
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/chat-messages", handler::handle);
|
||||
server.start();
|
||||
}
|
||||
|
||||
private void writeJson(HttpExchange exchange, int status, String responseBody) throws IOException {
|
||||
byte[] bytes = responseBody.getBytes(StandardCharsets.UTF_8);
|
||||
exchange.getResponseHeaders().add("Content-Type", "application/json");
|
||||
exchange.sendResponseHeaders(status, bytes.length);
|
||||
exchange.getResponseBody().write(bytes);
|
||||
exchange.close();
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface ExchangeHandler {
|
||||
|
||||
void handle(HttpExchange exchange) throws IOException;
|
||||
}
|
||||
}
|
||||
@ -8,6 +8,7 @@ import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiChapterParseResultDO;
|
||||
import cn.iocoder.muse.module.ai.dal.dataobject.muse.MuseAiParseJobDO;
|
||||
import cn.iocoder.muse.module.ai.dal.mysql.muse.MuseAiChapterParseResultMapper;
|
||||
import cn.iocoder.muse.module.ai.dal.mysql.muse.MuseAiParseJobMapper;
|
||||
import cn.iocoder.muse.module.ai.framework.ai.config.MuseAiProperties;
|
||||
import cn.iocoder.muse.module.infra.api.file.FileApi;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -65,7 +66,8 @@ class MuseAiImportParseServiceTest extends BaseMockitoUnitTest {
|
||||
ReflectionTestUtils.setField(service, "parseJobMapper", parseJobMapper);
|
||||
ReflectionTestUtils.setField(service, "chapterResultMapper", chapterResultMapper);
|
||||
ReflectionTestUtils.setField(service, "contentWorkOwnerFacade", contentWorkOwnerFacade);
|
||||
ReflectionTestUtils.setField(service, "llmParser", llmParser);
|
||||
// 单实现注入:size==1 时 resolveImportLlmParser 直接返回,不读 provider 配置,兼容既有用例。
|
||||
ReflectionTestUtils.setField(service, "importLlmParsers", List.of(llmParser));
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -271,6 +273,77 @@ class MuseAiImportParseServiceTest extends BaseMockitoUnitTest {
|
||||
verify(parseJobMapper).updateReviewCounters(JOB_ID, 1, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_selectDifyParserWhenProviderIsDify() {
|
||||
MuseAiImportLlmParser newApiParser = org.mockito.Mockito.mock(MuseAiImportLlmParser.class);
|
||||
MuseAiImportLlmParser difyParser = org.mockito.Mockito.mock(MuseAiImportLlmParser.class);
|
||||
org.mockito.Mockito.lenient().when(newApiParser.providerKey()).thenReturn("new-api");
|
||||
org.mockito.Mockito.lenient().when(difyParser.providerKey()).thenReturn("dify");
|
||||
ReflectionTestUtils.setField(service, "importLlmParsers", List.of(newApiParser, difyParser));
|
||||
ReflectionTestUtils.setField(service, "museAiProperties", importProviderProperties("dify"));
|
||||
stubFullBookInsert();
|
||||
when(difyParser.parse(any(MuseAiImportLlmParser.LlmParseCommand.class))).thenReturn(
|
||||
MuseAiImportLlmParser.LlmParseResult.success(
|
||||
List.of(new MuseAiImportLlmParser.LlmChapter("Dify 章节", "Dify 切出的正文。")),
|
||||
Map.of("parser", "dify_import_llm")));
|
||||
|
||||
MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("txt", "full_book"));
|
||||
|
||||
assertEquals("completed", result.status());
|
||||
verify(difyParser).parse(any(MuseAiImportLlmParser.LlmParseCommand.class));
|
||||
verify(newApiParser, never()).parse(any(MuseAiImportLlmParser.LlmParseCommand.class));
|
||||
ArgumentCaptor<MuseAiChapterParseResultDO> chapterCaptor =
|
||||
ArgumentCaptor.forClass(MuseAiChapterParseResultDO.class);
|
||||
verify(chapterResultMapper).insert(chapterCaptor.capture());
|
||||
org.junit.jupiter.api.Assertions.assertTrue(
|
||||
chapterCaptor.getValue().getQualityResult().contains("dify_import_llm"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_selectNewApiParserWhenProviderIsNewApi() {
|
||||
MuseAiImportLlmParser newApiParser = org.mockito.Mockito.mock(MuseAiImportLlmParser.class);
|
||||
MuseAiImportLlmParser difyParser = org.mockito.Mockito.mock(MuseAiImportLlmParser.class);
|
||||
org.mockito.Mockito.lenient().when(newApiParser.providerKey()).thenReturn("new-api");
|
||||
org.mockito.Mockito.lenient().when(difyParser.providerKey()).thenReturn("dify");
|
||||
ReflectionTestUtils.setField(service, "importLlmParsers", List.of(newApiParser, difyParser));
|
||||
// 用下划线写法验证服务层归一化:new_api 应等价 new-api。
|
||||
ReflectionTestUtils.setField(service, "museAiProperties", importProviderProperties("new_api"));
|
||||
stubFullBookInsert();
|
||||
when(newApiParser.parse(any(MuseAiImportLlmParser.LlmParseCommand.class))).thenReturn(
|
||||
MuseAiImportLlmParser.LlmParseResult.success(
|
||||
List.of(new MuseAiImportLlmParser.LlmChapter("New-API 章节", "New-API 切出的正文。")),
|
||||
Map.of("parser", "new_api_import_llm")));
|
||||
|
||||
MuseAiImportParseApi.ParseJobProjection result = service.createParseJob(createCommand("txt", "full_book"));
|
||||
|
||||
assertEquals("completed", result.status());
|
||||
verify(newApiParser).parse(any(MuseAiImportLlmParser.LlmParseCommand.class));
|
||||
verify(difyParser, never()).parse(any(MuseAiImportLlmParser.LlmParseCommand.class));
|
||||
ArgumentCaptor<MuseAiChapterParseResultDO> chapterCaptor =
|
||||
ArgumentCaptor.forClass(MuseAiChapterParseResultDO.class);
|
||||
verify(chapterResultMapper).insert(chapterCaptor.capture());
|
||||
org.junit.jupiter.api.Assertions.assertTrue(
|
||||
chapterCaptor.getValue().getQualityResult().contains("new_api_import_llm"));
|
||||
}
|
||||
|
||||
private void stubFullBookInsert() {
|
||||
when(parseJobMapper.selectByCommandId("cmd-parse-1")).thenReturn(null);
|
||||
when(parseJobMapper.selectByImportTaskIdAndOwner(IMPORT_TASK_ID, USER_ID)).thenReturn(null);
|
||||
when(fileApi.getFileBytes("100/content/import/book.md")).thenReturn(
|
||||
"一段没有标题的正文。".getBytes(StandardCharsets.UTF_8));
|
||||
doAnswer(invocation -> {
|
||||
MuseAiParseJobDO job = invocation.getArgument(0);
|
||||
job.setId(JOB_ID);
|
||||
return 1;
|
||||
}).when(parseJobMapper).insert(any(MuseAiParseJobDO.class));
|
||||
}
|
||||
|
||||
private MuseAiProperties importProviderProperties(String provider) {
|
||||
MuseAiProperties properties = new MuseAiProperties();
|
||||
properties.getImport().setProvider(provider);
|
||||
return properties;
|
||||
}
|
||||
|
||||
private MuseAiImportParseApi.CreateParseJobCommand createCommand(String format) {
|
||||
return createCommand(format, "chapter");
|
||||
}
|
||||
|
||||
@ -37,8 +37,9 @@ MUSE_AI_DIFY_CONSOLE_BASE_URLS=http://100.64.0.8:18080
|
||||
MUSE_AI_DIFY_WRITING_APP_ID=75f105e8-dbca-4aa6-a296-2113a65ad005
|
||||
MUSE_AI_DIFY_WRITING_API_KEY=app-Un5DDoOiK7eQc617ahdn7ku2
|
||||
MUSE_AI_DIFY_WRITING_CREDENTIAL_REF=dify-writing-s1
|
||||
MUSE_AI_DIFY_PARSER_APP_ID=fed4d25c-1289-4090-9771-a811c89aba44
|
||||
MUSE_AI_DIFY_PARSER_API_KEY=app-It4V7WukHohP8rhWV0gOn2H5
|
||||
# S8: 解析器用 chat app(passthrough M3,克隆自写作 app;规格允许 app/workflow 二选一,创始人定 chat app),弃用空 workflow 壳 fed4d25c。
|
||||
MUSE_AI_DIFY_PARSER_APP_ID=17438ceb-6e47-4c2b-adeb-eda21d6e6a88
|
||||
MUSE_AI_DIFY_PARSER_API_KEY=app-9mLRF1ZhFLX1I81pouHpvBi8
|
||||
MUSE_AI_DIFY_PARSER_CREDENTIAL_REF=dify-parser-s1
|
||||
MUSE_AI_DIFY_CONNECT_TIMEOUT_SECONDS=5
|
||||
MUSE_AI_DIFY_FIRST_BYTE_TIMEOUT_SECONDS=15
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user