415 lines
17 KiB
Python
415 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""分层诊断运行器与诊断产物头校验。
|
||
|
||
确定性层支持 regex、handler 与 density；语义层由外部 detector/人工产出 finding，
|
||
再通过当前正文、active 规则、版本与 layer 绑定门。所有路径只产发现，不修改正文。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import re
|
||
from collections import defaultdict
|
||
|
||
from .carriers import carrier_at, carrier_ranges, carve_out_applies, scope_allows
|
||
from .schemas import validate
|
||
|
||
|
||
class ArtifactIncomplete(ValueError):
|
||
"""诊断产物头或外部发现不满足合同。"""
|
||
|
||
|
||
def text_hash(text: str) -> str:
|
||
return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
|
||
|
||
|
||
def context_window(text: str, start: int, end: int, width: int = 12) -> str:
|
||
"""span 前后各 width 字符，换行转空格。"""
|
||
lo, hi = max(0, start - width), min(len(text), end + width)
|
||
return text[lo:hi].replace("\n", " ")
|
||
|
||
|
||
def _decision(rule: dict, carrier: str) -> tuple[str, str, str]:
|
||
"""给出保守的诊断建议；命中不自动等于修改命令。"""
|
||
if carve_out_applies(rule, carrier):
|
||
return "ask", "carve_out_candidate", "medium"
|
||
disposition = rule["default_disposition"]
|
||
if disposition == "blocking" and rule["layer"] == "mechanical":
|
||
return "repair", "none", "high"
|
||
if disposition == "advisory":
|
||
return "ask", "unknown", "low"
|
||
return "ask", "pending_arbitration", "high"
|
||
|
||
|
||
def _finding(rule: dict, *, finding_id: str, text: str, spans: list[str], start: int, end: int,
|
||
carrier: str, evidence: str, width: int) -> dict:
|
||
decision, possible_function, confidence = _decision(rule, carrier)
|
||
carve_outs = list(rule.get("carve_out", [])) if carve_out_applies(rule, carrier) else []
|
||
return {
|
||
"id": finding_id,
|
||
"text_hash": text_hash(text),
|
||
"rule_id": rule["id"],
|
||
"rule_version": rule["version"],
|
||
"spans": spans,
|
||
"context_window": context_window(text, start, end, width),
|
||
"layer": rule["layer"],
|
||
"evidence": evidence,
|
||
"possible_function": possible_function,
|
||
"confidence": confidence,
|
||
"decision_proposal": decision,
|
||
"carrier": carrier,
|
||
"default_disposition": rule["default_disposition"],
|
||
"carve_out_candidates": carve_outs,
|
||
}
|
||
|
||
|
||
def _regex_findings(text: str, rule: dict, ranges, width: int) -> list[dict]:
|
||
rows = []
|
||
pattern = re.compile(rule["trigger"]["pattern"], re.MULTILINE)
|
||
for match in pattern.finditer(text):
|
||
carrier = carrier_at(text, match.start(), match.end(), ranges)
|
||
if not scope_allows(rule["carrier_scope"], carrier):
|
||
continue
|
||
rows.append(_finding(
|
||
rule,
|
||
finding_id="",
|
||
text=text,
|
||
spans=[match.group(0)],
|
||
start=match.start(),
|
||
end=match.end(),
|
||
carrier=carrier,
|
||
evidence=f"确定性正则命中：{rule['trigger']['pattern']}",
|
||
width=width,
|
||
))
|
||
return rows
|
||
|
||
|
||
def _sentence_rows(text: str) -> list[tuple[int, int, str]]:
|
||
pattern = re.compile(r"[^。！？!?\n]+[。！？!?]?", re.MULTILINE)
|
||
return [
|
||
(match.start(), match.end(), match.group(0).strip())
|
||
for match in pattern.finditer(text)
|
||
if match.group(0).strip()
|
||
]
|
||
|
||
|
||
def _short_sentence_runs(text: str, rule: dict, ranges, width: int) -> list[dict]:
|
||
trigger = rule["trigger"]
|
||
max_chars = int(trigger.get("max_chars", 8))
|
||
min_run = int(trigger.get("min_run", 4))
|
||
rows = _sentence_rows(text)
|
||
findings = []
|
||
run: list[tuple[int, int, str, str]] = []
|
||
|
||
def flush() -> None:
|
||
if len(run) < min_run:
|
||
run.clear()
|
||
return
|
||
start, end = run[0][0], run[-1][1]
|
||
carrier = run[0][3]
|
||
findings.append(_finding(
|
||
rule,
|
||
finding_id="",
|
||
text=text,
|
||
spans=[item[2] for item in run],
|
||
start=start,
|
||
end=end,
|
||
carrier=carrier,
|
||
evidence=f"handler=short_sentence_run，连续 {len(run)} 句不超过 {max_chars} 字",
|
||
width=width,
|
||
))
|
||
run.clear()
|
||
|
||
for start, end, sentence in rows:
|
||
carrier = carrier_at(text, start, end, ranges)
|
||
length = len(re.sub(r"[\s。！？!?]", "", sentence))
|
||
if length <= max_chars and scope_allows(rule["carrier_scope"], carrier):
|
||
if run and run[-1][3] != carrier:
|
||
flush()
|
||
run.append((start, end, sentence, carrier))
|
||
else:
|
||
flush()
|
||
flush()
|
||
return findings
|
||
|
||
|
||
def _repeated_sentence_starts(text: str, rule: dict, ranges, width: int) -> list[dict]:
|
||
trigger = rule["trigger"]
|
||
min_run = int(trigger.get("min_run", 3))
|
||
max_chars = int(trigger.get("max_chars", 4))
|
||
grouped: dict[str, list[tuple[int, int, str, str]]] = defaultdict(list)
|
||
for start, end, sentence in _sentence_rows(text):
|
||
carrier = carrier_at(text, start, end, ranges)
|
||
if not scope_allows(rule["carrier_scope"], carrier):
|
||
continue
|
||
normalized = re.sub(r"^[\s「『“]+", "", sentence)
|
||
prefix = normalized[:max_chars]
|
||
if prefix:
|
||
grouped[prefix].append((start, end, sentence, carrier))
|
||
findings = []
|
||
for prefix, rows in sorted(grouped.items()):
|
||
if len(rows) < min_run:
|
||
continue
|
||
findings.append(_finding(
|
||
rule,
|
||
finding_id="",
|
||
text=text,
|
||
spans=[item[2] for item in rows],
|
||
start=rows[0][0],
|
||
end=rows[-1][1],
|
||
carrier=rows[0][3] if len({item[3] for item in rows}) == 1 else "mixed",
|
||
evidence=f"handler=repeated_sentence_start，句首「{prefix}」重复 {len(rows)} 次",
|
||
width=width,
|
||
))
|
||
return findings
|
||
|
||
|
||
# 摄像头式动作清单：分句切分后，短小且命中动作词、又无因果/心理连接的分句成串出现才算异常。
|
||
_CLAUSE_SPLIT_PATTERN = re.compile(r"[^，,。！？!?；;：:\n]+")
|
||
_ACTION_CONNECTIVE_PATTERN = re.compile(
|
||
r"因为|所以|于是|结果|为了|忽然|突然|竟然|不料|心中|心里|觉得|感到|想起|暗想"
|
||
)
|
||
|
||
|
||
def _camera_action_lists(text: str, rule: dict, ranges, width: int) -> list[dict]:
|
||
"""连续短动作分句清单（P-11）：只报无选择、无因果、无情绪变化的逐帧罗列。
|
||
|
||
动作词表放在规则 trigger.pattern 里，handler 只做结构判定；
|
||
战斗动作链是否保留交给 carve_out 与人工复核，不在这里裁决。
|
||
"""
|
||
trigger = rule["trigger"]
|
||
max_chars = int(trigger.get("max_chars", 6))
|
||
min_run = int(trigger.get("min_run", 4))
|
||
pattern = trigger.get("pattern")
|
||
if not isinstance(pattern, str) or not pattern.strip():
|
||
raise ArtifactIncomplete(f"规则 {rule['id']}: camera_action_list 缺动作词表 pattern")
|
||
verb = re.compile(pattern)
|
||
findings: list[dict] = []
|
||
run: list[tuple[int, int, str, str]] = []
|
||
|
||
def flush() -> None:
|
||
if len(run) >= min_run:
|
||
findings.append(_finding(
|
||
rule,
|
||
finding_id="",
|
||
text=text,
|
||
spans=[item[2] for item in run],
|
||
start=run[0][0],
|
||
end=run[-1][1],
|
||
carrier=run[0][3] if len({item[3] for item in run}) == 1 else "mixed",
|
||
evidence=f"handler=camera_action_list，连续 {len(run)} 个短动作分句无因果连接",
|
||
width=width,
|
||
))
|
||
run.clear()
|
||
|
||
for match in _CLAUSE_SPLIT_PATTERN.finditer(text):
|
||
clause = re.sub(r"\s", "", match.group(0))
|
||
carrier = carrier_at(text, match.start(), match.end(), ranges)
|
||
qualifies = (
|
||
bool(clause)
|
||
and len(clause) <= max_chars
|
||
and verb.search(clause) is not None
|
||
and _ACTION_CONNECTIVE_PATTERN.search(clause) is None
|
||
and scope_allows(rule["carrier_scope"], carrier)
|
||
)
|
||
if qualifies:
|
||
if run and run[-1][3] != carrier:
|
||
flush()
|
||
run.append((match.start(), match.end(), clause, carrier))
|
||
else:
|
||
flush()
|
||
flush()
|
||
return findings
|
||
|
||
|
||
def _uniform_paragraph_lengths(text: str, rule: dict, ranges, width: int) -> list[dict]:
|
||
"""段长过度均匀（P-05 先行子集）：连续段落长度贴近均值才提示。
|
||
|
||
只量化段长分布；目录要求最终以作品/场景基线定阈值，tolerance 是冷启动
|
||
默认值，属于可审配置而非固定长短句比例目标。段尾与句法整齐暂不量化。
|
||
"""
|
||
trigger = rule["trigger"]
|
||
min_run = int(trigger.get("min_run", 4))
|
||
tolerance = int(trigger.get("tolerance", 15)) / 100
|
||
paragraphs: list[tuple[int, int, str, int, str, bool]] = []
|
||
for match in re.finditer(r"[^\n]+", text):
|
||
body = re.sub(r"\s", "", match.group(0))
|
||
carrier = carrier_at(text, match.start(), match.end(), ranges)
|
||
allowed = bool(body) and scope_allows(rule["carrier_scope"], carrier)
|
||
paragraphs.append((match.start(), match.end(), body, len(body), carrier, allowed))
|
||
findings: list[dict] = []
|
||
index = 0
|
||
total = len(paragraphs)
|
||
while index < total:
|
||
if not paragraphs[index][5]:
|
||
index += 1
|
||
continue
|
||
end = index
|
||
while end + 1 < total and paragraphs[end + 1][5]:
|
||
window = paragraphs[index:end + 2]
|
||
mean = sum(item[3] for item in window) / len(window)
|
||
if all(abs(item[3] - mean) <= mean * tolerance for item in window):
|
||
end += 1
|
||
else:
|
||
break
|
||
count = end - index + 1
|
||
if count >= min_run:
|
||
window = paragraphs[index:end + 1]
|
||
findings.append(_finding(
|
||
rule,
|
||
finding_id="",
|
||
text=text,
|
||
spans=[item[2] for item in window],
|
||
start=window[0][0],
|
||
end=window[-1][1],
|
||
carrier=window[0][4] if len({item[4] for item in window}) == 1 else "mixed",
|
||
evidence=(
|
||
f"handler=uniform_paragraph_length，连续 {count} 段段长贴近均值"
|
||
f"（容差 {int(tolerance * 100)}%）"
|
||
),
|
||
width=width,
|
||
))
|
||
index = end + 1
|
||
else:
|
||
index += 1
|
||
return findings
|
||
|
||
|
||
def _handler_findings(text: str, rule: dict, ranges, width: int) -> list[dict]:
|
||
handler = rule["trigger"]["handler"]
|
||
if handler == "short_sentence_run":
|
||
return _short_sentence_runs(text, rule, ranges, width)
|
||
if handler == "repeated_sentence_start":
|
||
return _repeated_sentence_starts(text, rule, ranges, width)
|
||
if handler == "camera_action_list":
|
||
return _camera_action_lists(text, rule, ranges, width)
|
||
if handler == "uniform_paragraph_length":
|
||
return _uniform_paragraph_lengths(text, rule, ranges, width)
|
||
raise ArtifactIncomplete(f"规则 {rule['id']} 使用未知 handler: {handler}")
|
||
|
||
|
||
def _density_findings(text: str, rule: dict, ranges, width: int) -> list[dict]:
|
||
trigger = rule["trigger"]
|
||
window_chars = int(trigger["window_chars"])
|
||
min_hits = int(trigger["min_hits"])
|
||
matches = []
|
||
for match in re.finditer(trigger["pattern"], text, flags=re.MULTILINE):
|
||
carrier = carrier_at(text, match.start(), match.end(), ranges)
|
||
if scope_allows(rule["carrier_scope"], carrier):
|
||
matches.append((match.start(), match.end(), match.group(0), carrier))
|
||
findings = []
|
||
consumed_until = -1
|
||
for index, item in enumerate(matches):
|
||
if item[0] < consumed_until:
|
||
continue
|
||
window_end = item[0] + window_chars
|
||
cluster = [candidate for candidate in matches[index:] if candidate[0] < window_end]
|
||
if len(cluster) < min_hits:
|
||
continue
|
||
start, end = cluster[0][0], cluster[-1][1]
|
||
carrier = cluster[0][3] if len({entry[3] for entry in cluster}) == 1 else "mixed"
|
||
findings.append(_finding(
|
||
rule,
|
||
finding_id="",
|
||
text=text,
|
||
spans=[entry[2] for entry in cluster],
|
||
start=start,
|
||
end=end,
|
||
carrier=carrier,
|
||
evidence=f"density 窗口 {window_chars} 字内命中 {len(cluster)} 次（阈值 {min_hits}）",
|
||
width=width,
|
||
))
|
||
consumed_until = end
|
||
return findings
|
||
|
||
|
||
def run_deterministic_rules(text: str, rules: list, rule_library_version: str, mode: str,
|
||
window_width: int = 12) -> dict:
|
||
"""执行 active 的 regex/handler/density 规则，返回完整诊断产物。"""
|
||
if not isinstance(text, str) or not text:
|
||
raise ArtifactIncomplete("诊断文本不能为空")
|
||
if mode not in {"Audit", "Patch"}:
|
||
raise ArtifactIncomplete(f"诊断 mode 非法: {mode}")
|
||
if isinstance(window_width, bool) or not isinstance(window_width, int) or window_width < 0:
|
||
raise ArtifactIncomplete("诊断 context window 必须是非负整数")
|
||
ranges = carrier_ranges(text)
|
||
findings = []
|
||
for rule in rules:
|
||
if rule["status"] != "active":
|
||
continue
|
||
trigger_type = rule["trigger"]["type"]
|
||
if trigger_type == "regex":
|
||
rows = _regex_findings(text, rule, ranges, window_width)
|
||
elif trigger_type == "handler":
|
||
rows = _handler_findings(text, rule, ranges, window_width)
|
||
elif trigger_type == "density":
|
||
rows = _density_findings(text, rule, ranges, window_width)
|
||
else:
|
||
continue
|
||
for row in rows:
|
||
row["id"] = f"f{len(findings) + 1}"
|
||
findings.append(row)
|
||
return {
|
||
"text_hash": text_hash(text),
|
||
"rule_library_version": rule_library_version,
|
||
"mode": mode,
|
||
"findings": findings,
|
||
"executed_layers": sorted({finding["layer"] for finding in findings}),
|
||
}
|
||
|
||
|
||
def run_regex_rules(text: str, rules: list, rule_library_version: str, mode: str,
|
||
window_width: int = 12) -> dict:
|
||
"""兼容旧调用名；现在会执行全部确定性触发器。"""
|
||
return run_deterministic_rules(text, rules, rule_library_version, mode, window_width)
|
||
|
||
|
||
def merge_model_findings(artifact: dict, external: list, *, rules: dict | None = None,
|
||
text: str | None = None):
|
||
"""注入 model_judgment finding，并绑定当前正文与 active 规则。"""
|
||
base = len(artifact["findings"])
|
||
for index, item in enumerate(external):
|
||
validate(item, "finding")
|
||
if item["text_hash"] != artifact["text_hash"]:
|
||
raise ArtifactIncomplete(f"外部发现 {item.get('id')} 的 text_hash 与诊断文本不一致")
|
||
finding = dict(item)
|
||
if rules is not None:
|
||
rule = rules.get(finding["rule_id"])
|
||
if rule is None or rule.get("status") != "active":
|
||
raise ArtifactIncomplete(f"外部发现 {finding.get('id')} 未绑定 active 规则")
|
||
if rule["trigger"]["type"] != "model_judgment":
|
||
raise ArtifactIncomplete(f"外部发现 {finding.get('id')} 不能冒充确定性规则结果")
|
||
if finding["rule_version"] != rule["version"] or finding["layer"] != rule["layer"]:
|
||
raise ArtifactIncomplete(f"外部发现 {finding.get('id')} 的规则版本或 layer 不一致")
|
||
if text is not None:
|
||
missing = [span for span in finding["spans"] if span not in text]
|
||
if missing:
|
||
raise ArtifactIncomplete(f"外部发现 {finding.get('id')} 的 span 不在当前正文")
|
||
if "carrier" not in finding:
|
||
first = finding["spans"][0]
|
||
start = text.find(first)
|
||
finding["carrier"] = carrier_at(text, start, start + len(first)) if start >= 0 else "unknown"
|
||
finding["id"] = finding.get("id") or f"f{base + index + 1}"
|
||
artifact["findings"].append(finding)
|
||
return artifact
|
||
|
||
|
||
def validate_artifact(artifact: dict):
|
||
"""产物头完整性门禁：缺项即视为诊断未发生。"""
|
||
for key in ("text_hash", "rule_library_version", "mode", "findings"):
|
||
if key not in artifact or artifact[key] in (None, ""):
|
||
raise ArtifactIncomplete(f"诊断产物缺 {key}，视为诊断未发生")
|
||
if artifact["mode"] not in {"Audit", "Patch"}:
|
||
raise ArtifactIncomplete(f"诊断产物 mode 非法: {artifact['mode']}")
|
||
if not isinstance(artifact["findings"], list):
|
||
raise ArtifactIncomplete("诊断产物 findings 必须是数组")
|
||
for finding in artifact["findings"]:
|
||
validate(finding, "finding")
|
||
return True
|
||
|
||
|
||
__all__ = [
|
||
"ArtifactIncomplete", "text_hash", "context_window", "run_deterministic_rules",
|
||
"run_regex_rules", "merge_model_findings", "validate_artifact",
|
||
]
|