810 lines
33 KiB
Bash
Executable File
810 lines
33 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# 激活 prepare-staging-rc.sh 已构建的 commit-pinned RC。
|
||
# 本机只负责经 Tailscale 直连下发；所有起停、证据和回滚都在 mini-desktop 完成。
|
||
set -euo pipefail
|
||
|
||
RC_COMMIT="${RC_COMMIT:-}"
|
||
RC_HOST="${RC_HOST:-root@100.64.0.7}"
|
||
RC_ROOT="${RC_ROOT:-/root/game-staging/releases}"
|
||
GENERATION_ATTESTATION_ROOT="${GENERATION_ATTESTATION_ROOT:-/root/game-staging/attestations}"
|
||
BOOTSTRAP_ADMIN_TOKEN_FILE="${BOOTSTRAP_ADMIN_TOKEN_FILE:-}"
|
||
ACTIVATION_TIMEOUT_SEC="${ACTIVATION_TIMEOUT_SEC:-180}"
|
||
ACTIVATION_MODE="activate"
|
||
if [ "${1:-}" = "--verify-r1-attestation" ]; then
|
||
ACTIVATION_MODE="verify-r1-attestation"
|
||
elif [ -n "${1:-}" ]; then
|
||
printf '未知参数：%s\n' "$1" >&2
|
||
exit 2
|
||
fi
|
||
if [ -n "${GENERATION_GATE_EVIDENCE_REF:-}" ] && [ -n "${GENERATION_GATE_REF:-}" ] \
|
||
&& [ "$GENERATION_GATE_EVIDENCE_REF" != "$GENERATION_GATE_REF" ]; then
|
||
printf 'GENERATION_GATE_EVIDENCE_REF 与 GENERATION_GATE_REF 冲突，拒绝选择其一继续。\n' >&2
|
||
exit 2
|
||
fi
|
||
GENERATION_GATE_EVIDENCE_REF="${GENERATION_GATE_EVIDENCE_REF:-${GENERATION_GATE_REF:-}}"
|
||
# 新变量表达真实语义；旧变量仅兼容受控 hash + RC 内相对路径格式。
|
||
if [ -n "${STORAGE_AUDIT_REF:-}" ] && [ -n "${OBJECT_SNAPSHOT_REF:-}" ] \
|
||
&& [ "$STORAGE_AUDIT_REF" != "$OBJECT_SNAPSHOT_REF" ]; then
|
||
printf 'STORAGE_AUDIT_REF 与旧 OBJECT_SNAPSHOT_REF 冲突，拒绝选择其一继续。\n' >&2
|
||
exit 2
|
||
fi
|
||
STORAGE_AUDIT_REF="${STORAGE_AUDIT_REF:-${OBJECT_SNAPSHOT_REF:-}}"
|
||
SSH_BIN="${SSH_BIN:-ssh}"
|
||
|
||
if [[ ! "$RC_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then
|
||
printf 'RC_COMMIT 必须是完整 40 位小写提交哈希。\n' >&2
|
||
exit 2
|
||
fi
|
||
if [[ ! "$ACTIVATION_TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$ACTIVATION_TIMEOUT_SEC" -lt 30 ]; then
|
||
printf 'ACTIVATION_TIMEOUT_SEC 必须是至少 30 秒的整数。\n' >&2
|
||
exit 2
|
||
fi
|
||
validate_evidence_ref() {
|
||
local ref="$1" path component
|
||
[[ "$ref" =~ ^sha256:[0-9a-f]{64}:[A-Za-z0-9._/-]+$ ]] || return 1
|
||
path="${ref#sha256:}"
|
||
path="${path#*:}"
|
||
[ -n "$path" ] && [[ "$path" != /* ]] || return 1
|
||
IFS='/' read -r -a components <<<"$path"
|
||
for component in "${components[@]}"; do
|
||
[ -n "$component" ] && [ "$component" != '.' ] && [ "$component" != '..' ] \
|
||
&& [ "$component" != '.git' ] || return 1
|
||
done
|
||
}
|
||
if [ -z "$GENERATION_GATE_EVIDENCE_REF" ] || ! validate_evidence_ref "$GENERATION_GATE_EVIDENCE_REF"; then
|
||
printf 'GENERATION_GATE_EVIDENCE_REF 必填，格式为 sha256:<64hex>:<安全相对路径>。\n' >&2
|
||
exit 2
|
||
fi
|
||
if [ -n "$STORAGE_AUDIT_REF" ] \
|
||
&& [[ ! "$STORAGE_AUDIT_REF" =~ ^sha256:[0-9a-f]{64}:evidence/storage-audit\.tsv$ ]]; then
|
||
printf 'STORAGE_AUDIT_REF 非法：只接受 sha256:<64hex>:evidence/storage-audit.tsv；留空则远端实时生成。\n' >&2
|
||
exit 2
|
||
fi
|
||
if [ "$ACTIVATION_MODE" = activate ] && [[ ! "$BOOTSTRAP_ADMIN_TOKEN_FILE" =~ ^/[A-Za-z0-9._/-]+$ ]]; then
|
||
printf 'BOOTSTRAP_ADMIN_TOKEN_FILE 必填，且必须是 mini-desktop 上的安全绝对路径。\n' >&2
|
||
exit 2
|
||
fi
|
||
|
||
printf '激活 staging RC：host=%s commit=%s\n' "$RC_HOST" "$RC_COMMIT"
|
||
if [ "$ACTIVATION_MODE" = activate ]; then
|
||
printf '注意：该操作会执行数据库迁移并切换五个 staging 服务。\n'
|
||
else
|
||
printf '仅校验独立 R1 attestation，不访问数据库、不切换服务。\n'
|
||
fi
|
||
|
||
"$SSH_BIN" -o ProxyCommand=none "$RC_HOST" bash -s -- \
|
||
"$RC_COMMIT" "$RC_ROOT" "$ACTIVATION_TIMEOUT_SEC" "$STORAGE_AUDIT_REF" \
|
||
"$GENERATION_GATE_EVIDENCE_REF" "$GENERATION_ATTESTATION_ROOT" "$ACTIVATION_MODE" \
|
||
"$BOOTSTRAP_ADMIN_TOKEN_FILE" <<'REMOTE_SCRIPT'
|
||
set -Eeuo pipefail
|
||
|
||
commit="$1"
|
||
root="$2"
|
||
timeout_sec="$3"
|
||
expected_storage_audit_ref="$4"
|
||
expected_generation_gate_ref="$5"
|
||
attestation_root="$6"
|
||
activation_mode="$7"
|
||
bootstrap_admin_token_file="$8"
|
||
release="$root/$commit"
|
||
active_pointer="$root/active"
|
||
stable_prefix="gda-staging"
|
||
run_id="$(date -u +%Y%m%dT%H%M%SZ)"
|
||
short="${commit:0:12}"
|
||
candidate_prefix="gda-rc-${short}-${run_id}"
|
||
evidence="$release/evidence/activation-$run_id"
|
||
activation_log="$evidence/activation.log"
|
||
env_file="/root/huijing-dev.env"
|
||
switch_started=0
|
||
pointer_changed=0
|
||
previous_was_active=0
|
||
previous="none"
|
||
previous_valid=0
|
||
migration_started=0
|
||
database_restore_required=0
|
||
database_restore_rc=not-run
|
||
bootstrap_token_validated=0
|
||
account_provision_run_id="${run_id}-dogfood"
|
||
account_provision_cleanup_manifest="$evidence/account-provision/provision-cleanup.tsv"
|
||
runtime_cleanup_rc=not-run
|
||
|
||
umask 077
|
||
if [ "$activation_mode" = activate ]; then
|
||
mkdir -p "$evidence"
|
||
chmod 700 "$evidence"
|
||
exec > >(tee -a "$activation_log") 2>&1
|
||
else
|
||
evidence="$(mktemp -d)"
|
||
trap 'rm -rf "$evidence"' EXIT
|
||
fi
|
||
|
||
log_step() {
|
||
printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$1"
|
||
}
|
||
|
||
delete_bootstrap_token() {
|
||
[ "$bootstrap_token_validated" -eq 1 ] || return 0
|
||
if [ -f "$bootstrap_admin_token_file" ] && [ ! -L "$bootstrap_admin_token_file" ]; then
|
||
rm -f -- "$bootstrap_admin_token_file"
|
||
log_step "bootstrap admin token 文件已安全删除"
|
||
else
|
||
log_step "bootstrap admin token 文件状态异常，拒绝跟随或删除 symlink"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
cleanup_dogfood_provision_runtime() {
|
||
[ -f "$account_provision_cleanup_manifest" ] || return 0
|
||
log_step "定向清理本次 dogfood 注册/登录限流桶与 OAuth token 缓存"
|
||
REDIS_PASSWORD="${REDIS_PASSWORD:-}" \
|
||
"$release/deploy/provision-dogfood-users.sh" --cleanup \
|
||
"$account_provision_cleanup_manifest" "$account_provision_run_id" \
|
||
>"$evidence/account-provision-runtime-cleanup.log" 2>&1
|
||
}
|
||
|
||
unit_name() {
|
||
printf '%s-%s' "$1" "$2"
|
||
}
|
||
|
||
stop_stack() {
|
||
local prefix="$1" service unit
|
||
for service in backend studio admin cheap-agent cheap-worker; do
|
||
unit="$(unit_name "$prefix" "$service")"
|
||
systemctl stop "$unit" >/dev/null 2>&1 || true
|
||
systemctl reset-failed "$unit" >/dev/null 2>&1 || true
|
||
done
|
||
}
|
||
|
||
capture_journals() {
|
||
local prefix="$1" label="$2" service unit
|
||
mkdir -p "$evidence/journal-$label"
|
||
for service in backend studio admin cheap-agent cheap-worker; do
|
||
unit="$(unit_name "$prefix" "$service")"
|
||
journalctl -u "$unit" --no-pager -n 300 >"$evidence/journal-$label/$service.log" 2>&1 || true
|
||
done
|
||
}
|
||
|
||
port_listening() {
|
||
local port="$1"
|
||
ss -H -ltn | awk -v suffix=":$port" '$4 ~ suffix"$" {found=1} END {exit !found}'
|
||
}
|
||
|
||
wait_port() {
|
||
local name="$1" port="$2" deadline=$((SECONDS + timeout_sec))
|
||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||
if port_listening "$port"; then
|
||
log_step "$name readiness PASS port=$port"
|
||
return 0
|
||
fi
|
||
sleep 2
|
||
done
|
||
log_step "$name readiness FAIL port=$port"
|
||
return 1
|
||
}
|
||
|
||
wait_url() {
|
||
local name="$1" url="$2" deadline=$((SECONDS + timeout_sec)) code
|
||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||
code="$(curl -sS --noproxy '*' --connect-timeout 2 --max-time 5 -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || true)"
|
||
if [ "$code" = "200" ]; then
|
||
log_step "$name readiness PASS http=200"
|
||
return 0
|
||
fi
|
||
sleep 2
|
||
done
|
||
log_step "$name readiness FAIL url=$url"
|
||
return 1
|
||
}
|
||
|
||
wait_json_health() {
|
||
local name="$1" url="$2" field="$3" expected="$4" deadline=$((SECONDS + timeout_sec)) body
|
||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||
body="$(curl -sS --noproxy '*' --connect-timeout 2 --max-time 5 "$url" 2>/dev/null || true)"
|
||
if HEALTH_FIELD="$field" HEALTH_EXPECTED="$expected" python3 -c '
|
||
import json,os,sys
|
||
value=json.load(sys.stdin)
|
||
for part in os.environ["HEALTH_FIELD"].split("."):
|
||
value=value.get(part) if isinstance(value,dict) else None
|
||
assert str(value).lower() == os.environ["HEALTH_EXPECTED"].lower()
|
||
' <<<"$body" >/dev/null 2>&1; then
|
||
log_step "$name readiness PASS"
|
||
return 0
|
||
fi
|
||
sleep 2
|
||
done
|
||
log_step "$name readiness FAIL url=$url"
|
||
return 1
|
||
}
|
||
|
||
run_unit() {
|
||
local unit="$1" launcher="$2"
|
||
systemctl reset-failed "$unit" >/dev/null 2>&1 || true
|
||
systemd-run --quiet --collect --unit="$unit" \
|
||
--property=Restart=on-failure --property=RestartSec=2s \
|
||
--property=TimeoutStopSec=30s "$launcher"
|
||
}
|
||
|
||
write_launchers() {
|
||
local target="$1" label="$2" backend_port="$3" studio_port="$4" admin_port="$5"
|
||
local agent_port="$6" worker_port="$7" discovery_enabled="$8"
|
||
local launch_dir="$evidence/launchers-$label" quoted_target
|
||
quoted_target="$(printf '%q' "$target")"
|
||
mkdir -p "$launch_dir"
|
||
|
||
cat >"$launch_dir/backend.sh" <<EOF
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
set -a
|
||
source /root/huijing-dev.env
|
||
set +a
|
||
export SPRING_PROFILES_ACTIVE=staging
|
||
export AIGC_WORKER_URL=http://127.0.0.1:${worker_port}/generate
|
||
exec java -jar ${quoted_target}/game-cloud/huijing-server/target/huijing-server.jar \\
|
||
--server.port=${backend_port} --spring.cloud.nacos.discovery.enabled=${discovery_enabled} \\
|
||
--spring.flyway.enabled=false
|
||
EOF
|
||
|
||
cat >"$launch_dir/studio.sh" <<EOF
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
cd ${quoted_target}/game-studio
|
||
exec node node_modules/vite/bin/vite.js preview \\
|
||
--host 0.0.0.0 --port ${studio_port} --strictPort --outDir dist
|
||
EOF
|
||
|
||
cat >"$launch_dir/admin.sh" <<EOF
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
cd ${quoted_target}/game-admin
|
||
exec node node_modules/vite/bin/vite.js preview \\
|
||
--host 0.0.0.0 --port ${admin_port} --strictPort --outDir dist-stage
|
||
EOF
|
||
|
||
cat >"$launch_dir/cheap-agent.sh" <<EOF
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
set -a
|
||
source /root/huijing-dev.env
|
||
set +a
|
||
export NO_PROXY=100.64.0.8,localhost,127.0.0.1
|
||
export no_proxy=100.64.0.8,localhost,127.0.0.1
|
||
unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
|
||
export CHEAP_SERVICE_PORT=${agent_port}
|
||
cd ${quoted_target}/cheap-worker
|
||
exec .venv/bin/python cheap_service_app.py
|
||
EOF
|
||
|
||
cat >"$launch_dir/cheap-worker.sh" <<EOF
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
set -a
|
||
source /root/huijing-dev.env
|
||
set +a
|
||
: "\${AIGC_CALLBACK_SECRET:?AIGC_CALLBACK_SECRET 未配置}"
|
||
export NO_PROXY=100.64.0.8,localhost,127.0.0.1
|
||
export no_proxy=100.64.0.8,localhost,127.0.0.1
|
||
unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
|
||
export CHEAP_SERVICE_URL=http://127.0.0.1:${agent_port}
|
||
cd ${quoted_target}/cheap-worker
|
||
exec .venv/bin/python worker_service.py --port ${worker_port} --callback-secret "\$AIGC_CALLBACK_SECRET"
|
||
EOF
|
||
|
||
chmod 700 "$launch_dir"/*.sh
|
||
printf '%s' "$launch_dir"
|
||
}
|
||
|
||
start_stack() {
|
||
local target="$1" label="$2" prefix="$3" backend_port="$4" studio_port="$5" admin_port="$6"
|
||
local agent_port="$7" worker_port="$8" discovery_enabled="$9" launch_dir
|
||
launch_dir="$(write_launchers "$target" "$label" "$backend_port" "$studio_port" "$admin_port" \
|
||
"$agent_port" "$worker_port" "$discovery_enabled")"
|
||
|
||
run_unit "$(unit_name "$prefix" cheap-agent)" "$launch_dir/cheap-agent.sh"
|
||
wait_port "$label cheap-agent" "$agent_port"
|
||
run_unit "$(unit_name "$prefix" cheap-worker)" "$launch_dir/cheap-worker.sh"
|
||
wait_json_health "$label cheap-worker" "http://127.0.0.1:$worker_port/health" ok true
|
||
run_unit "$(unit_name "$prefix" backend)" "$launch_dir/backend.sh"
|
||
wait_json_health "$label backend" "http://127.0.0.1:$backend_port/actuator/health" status UP
|
||
run_unit "$(unit_name "$prefix" studio)" "$launch_dir/studio.sh"
|
||
wait_url "$label studio" "http://127.0.0.1:$studio_port/"
|
||
run_unit "$(unit_name "$prefix" admin)" "$launch_dir/admin.sh"
|
||
wait_url "$label admin" "http://127.0.0.1:$admin_port/"
|
||
}
|
||
|
||
start_candidate_probe() {
|
||
local target="$1" label="$2" prefix="$3" studio_port="$4" admin_port="$5"
|
||
local agent_port="$6" worker_port="$7" launch_dir
|
||
launch_dir="$(write_launchers "$target" "$label" 18080 "$studio_port" "$admin_port" \
|
||
"$agent_port" "$worker_port" false)"
|
||
|
||
# 当前应用没有可证明关闭全部 @Scheduled 的全局开关，候选 backend 不得连接共享数据面启动。
|
||
printf 'candidate_backend=SKIPPED_NO_ISOLATED_DATA_PLANE\n' >"$evidence/candidate-backend.status"
|
||
run_unit "$(unit_name "$prefix" cheap-agent)" "$launch_dir/cheap-agent.sh"
|
||
wait_port "$label cheap-agent" "$agent_port"
|
||
run_unit "$(unit_name "$prefix" cheap-worker)" "$launch_dir/cheap-worker.sh"
|
||
wait_json_health "$label cheap-worker" "http://127.0.0.1:$worker_port/health" ok true
|
||
run_unit "$(unit_name "$prefix" studio)" "$launch_dir/studio.sh"
|
||
wait_url "$label studio" "http://127.0.0.1:$studio_port/"
|
||
run_unit "$(unit_name "$prefix" admin)" "$launch_dir/admin.sh"
|
||
wait_url "$label admin" "http://127.0.0.1:$admin_port/"
|
||
}
|
||
|
||
valid_prepared_release() {
|
||
local target="$1" expected="${2:-}"
|
||
[ -d "$target/.git" ] || return 1
|
||
[ -f "$target/evidence/artifact-paths.env" ] || return 1
|
||
[ -f "$target/game-cloud/huijing-server/target/huijing-server.jar" ] || return 1
|
||
[ -d "$target/game-studio/dist" ] || return 1
|
||
[ -d "$target/game-admin/dist-stage" ] || return 1
|
||
[ -x "$target/cheap-worker/.venv/bin/python" ] || return 1
|
||
if [ -n "$expected" ]; then
|
||
[ "$(git -C "$target" rev-parse HEAD)" = "$expected" ] || return 1
|
||
fi
|
||
}
|
||
|
||
verify_generation_gate_evidence() {
|
||
local ref="$1" expected_commit="$2" hash_and_path expected_hash relative_path actual_hash
|
||
local source_path canonical_root canonical_source component walk mode
|
||
hash_and_path="${ref#sha256:}"
|
||
expected_hash="${hash_and_path%%:*}"
|
||
relative_path="${hash_and_path#*:}"
|
||
|
||
# attestation 是 R1 执行方在 commit 之外生成的内容寻址文件；hash 在外部引用中，避免“证明文件属于被证明 commit”的自引用。
|
||
[ "${relative_path##*/}" = "$expected_hash.attestation" ] || {
|
||
log_step "生成门 attestation 文件名未绑定 SHA-256 path=$relative_path"
|
||
return 1
|
||
}
|
||
[ -d "$attestation_root" ] && [ ! -L "$attestation_root" ] || {
|
||
log_step "attestation 根目录缺失或为 symlink root=$attestation_root"
|
||
return 1
|
||
}
|
||
canonical_root="$(realpath "$attestation_root")" || return 1
|
||
[ "$canonical_root" = "$attestation_root" ] || {
|
||
log_step "attestation 根目录必须使用 canonical 绝对路径 root=$attestation_root"
|
||
return 1
|
||
}
|
||
walk="$attestation_root"
|
||
IFS='/' read -r -a path_components <<<"$relative_path"
|
||
for component in "${path_components[@]}"; do
|
||
walk="$walk/$component"
|
||
[ ! -L "$walk" ] || {
|
||
log_step "attestation 路径不得包含 symlink path=$relative_path"
|
||
return 1
|
||
}
|
||
done
|
||
source_path="$attestation_root/$relative_path"
|
||
[ -f "$source_path" ] || {
|
||
log_step "attestation 不是普通文件 path=$relative_path"
|
||
return 1
|
||
}
|
||
canonical_source="$(realpath "$source_path")" || return 1
|
||
[ "$canonical_source" = "$source_path" ] || return 1
|
||
mode="$(stat -c '%a' "$source_path" 2>/dev/null || stat -f '%Lp' "$source_path")"
|
||
[ "$mode" = 600 ] || {
|
||
log_step "attestation 权限必须为 0600 actual=$mode path=$relative_path"
|
||
return 1
|
||
}
|
||
actual_hash="$(sha256sum "$source_path" | awk '{print $1}')"
|
||
[ "$actual_hash" = "$expected_hash" ] || {
|
||
log_step "生成门证据 SHA-256 不匹配 path=$relative_path"
|
||
return 1
|
||
}
|
||
|
||
# 证据由 R1 执行方生成，激活器只验证，不创建或改写任何“通过”结论。
|
||
python3 - "$source_path" "$expected_commit" <<'PY'
|
||
import sys
|
||
|
||
path, expected_commit = sys.argv[1:]
|
||
values = {}
|
||
with open(path, encoding="utf-8") as handle:
|
||
for line_number, raw in enumerate(handle, 1):
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
if "=" not in line:
|
||
raise SystemExit(f"生成门证据第 {line_number} 行不是 key=value")
|
||
key, value = line.split("=", 1)
|
||
key, value = key.strip(), value.strip()
|
||
if not key or key in values:
|
||
raise SystemExit(f"生成门证据键为空或重复: {key!r}")
|
||
values[key] = value
|
||
|
||
required = {
|
||
"schema": "generation-r1-gate/1",
|
||
"rc_commit": expected_commit,
|
||
"r1_status": "PASS",
|
||
"actor_prompt_eval": "PASS",
|
||
"judge_prompt_eval": "PASS",
|
||
"game_content_gold": "PASS",
|
||
"live_prompt": "PASS",
|
||
"full_gate": "PASS",
|
||
"fresh_25": "PASS",
|
||
"historical_11": "PASS",
|
||
"production_shadow_20": "PASS",
|
||
"runtime_eligible": "true",
|
||
}
|
||
for key, expected in required.items():
|
||
actual = values.get(key)
|
||
if actual != expected:
|
||
raise SystemExit(f"生成门证据未通过 {key}: expected={expected} actual={actual!r}")
|
||
PY
|
||
|
||
printf 'generation_gate_ref=%s\ngeneration_gate_path=%s\n' "$ref" "$relative_path" \
|
||
>"$evidence/generation-gate-result.txt"
|
||
log_step "生成 R1 门证据校验 PASS path=$relative_path"
|
||
}
|
||
|
||
if [ "$activation_mode" = verify-r1-attestation ]; then
|
||
verify_generation_gate_evidence "$expected_generation_gate_ref" "$commit"
|
||
printf 'GENERATION_ATTESTATION_PASS commit=%s ref=%s\n' "$commit" "$expected_generation_gate_ref"
|
||
exit 0
|
||
fi
|
||
|
||
exec 9>"$root/.activation.lock"
|
||
if ! flock -n 9; then
|
||
log_step "已有 staging activation 正在执行，拒绝并发部署"
|
||
exit 75
|
||
fi
|
||
|
||
flyway_history() {
|
||
docker exec game-staging-mysql sh -lc \
|
||
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysql -uroot -N -B ruoyi-vue-pro -e "SELECT installed_rank,COALESCE(version,\"\"),description,type,script,checksum,success FROM flyway_schema_history ORDER BY installed_rank"'
|
||
}
|
||
|
||
run_flyway_migrations() {
|
||
local runtime_dir="$evidence/flyway-runtime" source_file="$evidence/StagingFlywayMigration.java"
|
||
local class_dir="$evidence/flyway-classes" jar="$release/game-cloud/huijing-server/target/huijing-server.jar"
|
||
mkdir -p "$runtime_dir" "$class_dir"
|
||
java -Djarmode=tools -jar "$jar" extract --destination "$runtime_dir" >/dev/null
|
||
cat >"$source_file" <<'JAVA'
|
||
import org.flywaydb.core.Flyway;
|
||
|
||
/** 只执行 Flyway，不启动 Spring、消费者、定时任务或 Web 服务。 */
|
||
public final class StagingFlywayMigration {
|
||
public static void main(String[] args) {
|
||
if (args.length != 2) {
|
||
throw new IllegalArgumentException("需要 migration 与 staging migration 两个目录");
|
||
}
|
||
String url = System.getenv("STAGING_FLYWAY_URL");
|
||
String user = System.getenv("STAGING_FLYWAY_USER");
|
||
String password = System.getenv("STAGING_FLYWAY_PASSWORD");
|
||
if (url == null || user == null || password == null) {
|
||
throw new IllegalStateException("缺少 Flyway 数据库凭据");
|
||
}
|
||
Flyway flyway = Flyway.configure()
|
||
.dataSource(url, user, password)
|
||
.locations("filesystem:" + args[0], "filesystem:" + args[1])
|
||
.failOnMissingLocations(true)
|
||
.load();
|
||
flyway.validate();
|
||
var result = flyway.migrate();
|
||
System.out.printf("FLYWAY_MIGRATION_PASS executed=%d target=%s%n",
|
||
result.migrationsExecuted, result.targetSchemaVersion);
|
||
}
|
||
}
|
||
JAVA
|
||
javac -cp "$runtime_dir/lib/*" -d "$class_dir" "$source_file"
|
||
STAGING_FLYWAY_URL='jdbc:mysql://127.0.0.1:13306/ruoyi-vue-pro?useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true' \
|
||
STAGING_FLYWAY_USER=root STAGING_FLYWAY_PASSWORD="$MYSQL_ROOT_PASSWORD" \
|
||
java -cp "$class_dir:$runtime_dir/lib/*" StagingFlywayMigration \
|
||
"$release/game-cloud/huijing-server/src/main/resources/db/migration" \
|
||
"$release/game-cloud/huijing-server/src/main/resources/db/staging" \
|
||
>"$evidence/flyway-migration.log" 2>&1
|
||
grep -q '^FLYWAY_MIGRATION_PASS ' "$evidence/flyway-migration.log"
|
||
}
|
||
|
||
restore_database_snapshot() {
|
||
local dump="$evidence/pre-migration-ruoyi-vue-pro.sql.gz"
|
||
[ -s "$dump" ] || return 1
|
||
stop_stack "$stable_prefix"
|
||
docker exec game-staging-mysql sh -lc \
|
||
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysql -uroot -e "DROP DATABASE IF EXISTS `ruoyi-vue-pro`"' || return 1
|
||
gzip -dc "$dump" | docker exec -i game-staging-mysql sh -lc \
|
||
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysql -uroot' || return 1
|
||
flyway_history >"$evidence/flyway-after-compensation.tsv" || return 1
|
||
cmp -s "$evidence/flyway-before.tsv" "$evidence/flyway-after-compensation.tsv" || return 1
|
||
printf 'DATABASE_COMPENSATION=PASS snapshot=%s\n' "$dump" >"$evidence/database-compensation.status"
|
||
}
|
||
|
||
restore_active_pointer() {
|
||
local target="$1" temp="$root/.active-rollback-$run_id"
|
||
rm -f "$temp"
|
||
if [ "$target" = "none" ]; then
|
||
rm -f "$active_pointer"
|
||
else
|
||
ln -s "$target" "$temp"
|
||
mv -Tf "$temp" "$active_pointer"
|
||
fi
|
||
}
|
||
|
||
failure_handler() {
|
||
local rc=$?
|
||
trap - ERR
|
||
set +e
|
||
log_step "ACTIVATION FAIL rc=${rc}，开始保留证据并恢复旧栈"
|
||
capture_journals "$candidate_prefix" candidate-failed
|
||
capture_journals "$stable_prefix" stable-failed
|
||
stop_stack "$candidate_prefix"
|
||
cleanup_dogfood_provision_runtime
|
||
runtime_cleanup_rc=$?
|
||
printf 'runtime_cleanup_rc=%s\n' "$runtime_cleanup_rc" >"$evidence/account-provision-runtime-cleanup.status"
|
||
if [ "$database_restore_required" -eq 1 ]; then
|
||
log_step "迁移后失败，停止新栈并恢复迁移前数据库快照"
|
||
restore_database_snapshot
|
||
database_restore_rc=$?
|
||
printf 'database_restore_rc=%s\n' "$database_restore_rc" >"$evidence/database-restore-result.txt"
|
||
if [ "$database_restore_rc" -ne 0 ]; then
|
||
log_step "数据库补偿失败，禁止宣称服务回滚成功"
|
||
printf 'FIRST_RC_FULL_ROLLBACK=UNAVAILABLE database_restore=FAILED\n' >"$evidence/rollback-result.txt"
|
||
exit 71
|
||
fi
|
||
fi
|
||
if [ "$switch_started" -eq 1 ]; then
|
||
stop_stack "$stable_prefix"
|
||
if [ "$previous_was_active" -eq 1 ] && [ "$previous_valid" -eq 1 ]; then
|
||
log_step "恢复旧版本服务 target=$previous"
|
||
start_stack "$previous" rollback "$stable_prefix" 48080 4173 4174 8300 9501 true
|
||
rollback_rc=$?
|
||
printf 'rollback_service_rc=%s\n' "$rollback_rc" >"$evidence/rollback-result.txt"
|
||
else
|
||
printf 'rollback_service_rc=not-available\nFIRST_RC_FULL_ROLLBACK=UNAVAILABLE\n' >"$evidence/rollback-result.txt"
|
||
fi
|
||
fi
|
||
if [ "$pointer_changed" -eq 1 ]; then
|
||
restore_active_pointer "$previous"
|
||
fi
|
||
if [ "$bootstrap_token_validated" -eq 1 ]; then
|
||
log_step "激活失败，保留 0600 bootstrap admin token 供补偿后重试或人工吊销"
|
||
fi
|
||
if [ "$runtime_cleanup_rc" != not-run ] && [ "$runtime_cleanup_rc" -ne 0 ]; then
|
||
log_step "dogfood 运行态补偿失败，禁止直接重试激活"
|
||
rc=72
|
||
fi
|
||
printf 'failed commit=%s at=%s evidence=%s\n' "$commit" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$evidence" \
|
||
>"$release/RC_STATUS"
|
||
chmod -R go-rwx "$evidence" "$release/RC_STATUS"
|
||
log_step "ACTIVATION_EVIDENCE=$evidence"
|
||
exit "$rc"
|
||
}
|
||
trap failure_handler ERR
|
||
|
||
log_step "校验 commit-pinned RC 与配置"
|
||
valid_prepared_release "$release" "$commit"
|
||
grep -q "^ready commit=$commit " "$release/RC_STATUS"
|
||
[ -z "$(git -C "$release" status --porcelain --untracked-files=no)" ]
|
||
|
||
# 生成门是数据库访问、Nacos/服务切换之前的硬前置；引用缺失、漂移或 R1 任一项不绿立即失败。
|
||
verify_generation_gate_evidence "$expected_generation_gate_ref" "$commit"
|
||
|
||
[ -f "$env_file" ]
|
||
case "$(stat -c '%a' "$env_file")" in
|
||
*00) ;;
|
||
*) log_step "拒绝读取权限过宽的 $env_file"; false ;;
|
||
esac
|
||
|
||
# 只校验键存在和非空，绝不输出值；后续独立 Flyway 进程从当前受控环境读取数据库凭据。
|
||
set -a
|
||
source "$env_file"
|
||
set +a
|
||
: "${MYSQL_ROOT_PASSWORD:?}" "${REDIS_PASSWORD:?}" "${NACOS_PASSWORD:?}"
|
||
: "${AIGC_CALLBACK_SECRET:?}" "${NEWAPI_KEY:?}"
|
||
[ -n "${DOGFOOD_PASSWORD_SEED:-}" ] || {
|
||
log_step "缺 DOGFOOD_PASSWORD_SEED，无法确定性预置内测账号"
|
||
false
|
||
}
|
||
[ "${AIGC_EXECUTOR_ENABLED:-}" = "true" ]
|
||
|
||
cd "$release"
|
||
sha256sum -c evidence/artifacts.sha256 >"$evidence/artifact-verification.txt"
|
||
|
||
log_step "生成并校验 DB-only 存储审计"
|
||
storage_audit_output="$(RC_RELEASE="$release" EXPECTED_STORAGE_AUDIT_REF="$expected_storage_audit_ref" \
|
||
bash "$release/deploy/generate-staging-storage-audit.sh")"
|
||
storage_audit_ref="$(printf '%s\n' "$storage_audit_output" | sed -n 's/^STORAGE_AUDIT_PASS ref=//p')"
|
||
[[ "$storage_audit_ref" =~ ^sha256:[0-9a-f]{64}:evidence/storage-audit\.tsv$ ]]
|
||
cp "$release/evidence/storage-audit.tsv" "$evidence/storage-audit.tsv"
|
||
cp "$release/evidence/storage-audit.ref" "$evidence/storage-audit.ref"
|
||
printf 'storage_audit_ref=%s\n' "$storage_audit_ref" >"$evidence/storage-audit-result.txt"
|
||
log_step "$storage_audit_output"
|
||
|
||
if [ -L "$active_pointer" ]; then
|
||
previous="$(readlink -f "$active_pointer" || true)"
|
||
[ -n "$previous" ] || previous="none"
|
||
elif [ -L "$root/current" ]; then
|
||
# 首次迁移兼容旧 current 指针；成功后只维护 canonical active。
|
||
previous="$(readlink -f "$root/current" || true)"
|
||
[ -n "$previous" ] || previous="none"
|
||
fi
|
||
if [ "$previous" != "none" ] && valid_prepared_release "$previous"; then
|
||
previous_valid=1
|
||
fi
|
||
for service in backend studio admin cheap-agent cheap-worker; do
|
||
if systemctl is-active --quiet "$(unit_name "$stable_prefix" "$service")"; then
|
||
previous_was_active=1
|
||
fi
|
||
done
|
||
if [ "$previous_was_active" -eq 1 ] && [ "$previous_valid" -ne 1 ]; then
|
||
log_step "已有受管服务运行，但 active/current 未指向可回滚 RC，拒绝切换"
|
||
false
|
||
fi
|
||
printf 'previous=%s\nprevious_was_active=%s\n' "$previous" "$previous_was_active" >"$evidence/activation-context.txt"
|
||
|
||
log_step "启动不接共享数据面的候选探针；backend 因缺全局禁调度开关明确跳过"
|
||
for port in 14173 14174 18300 19501; do
|
||
if port_listening "$port"; then
|
||
log_step "候选端口已占用 port=$port"
|
||
false
|
||
fi
|
||
done
|
||
start_candidate_probe "$release" candidate "$candidate_prefix" 14173 14174 18300 19501
|
||
capture_journals "$candidate_prefix" candidate-ready
|
||
|
||
log_step "候选非数据面探针全绿，停止旧栈后执行独立 Flyway 迁移"
|
||
switch_started=1
|
||
capture_journals "$stable_prefix" pre-switch-old
|
||
stop_stack "$stable_prefix"
|
||
|
||
# 旧栈停写后再生成补偿快照，避免候选探针期间的新写入在失败恢复时丢失。
|
||
log_step "迁移前数据库快照"
|
||
docker inspect game-staging-mysql >/dev/null
|
||
docker exec game-staging-mysql sh -lc \
|
||
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysqldump -uroot --single-transaction --routines --events --databases ruoyi-vue-pro' \
|
||
| gzip -9 >"$evidence/pre-migration-ruoyi-vue-pro.sql.gz"
|
||
test -s "$evidence/pre-migration-ruoyi-vue-pro.sql.gz"
|
||
chmod 600 "$evidence/pre-migration-ruoyi-vue-pro.sql.gz"
|
||
|
||
log_step "在隔离临时 MySQL 容器中验证快照可恢复"
|
||
mysql_image="$(docker inspect game-staging-mysql --format '{{.Config.Image}}')"
|
||
DUMP_FILE="$evidence/pre-migration-ruoyi-vue-pro.sql.gz" \
|
||
EXPECTED_AUDIT_FILE="$evidence/storage-audit.tsv" MYSQL_IMAGE="$mysql_image" \
|
||
bash "$release/deploy/verify-mysql-dump-restore.sh" >"$evidence/mysql-restore-verify.log" 2>&1
|
||
grep -q '^RESTORE_VERIFY_PASS ' "$evidence/mysql-restore-verify.log"
|
||
log_step "数据库快照隔离恢复验证 PASS"
|
||
|
||
flyway_history >"$evidence/flyway-before.tsv"
|
||
migration_started=1
|
||
database_restore_required=1
|
||
run_flyway_migrations
|
||
flyway_history >"$evidence/flyway-after-migration.tsv"
|
||
flyway_failed_count="$(docker exec game-staging-mysql sh -lc \
|
||
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysql -uroot -N -B ruoyi-vue-pro -e "SELECT COUNT(*) FROM flyway_schema_history WHERE success <> 1"')"
|
||
[ "$flyway_failed_count" = "0" ]
|
||
log_step "独立 Flyway 迁移 PASS，开始切换正式五端口"
|
||
for port in 48080 4173 4174 8300 9501; do
|
||
if port_listening "$port"; then
|
||
log_step "正式端口仍被非受管进程占用 port=$port"
|
||
false
|
||
fi
|
||
done
|
||
|
||
start_stack "$release" stable "$stable_prefix" 48080 4173 4174 8300 9501 true
|
||
|
||
log_step "校验 bootstrap admin token 文件"
|
||
[ -f "$bootstrap_admin_token_file" ] && [ ! -L "$bootstrap_admin_token_file" ]
|
||
bootstrap_token_mode="$(stat -c '%a' "$bootstrap_admin_token_file" 2>/dev/null || stat -f '%Lp' "$bootstrap_admin_token_file")"
|
||
[ "$bootstrap_token_mode" = 600 ] || {
|
||
log_step "bootstrap admin token 文件权限必须为 0600 actual=$bootstrap_token_mode"
|
||
false
|
||
}
|
||
bootstrap_token_validated=1
|
||
|
||
# 默认 admin 尚有效时先完成 40 人预置；事务返回即表示 Nacos 已逐字恢复，之后才允许禁用 admin。
|
||
log_step "使用 bootstrap admin token 执行 40 人账号预置事务"
|
||
account_provision_evidence="$evidence/account-provision"
|
||
ADMIN_TOKEN_FILE="$bootstrap_admin_token_file" \
|
||
DOGFOOD_PROVISION_ACK='PROVISION_DOGFOOD_WITH_NACOS_TRANSACTION' \
|
||
RUN_ID="$account_provision_run_id" \
|
||
BASE=http://127.0.0.1:48080 MYSQL_SSH_HOST='' \
|
||
DOGFOOD_ACCOUNTS_FILE="$evidence/dogfood-accounts.tsv" \
|
||
EVIDENCE_DIR="$account_provision_evidence" \
|
||
bash "$release/deploy/provision-dogfood-with-nacos-transaction.sh" \
|
||
>"$evidence/account-provision-transaction.log" 2>&1
|
||
grep -q '^pass restored=true ' "$account_provision_evidence/transaction.status"
|
||
grep -q 'PROVISION_PASS accounts=40 creators=10 role_mismatches=0 claimed_accounts=40 duplicate_claims=0' \
|
||
"$account_provision_evidence/provision.log" || {
|
||
log_step "40 人终验缺失；若额度池不足，请先运行受控 new-api import 补建 neice_051-054"
|
||
false
|
||
}
|
||
chmod 600 "$evidence/account-provision-transaction.log" "$evidence/dogfood-accounts.tsv"
|
||
log_step "40 人账号与额度终验 PASS，Nacos 已恢复"
|
||
|
||
log_step "轮换并验证狗粮审核账号"
|
||
reviewer_credentials="$evidence/reviewer-credentials.json"
|
||
reviewer_evidence="$evidence/reviewer-evidence.json"
|
||
reviewer_runtime_evidence="$evidence/reviewer-runtime-verification.json"
|
||
python3 "$release/deploy/manage-dogfood-reviewers.py" \
|
||
--ack ROTATE_DOGFOOD_REVIEWERS \
|
||
--admin-token-file "$bootstrap_admin_token_file" \
|
||
--credentials-out "$reviewer_credentials" \
|
||
--evidence-out "$reviewer_evidence" \
|
||
>"$evidence/reviewer-rotation.log" 2>&1
|
||
|
||
# 工具已在轮换时验证默认公开密码失败；这里再验证新账号可登录，且被禁用 admin 的 bootstrap token 已失效。
|
||
python3 - "$reviewer_credentials" "$reviewer_evidence" "$bootstrap_admin_token_file" \
|
||
"$reviewer_runtime_evidence" <<'PY'
|
||
import json,os,stat,sys
|
||
from urllib.error import HTTPError,URLError
|
||
from urllib.request import Request,urlopen
|
||
|
||
credentials_path,evidence_path,token_path,output_path=map(os.path.abspath,sys.argv[1:])
|
||
|
||
def read_private(path):
|
||
flags=os.O_RDONLY|getattr(os,"O_NOFOLLOW",0)
|
||
fd=os.open(path,flags)
|
||
with os.fdopen(fd,"r",encoding="utf-8") as handle:
|
||
mode=stat.S_IMODE(os.fstat(handle.fileno()).st_mode)
|
||
if mode != 0o600 or not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
|
||
raise RuntimeError(f"敏感文件必须是 0600 普通文件 path={path}")
|
||
return handle.read()
|
||
|
||
credentials=json.loads(read_private(credentials_path))
|
||
rotation=json.loads(read_private(evidence_path))
|
||
bootstrap_token=read_private(token_path).strip()
|
||
if credentials.get("status") != "COMPLETE" or rotation.get("status") != "PASS":
|
||
raise RuntimeError("审核账号轮换输出未收口")
|
||
accounts=credentials.get("accounts") or {}
|
||
rotation_accounts={item.get("username"):item for item in rotation.get("accounts") or []}
|
||
if rotation_accounts.get("admin",{}).get("defaultPasswordLoginCode") not in (1002000000,1002000001):
|
||
raise RuntimeError("默认公开 admin 密码没有确定拒绝证据")
|
||
|
||
def request_json(request):
|
||
try:
|
||
with urlopen(request,timeout=10) as response:
|
||
return response.status,json.loads(response.read().decode("utf-8"))
|
||
except HTTPError as error:
|
||
body=error.read().decode("utf-8")
|
||
return error.code,json.loads(body) if body else {}
|
||
except (URLError,TimeoutError,json.JSONDecodeError) as error:
|
||
raise RuntimeError("审核账号运行态验证请求失败") from error
|
||
|
||
verified=[]
|
||
for username in ("dogfood_reviewer","dogfood_readonly"):
|
||
password=accounts.get(username)
|
||
if not isinstance(password,str) or not password:
|
||
raise RuntimeError(f"审核账号凭据缺失 username={username}")
|
||
payload=json.dumps({"username":username,"password":password}).encode("utf-8")
|
||
status,body=request_json(Request("http://127.0.0.1:48080/admin-api/system/auth/login",
|
||
data=payload,method="POST",headers={"Content-Type":"application/json","tenant-id":"1"}))
|
||
if status != 200 or body.get("code") != 0 or not (body.get("data") or {}).get("accessToken"):
|
||
raise RuntimeError(f"审核账号无法登录 username={username}")
|
||
verified.append(username)
|
||
|
||
status,body=request_json(Request(
|
||
"http://127.0.0.1:48080/admin-api/system/auth/get-permission-info",
|
||
method="GET",headers={"Authorization":f"Bearer {bootstrap_token}","tenant-id":"1"}))
|
||
if status != 401 or body.get("code") != 401:
|
||
raise RuntimeError(f"bootstrap admin 旧 token 失效证据不严格 status={status} code={body.get('code')}")
|
||
|
||
flags=os.O_WRONLY|os.O_CREAT|os.O_EXCL|getattr(os,"O_NOFOLLOW",0)
|
||
fd=os.open(output_path,flags,0o600)
|
||
with os.fdopen(fd,"w",encoding="utf-8") as handle:
|
||
json.dump({"status":"PASS","loginVerified":verified,
|
||
"defaultAdminPasswordRejected":True,"bootstrapTokenInvalidated":True},
|
||
handle,ensure_ascii=False,indent=2)
|
||
handle.write("\n")
|
||
PY
|
||
chmod 600 "$reviewer_credentials" "$reviewer_evidence" "$reviewer_runtime_evidence" \
|
||
"$evidence/reviewer-rotation.log"
|
||
log_step "狗粮审核账号轮换与运行态验证 PASS"
|
||
|
||
BASE=http://127.0.0.1:48080 STUDIO_BASE=http://127.0.0.1:4173 \
|
||
bash "$release/deploy/smoke-test.sh" >"$evidence/stable-smoke.log" 2>&1
|
||
capture_journals "$stable_prefix" stable-ready
|
||
|
||
log_step "正式栈全绿，原子切换 active 指针"
|
||
active_temp="$root/.active-$run_id"
|
||
ln -s "$release" "$active_temp"
|
||
mv -Tf "$active_temp" "$active_pointer"
|
||
pointer_changed=1
|
||
printf 'active commit=%s at=%s evidence=%s\n' "$commit" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$evidence" \
|
||
>"$release/RC_STATUS"
|
||
|
||
stop_stack "$candidate_prefix"
|
||
rm -f -- "$account_provision_cleanup_manifest"
|
||
chmod -R go-rwx "$evidence" "$release/RC_STATUS"
|
||
delete_bootstrap_token
|
||
trap - ERR
|
||
log_step "RC_ACTIVATION_PASS commit=$commit active=$release"
|
||
log_step "ACTIVATION_EVIDENCE=$evidence"
|
||
REMOTE_SCRIPT
|