修复编辑器定价持久化与资源预览
将模型定价改为 SpacetimeDB 强类型持久化并兼容旧配置种子迁移 修复快速编辑支付弹窗期间框选显示和交互冻结 补齐后台图片放大以及视频音频资源预览 收紧主站和 External 资源换签授权并记录管理员跨用户审计 固化外部生成入队价格、attempt 钱包结算和最终 lease 失败收口 加固运行时身份轮换、bootstrap secret 与生产构建发布门禁 同步生成绑定、定向测试、运维脚本和项目文档
This commit is contained in:
@@ -207,6 +207,32 @@ ensure_env_value_migrates_old_default() {
|
||||
write_env_value "${file_path}" "${key}" "${new_default}"
|
||||
}
|
||||
|
||||
ensure_runtime_bootstrap_secret_file_env() {
|
||||
local file_path="$1"
|
||||
local key="GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE"
|
||||
local direct_key="GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET"
|
||||
local canonical_path="/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt"
|
||||
local current_value direct_value
|
||||
|
||||
direct_value="$(read_env_value "${file_path}" "${direct_key}")"
|
||||
if [[ -n "${direct_value}" ]]; then
|
||||
unset direct_value
|
||||
echo "[production-api-deploy] ${file_path} 不得保存 ${direct_key} 明文;生产环境只允许使用 ${key}。" >&2
|
||||
exit 1
|
||||
fi
|
||||
unset direct_value
|
||||
|
||||
current_value="$(read_env_value "${file_path}" "${key}")"
|
||||
if [[ -z "${current_value}" ]]; then
|
||||
ensure_env_value "${file_path}" "${key}" "${canonical_path}"
|
||||
return
|
||||
fi
|
||||
if [[ "${current_value}" != "${canonical_path}" ]]; then
|
||||
echo "[production-api-deploy] ${key} 必须使用 Stdb publish 同步写入的固定路径 ${canonical_path}: ${file_path}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_privileged() {
|
||||
if [[ "$(id -u)" -eq 0 ]]; then
|
||||
"$@"
|
||||
@@ -235,6 +261,24 @@ ensure_runtime_dir() {
|
||||
run_privileged install -d -o genarrative -g genarrative -m "${mode}" "${path}"
|
||||
}
|
||||
|
||||
migrate_legacy_editor_generation_pricing_override() {
|
||||
local current_link="$1"
|
||||
local target_dir="/var/lib/genarrative/editor-generation-pricing"
|
||||
local target_file="${target_dir}/editor-generation-pricing.override.json"
|
||||
local legacy_file="${current_link}/.app/editor-generation-pricing.override.json"
|
||||
|
||||
if [[ -f "${target_file}" ]]; then
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${legacy_file}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[production-api-deploy] 迁移旧模型定价 override 到运行态目录: ${legacy_file} -> ${target_file}"
|
||||
ensure_runtime_dir "${target_dir}" "0750"
|
||||
run_privileged install -o genarrative -g genarrative -m 0640 "${legacy_file}" "${target_file}"
|
||||
}
|
||||
|
||||
ensure_runtime_env_and_dirs() {
|
||||
local api_env_file="$1"
|
||||
local tracking_enabled tracking_outbox_dir wallet_refund_enabled wallet_refund_outbox_dir
|
||||
@@ -255,6 +299,7 @@ ensure_runtime_env_and_dirs() {
|
||||
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_runtime_bootstrap_secret_file_env "${api_env_file}"
|
||||
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000" "180000"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
@@ -285,6 +330,7 @@ ensure_worker_runtime_env_defaults() {
|
||||
ensure_env_value_migrates_old_default "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_runtime_bootstrap_secret_file_env "${worker_env_file}"
|
||||
}
|
||||
|
||||
extract_pingora_env_files_from_unit() {
|
||||
@@ -1038,6 +1084,7 @@ fi
|
||||
|
||||
ensure_runtime_env_and_dirs "${API_ENV_FILE}"
|
||||
ensure_worker_runtime_env_defaults "${WORKER_ENV_FILE}"
|
||||
migrate_legacy_editor_generation_pricing_override "${CURRENT_LINK}"
|
||||
|
||||
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
|
||||
PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")"
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
callSpacetimeProcedureViaCli,
|
||||
ensureProcedureOk,
|
||||
validateSpacetimeDatabaseName,
|
||||
} from '../spacetime-migration-common.mjs';
|
||||
import { spawn } from 'node:child_process';
|
||||
|
||||
const PROCEDURE = 'rotate_editor_generation_runtime_service_identity_and_return';
|
||||
|
||||
function usage() {
|
||||
console.error(
|
||||
'用法: production-runtime-writer-identity-rotate.mjs --database <database> --server-url <http(s)://host:port> --operator-identity <64-hex> --operator-user-id <user-id> --next-writer-identity <64-hex> --confirm-next-writer-identity <64-hex> --note <audit-note>',
|
||||
);
|
||||
}
|
||||
|
||||
function readValue(argv, index, name) {
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith('--')) {
|
||||
throw new Error(`${name} 缺少参数值。`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeIdentity(value, name) {
|
||||
const identity = value.trim().replace(/^0x/u, '').toLowerCase();
|
||||
if (!/^[0-9a-f]{64}$/u.test(identity)) {
|
||||
throw new Error(`${name} 必须是 64 位十六进制 SpacetimeDB identity。`);
|
||||
}
|
||||
return identity;
|
||||
}
|
||||
|
||||
async function readCurrentCliIdentity() {
|
||||
const output = await new Promise((resolve, reject) => {
|
||||
const child = spawn('spacetime', ['login', 'show'], {
|
||||
shell: false,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let text = '';
|
||||
child.stdout.on('data', (chunk) => {
|
||||
text += chunk.toString();
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
text += chunk.toString();
|
||||
});
|
||||
child.on('error', reject);
|
||||
child.on('exit', (code, signal) => {
|
||||
if (signal) {
|
||||
reject(new Error(`spacetime login show 被信号中断: ${signal}`));
|
||||
} else if (code !== 0) {
|
||||
reject(new Error(`无法读取当前 SpacetimeDB CLI identity,退出码 ${code}`));
|
||||
} else {
|
||||
resolve(text);
|
||||
}
|
||||
});
|
||||
});
|
||||
const identities = [...output.matchAll(/\b[0-9a-f]{64}\b/giu)].map((match) =>
|
||||
match[0].toLowerCase(),
|
||||
);
|
||||
if (identities.length !== 1) {
|
||||
throw new Error('spacetime login show 未返回唯一的当前 CLI identity。');
|
||||
}
|
||||
return identities[0];
|
||||
}
|
||||
|
||||
function parseOptions(argv) {
|
||||
const options = {
|
||||
database: '',
|
||||
serverUrl: '',
|
||||
operatorIdentity: '',
|
||||
operatorUserId: '',
|
||||
nextWriterIdentity: '',
|
||||
confirmNextWriterIdentity: '',
|
||||
note: '',
|
||||
passthrough: [],
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--help' || arg === '-h') {
|
||||
usage();
|
||||
process.exit(0);
|
||||
}
|
||||
if (arg === '--database') {
|
||||
options.database = readValue(argv, index, arg);
|
||||
} else if (arg === '--server-url') {
|
||||
options.serverUrl = readValue(argv, index, arg);
|
||||
} else if (arg === '--operator-identity') {
|
||||
options.operatorIdentity = readValue(argv, index, arg);
|
||||
} else if (arg === '--operator-user-id') {
|
||||
options.operatorUserId = readValue(argv, index, arg);
|
||||
} else if (arg === '--next-writer-identity') {
|
||||
options.nextWriterIdentity = readValue(argv, index, arg);
|
||||
} else if (arg === '--confirm-next-writer-identity') {
|
||||
options.confirmNextWriterIdentity = readValue(argv, index, arg);
|
||||
} else if (arg === '--note') {
|
||||
options.note = readValue(argv, index, arg);
|
||||
} else {
|
||||
throw new Error(`未知参数: ${arg}`);
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
try {
|
||||
const options = parseOptions(process.argv.slice(2));
|
||||
validateSpacetimeDatabaseName(options.database);
|
||||
if (!/^https?:\/\/[A-Za-z0-9.-]+(?::[0-9]{1,5})?$/u.test(options.serverUrl)) {
|
||||
throw new Error('--server-url 必须是无路径、无查询参数的显式 http(s) URL。');
|
||||
}
|
||||
|
||||
const operatorIdentity = normalizeIdentity(options.operatorIdentity, '--operator-identity');
|
||||
const nextWriterIdentity = normalizeIdentity(
|
||||
options.nextWriterIdentity,
|
||||
'--next-writer-identity',
|
||||
);
|
||||
const confirmedIdentity = normalizeIdentity(
|
||||
options.confirmNextWriterIdentity,
|
||||
'--confirm-next-writer-identity',
|
||||
);
|
||||
if (nextWriterIdentity !== confirmedIdentity) {
|
||||
throw new Error('--confirm-next-writer-identity 必须与 --next-writer-identity 完全一致。');
|
||||
}
|
||||
if (nextWriterIdentity === operatorIdentity) {
|
||||
throw new Error('runtime writer identity 不能设为 migration operator identity。');
|
||||
}
|
||||
|
||||
const currentCliIdentity = await readCurrentCliIdentity();
|
||||
if (operatorIdentity !== currentCliIdentity) {
|
||||
throw new Error('--operator-identity 必须与当前 SpacetimeDB CLI 登录 identity 一致。');
|
||||
}
|
||||
|
||||
const operatorUserId = options.operatorUserId.trim();
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$/u.test(operatorUserId)) {
|
||||
throw new Error('--operator-user-id 格式不合法。');
|
||||
}
|
||||
const note = options.note.trim();
|
||||
if (!note || note.length > 500) {
|
||||
throw new Error('--note 必填且不能超过 500 个字符。');
|
||||
}
|
||||
|
||||
const result = await callSpacetimeProcedureViaCli(
|
||||
{ database: options.database, serverUrl: options.serverUrl, passthrough: [] },
|
||||
PROCEDURE,
|
||||
{
|
||||
next_writer_identity_hex: nextWriterIdentity,
|
||||
operator_user_id: operatorUserId,
|
||||
note,
|
||||
},
|
||||
);
|
||||
ensureProcedureOk(result);
|
||||
console.log(
|
||||
`[runtime-writer-identity-rotation] 已完成 identity rotation: database=${options.database}, operator_identity=${operatorIdentity}, next_writer_identity=${nextWriterIdentity}; 审计记录已写入 editor_generation_runtime_identity_rotation。`,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[runtime-writer-identity-rotation] 操作失败;必须使用已授权的 migration operator CLI 登录态。${error instanceof Error ? ` ${error.message}` : String(error)}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -1,11 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/deploy/production-stdb-publish.sh --source-dir build/<version> --database <database> [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--clear-database] [--backup-mode async|sync|skip]
|
||||
./scripts/deploy/production-stdb-publish.sh --source-dir build/<version> --database <database> --migration-bootstrap-secret-file <protected-file> [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--api-health-url http://127.0.0.1:8082/healthz] [--api-readiness-timeout-seconds 60] [--clear-database] [--backup-mode async|sync|skip]
|
||||
|
||||
说明:
|
||||
进入维护模式,校验 spacetime_module.wasm.sha256,并在生产实例本机执行 spacetime publish。
|
||||
@@ -14,6 +15,9 @@ usage() {
|
||||
发布时固定追加 --no-config,只使用显式参数,避免工作区或用户目录里的 spacetime 配置干扰目标。
|
||||
async 模式会在 publish 前先做本地冷备份,再在 publish 完成后后台上传 OSS,避免低带宽上传阻塞部署。
|
||||
如需强制等待备份完成并在失败时阻断 publish,传入 --backup-mode sync。
|
||||
发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径,再重启并验活重启前 active 的服务。
|
||||
migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供,不从构建 artifact 读取。
|
||||
如果 API 重启前为 active,会在退出维护模式前等待本机 /healthz readiness 通过。
|
||||
失败时保留维护模式。
|
||||
EOF
|
||||
}
|
||||
@@ -44,6 +48,9 @@ SERVER_ALIAS="local"
|
||||
SERVER_URL="http://127.0.0.1:3101"
|
||||
SPACETIME_ROOT_DIR="/stdb"
|
||||
RUN_AS_USER="spacetimedb"
|
||||
MIGRATION_BOOTSTRAP_SECRET_FILE=""
|
||||
API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-server.env}"
|
||||
WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
|
||||
CLEAR_DATABASE=0
|
||||
BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
|
||||
DEPLOY_COMPLETED=0
|
||||
@@ -54,6 +61,229 @@ ASYNC_BACKUP_ARCHIVE=""
|
||||
ASYNC_BACKUP_MANIFEST=""
|
||||
ASYNC_BACKUP_LOG=""
|
||||
SPACETIME_READY_TIMEOUT_SECONDS="${GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS:-60}"
|
||||
API_HEALTH_URL="${GENARRATIVE_STDB_PUBLISH_API_HEALTH_URL:-http://127.0.0.1:8082/healthz}"
|
||||
API_READINESS_TIMEOUT_SECONDS="${GENARRATIVE_STDB_PUBLISH_API_READINESS_TIMEOUT_SECONDS:-60}"
|
||||
API_WAS_ACTIVE=0
|
||||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE="${GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE:-/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt}"
|
||||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE="/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt"
|
||||
|
||||
run_privileged() {
|
||||
if [[ "$(id -u)" -eq 0 ]]; then
|
||||
"$@"
|
||||
elif command -v sudo >/dev/null 2>&1; then
|
||||
sudo -n "$@"
|
||||
else
|
||||
echo "[production-stdb-publish] 当前用户不是 root,且 sudo 不可用;无法完成受保护运行态文件或服务操作。" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_runtime_bootstrap_secret_env_file() {
|
||||
local env_file="$1"
|
||||
local required="$2"
|
||||
|
||||
if [[ ! -e "${env_file}" ]]; then
|
||||
if [[ "${required}" == "true" ]]; then
|
||||
echo "[production-stdb-publish] 运行时环境文件不存在,无法补齐 bootstrap secret FILE: ${env_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[production-stdb-publish] worker 环境文件不存在,跳过 bootstrap secret FILE 补齐: ${env_file}"
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${env_file}" || -L "${env_file}" ]]; then
|
||||
echo "[production-stdb-publish] 运行时环境文件必须是普通文件且不能是符号链接: ${env_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_privileged python3 - "${env_file}" "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}" <<'PY'
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
env_path = Path(sys.argv[1])
|
||||
canonical_path = sys.argv[2]
|
||||
file_key = "GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE"
|
||||
direct_key = "GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET"
|
||||
|
||||
|
||||
def normalize_value(value: str) -> str:
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"):
|
||||
value = value[1:-1]
|
||||
return value.strip()
|
||||
|
||||
|
||||
lines = env_path.read_text(encoding="utf-8").splitlines()
|
||||
next_lines: list[str] = []
|
||||
file_key_written = False
|
||||
changed = False
|
||||
|
||||
for raw_line in lines:
|
||||
stripped = raw_line.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||||
next_lines.append(raw_line)
|
||||
continue
|
||||
|
||||
key, value = stripped.split("=", 1)
|
||||
if key == direct_key and normalize_value(value):
|
||||
print(
|
||||
f"[production-stdb-publish] {env_path} 不得保存 {direct_key} 明文;生产环境只允许使用 {file_key}。",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
if key != file_key:
|
||||
next_lines.append(raw_line)
|
||||
continue
|
||||
|
||||
current_value = normalize_value(value)
|
||||
if current_value and current_value != canonical_path:
|
||||
print(
|
||||
f"[production-stdb-publish] {file_key} 必须使用固定路径 {canonical_path}: {env_path}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
if file_key_written:
|
||||
changed = True
|
||||
continue
|
||||
|
||||
next_lines.append(f"{file_key}={canonical_path}")
|
||||
file_key_written = True
|
||||
changed = changed or raw_line != next_lines[-1]
|
||||
|
||||
if not file_key_written:
|
||||
next_lines.append(f"{file_key}={canonical_path}")
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
metadata = env_path.stat()
|
||||
fd, temp_name = tempfile.mkstemp(prefix=f".{env_path.name}.", dir=env_path.parent)
|
||||
try:
|
||||
os.fchmod(fd, stat.S_IMODE(metadata.st_mode))
|
||||
os.fchown(fd, metadata.st_uid, metadata.st_gid)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as temp_file:
|
||||
temp_file.write("\n".join(next_lines) + "\n")
|
||||
os.replace(temp_name, env_path)
|
||||
finally:
|
||||
if os.path.exists(temp_name):
|
||||
os.unlink(temp_name)
|
||||
PY
|
||||
echo "[production-stdb-publish] 已确认 bootstrap secret FILE 配置: ${env_file}"
|
||||
}
|
||||
|
||||
get_runtime_service_active_state() {
|
||||
local service_name="$1"
|
||||
local state=""
|
||||
local exit_code=0
|
||||
|
||||
if state="$(run_privileged systemctl is-active "${service_name}")"; then
|
||||
if [[ "${state}" != "active" ]]; then
|
||||
echo "[production-stdb-publish] systemctl is-active 返回成功但状态异常: ${service_name}, state=${state}" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "${state}"
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [[ "${exit_code}" -eq 3 && "${state}" =~ ^(inactive|failed|activating|deactivating|reloading|maintenance|refreshing)$ ]]; then
|
||||
printf '%s\n' "${state}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "[production-stdb-publish] 查询运行时服务状态失败: ${service_name}, exit=${exit_code}, state=${state:-<empty>}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_runtime_service_and_require_active() {
|
||||
local service_name="$1"
|
||||
local state=""
|
||||
|
||||
echo "[production-stdb-publish] 重启运行时服务以加载新引导密钥: ${service_name}"
|
||||
if ! run_privileged systemctl restart "${service_name}"; then
|
||||
echo "[production-stdb-publish] 运行时服务重启失败: ${service_name}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! state="$(get_runtime_service_active_state "${service_name}")"; then
|
||||
return 1
|
||||
fi
|
||||
if [[ "${state}" != "active" ]]; then
|
||||
echo "[production-stdb-publish] 运行时服务重启后未恢复 active: ${service_name}, state=${state}" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "[production-stdb-publish] 运行时服务重启后已恢复 active: ${service_name}"
|
||||
}
|
||||
|
||||
restart_runtime_services_after_bootstrap_secret_install() {
|
||||
local api_state=""
|
||||
local controller_state=""
|
||||
local worker_service=""
|
||||
local worker_state=""
|
||||
local worker_units_output=""
|
||||
local list_units_exit_code=0
|
||||
local -a active_worker_services=()
|
||||
|
||||
if ! api_state="$(get_runtime_service_active_state genarrative-api.service)"; then
|
||||
return 1
|
||||
fi
|
||||
if ! controller_state="$(get_runtime_service_active_state genarrative-external-generation-controller.service)"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if worker_units_output="$(
|
||||
run_privileged systemctl list-units \
|
||||
--type=service \
|
||||
--state=active \
|
||||
--no-legend \
|
||||
--plain \
|
||||
'genarrative-external-generation-worker@*.service'
|
||||
)"; then
|
||||
:
|
||||
else
|
||||
list_units_exit_code=$?
|
||||
echo "[production-stdb-publish] 查询 active worker 服务失败,阻断退出维护模式: exit=${list_units_exit_code}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
while read -r worker_service _; do
|
||||
if [[ "${worker_service}" =~ ^genarrative-external-generation-worker@[A-Za-z0-9_.@:-]+\.service$ ]]; then
|
||||
active_worker_services+=("${worker_service}")
|
||||
fi
|
||||
done <<<"${worker_units_output}"
|
||||
|
||||
if [[ "${api_state}" == "active" ]]; then
|
||||
API_WAS_ACTIVE=1
|
||||
if ! restart_runtime_service_and_require_active genarrative-api.service; then
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if [[ "${controller_state}" == "active" ]]; then
|
||||
if ! restart_runtime_service_and_require_active genarrative-external-generation-controller.service; then
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${#active_worker_services[@]}" -gt 0 ]]; then
|
||||
echo "[production-stdb-publish] 重启运行时 worker 以加载新引导密钥: ${active_worker_services[*]}"
|
||||
if ! run_privileged systemctl restart "${active_worker_services[@]}"; then
|
||||
echo "[production-stdb-publish] 运行时 worker 重启失败: ${active_worker_services[*]}" >&2
|
||||
return 1
|
||||
fi
|
||||
for worker_service in "${active_worker_services[@]}"; do
|
||||
if ! worker_state="$(get_runtime_service_active_state "${worker_service}")"; then
|
||||
return 1
|
||||
fi
|
||||
if [[ "${worker_state}" != "active" ]]; then
|
||||
echo "[production-stdb-publish] 运行时 worker 重启后未恢复 active: ${worker_service}, state=${worker_state}" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "[production-stdb-publish] 运行时 worker 重启后已恢复 active: ${worker_service}"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -86,6 +316,26 @@ while [[ $# -gt 0 ]]; do
|
||||
RUN_AS_USER="${2:?缺少 --run-as-user 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--migration-bootstrap-secret-file)
|
||||
MIGRATION_BOOTSTRAP_SECRET_FILE="${2:?缺少 --migration-bootstrap-secret-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--api-env-file)
|
||||
API_ENV_FILE="${2:?缺少 --api-env-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--worker-env-file)
|
||||
WORKER_ENV_FILE="${2:?缺少 --worker-env-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--api-health-url)
|
||||
API_HEALTH_URL="${2:?缺少 --api-health-url 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--api-readiness-timeout-seconds)
|
||||
API_READINESS_TIMEOUT_SECONDS="${2:?缺少 --api-readiness-timeout-seconds 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--clear-database)
|
||||
CLEAR_DATABASE=1
|
||||
shift
|
||||
@@ -119,6 +369,18 @@ if [[ ! "${SPACETIME_ROOT_DIR}" == /* || "${SPACETIME_ROOT_DIR}" == *".."* ]]; t
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" != "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}" ]]; then
|
||||
echo "[production-stdb-publish] GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE 必须使用固定路径 ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do
|
||||
if [[ "${runtime_env_file}" != /* || "${runtime_env_file}" == *".."* || "${runtime_env_file}" == "/" ]]; then
|
||||
echo "[production-stdb-publish] 运行时环境文件必须是非根绝对路径且不能包含 ..: ${runtime_env_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then
|
||||
echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2
|
||||
exit 1
|
||||
@@ -134,6 +396,16 @@ if [[ ! "${SPACETIME_READY_TIMEOUT_SECONDS}" =~ ^[0-9]+$ || "${SPACETIME_READY_T
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${API_READINESS_TIMEOUT_SECONDS}" =~ ^[0-9]+$ || "${API_READINESS_TIMEOUT_SECONDS}" -le 0 ]]; then
|
||||
echo "[production-stdb-publish] API readiness timeout 必须是正整数: ${API_READINESS_TIMEOUT_SECONDS}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${API_HEALTH_URL}" =~ ^http://(127\.0\.0\.1|localhost)(:[0-9]{1,5})?/healthz$ ]]; then
|
||||
echo "[production-stdb-publish] API readiness 必须使用本机 HTTP /healthz: ${API_HEALTH_URL}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "${SOURCE_DIR}" ]]; then
|
||||
echo "[production-stdb-publish] 发布目录不存在: ${SOURCE_DIR}" >&2
|
||||
exit 1
|
||||
@@ -141,11 +413,31 @@ fi
|
||||
|
||||
SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)"
|
||||
|
||||
if [[ ! -f "${SOURCE_DIR}/spacetime_module.wasm" || ! -f "${SOURCE_DIR}/spacetime_module.wasm.sha256" ]]; then
|
||||
echo "[production-stdb-publish] 缺少 spacetime_module.wasm 或 spacetime_module.wasm.sha256: ${SOURCE_DIR}" >&2
|
||||
if [[ ! -f "${SOURCE_DIR}/spacetime_module.wasm" || ! -f "${SOURCE_DIR}/spacetime_module.wasm.sha256" || ! -f "${SOURCE_DIR}/release-manifest.json" ]]; then
|
||||
echo "[production-stdb-publish] 缺少 spacetime_module.wasm、checksum 或 release-manifest.json: ${SOURCE_DIR}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || ! -f "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || -L "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || ! -r "${MIGRATION_BOOTSTRAP_SECRET_FILE}" ]]; then
|
||||
echo "[production-stdb-publish] --migration-bootstrap-secret-file 必须是可读、非符号链接的受保护普通文件。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MIGRATION_BOOTSTRAP_SECRET="$(cat "${MIGRATION_BOOTSTRAP_SECRET_FILE}")"
|
||||
MIGRATION_BOOTSTRAP_SECRET="${MIGRATION_BOOTSTRAP_SECRET%$'\r'}"
|
||||
if [[ ! "${MIGRATION_BOOTSTRAP_SECRET}" =~ ^[0-9a-fA-F]{64}$ ]]; then
|
||||
echo "[production-stdb-publish] 运行时服务身份引导密钥必须是 64 位十六进制高熵值。" >&2
|
||||
exit 1
|
||||
fi
|
||||
MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE="$(printf '%s' "${MIGRATION_BOOTSTRAP_SECRET}" | sha256sum)"
|
||||
MIGRATION_BOOTSTRAP_SECRET_SHA256="${MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE%% *}"
|
||||
EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256="$(node -e 'const fs=require("node:fs"); const manifest=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.stdout.write(manifest.migration_bootstrap_secret_sha256 || "");' "${SOURCE_DIR}/release-manifest.json")"
|
||||
if [[ ! "${EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256}" =~ ^[0-9a-f]{64}$ || "${MIGRATION_BOOTSTRAP_SECRET_SHA256}" != "${EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256}" ]]; then
|
||||
echo "[production-stdb-publish] Secret File 与构建 WASM 的 bootstrap secret 摘要不一致。" >&2
|
||||
exit 1
|
||||
fi
|
||||
unset MIGRATION_BOOTSTRAP_SECRET MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE
|
||||
|
||||
on_exit() {
|
||||
local exit_code=$?
|
||||
if [[ "${BACKUP_MODE}" == "async" && -n "${ASYNC_BACKUP_STATUS_FILE}" && -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
|
||||
@@ -237,6 +529,31 @@ wait_for_spacetime_ready() {
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_api_healthz_ready() {
|
||||
if [[ "${API_WAS_ACTIVE}" -ne 1 ]]; then
|
||||
echo "[production-stdb-publish] API 重启前不是 active,跳过 /healthz readiness"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local deadline=$((SECONDS + API_READINESS_TIMEOUT_SECONDS))
|
||||
local last_status=""
|
||||
|
||||
echo "[production-stdb-publish] 等待 API /healthz readiness: ${API_HEALTH_URL},timeout=${API_READINESS_TIMEOUT_SECONDS}s"
|
||||
while (( SECONDS < deadline )); do
|
||||
if last_status="$(curl -fsS --max-time 2 "${API_HEALTH_URL}" 2>&1)"; then
|
||||
echo "[production-stdb-publish] API /healthz readiness 已通过: ${API_HEALTH_URL}"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "[production-stdb-publish] API /healthz readiness 未在超时内通过: ${API_HEALTH_URL}" >&2
|
||||
if [[ -n "${last_status}" ]]; then
|
||||
echo "[production-stdb-publish] 最后一次 /healthz 检查输出: ${last_status}" >&2
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
|
||||
|
||||
case "${BACKUP_MODE}" in
|
||||
@@ -328,6 +645,32 @@ else
|
||||
spacetime "${PUBLISH_ARGS[@]}"
|
||||
fi
|
||||
|
||||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR="$(dirname "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}")"
|
||||
if [[ -L "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR}" || -L "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" ]]; then
|
||||
echo "[production-stdb-publish] 运行时服务身份引导密钥路径不能是符号链接: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! id genarrative >/dev/null 2>&1; then
|
||||
echo "[production-stdb-publish] 缺少运行时服务用户 genarrative,无法安全安装引导密钥。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_privileged install -d -o root -g genarrative -m 0750 "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR}"
|
||||
run_privileged install -o root -g genarrative -m 0440 \
|
||||
"${MIGRATION_BOOTSTRAP_SECRET_FILE}" \
|
||||
"${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
|
||||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA="$(run_privileged stat -c '%U:%G:%a' "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}")"
|
||||
if [[ "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA}" != "root:genarrative:440" ]]; then
|
||||
echo "[production-stdb-publish] 运行时服务身份引导密钥权限不符合 root:genarrative:0440: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA}" >&2
|
||||
exit 1
|
||||
fi
|
||||
run_privileged runuser -u genarrative -- test -r "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
|
||||
echo "[production-stdb-publish] 已安装运行时服务身份引导密钥: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
|
||||
ensure_runtime_bootstrap_secret_env_file "${API_ENV_FILE}" true
|
||||
ensure_runtime_bootstrap_secret_env_file "${WORKER_ENV_FILE}" false
|
||||
restart_runtime_services_after_bootstrap_secret_install
|
||||
wait_for_api_healthz_ready
|
||||
|
||||
"${SCRIPT_DIR}/maintenance-off.sh"
|
||||
DEPLOY_COMPLETED=1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user