收紧外部生成任务租约和超时保护
缩短 worker lease 并新增普通和长任务执行预算配置 让 worker 超过执行预算后写失败并释放槽位 降低 BgFilter 默认超时并加入连续失败熔断兜底 补充资产计费 future 取消时的退款补偿保护 同步生产 env 示例、部署脚本和运维文档
This commit is contained in:
@@ -5,7 +5,7 @@ set -euo pipefail
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
|
||||
|
||||
说明:
|
||||
进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
|
||||
@@ -183,10 +183,30 @@ ensure_env_value() {
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[production-api-deploy] 补齐 api-server 环境变量: ${key} -> ${file_path}"
|
||||
echo "[production-api-deploy] 补齐运行态环境变量: ${key} -> ${file_path}"
|
||||
write_env_value "${file_path}" "${key}" "${default_value}"
|
||||
}
|
||||
|
||||
ensure_env_value_migrates_old_default() {
|
||||
local file_path="$1"
|
||||
local key="$2"
|
||||
local old_default="$3"
|
||||
local new_default="$4"
|
||||
local current_value
|
||||
|
||||
current_value="$(read_env_value "${file_path}" "${key}")"
|
||||
if [[ -z "${current_value}" ]]; then
|
||||
ensure_env_value "${file_path}" "${key}" "${new_default}"
|
||||
return
|
||||
fi
|
||||
if [[ "${current_value}" != "${old_default}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[production-api-deploy] 迁移运行态环境变量旧默认值: ${key} ${old_default} -> ${new_default} (${file_path})"
|
||||
write_env_value "${file_path}" "${key}" "${new_default}"
|
||||
}
|
||||
|
||||
run_privileged() {
|
||||
if [[ "$(id -u)" -eq 0 ]]; then
|
||||
"$@"
|
||||
@@ -232,6 +252,12 @@ ensure_runtime_env_and_dirs() {
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE" "100"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS" "1000"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES" "67108864"
|
||||
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
|
||||
tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")"
|
||||
tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")"
|
||||
@@ -245,6 +271,22 @@ ensure_runtime_env_and_dirs() {
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_worker_runtime_env_defaults() {
|
||||
local worker_env_file="$1"
|
||||
|
||||
if [[ -z "${worker_env_file}" ]]; then
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${worker_env_file}" ]]; then
|
||||
echo "[production-api-deploy] worker 环境文件不存在，跳过运行态默认值补齐: ${worker_env_file}"
|
||||
return
|
||||
fi
|
||||
|
||||
ensure_env_value_migrates_old_default "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
}
|
||||
|
||||
extract_pingora_env_files_from_unit() {
|
||||
local service_name="$1"
|
||||
local unit_content
|
||||
@@ -585,6 +627,7 @@ WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
|
||||
WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
|
||||
HEALTH_URL="http://127.0.0.1:8082/readyz"
|
||||
API_ENV_FILE="/etc/genarrative/api-server.env"
|
||||
WORKER_ENV_FILE="/etc/genarrative/external-generation-worker.env"
|
||||
DATABASE=""
|
||||
SPACETIME_SERVER_URL=""
|
||||
DEPLOY_COMPLETED=0
|
||||
@@ -653,6 +696,10 @@ while [[ $# -gt 0 ]]; do
|
||||
API_ENV_FILE="${2:?缺少 --api-env-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--worker-env-file)
|
||||
WORKER_ENV_FILE="${2:?缺少 --worker-env-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--database)
|
||||
DATABASE="${2:?缺少 --database 的值}"
|
||||
shift 2
|
||||
@@ -673,6 +720,9 @@ require_argument "${SOURCE_DIR}" "--source-dir"
|
||||
require_absolute_path "${RELEASE_ROOT}" "--release-root"
|
||||
require_absolute_path "${CURRENT_LINK}" "--current-link"
|
||||
require_absolute_path "${API_ENV_FILE}" "--api-env-file"
|
||||
if [[ -n "${WORKER_ENV_FILE}" ]]; then
|
||||
require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file"
|
||||
fi
|
||||
|
||||
if [[ -n "${DATABASE}" ]]; then
|
||||
validate_spacetime_database_name "${DATABASE}"
|
||||
@@ -979,6 +1029,7 @@ if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
|
||||
fi
|
||||
|
||||
ensure_runtime_env_and_dirs "${API_ENV_FILE}"
|
||||
ensure_worker_runtime_env_defaults "${WORKER_ENV_FILE}"
|
||||
|
||||
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
|
||||
PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")"
|
||||
|
||||
@@ -427,12 +427,34 @@ ensure_env_value() {
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[server-provision] 补齐 api-server 环境变量: ${key} -> ${file}"
|
||||
echo "[server-provision] 补齐运行态环境变量: ${key} -> ${file}"
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
write_env_value "${file}" "${key}" "${default_value}"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_env_value_migrates_old_default() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local old_default="$3"
|
||||
local new_default="$4"
|
||||
local current_value
|
||||
|
||||
current_value="$(read_env_value "${file}" "${key}")"
|
||||
if [[ -z "${current_value}" ]]; then
|
||||
ensure_env_value "${file}" "${key}" "${new_default}"
|
||||
return
|
||||
fi
|
||||
if [[ "${current_value}" != "${old_default}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[server-provision] 迁移运行态环境变量旧默认值: ${key} ${old_default} -> ${new_default} (${file})"
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
write_env_value "${file}" "${key}" "${new_default}"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_api_runtime_env_defaults() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ ensure api-server runtime env defaults in ${API_ENV_FILE}"
|
||||
@@ -454,6 +476,27 @@ ensure_api_runtime_env_defaults() {
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE" "100"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS" "1000"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES" "67108864"
|
||||
ensure_env_value_migrates_old_default "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
}
|
||||
|
||||
ensure_worker_runtime_env_defaults() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ ensure external-generation worker runtime env defaults in ${WORKER_ENV_FILE}"
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
|
||||
echo "[server-provision] worker 环境文件不存在，无法补齐运行态变量: ${WORKER_ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ensure_env_value_migrates_old_default "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
}
|
||||
|
||||
parse_json_string_field() {
|
||||
@@ -964,6 +1007,7 @@ if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
|
||||
else
|
||||
echo "[server-provision] 已存在 worker 环境文件，保留不覆盖: ${WORKER_ENV_FILE}"
|
||||
fi
|
||||
ensure_worker_runtime_env_defaults
|
||||
|
||||
if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then
|
||||
echo "+ create ${CONTROLLER_ENV_FILE} from example"
|
||||
|
||||
Reference in New Issue
Block a user