修复外部生成任务大载荷导致的内存膨胀

阻止图片编辑器将 Data URL 和超限 JSON 写入生成任务，复用 objectKey 等轻量引用新增外部生成任务摘要投影与分批 payload 压缩回填流程，避免正式列表读取大字段收紧 SpacetimeDB 发布为永不删数据并移除 Jenkins 普通清库入口，补齐维护窗口门禁更新生成绑定、运维文档和回归测试，并完成生产数据副本迁移与压缩验证
2026-07-11 13:55:05 +08:00
parent f848cbf78b
commit 324f99efdf
42 changed files with 3528 additions and 317 deletions
@@ -688,6 +688,7 @@ DEPLOY_COMPLETED=0
 PINGORA_INCLUDED=0
 REQUIRE_PINGORA_GATEWAY=0
 MAINTENANCE_ENABLED_BY_DEPLOY=0
+MAINTENANCE_FILE="${GENARRATIVE_MAINTENANCE_FILE:-/var/lib/genarrative/maintenance/enabled}"
 CURRENT_LINK_SWITCHED=0
 RELEASE_DIR=""
 STAGING_RELEASE_DIR=""
@@ -847,8 +848,12 @@ on_exit() {

 trap on_exit EXIT

+if [[ ! -f "${MAINTENANCE_FILE}" ]]; then
+  MAINTENANCE_ENABLED_BY_DEPLOY=1
+else
+  echo "[production-api-deploy] 继承已有维护模式；部署失败时不得误退出上游维护窗口: ${MAINTENANCE_FILE}"
+fi
 bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}"
-MAINTENANCE_ENABLED_BY_DEPLOY=1

 echo "[production-api-deploy] 校验 api-server"
 (
@@ -6,16 +6,18 @@ umask 077
 usage() {
  cat <<'EOF'
 用法:
-  ./scripts/deploy/production-stdb-publish.sh --source-dir build/<version> --database <database> --migration-bootstrap-secret-file <protected-file> [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--api-health-url http://127.0.0.1:8082/healthz] [--api-readiness-timeout-seconds 60] [--clear-database] [--backup-mode async|sync|skip]
+  ./scripts/deploy/production-stdb-publish.sh --source-dir build/<version> --database <database> --migration-bootstrap-secret-file <protected-file> [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--api-health-url http://127.0.0.1:8082/healthz] [--api-readiness-timeout-seconds 60] [--keep-maintenance-mode] [--backup-mode async|sync|skip]

 说明:
  进入维护模式，校验 spacetime_module.wasm.sha256，并在生产实例本机执行 spacetime publish。
+  publish 固定使用 --delete-data=never 与 scoped --yes=migrate,break-clients；任何需要删除数据的 schema 冲突都会阻断发布。
  默认使用 http://127.0.0.1:3101，避免与部署机本机 Git/Web 服务的 3000 端口冲突。
  默认使用 /stdb 作为 spacetime CLI root-dir，并以 spacetimedb 用户发布，避免 root CLI 身份污染自托管实例。
  发布时固定追加 --no-config，只使用显式参数，避免工作区或用户目录里的 spacetime 配置干扰目标。
  async 模式会在 publish 前先做本地冷备份，再在 publish 完成后后台上传 OSS，避免低带宽上传阻塞部署。
  如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。
  发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。
+  --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。
  migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。
  如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。
  失败时保留维护模式。
@@ -51,7 +53,7 @@ RUN_AS_USER="spacetimedb"
 MIGRATION_BOOTSTRAP_SECRET_FILE=""
 API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-server.env}"
 WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
-CLEAR_DATABASE=0
+KEEP_MAINTENANCE_MODE=0
 BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
 DEPLOY_COMPLETED=0
 PUBLISH_TMP_DIR=""
@@ -285,6 +287,54 @@ restart_runtime_services_after_bootstrap_secret_install() {
  fi
 }

+stop_runtime_services_for_rollout_gate() {
+  local api_state=""
+  local controller_state=""
+  local worker_service=""
+  local worker_units_output=""
+  local -a services_to_stop=()
+
+  api_state="$(get_runtime_service_active_state genarrative-api.service)"
+  controller_state="$(get_runtime_service_active_state genarrative-external-generation-controller.service)"
+  if ! worker_units_output="$(
+    run_privileged systemctl list-units \
+      --type=service \
+      --state=active \
+      --no-legend \
+      --plain \
+      'genarrative-external-generation-worker@*.service'
+  )"; then
+    echo "[production-stdb-publish] 查询 active worker 服务失败，无法建立受控维护窗口。" >&2
+    return 1
+  fi
+
+  if [[ "${controller_state}" == "active" ]]; then
+    services_to_stop+=(genarrative-external-generation-controller.service)
+  fi
+  while read -r worker_service _; do
+    if [[ "${worker_service}" =~ ^genarrative-external-generation-worker@[A-Za-z0-9_.@:-]+\.service$ ]]; then
+      services_to_stop+=("${worker_service}")
+    fi
+  done <<<"${worker_units_output}"
+  if [[ "${api_state}" == "active" ]]; then
+    services_to_stop+=(genarrative-api.service)
+  fi
+
+  if [[ "${#services_to_stop[@]}" -eq 0 ]]; then
+    echo "[production-stdb-publish] 受控维护窗口开始前没有 active API/controller/worker。"
+    return 0
+  fi
+
+  echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}"
+  run_privileged systemctl stop "${services_to_stop[@]}"
+  for worker_service in "${services_to_stop[@]}"; do
+    if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then
+      echo "[production-stdb-publish] 运行时服务停止后仍为 active: ${worker_service}" >&2
+      return 1
+    fi
+  done
+}
+
 while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help)
@@ -336,8 +386,8 @@ while [[ $# -gt 0 ]]; do
      API_READINESS_TIMEOUT_SECONDS="${2:?缺少 --api-readiness-timeout-seconds 的值}"
      shift 2
      ;;
-    --clear-database)
-      CLEAR_DATABASE=1
+    --keep-maintenance-mode)
+      KEEP_MAINTENANCE_MODE=1
      shift
      ;;
    --skip-backup)
@@ -455,6 +505,7 @@ on_exit() {
 trap on_exit EXIT

 prepare_async_backup() {
+  local -a restart_service_args=()
  ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
  if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
    ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
@@ -464,6 +515,10 @@ prepare_async_backup() {
    exit 1
  fi

+  if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
+    restart_service_args+=(--restart-service-after genarrative-api.service)
+  fi
+
  ASYNC_BACKUP_STATUS_FILE="$(mktemp /tmp/genarrative-stdb-backup-status.XXXXXX.json)"
  echo "[production-stdb-publish] publish 前生成本地冷备份，随后会异步上传 OSS"
  node -- "${ASYNC_BACKUP_SCRIPT}" \
@@ -471,7 +526,7 @@ prepare_async_backup() {
    --data-dir "${SPACETIME_ROOT_DIR}" \
    --database "${DATABASE}" \
    --stop-service spacetimedb.service \
-    --restart-service-after genarrative-api.service \
+    "${restart_service_args[@]}" \
    --defer-upload \
    --result-file "${ASYNC_BACKUP_STATUS_FILE}"
 }
@@ -555,12 +610,16 @@ wait_for_api_healthz_ready() {
 }

 "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
+if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
+  stop_runtime_services_for_rollout_gate
+fi

 case "${BACKUP_MODE}" in
  async)
    prepare_async_backup
    ;;
  sync)
+    SYNC_BACKUP_RESTART_SERVICE_ARGS=()
    BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
    if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
      BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
@@ -569,6 +628,9 @@ case "${BACKUP_MODE}" in
      echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2
      exit 1
    fi
+    if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
+      SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service)
+    fi

    echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布"
    node -- "${BACKUP_SCRIPT}" \
@@ -576,7 +638,7 @@ case "${BACKUP_MODE}" in
      --data-dir "${SPACETIME_ROOT_DIR}" \
      --database "${DATABASE}" \
      --stop-service spacetimedb.service \
-      --restart-service-after genarrative-api.service
+      "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}"
    ;;
  skip)
    echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份"
@@ -596,7 +658,8 @@ PUBLISH_ARGS=(
  publish
  "${DATABASE}"
  --bin-path "${SOURCE_DIR}/spacetime_module.wasm"
-  --yes
+  --delete-data=never
+  --yes=migrate,break-clients
  --no-config
 )

@@ -606,10 +669,6 @@ else
  PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
 fi

-if [[ "${CLEAR_DATABASE}" -eq 1 ]]; then
-  PUBLISH_ARGS+=(--clear-database)
-fi
-
 if [[ -n "${SERVER_URL}" ]]; then
  echo "[production-stdb-publish] 发布 SpacetimeDB module: ${DATABASE} -> ${SERVER_URL}, root=${SPACETIME_ROOT_DIR}"
 else
@@ -629,7 +688,8 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
    publish
    "${DATABASE}"
    --bin-path "${PUBLISH_TMP_DIR}/spacetime_module.wasm"
-    --yes
+    --delete-data=never
+    --yes=migrate,break-clients
    --no-config
  )
  if [[ -n "${SERVER_URL}" ]]; then
@@ -637,9 +697,6 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
  else
    PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
  fi
-  if [[ "${CLEAR_DATABASE}" -eq 1 ]]; then
-    PUBLISH_ARGS+=(--clear-database)
-  fi
  runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}"
 else
  spacetime "${PUBLISH_ARGS[@]}"
@@ -668,6 +725,11 @@ run_privileged runuser -u genarrative -- test -r "${RUNTIME_SERVICE_BOOTSTRAP_SE
 echo "[production-stdb-publish] 已安装运行时服务身份引导密钥: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
 ensure_runtime_bootstrap_secret_env_file "${API_ENV_FILE}" true
 ensure_runtime_bootstrap_secret_env_file "${WORKER_ENV_FILE}" false
+if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
+  echo "[production-stdb-publish] module 发布完成；按参数保持维护模式和旧运行时服务停止状态，等待人工维护与 API deploy。"
+  DEPLOY_COMPLETED=1
+  exit 0
+fi
 restart_runtime_services_after_bootstrap_secret_install
 wait_for_api_healthz_ready