修复数据库备份异步上传收尾
Project CI / Repository checks (push) Successful in 44s
Project CI / Frontend tests (push) Successful in 2m41s
Project CI / Native shell tests (push) Failing after 19s
Project CI / Backend tests (push) Successful in 3m43s

使用独立systemd服务承接Jenkins退出后的OSS上传
串行补偿延期归档并仅在完整验真后清理
补充备份回归门禁与生产运维约定
This commit is contained in:
2026-07-31 12:08:45 +08:00
parent a36290b2e5
commit 0386a5df7c
7 changed files with 325 additions and 10 deletions
+56 -9
View File
@@ -532,6 +532,11 @@ prepare_async_backup() {
}
start_async_backup_upload() {
local log_dir=""
local node_binary=""
local unit_name=""
local unit_suffix=""
if [[ -z "${ASYNC_BACKUP_STATUS_FILE}" || ! -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
echo "[production-stdb-publish] 警告:未找到可上传的本地备份状态文件,跳过异步上传" >&2
return 0
@@ -543,16 +548,58 @@ start_async_backup_upload() {
echo "[production-stdb-publish] 警告:备份状态文件缺少 archivePath 或 manifestPath,跳过异步上传" >&2
return 0
fi
if [[ "${ASYNC_BACKUP_ARCHIVE}" != /* || ! -f "${ASYNC_BACKUP_ARCHIVE}" || -L "${ASYNC_BACKUP_ARCHIVE}" ]]; then
echo "[production-stdb-publish] 警告:异步上传归档必须是现存、非符号链接的普通绝对路径文件,保留状态文件等待处理: ${ASYNC_BACKUP_ARCHIVE}" >&2
return 1
fi
if [[ "${ASYNC_BACKUP_MANIFEST}" != /* || ! -f "${ASYNC_BACKUP_MANIFEST}" || -L "${ASYNC_BACKUP_MANIFEST}" ]]; then
echo "[production-stdb-publish] 警告:异步上传 manifest 必须是现存、非符号链接的普通绝对路径文件,保留状态文件等待处理: ${ASYNC_BACKUP_MANIFEST}" >&2
return 1
fi
if ! command -v systemd-run >/dev/null 2>&1; then
echo "[production-stdb-publish] 警告:systemd-run 不可用,无法启动独立上传服务;保留状态文件等待处理" >&2
return 1
fi
mkdir -p "$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
ASYNC_BACKUP_LOG="$(dirname "${ASYNC_BACKUP_ARCHIVE}")/${DATABASE}-upload.log"
echo "[production-stdb-publish] 后台上传本地备份到 OSS: ${ASYNC_BACKUP_ARCHIVE}"
nohup node -- "${ASYNC_BACKUP_SCRIPT}" \
--env-file /etc/genarrative/api-server.env \
--upload-archive "${ASYNC_BACKUP_ARCHIVE}" \
--manifest-file "${ASYNC_BACKUP_MANIFEST}" \
>"${ASYNC_BACKUP_LOG}" 2>&1 &
echo "[production-stdb-publish] OSS 后台上传日志: ${ASYNC_BACKUP_LOG}"
node_binary="$(command -v node || true)"
if [[ "${node_binary}" != /* || ! -x "${node_binary}" ]]; then
echo "[production-stdb-publish] 警告:未找到可供 systemd 服务执行的绝对 node 路径;保留状态文件等待处理" >&2
return 1
fi
log_dir="$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
unit_suffix="$(date -u +%Y%m%dT%H%M%S%N)-$$-${RANDOM}"
unit_name="genarrative-stdb-backup-upload-${unit_suffix}.service"
if ! ASYNC_BACKUP_LOG="$(mktemp "${log_dir}/${DATABASE}-upload-${unit_suffix}.XXXXXX.log")"; then
echo "[production-stdb-publish] 警告:无法创建独立 OSS 上传日志,保留状态文件和本地归档等待处理" >&2
return 1
fi
if ! chmod 0600 "${ASYNC_BACKUP_LOG}"; then
echo "[production-stdb-publish] 警告:无法收紧独立 OSS 上传日志权限,保留状态文件和本地归档等待处理: ${ASYNC_BACKUP_LOG}" >&2
return 1
fi
echo "[production-stdb-publish] 通过独立 systemd 服务串行上传 deferred/pending 本地备份到 OSS: ${log_dir}"
if ! run_privileged systemd-run \
--no-ask-password \
--unit="${unit_name}" \
--description="Genarrative SpacetimeDB backup upload ${DATABASE}" \
--collect \
--service-type=exec \
--property="Restart=no" \
--property="UMask=0077" \
--property="StandardOutput=append:${ASYNC_BACKUP_LOG}" \
--property="StandardError=append:${ASYNC_BACKUP_LOG}" \
-- "${node_binary}" -- "${ASYNC_BACKUP_SCRIPT}" \
--env-file /etc/genarrative/api-server.env \
--database "${DATABASE}" \
--upload-deferred-dir "${log_dir}"; then
echo "[production-stdb-publish] 警告:独立 OSS 上传服务启动失败,保留状态文件和本地归档等待处理;启动日志: ${ASYNC_BACKUP_LOG}" >&2
return 1
fi
echo "[production-stdb-publish] OSS 上传服务已启动: ${unit_name}"
echo "[production-stdb-publish] OSS 上传日志: ${ASYNC_BACKUP_LOG}"
rm -f "${ASYNC_BACKUP_STATUS_FILE}"
ASYNC_BACKUP_STATUS_FILE=""
}