修复数据库备份异步上传收尾

使用独立systemd服务承接Jenkins退出后的OSS上传
串行补偿延期归档并仅在完整验真后清理
补充备份回归门禁与生产运维约定

scripts/check-database-backup-to-oss.mjs

@@ -14,6 +14,7 @@ import {
   cleanupHistoryCandidates,
   collectDirectFileEntries,
   createUploadBandwidthLimiter,
+  discoverDeferredArchiveUploads,
   discoverHistoryPlan,
   restoreDirectFilesBackup,
   restoreDirectFilesLatest,
@@ -46,6 +47,7 @@ if (failures.length > 0) {
 console.log('[check:database-backup] OK');
 
 async function main() {
+  assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
   assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
   assertInsufficientSpaceStopsBeforeServiceChanges();
   assertArchiveFailureStillRestoresDependentServices();
@@ -73,6 +75,87 @@ async function main() {
   await assertDirectFilesRestoreDownloadsCatalogAndObjects();
 }
 
+function assertDeferredArchiveDiscoveryIsBoundedAndDeterministic() {
+  const root = path.join(tmpRoot, 'deferred-archive-discovery');
+  mkdirSync(root, {recursive: true});
+  const createCandidate = ({name, status, database = 'test-db', withArchive = true}) => {
+    const archivePath = path.join(root, `${name}.tar.gz`);
+    const manifestPath = `${archivePath}.manifest.json`;
+    if (withArchive) {
+      writeFileSync(archivePath, name);
+    }
+    writeFileSync(manifestPath, `${JSON.stringify({
+      backupKind: 'spacetimedb-data-dir',
+      database,
+      archivePath,
+      uploadStatus: status,
+    })}\n`);
+    return {archivePath, manifestPath};
+  };
+  const later = createCandidate({name: 'test-db-20260731T020000Z', status: 'pending'});
+  const earlier = createCandidate({name: 'test-db-20260731T010000Z', status: 'deferred'});
+  const uploaded = createCandidate({name: 'test-db-20260731T000000Z', status: 'uploaded'});
+  createCandidate({name: 'other-db-20260731T000000Z', status: 'deferred', database: 'other-db'});
+  const missing = createCandidate({name: 'test-db-20260730T230000Z', status: 'deferred', withArchive: false});
+
+  const result = discoverDeferredArchiveUploads({workDir: root, database: 'test-db'});
+  assertEqual(
+    result.archives.map(({archivePath}) => archivePath).join(','),
+    [earlier.archivePath, later.archivePath].join(','),
+    'deferred/pending 扫描必须只返回同库现存归档，并按文件名稳定排序。',
+  );
+  assertEqual(result.missingArchives.length, 1, '缺失归档的 deferred 清单必须单独报告。');
+  assertEqual(result.missingArchives[0].manifestPath, missing.manifestPath, '缺失归档报告必须保留精确 manifest。');
+  const cleanupResult = discoverDeferredArchiveUploads({workDir: root, database: 'test-db', includeUploaded: true});
+  assertEqual(
+    cleanupResult.archives.map(({archivePath}) => archivePath).join(','),
+    [uploaded.archivePath, earlier.archivePath, later.archivePath].join(','),
+    '未要求保留本地归档时，补偿扫描必须同时收敛上传后未清理的本地归档。',
+  );
+  const cliDryRun = spawnSync(process.execPath, [
+    BACKUP_SCRIPT,
+    '--upload-deferred-dir', root,
+    '--database', 'test-db',
+    '--bucket', 'test-bucket',
+    '--endpoint', 'oss-cn-shanghai.aliyuncs.com',
+    '--access-key-id', 'test-id',
+    '--access-key-secret', 'test-secret',
+    '--keep-local',
+    '--dry-run',
+  ], {encoding: 'utf8'});
+  assertStatus(cliDryRun, 0, 'deferred 补偿扫描 dry-run 必须可通过统一 CLI 入口执行。');
+  assertIncludes(cliDryRun.stdout, 'count=2', 'deferred 补偿扫描 CLI 必须报告待处理归档数量。');
+  assertTrue(existsSync(earlier.archivePath) && existsSync(later.archivePath), 'dry-run 不得删除 deferred 本地归档。');
+
+  const unsafeRoot = path.join(tmpRoot, 'deferred-archive-unsafe');
+  mkdirSync(unsafeRoot, {recursive: true});
+  const escapedArchive = path.join(tmpRoot, 'outside.tar.gz');
+  writeFileSync(escapedArchive, 'outside');
+  writeFileSync(
+    path.join(unsafeRoot, 'test-db-unsafe.tar.gz.manifest.json'),
+    `${JSON.stringify({database: 'test-db', archivePath: escapedArchive, uploadStatus: 'deferred'})}\n`,
+  );
+  assertThrows(
+    () => discoverDeferredArchiveUploads({workDir: unsafeRoot, database: 'test-db'}),
+    '路径与清单不匹配',
+    'deferred 扫描必须拒绝目录外归档或 manifest 名不匹配。',
+  );
+
+  const symlinkRoot = path.join(tmpRoot, 'deferred-archive-symlink');
+  mkdirSync(symlinkRoot, {recursive: true});
+  const symlinkArchive = path.join(symlinkRoot, 'test-db-symlink.tar.gz');
+  symlinkSync(escapedArchive, symlinkArchive);
+  writeFileSync(
+    `${symlinkArchive}.manifest.json`,
+    `${JSON.stringify({database: 'test-db', archivePath: symlinkArchive, uploadStatus: 'deferred'})}\n`,
+  );
+  assertThrows(
+    () => discoverDeferredArchiveUploads({workDir: symlinkRoot, database: 'test-db'}),
+    '非符号链接的普通文件',
+    'deferred 扫描必须拒绝符号链接归档。',
+  );
+}
+
 function readGzipJson(filePath) {
   return JSON.parse(gunzipSync(readFileSync(filePath)).toString('utf8'));
 }

scripts/check-production-ops-guardrails.mjs

@@ -206,6 +206,40 @@ const checks = [
     includes: '按参数保持维护模式和旧运行时服务停止状态',
     reason: '受控维护发布成功后不得自动重启旧运行时或退出维护模式。',
   },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: 'systemd-run',
+    reason:
+      '生产 Stdb publish 的异步 OSS 上传必须交给 systemd transient service，避免 Jenkins 结束时清理上传进程。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: '--unit="${unit_name}"',
+    reason: '生产 Stdb publish 必须为异步 OSS 上传创建独立、可追踪的 transient unit。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: '--collect',
+    reason: '异步 OSS 上传的 transient unit 结束后必须允许 systemd 回收。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: '--service-type=exec',
+    reason:
+      '异步 OSS 上传必须等待 systemd 确认上传进程 exec 成功，不能把启动失败误判为已接管。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: '--upload-deferred-dir "${log_dir}"',
+    reason:
+      '独立 OSS 上传服务必须补偿扫描历史 deferred/pending 归档，不能只处理当次归档。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    excludes: 'nohup ',
+    reason:
+      '生产 Stdb publish 不得恢复会继承 Jenkins 进程生命周期的 nohup 异步上传。',
+  },
   {
     file: 'scripts/deploy/production-api-deploy.sh',
     includes: 'ensure_runtime_bootstrap_secret_file_env',
@@ -7148,6 +7182,38 @@ for (const check of checks) {
   }
 }
 
+const stdbPublishContent = readFileSync(
+  'scripts/deploy/production-stdb-publish.sh',
+  'utf8',
+);
+const asyncBackupUploadStart = stdbPublishContent.indexOf(
+  'start_async_backup_upload() {',
+);
+const asyncBackupUploadEnd = stdbPublishContent.indexOf(
+  '\nwait_for_spacetime_ready() {',
+  asyncBackupUploadStart,
+);
+const asyncBackupUploadFunction =
+  asyncBackupUploadStart >= 0 && asyncBackupUploadEnd > asyncBackupUploadStart
+    ? stdbPublishContent.slice(asyncBackupUploadStart, asyncBackupUploadEnd)
+    : '';
+const systemdRunFailureGuard = asyncBackupUploadFunction.match(
+  /if\s+!\s+(?:run_privileged\s+)?systemd-run\b[\s\S]*?\bthen\b(?:(?!\n\s*fi\b)[\s\S])*?\breturn\s+[1-9][0-9]*\b(?:(?!\n\s*fi\b)[\s\S])*?\n\s*fi\b/u,
+);
+const asyncBackupStatusClearOffset = asyncBackupUploadFunction.indexOf(
+  'rm -f "${ASYNC_BACKUP_STATUS_FILE}"',
+);
+if (
+  !systemdRunFailureGuard ||
+  asyncBackupStatusClearOffset <
+    (systemdRunFailureGuard.index ?? 0) + systemdRunFailureGuard[0].length
+) {
+  failed = true;
+  console.error(
+    '[check:production-ops] production-stdb-publish 的 systemd-run 启动失败分支必须先返回非零，并且只能在 transient unit 启动成功后清理异步备份 status 文件。',
+  );
+}
+
 for (const file of jenkinsSourceCheckoutFiles) {
   const content = readFileSync(file, 'utf8');
   if (!content.includes(jenkinsLoopbackGitRemote)) {

scripts/database-backup-to-oss.mjs

@@ -64,6 +64,7 @@ function usage() {
   npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
   node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload]
   node -- scripts/database-backup-to-oss.mjs --upload-archive <path>
+  node -- scripts/database-backup-to-oss.mjs --upload-deferred-dir <path>
   node -- scripts/database-backup-to-oss.mjs --publish-manifest <path>
   node -- scripts/database-backup-to-oss.mjs --restore-files-state <path> --restore-dir <path>
   node -- scripts/database-backup-to-oss.mjs --restore-files-latest --restore-dir <path> [--dry-run]
@@ -74,6 +75,7 @@ function usage() {
   --storage-format files 不打包：按原相对路径建立 catalog，文件内容以 SHA-256 不可变对象上传；重复运行只上传新增或变化内容。
   archive history 必须有已验真的 full baseline state；files history 必须复用同一 work-dir 中已发布的 full catalog state。
   --defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。
+  --upload-deferred-dir 串行收敛目录内 deferred/pending 及已上传未清理归档；只有 OSS 上传与验真完成后才按 keep-local 规则删除。
   默认读取 .env / .env.local / .env.secrets.local；生产服务可传 --env-file /etc/genarrative/api-server.env。
   shell 环境变量优先级最高，不会被 env 文件覆盖。
 
@@ -157,6 +159,7 @@ function parseArgs(argv) {
     dryRun: false,
     deferUpload: false,
     uploadArchive: '',
+    uploadDeferredDir: '',
     manifestFile: '',
     objectKey: '',
     resultFile: '',
@@ -237,6 +240,9 @@ function parseArgs(argv) {
       case '--upload-archive':
         options.uploadArchive = readValue();
         break;
+      case '--upload-deferred-dir':
+        options.uploadDeferredDir = readValue();
+        break;
       case '--manifest-file':
         options.manifestFile = readValue();
         break;
@@ -2811,6 +2817,50 @@ export async function uploadHistoryArchiveWithCleanup({
   return {result, uploadedManifest, cleanup, state};
 }
 
+export function discoverDeferredArchiveUploads({workDir, database, includeUploaded = false}) {
+  const resolvedWorkDir = resolvePath(workDir);
+  if (!existsSync(resolvedWorkDir)) {
+    return {archives: [], missingArchives: []};
+  }
+
+  const archives = [];
+  const missingArchives = [];
+  const manifestSuffix = '.tar.gz.manifest.json';
+  const expectedDatabase = String(database || '').trim();
+  const entries = readdirSync(resolvedWorkDir, {withFileTypes: true})
+    .filter((candidate) => candidate.isFile() && candidate.name.endsWith(manifestSuffix))
+    .sort((left, right) => left.name.localeCompare(right.name, 'en'));
+  for (const entry of entries) {
+    const manifestPath = join(resolvedWorkDir, entry.name);
+    const manifest = readManifest(manifestPath);
+    const uploadStatus = String(manifest.uploadStatus || '').trim();
+    if (!['deferred', 'pending'].includes(uploadStatus) && !(includeUploaded && uploadStatus === 'uploaded')) {
+      continue;
+    }
+    if (expectedDatabase && String(manifest.database || '').trim() !== expectedDatabase) {
+      continue;
+    }
+    if (!manifest.archivePath) {
+      throw new Error(`deferred 备份清单缺少 archivePath: ${manifestPath}`);
+    }
+    const archivePath = resolvePath(manifest.archivePath);
+    if (dirname(archivePath) !== resolvedWorkDir || manifestPath !== `${archivePath}.manifest.json`) {
+      throw new Error(`deferred 备份路径与清单不匹配: ${manifestPath}`);
+    }
+    const candidate = {archivePath, manifestPath, manifest};
+    if (existsSync(archivePath)) {
+      const archiveStat = lstatSync(archivePath);
+      if (!archiveStat.isFile() || archiveStat.isSymbolicLink()) {
+        throw new Error(`deferred 备份归档必须是非符号链接的普通文件: ${archivePath}`);
+      }
+      archives.push(candidate);
+    } else {
+      missingArchives.push(candidate);
+    }
+  }
+  return {archives, missingArchives};
+}
+
 async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix, bandwidthLimiter}) {
   const archivePath = resolvePath(args.uploadArchive);
   if (!existsSync(archivePath)) {
@@ -2897,6 +2947,37 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId,
   }
 }
 
+async function uploadDeferredArchives({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix, database, bandwidthLimiter}) {
+  const workDir = resolvePath(args.uploadDeferredDir);
+  const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
+  const {archives, missingArchives} = discoverDeferredArchiveUploads({
+    workDir,
+    database,
+    includeUploaded: !keepLocal,
+  });
+  for (const {manifestPath} of missingArchives) {
+    console.warn(`[database-backup] deferred 清单对应的本地归档不存在，跳过: ${manifestPath}`);
+  }
+  if (archives.length === 0) {
+    console.log(`[database-backup] 没有可补偿的本地归档: ${workDir}`);
+    return;
+  }
+  console.log(`[database-backup] 开始串行上传待补偿本地归档: count=${archives.length}`);
+  for (const {archivePath, manifestPath} of archives) {
+    await uploadExistingArchive({
+      args: {...args, uploadArchive: archivePath, manifestFile: manifestPath},
+      env,
+      bucket,
+      endpoint,
+      accessKeyId,
+      accessKeySecret,
+      objectPrefix,
+      bandwidthLimiter,
+    });
+  }
+  console.log(`[database-backup] 待补偿本地归档上传完成: count=${archives.length}`);
+}
+
 async function publishExistingManifest({args, bucket, endpoint, accessKeyId, accessKeySecret, bandwidthLimiter}) {
   const manifestPath = resolvePath(args.publishManifest);
   const manifest = readManifest(manifestPath);
@@ -3108,6 +3189,7 @@ async function main() {
   ));
   const workDir = resolvePath(firstNonEmpty(
     args.workDir,
+    args.uploadDeferredDir,
     env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR,
     isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR,
   ));
@@ -3171,6 +3253,9 @@ async function main() {
   if (args.restoreDir) {
     throw new Error('--restore-dir 只能与 --restore-files-state 或 --restore-files-latest 一起使用。');
   }
+  if (args.uploadArchive && args.uploadDeferredDir) {
+    throw new Error('--upload-archive 与 --upload-deferred-dir 不能同时使用。');
+  }
 
   if (!args.dryRun) {
     const lockPath = acquireBackupLock({workDir, database});
@@ -3196,6 +3281,21 @@ async function main() {
     return;
   }
 
+  if (args.uploadDeferredDir) {
+    await uploadDeferredArchives({
+      args,
+      env,
+      bucket,
+      endpoint,
+      accessKeyId,
+      accessKeySecret,
+      objectPrefix,
+      database,
+      bandwidthLimiter: uploadBandwidthLimiter,
+    });
+    return;
+  }
+
   if (storageFormat === 'files') {
     if (args.deferUpload) {
       throw new Error('files 模式无需本地归档且不支持 --defer-upload；失败后使用同一 work-dir 重跑即可续传。');

scripts/deploy/production-stdb-publish.sh

@@ -532,6 +532,11 @@ prepare_async_backup() {
 }
 
 start_async_backup_upload() {
+  local log_dir=""
+  local node_binary=""
+  local unit_name=""
+  local unit_suffix=""
+
   if [[ -z "${ASYNC_BACKUP_STATUS_FILE}" || ! -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
     echo "[production-stdb-publish] 警告：未找到可上传的本地备份状态文件，跳过异步上传" >&2
     return 0
@@ -543,16 +548,58 @@ start_async_backup_upload() {
     echo "[production-stdb-publish] 警告：备份状态文件缺少 archivePath 或 manifestPath，跳过异步上传" >&2
     return 0
   fi
+  if [[ "${ASYNC_BACKUP_ARCHIVE}" != /* || ! -f "${ASYNC_BACKUP_ARCHIVE}" || -L "${ASYNC_BACKUP_ARCHIVE}" ]]; then
+    echo "[production-stdb-publish] 警告：异步上传归档必须是现存、非符号链接的普通绝对路径文件，保留状态文件等待处理: ${ASYNC_BACKUP_ARCHIVE}" >&2
+    return 1
+  fi
+  if [[ "${ASYNC_BACKUP_MANIFEST}" != /* || ! -f "${ASYNC_BACKUP_MANIFEST}" || -L "${ASYNC_BACKUP_MANIFEST}" ]]; then
+    echo "[production-stdb-publish] 警告：异步上传 manifest 必须是现存、非符号链接的普通绝对路径文件，保留状态文件等待处理: ${ASYNC_BACKUP_MANIFEST}" >&2
+    return 1
+  fi
+  if ! command -v systemd-run >/dev/null 2>&1; then
+    echo "[production-stdb-publish] 警告：systemd-run 不可用，无法启动独立上传服务；保留状态文件等待处理" >&2
+    return 1
+  fi
 
-  mkdir -p "$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
-  ASYNC_BACKUP_LOG="$(dirname "${ASYNC_BACKUP_ARCHIVE}")/${DATABASE}-upload.log"
-  echo "[production-stdb-publish] 后台上传本地备份到 OSS: ${ASYNC_BACKUP_ARCHIVE}"
-  nohup node -- "${ASYNC_BACKUP_SCRIPT}" \
-    --env-file /etc/genarrative/api-server.env \
-    --upload-archive "${ASYNC_BACKUP_ARCHIVE}" \
-    --manifest-file "${ASYNC_BACKUP_MANIFEST}" \
-    >"${ASYNC_BACKUP_LOG}" 2>&1 &
-  echo "[production-stdb-publish] OSS 后台上传日志: ${ASYNC_BACKUP_LOG}"
+  node_binary="$(command -v node || true)"
+  if [[ "${node_binary}" != /* || ! -x "${node_binary}" ]]; then
+    echo "[production-stdb-publish] 警告：未找到可供 systemd 服务执行的绝对 node 路径；保留状态文件等待处理" >&2
+    return 1
+  fi
+
+  log_dir="$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
+  unit_suffix="$(date -u +%Y%m%dT%H%M%S%N)-$$-${RANDOM}"
+  unit_name="genarrative-stdb-backup-upload-${unit_suffix}.service"
+  if ! ASYNC_BACKUP_LOG="$(mktemp "${log_dir}/${DATABASE}-upload-${unit_suffix}.XXXXXX.log")"; then
+    echo "[production-stdb-publish] 警告：无法创建独立 OSS 上传日志，保留状态文件和本地归档等待处理" >&2
+    return 1
+  fi
+  if ! chmod 0600 "${ASYNC_BACKUP_LOG}"; then
+    echo "[production-stdb-publish] 警告：无法收紧独立 OSS 上传日志权限，保留状态文件和本地归档等待处理: ${ASYNC_BACKUP_LOG}" >&2
+    return 1
+  fi
+
+  echo "[production-stdb-publish] 通过独立 systemd 服务串行上传 deferred/pending 本地备份到 OSS: ${log_dir}"
+  if ! run_privileged systemd-run \
+    --no-ask-password \
+    --unit="${unit_name}" \
+    --description="Genarrative SpacetimeDB backup upload ${DATABASE}" \
+    --collect \
+    --service-type=exec \
+    --property="Restart=no" \
+    --property="UMask=0077" \
+    --property="StandardOutput=append:${ASYNC_BACKUP_LOG}" \
+    --property="StandardError=append:${ASYNC_BACKUP_LOG}" \
+    -- "${node_binary}" -- "${ASYNC_BACKUP_SCRIPT}" \
+      --env-file /etc/genarrative/api-server.env \
+      --database "${DATABASE}" \
+      --upload-deferred-dir "${log_dir}"; then
+    echo "[production-stdb-publish] 警告：独立 OSS 上传服务启动失败，保留状态文件和本地归档等待处理；启动日志: ${ASYNC_BACKUP_LOG}" >&2
+    return 1
+  fi
+
+  echo "[production-stdb-publish] OSS 上传服务已启动: ${unit_name}"
+  echo "[production-stdb-publish] OSS 上传日志: ${ASYNC_BACKUP_LOG}"
   rm -f "${ASYNC_BACKUP_STATUS_FILE}"
   ASYNC_BACKUP_STATUS_FILE=""
 }