限制 dev 发布目录与 Jenkins 构建暂存的历史堆积

- production-api-deploy.sh 新增 --keep-releases（默认 2）：发布成功后保留 current 目标与最近 1 个历史 release，只清理同时含 api-server 或 web 标记的旧目录，清理失败只告警、不改变发布结论
- production-api-deploy.sh 新增 prune_old_releases 与参数校验：非整数或小于 1 的 --keep-releases 在任何动作之前失败
- check-production-api-deploy 增加默认值、显式值和非法值三类夹具，并断言非法参数不会提升 release
- check-production-ops 增加 release 保留策略与三个部署 Job 暂存清理的合同
- Api Deploy、Web Deploy、Stdb Publish 三个 Jenkinsfile 在部署或发布成功后只保留最近 2 个 build/<version> 暂存，失败时保留暂存便于诊断和重跑
- 内联清理片段避开 Groovy 反斜杠转义，并在 set -euo pipefail 下对空 build 目录安全（if [ -d build ] 守卫加兜底）
- 开发运维文档补充 release 与 Jenkins 暂存的保留上限口径
- pitfalls 记录 dev 根盘写满的两处根因（发布与暂存无上限、JNLP 幽灵 agent 刷 syslog）以及 Jenkins 内联 shell 的两个坑

scripts/check-production-api-deploy.mjs

@@ -9,6 +9,7 @@ import {
   readFileSync,
   readlinkSync,
   rmSync,
+  utimesSync,
   writeFileSync,
 } from 'node:fs';
 import { tmpdir } from 'node:os';
@@ -95,6 +96,9 @@ function main() {
   assertRealWechatPayUsesLastRefundReconciliationAssignment();
   assertDeployCleansStagingReleaseOnFailure();
   assertDeployRejectsFinalReleaseRaceAndCleansStaging();
+  assertDeployPrunesOldReleases();
+  assertDeployHonorsExplicitKeepReleases();
+  assertDeployRejectsInvalidKeepReleases();
   assertMissingBackupScriptFails();
   assertMissingHealthPatrolScriptFails();
   assertMissingPingoraCurrentReleaseAuditFails();
@@ -1754,6 +1758,106 @@ function assertDeployRejectsFinalReleaseRaceAndCleansStaging() {
   assertMaintenanceCleared(fixture, '目标 release 竞态失败');
 }
 
+function assertDeployPrunesOldReleases() {
+  const fixture = prepareFixture('prune-old-releases');
+  const nowSeconds = Date.now() / 1000;
+  const seededReleases = [
+    ['20260610-oldest', nowSeconds - 3000],
+    ['20260611-middle', nowSeconds - 2000],
+    ['20260613-previous', nowSeconds - 1000],
+  ];
+  for (const [name, mtimeSeconds] of seededReleases) {
+    const dir = path.join(fixture.releaseRoot, name);
+    mkdirSync(path.join(dir, 'web'), { recursive: true });
+    writeFileSync(path.join(dir, 'web', 'index.html'), `${name}\n`, 'utf8');
+    utimesSync(dir, mtimeSeconds, mtimeSeconds);
+  }
+  // 发布根目录下不含 api-server/web 的目录不属于发布产物，清理时必须保留。
+  const unrelatedDir = path.join(fixture.releaseRoot, 'dev-mcp-host-cache');
+  mkdirSync(unrelatedDir, { recursive: true });
+  writeFileSync(path.join(unrelatedDir, 'keep.txt'), 'keep\n', 'utf8');
+  utimesSync(unrelatedDir, nowSeconds - 4000, nowSeconds - 4000);
+
+  const result = runDeploy(fixture);
+
+  assertStatus(result, 0, '存在历史 release 时完整 fixture 仍应部署成功。');
+  if (result.status !== 0) {
+    return;
+  }
+  assertFileExists(
+    path.join(fixture.releaseRoot, fixture.version),
+    '本此发布目录必须保留。',
+  );
+  assertFileExists(
+    path.join(fixture.releaseRoot, '20260613-previous'),
+    '默认 --keep-releases 2 必须保留最近一个历史 release 以便回滚。',
+  );
+  for (const name of ['20260611-middle', '20260610-oldest']) {
+    if (existsSync(path.join(fixture.releaseRoot, name))) {
+      failures.push(
+        `默认 --keep-releases 2 必须清理更早的历史 release: ${name}`,
+      );
+    }
+  }
+  assertFileExists(
+    path.join(unrelatedDir, 'keep.txt'),
+    '清理历史 release 不得删除发布根目录下不含 api-server/web 的目录。',
+  );
+  assertIncludes(
+    result.stdout,
+    '清理历史 release:',
+    '清理历史 release 时必须输出被清理的目录。',
+  );
+}
+
+function assertDeployHonorsExplicitKeepReleases() {
+  const fixture = prepareFixture('keep-single-release');
+  const nowSeconds = Date.now() / 1000;
+  const previousReleaseDir = path.join(
+    fixture.releaseRoot,
+    '20260613-previous',
+  );
+  mkdirSync(path.join(previousReleaseDir, 'web'), { recursive: true });
+  writeFileSync(
+    path.join(previousReleaseDir, 'web', 'index.html'),
+    'previous\n',
+    'utf8',
+  );
+  utimesSync(previousReleaseDir, nowSeconds - 1000, nowSeconds - 1000);
+
+  const result = runDeploy(fixture, { keepReleases: 1 });
+
+  assertStatus(result, 0, '--keep-releases 1 时完整 fixture 仍应部署成功。');
+  if (result.status !== 0) {
+    return;
+  }
+  assertFileExists(
+    path.join(fixture.releaseRoot, fixture.version),
+    '--keep-releases 1 必须保留本次发布目录。',
+  );
+  if (existsSync(previousReleaseDir)) {
+    failures.push('--keep-releases 1 必须清理除 current 以外的历史 release。');
+  }
+}
+
+function assertDeployRejectsInvalidKeepReleases() {
+  const fixture = prepareFixture('invalid-keep-releases');
+  const result = runDeploy(fixture, { keepReleases: '0' });
+
+  assertStatus(result, 1, '--keep-releases 0 必须被拒绝。');
+  assertIncludes(
+    result.stderr,
+    '--keep-releases 必须是大于等于 1 的整数',
+    '--keep-releases 非法时必须给出明确错误。',
+  );
+  assertNotIncludes(
+    readOptionalCommandsLog(fixture),
+    'systemctl restart genarrative-api.service',
+    '--keep-releases 非法时不得重启 API 服务。',
+  );
+  assertNoReleasePromoted(fixture, '--keep-releases 非法时不得提升 release');
+}
+
 function assertMissingPingoraDirectCheckFails() {
   const fixture = prepareFixture('missing-direct-live');
   rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-direct-live.mjs'));
@@ -2785,6 +2889,9 @@ function runDeploy(fixture, options = {}) {
   if (options.keepMaintenance) {
     args.push('--keep-maintenance-mode');
   }
+  if (options.keepReleases !== undefined) {
+    args.push('--keep-releases', String(options.keepReleases));
+  }
   return spawnSync('bash', args, {
     cwd: process.cwd(),
     encoding: 'utf8',

scripts/check-production-ops-guardrails.mjs

@@ -428,6 +428,24 @@ const checks = [
       '--bgfilter-worker-env-file "${BGFILTER_WORKER_ENV_FILE:-/etc/genarrative/bgfilter-worker.env}"',
     reason: 'API Deploy Job 必须把 BgFilter worker env 路径传给发布脚本。',
   },
+  {
+    file: 'jenkins/Jenkinsfile.production-api-deploy',
+    includes: '[staging-cleanup]',
+    reason:
+      'API Deploy Job 必须清理历史 build/<version> 暂存，避免目标机被 copyArtifacts 暂存撑满。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-web-deploy',
+    includes: '[staging-cleanup]',
+    reason:
+      'Web Deploy Job 必须清理历史 build/<version> 暂存，避免目标机被 copyArtifacts 暂存撑满。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-stdb-module-publish',
+    includes: '[staging-cleanup]',
+    reason:
+      'Stdb Publish Job 必须清理历史 build/<version> 暂存，避免目标机被 copyArtifacts 暂存撑满。',
+  },
   {
     file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
     includes:
@@ -475,6 +493,17 @@ const checks = [
     reason:
       'API deploy 必须在 current 切换前复核真实微信支付退款 reconciliation。',
   },
+  {
+    file: 'scripts/deploy/production-api-deploy.sh',
+    includes: 'prune_old_releases',
+    reason:
+      'API deploy 必须在发布成功后按 --keep-releases 清理历史 release 目录，避免目标机根盘被历史发布撑满。',
+  },
+  {
+    file: 'scripts/deploy/production-api-deploy.sh',
+    includes: '--keep-releases 必须是大于等于 1 的整数',
+    reason: 'API deploy 必须拒绝非法的 --keep-releases 参数。',
+  },
   {
     file: 'jenkins/Jenkinsfile.production-stdb-module-publish',
     includes:

scripts/deploy/production-api-deploy.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 usage() {
   cat <<'EOF'
 用法:
-  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url <url>] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--controller-env-file /etc/genarrative/external-generation-controller.env] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
+  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url <url>] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--controller-env-file /etc/genarrative/external-generation-controller.env] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode] [--keep-releases 2]
 
 说明:
   进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
@@ -16,6 +16,7 @@ usage() {
   若发布包包含 pingora-gateway，或传入 --require-pingora-gateway，部署脚本会要求 release manifest、二进制与 checksum 一致，再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，启动或重启 Pingora 影子服务并复核 active。
   默认在 readiness 通过后退出维护模式；传入 --keep-maintenance-mode 时保留维护文件，供人工验收后再恢复公网。
   current 链接切换前失败时会退出本次打开的维护模式；current 链接切换后失败时保留维护模式，避免暴露半发布版本。
+  发布成功后按 --keep-releases（默认 2）保留 current 目标与最近的历史 release 目录，清理其它含 api-server 或 web 的旧发布目录，避免目标机根盘被历史发布撑满；该清理失败只告警，不改变本次发布结论。
 EOF
 }
 
@@ -1177,6 +1178,7 @@ DEPLOY_COMPLETED=0
 PINGORA_INCLUDED=0
 REQUIRE_PINGORA_GATEWAY=0
 KEEP_MAINTENANCE_MODE=0
+KEEP_RELEASES=2
 MAINTENANCE_ENABLED_BY_DEPLOY=0
 MAINTENANCE_FILE="${GENARRATIVE_MAINTENANCE_FILE:-/var/lib/genarrative/maintenance/enabled}"
 CURRENT_LINK_SWITCHED=0
@@ -1222,6 +1224,10 @@ while [[ $# -gt 0 ]]; do
       KEEP_MAINTENANCE_MODE=1
       shift
       ;;
+    --keep-releases)
+      KEEP_RELEASES="${2:?缺少 --keep-releases 的值}"
+      shift 2
+      ;;
     --worker-service-pattern)
       WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
       shift 2
@@ -1290,6 +1296,10 @@ done
 require_argument "${SOURCE_DIR}" "--source-dir"
 require_absolute_path "${RELEASE_ROOT}" "--release-root"
 require_absolute_path "${CURRENT_LINK}" "--current-link"
+if [[ ! "${KEEP_RELEASES}" =~ ^[0-9]+$ ]] || (( KEEP_RELEASES < 1 )); then
+  echo "[production-api-deploy] --keep-releases 必须是大于等于 1 的整数: ${KEEP_RELEASES}" >&2
+  exit 1
+fi
 require_absolute_path "${API_ENV_FILE}" "--api-env-file"
 if [[ -n "${WORKER_ENV_FILE}" ]]; then
   require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file"
@@ -1359,6 +1369,44 @@ cleanup_rendered_systemd_unit() {
   RENDERED_SYSTEMD_UNIT_FILE=""
 }
 
+# 保留 current 目标与最近 keep_total-1 个其它发布目录。
+# 只清理同时不是符号链接、名称不以点开头且含 api-server 或 web 的目录，
+# 避免误删发布根目录下不属于发布产物的目录。
+prune_old_releases() {
+  local keep_total="$1"
+  local current_target=""
+  local keep_others=$(( keep_total - 1 ))
+  local kept_others=0
+  local candidate resolved
+
+  if [[ -L "${CURRENT_LINK}" ]]; then
+    current_target="$(readlink -f "${CURRENT_LINK}" 2>/dev/null || true)"
+  fi
+
+  while IFS= read -r candidate; do
+    [[ -n "${candidate}" ]] || continue
+    case "${candidate}" in
+      *[[:space:]]*) continue ;;
+    esac
+    [[ -d "${candidate}" && ! -L "${candidate}" ]] || continue
+    [[ -e "${candidate}/api-server" || -e "${candidate}/web" ]] || continue
+    resolved="$(readlink -f "${candidate}")"
+    if [[ -n "${current_target}" && "${resolved}" == "${current_target}" ]]; then
+      continue
+    fi
+    if (( kept_others < keep_others )); then
+      kept_others=$(( kept_others + 1 ))
+      continue
+    fi
+    echo "[production-api-deploy] 清理历史 release: ${candidate}"
+    rm -rf --one-file-system "${resolved}"
+  done < <(
+    find "${RELEASE_ROOT}" -mindepth 1 -maxdepth 1 -type d -name '[!.]*' -printf '%T@ %p\n' 2>/dev/null |
+      sort -rn |
+      cut -d' ' -f2-
+  )
+}
+
 on_exit() {
   local exit_code=$?
   cleanup_rendered_systemd_unit
@@ -1697,6 +1745,8 @@ for _ in {1..30}; do
       bash "${SCRIPT_DIR}/maintenance-off.sh"
     fi
     DEPLOY_COMPLETED=1
+    prune_old_releases "${KEEP_RELEASES}" ||
+      echo "[production-api-deploy] 历史 release 清理失败（仅影响磁盘占用，不影响本次发布结论）" >&2
     echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server"
     exit 0
   fi