补齐 Pingora 直连彩排状态

新增 Pingora 直连彩排状态脚本,核验 health patrol、端口归属、systemd、realpath canary 和 current release 自审

将彩排状态脚本纳入 API release 构建、部署、Jenkins 归档和生产运维护栏

补充 release readiness、current release audit、cutover snapshot 与发布部署检查对彩排脚本的覆盖

更新 Pingora 运维文档和共享记忆,记录 dev health patrol 真实配置漂移与修复方式

在 dev 服务器真实部署验证 Nginx 仍接 80/443、Pingora shadow 与 realpath canary 高端口正常、health patrol 和彩排状态均为 OK
This commit is contained in:
2026-06-18 00:31:42 +08:00
parent 123d4ada01
commit 4bca480f3c
20 changed files with 1702 additions and 10 deletions
+2 -1
View File
@@ -552,6 +552,7 @@ copy_required_file "${SCRIPT_DIR}/spacetime-revoke-migration-operator.mjs" "${TA
copy_required_file "${SCRIPT_DIR}/database-backup-to-oss.mjs" "${TARGET_DIR}/scripts/database-backup-to-oss.mjs" "数据库 OSS 备份脚本"
copy_required_file "${SCRIPT_DIR}/ops/production-health-patrol.mjs" "${TARGET_DIR}/scripts/ops/production-health-patrol.mjs" "生产健康巡检脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-current-release-audit.mjs" "${TARGET_DIR}/scripts/ops/pingora-current-release-audit.mjs" "Pingora current release 自审脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-direct-rehearsal-status.mjs" "${TARGET_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" "Pingora 直连切换彩排状态脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-status-snapshot.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" "Pingora 直连切换状态快照脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-evidence-bundle.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" "Pingora 直连切换证据包脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-command-evidence.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" "Pingora 直连切换命令证据脚本"
@@ -584,7 +585,7 @@ cat >"${TARGET_DIR}/README.md" <<EOF
- \`migration-bootstrap-secret.txt\`:构建 \`spacetime_module.wasm\` 时注入的迁移引导密钥,仅用于创建首个迁移操作员;请作为敏感文件保存到 Jenkins Secret Text,授权完成后不要长期留在公开归档中。
- \`*.sha256\`:发布产物 checksum,用于部署前校验。
- \`release-manifest.json\`:发布版本、源码 commit 与产物清单。
- \`scripts/\`:维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、生产健康巡检脚本、Pingora release readiness 聚合门禁、直连启用 / 回退、realpath canary 启用 / 关闭、health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
- \`scripts/\`:维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、生产健康巡检脚本、Pingora release readiness 聚合门禁、直连启用 / 回退、realpath canary 启用 / 关闭、health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 直连彩排状态 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
- \`scripts/deploy/production-api-deploy.sh\`API Deploy 执行入口;同目录的 \`maintenance-on.sh\` / \`maintenance-off.sh\` 必须来自同一发布包。
- \`deploy/\`systemd、Nginx 和生产环境变量示例;\`deploy/nginx/genarrative-dev-http.conf\` 仅供无域名开发服初始化使用。
@@ -76,6 +76,11 @@ function assertScriptShape() {
'scripts/ops/pingora-current-release-audit.mjs',
'current release 自审必须检查自身已随包发布。',
);
assertIncludes(
content,
'scripts/ops/pingora-direct-rehearsal-status.mjs',
'current release 自审必须检查直连彩排状态脚本已随包发布。',
);
assertIncludes(
content,
'scripts/check-pingora-release-readiness.mjs',
@@ -453,6 +458,7 @@ function prepareReleaseRoot(releaseRoot, options) {
'scripts/database-backup-to-oss.mjs',
'scripts/ops/production-health-patrol.mjs',
'scripts/ops/pingora-current-release-audit.mjs',
'scripts/ops/pingora-direct-rehearsal-status.mjs',
'scripts/ops/pingora-cutover-status-snapshot.mjs',
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
'scripts/check-production-health-patrol-env.mjs',
@@ -65,6 +65,11 @@ function assertScriptShape() {
'scripts/ops/pingora-current-release-audit.mjs',
'状态快照必须把 current release 自审脚本纳入 release artifact 采集。',
);
assertIncludes(
content,
'scripts/ops/pingora-direct-rehearsal-status.mjs',
'状态快照必须把直连彩排状态脚本纳入 release artifact 采集。',
);
assertIncludes(
content,
'scripts/ops/pingora-cutover-evidence-audit.mjs',
@@ -216,6 +221,12 @@ function assertSnapshotSucceedsWithFakeSystemctlAndReleaseRoot() {
if (!canaryLogParityArtifact || canaryLogParityArtifact.status !== 'OK') {
failures.push('快照必须确认 current release 已包含 canary access log 对账脚本。');
}
const rehearsalStatusArtifact = payload.releaseArtifacts.artifacts.find(
(item) => item.path === 'scripts/ops/pingora-direct-rehearsal-status.mjs',
);
if (!rehearsalStatusArtifact || rehearsalStatusArtifact.status !== 'OK') {
failures.push('快照必须确认 current release 已包含直连彩排状态脚本。');
}
const evidenceAuditArtifact = payload.releaseArtifacts.artifacts.find(
(item) => item.path === 'scripts/ops/pingora-cutover-evidence-audit.mjs',
);
@@ -727,6 +738,7 @@ function prepareReleaseRoot(releaseRoot) {
for (const file of [
'scripts/database-backup-to-oss.mjs',
'scripts/ops/pingora-current-release-audit.mjs',
'scripts/ops/pingora-direct-rehearsal-status.mjs',
'scripts/ops/pingora-cutover-evidence-audit.mjs',
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
'scripts/ops/pingora-cutover-command-evidence.mjs',
File diff suppressed because it is too large Load Diff
@@ -80,6 +80,7 @@ function main() {
assertDefaultPlanIncludesHealthPatrolEnvSmoke();
assertDefaultPlanIncludesPingoraHealthPatrolEnvSwitchSmoke();
assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke();
assertDefaultPlanIncludesPingoraDirectRehearsalStatusSmoke();
assertDefaultPlanIncludesPingoraCutoverStatusSnapshotSmoke();
assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke();
assertDefaultPlanIncludesPingoraCutoverCommandEvidenceSmoke();
@@ -260,6 +261,22 @@ function assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke() {
);
}
function assertDefaultPlanIncludesPingoraDirectRehearsalStatusSmoke() {
const plan = readPlan(['--dry-run-plan']);
const step = findStep(plan, 'Pingora direct rehearsal 状态烟测');
if (!step) {
failures.push(
'默认 release readiness 计划必须包含 Pingora direct rehearsal 状态烟测。',
);
return;
}
assertIncludes(
step.args,
'check:pingora-direct-rehearsal-status',
'Pingora direct rehearsal 状态烟测必须调用 check:pingora-direct-rehearsal-status。',
);
}
function assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke() {
const plan = readPlan(['--dry-run-plan']);
const step = findStep(plan, 'Pingora cutover 证据包烟测');
@@ -1648,6 +1648,11 @@ function buildSteps(config) {
command: 'npm',
args: ['run', 'check:pingora-current-release-audit'],
},
{
name: 'Pingora direct rehearsal 状态烟测',
command: 'npm',
args: ['run', 'check:pingora-direct-rehearsal-status'],
},
{
name: 'Pingora cutover 状态快照烟测',
command: 'npm',
+32
View File
@@ -54,6 +54,7 @@ function main() {
assertMissingBackupScriptFails();
assertMissingHealthPatrolScriptFails();
assertMissingPingoraCurrentReleaseAuditFails();
assertMissingPingoraDirectRehearsalStatusFails();
assertMissingPingoraCutoverStatusSnapshotFails();
assertMissingPingoraCutoverEvidenceBundleFails();
assertMissingPingoraCutoverCommandEvidenceFails();
@@ -136,6 +137,10 @@ function assertDeployCopiesPingoraDirectReleaseDependencies() {
path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'),
'current release 必须包含 Pingora current release 自审脚本。',
);
assertFileExists(
path.join(releaseDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'),
'current release 必须包含 Pingora 直连彩排状态脚本。',
);
assertFileExists(
path.join(releaseDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'),
'current release 必须包含 Pingora 直连切换状态快照脚本。',
@@ -718,6 +723,29 @@ function assertMissingPingoraCurrentReleaseAuditFails() {
}
}
function assertMissingPingoraDirectRehearsalStatusFails() {
const fixture = prepareFixture('missing-pingora-direct-rehearsal-status');
rmSync(
path.join(
fixture.sourceDir,
'scripts/ops/pingora-direct-rehearsal-status.mjs',
),
);
const result = runDeploy(fixture);
if (result.status === 0) {
failures.push('发布产物缺少 Pingora 直连彩排状态脚本时部署必须失败。');
}
assertIncludes(
result.stderr,
'发布产物缺少 Pingora 直连彩排状态脚本',
'缺少 Pingora 直连彩排状态脚本时必须给出明确错误。',
);
if (!existsSync(fixture.maintenanceFile)) {
failures.push('部署失败时必须保持维护模式。');
}
}
function assertMissingPingoraCutoverStatusSnapshotFails() {
const fixture = prepareFixture('missing-pingora-cutover-status-snapshot');
rmSync(
@@ -1060,6 +1088,10 @@ function prepareFixture(name) {
'scripts/ops/pingora-current-release-audit.mjs',
path.join(sourceDir, 'scripts/ops/pingora-current-release-audit.mjs'),
);
copyFile(
'scripts/ops/pingora-direct-rehearsal-status.mjs',
path.join(sourceDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'),
);
copyFile(
'scripts/ops/pingora-cutover-status-snapshot.mjs',
path.join(sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'),
+8 -1
View File
@@ -155,6 +155,13 @@ function assertApiReleaseContainsPingoraDirectDependencies() {
),
'API release 必须包含 Pingora current release 自审脚本。',
);
assertFileExists(
path.join(
releaseDir,
'scripts/ops/pingora-direct-rehearsal-status.mjs',
),
'API release 必须包含 Pingora 直连彩排状态脚本。',
);
assertFileExists(
path.join(
releaseDir,
@@ -280,7 +287,7 @@ function assertApiReleaseContainsPingoraDirectDependencies() {
);
assertIncludes(
releaseReadme,
'health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审',
'health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 直连彩排状态',
'API release README 必须说明随包携带 Pingora 运行态复核脚本。',
);
assertIncludes(
@@ -66,6 +66,26 @@ const checks = [
includes: 'scripts/check-pingora-release-readiness.mjs',
reason: 'Pingora current release 自审必须确认 release readiness 聚合门禁脚本已随包发布。',
},
{
file: 'scripts/ops/pingora-current-release-audit.mjs',
includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
reason: 'Pingora current release 自审必须确认直连彩排状态脚本已随包发布。',
},
{
file: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
includes: '该脚本只读采集',
reason: 'Pingora 直连彩排状态脚本必须声明只读边界。',
},
{
file: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
includes: 'ss',
reason: 'Pingora 直连彩排状态脚本必须读取端口监听状态。',
},
{
file: 'scripts/check-pingora-direct-rehearsal-status.mjs',
includes: 'assertNginxPublicRealpathRehearsalPasses',
reason: 'Pingora 直连彩排状态自测必须覆盖 Nginx 公网 + realpath canary 正例。',
},
{
file: 'scripts/ops/pingora-current-release-audit.mjs',
includes: 'scripts/check-pingora-canary-live.mjs',
@@ -939,6 +959,11 @@ const checks = [
includes: 'scripts/ops/pingora-current-release-audit.mjs',
reason: '生产 API release 必须携带 Pingora current release 自审脚本。',
},
{
file: 'scripts/build-production-release.sh',
includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
reason: '生产 API release 必须携带 Pingora 直连彩排状态脚本。',
},
{
file: 'scripts/build-production-release.sh',
includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs',
@@ -1114,6 +1139,11 @@ const checks = [
includes: 'PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE',
reason: 'API deploy 必须从发布产物复制 Pingora current release 自审脚本。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE',
reason: 'API deploy 必须从发布产物复制 Pingora 直连彩排状态脚本。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE',
@@ -1139,6 +1169,11 @@ const checks = [
includes: 'scripts/ops/pingora-current-release-audit.mjs',
reason: 'API release 动态烟测必须断言 Pingora current release 自审脚本随包发布。',
},
{
file: 'scripts/check-production-api-release.mjs',
includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
reason: 'API release 动态烟测必须断言 Pingora 直连彩排状态脚本随包发布。',
},
{
file: 'scripts/check-production-api-release.mjs',
includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs',
@@ -1361,6 +1396,11 @@ const checks = [
includes: 'check:pingora-current-release-audit',
reason: 'Pingora current release 自审必须保留可重复 npm 自测入口。',
},
{
file: 'package.json',
includes: 'check:pingora-direct-rehearsal-status',
reason: 'Pingora 直连彩排状态脚本必须保留可重复 npm 自测入口。',
},
{
file: 'package.json',
includes: 'check:pingora-cutover-status-snapshot',
@@ -1452,6 +1492,11 @@ const checks = [
includes: 'check:pingora-current-release-audit',
reason: 'Pingora 聚合门禁必须执行 current release 自审烟测。',
},
{
file: 'scripts/check-pingora-release-readiness.mjs',
includes: 'check:pingora-direct-rehearsal-status',
reason: 'Pingora 聚合门禁必须执行直连彩排状态烟测。',
},
{
file: 'scripts/check-pingora-release-readiness.mjs',
includes: 'check:pingora-cutover-status-snapshot',
+7
View File
@@ -610,6 +610,7 @@ fi
BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs"
PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs"
PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
@@ -650,6 +651,12 @@ if [[ ! -f "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" ]]; then
fi
cp "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
if [[ ! -f "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" ]]; then
echo "[production-api-deploy] 发布产物缺少 Pingora 直连彩排状态脚本: ${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" >&2
exit 1
fi
cp "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then
echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2
exit 1
@@ -11,6 +11,7 @@ const REQUIRED_ARTIFACTS = [
{ path: 'scripts/database-backup-to-oss.mjs' },
{ path: 'scripts/ops/production-health-patrol.mjs' },
{ path: 'scripts/ops/pingora-current-release-audit.mjs' },
{ path: 'scripts/ops/pingora-direct-rehearsal-status.mjs' },
{ path: 'scripts/ops/pingora-cutover-status-snapshot.mjs' },
{ path: 'scripts/ops/pingora-cutover-evidence-bundle.mjs' },
{ path: 'scripts/check-production-health-patrol-env.mjs' },
@@ -21,6 +21,7 @@ const SERVICES = [
const RELEASE_ARTIFACTS = [
'scripts/ops/pingora-cutover-status-snapshot.mjs',
'scripts/ops/pingora-current-release-audit.mjs',
'scripts/ops/pingora-direct-rehearsal-status.mjs',
'scripts/ops/pingora-cutover-evidence-audit.mjs',
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
'scripts/ops/pingora-cutover-command-evidence.mjs',
File diff suppressed because it is too large Load Diff