补齐 Pingora 直连彩排状态
新增 Pingora 直连彩排状态脚本,核验 health patrol、端口归属、systemd、realpath canary 和 current release 自审 将彩排状态脚本纳入 API release 构建、部署、Jenkins 归档和生产运维护栏 补充 release readiness、current release audit、cutover snapshot 与发布部署检查对彩排脚本的覆盖 更新 Pingora 运维文档和共享记忆,记录 dev health patrol 真实配置漂移与修复方式 在 dev 服务器真实部署验证 Nginx 仍接 80/443、Pingora shadow 与 realpath canary 高端口正常、health patrol 和彩排状态均为 OK
This commit is contained in:
@@ -552,6 +552,7 @@ copy_required_file "${SCRIPT_DIR}/spacetime-revoke-migration-operator.mjs" "${TA
|
||||
copy_required_file "${SCRIPT_DIR}/database-backup-to-oss.mjs" "${TARGET_DIR}/scripts/database-backup-to-oss.mjs" "数据库 OSS 备份脚本"
|
||||
copy_required_file "${SCRIPT_DIR}/ops/production-health-patrol.mjs" "${TARGET_DIR}/scripts/ops/production-health-patrol.mjs" "生产健康巡检脚本"
|
||||
copy_required_file "${SCRIPT_DIR}/ops/pingora-current-release-audit.mjs" "${TARGET_DIR}/scripts/ops/pingora-current-release-audit.mjs" "Pingora current release 自审脚本"
|
||||
copy_required_file "${SCRIPT_DIR}/ops/pingora-direct-rehearsal-status.mjs" "${TARGET_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" "Pingora 直连切换彩排状态脚本"
|
||||
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-status-snapshot.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" "Pingora 直连切换状态快照脚本"
|
||||
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-evidence-bundle.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" "Pingora 直连切换证据包脚本"
|
||||
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-command-evidence.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" "Pingora 直连切换命令证据脚本"
|
||||
@@ -584,7 +585,7 @@ cat >"${TARGET_DIR}/README.md" <<EOF
|
||||
- \`migration-bootstrap-secret.txt\`:构建 \`spacetime_module.wasm\` 时注入的迁移引导密钥,仅用于创建首个迁移操作员;请作为敏感文件保存到 Jenkins Secret Text,授权完成后不要长期留在公开归档中。
|
||||
- \`*.sha256\`:发布产物 checksum,用于部署前校验。
|
||||
- \`release-manifest.json\`:发布版本、源码 commit 与产物清单。
|
||||
- \`scripts/\`:维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、生产健康巡检脚本、Pingora release readiness 聚合门禁、直连启用 / 回退、realpath canary 启用 / 关闭、health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
|
||||
- \`scripts/\`:维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、生产健康巡检脚本、Pingora release readiness 聚合门禁、直连启用 / 回退、realpath canary 启用 / 关闭、health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 直连彩排状态 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
|
||||
- \`scripts/deploy/production-api-deploy.sh\`:API Deploy 执行入口;同目录的 \`maintenance-on.sh\` / \`maintenance-off.sh\` 必须来自同一发布包。
|
||||
- \`deploy/\`:systemd、Nginx 和生产环境变量示例;\`deploy/nginx/genarrative-dev-http.conf\` 仅供无域名开发服初始化使用。
|
||||
|
||||
|
||||
@@ -76,6 +76,11 @@ function assertScriptShape() {
|
||||
'scripts/ops/pingora-current-release-audit.mjs',
|
||||
'current release 自审必须检查自身已随包发布。',
|
||||
);
|
||||
assertIncludes(
|
||||
content,
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
'current release 自审必须检查直连彩排状态脚本已随包发布。',
|
||||
);
|
||||
assertIncludes(
|
||||
content,
|
||||
'scripts/check-pingora-release-readiness.mjs',
|
||||
@@ -453,6 +458,7 @@ function prepareReleaseRoot(releaseRoot, options) {
|
||||
'scripts/database-backup-to-oss.mjs',
|
||||
'scripts/ops/production-health-patrol.mjs',
|
||||
'scripts/ops/pingora-current-release-audit.mjs',
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
'scripts/ops/pingora-cutover-status-snapshot.mjs',
|
||||
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
|
||||
'scripts/check-production-health-patrol-env.mjs',
|
||||
|
||||
@@ -65,6 +65,11 @@ function assertScriptShape() {
|
||||
'scripts/ops/pingora-current-release-audit.mjs',
|
||||
'状态快照必须把 current release 自审脚本纳入 release artifact 采集。',
|
||||
);
|
||||
assertIncludes(
|
||||
content,
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
'状态快照必须把直连彩排状态脚本纳入 release artifact 采集。',
|
||||
);
|
||||
assertIncludes(
|
||||
content,
|
||||
'scripts/ops/pingora-cutover-evidence-audit.mjs',
|
||||
@@ -216,6 +221,12 @@ function assertSnapshotSucceedsWithFakeSystemctlAndReleaseRoot() {
|
||||
if (!canaryLogParityArtifact || canaryLogParityArtifact.status !== 'OK') {
|
||||
failures.push('快照必须确认 current release 已包含 canary access log 对账脚本。');
|
||||
}
|
||||
const rehearsalStatusArtifact = payload.releaseArtifacts.artifacts.find(
|
||||
(item) => item.path === 'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
);
|
||||
if (!rehearsalStatusArtifact || rehearsalStatusArtifact.status !== 'OK') {
|
||||
failures.push('快照必须确认 current release 已包含直连彩排状态脚本。');
|
||||
}
|
||||
const evidenceAuditArtifact = payload.releaseArtifacts.artifacts.find(
|
||||
(item) => item.path === 'scripts/ops/pingora-cutover-evidence-audit.mjs',
|
||||
);
|
||||
@@ -727,6 +738,7 @@ function prepareReleaseRoot(releaseRoot) {
|
||||
for (const file of [
|
||||
'scripts/database-backup-to-oss.mjs',
|
||||
'scripts/ops/pingora-current-release-audit.mjs',
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
'scripts/ops/pingora-cutover-evidence-audit.mjs',
|
||||
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
|
||||
'scripts/ops/pingora-cutover-command-evidence.mjs',
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -80,6 +80,7 @@ function main() {
|
||||
assertDefaultPlanIncludesHealthPatrolEnvSmoke();
|
||||
assertDefaultPlanIncludesPingoraHealthPatrolEnvSwitchSmoke();
|
||||
assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke();
|
||||
assertDefaultPlanIncludesPingoraDirectRehearsalStatusSmoke();
|
||||
assertDefaultPlanIncludesPingoraCutoverStatusSnapshotSmoke();
|
||||
assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke();
|
||||
assertDefaultPlanIncludesPingoraCutoverCommandEvidenceSmoke();
|
||||
@@ -260,6 +261,22 @@ function assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke() {
|
||||
);
|
||||
}
|
||||
|
||||
function assertDefaultPlanIncludesPingoraDirectRehearsalStatusSmoke() {
|
||||
const plan = readPlan(['--dry-run-plan']);
|
||||
const step = findStep(plan, 'Pingora direct rehearsal 状态烟测');
|
||||
if (!step) {
|
||||
failures.push(
|
||||
'默认 release readiness 计划必须包含 Pingora direct rehearsal 状态烟测。',
|
||||
);
|
||||
return;
|
||||
}
|
||||
assertIncludes(
|
||||
step.args,
|
||||
'check:pingora-direct-rehearsal-status',
|
||||
'Pingora direct rehearsal 状态烟测必须调用 check:pingora-direct-rehearsal-status。',
|
||||
);
|
||||
}
|
||||
|
||||
function assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke() {
|
||||
const plan = readPlan(['--dry-run-plan']);
|
||||
const step = findStep(plan, 'Pingora cutover 证据包烟测');
|
||||
|
||||
@@ -1648,6 +1648,11 @@ function buildSteps(config) {
|
||||
command: 'npm',
|
||||
args: ['run', 'check:pingora-current-release-audit'],
|
||||
},
|
||||
{
|
||||
name: 'Pingora direct rehearsal 状态烟测',
|
||||
command: 'npm',
|
||||
args: ['run', 'check:pingora-direct-rehearsal-status'],
|
||||
},
|
||||
{
|
||||
name: 'Pingora cutover 状态快照烟测',
|
||||
command: 'npm',
|
||||
|
||||
@@ -54,6 +54,7 @@ function main() {
|
||||
assertMissingBackupScriptFails();
|
||||
assertMissingHealthPatrolScriptFails();
|
||||
assertMissingPingoraCurrentReleaseAuditFails();
|
||||
assertMissingPingoraDirectRehearsalStatusFails();
|
||||
assertMissingPingoraCutoverStatusSnapshotFails();
|
||||
assertMissingPingoraCutoverEvidenceBundleFails();
|
||||
assertMissingPingoraCutoverCommandEvidenceFails();
|
||||
@@ -136,6 +137,10 @@ function assertDeployCopiesPingoraDirectReleaseDependencies() {
|
||||
path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'),
|
||||
'current release 必须包含 Pingora current release 自审脚本。',
|
||||
);
|
||||
assertFileExists(
|
||||
path.join(releaseDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'),
|
||||
'current release 必须包含 Pingora 直连彩排状态脚本。',
|
||||
);
|
||||
assertFileExists(
|
||||
path.join(releaseDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'),
|
||||
'current release 必须包含 Pingora 直连切换状态快照脚本。',
|
||||
@@ -718,6 +723,29 @@ function assertMissingPingoraCurrentReleaseAuditFails() {
|
||||
}
|
||||
}
|
||||
|
||||
function assertMissingPingoraDirectRehearsalStatusFails() {
|
||||
const fixture = prepareFixture('missing-pingora-direct-rehearsal-status');
|
||||
rmSync(
|
||||
path.join(
|
||||
fixture.sourceDir,
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
),
|
||||
);
|
||||
const result = runDeploy(fixture);
|
||||
|
||||
if (result.status === 0) {
|
||||
failures.push('发布产物缺少 Pingora 直连彩排状态脚本时部署必须失败。');
|
||||
}
|
||||
assertIncludes(
|
||||
result.stderr,
|
||||
'发布产物缺少 Pingora 直连彩排状态脚本',
|
||||
'缺少 Pingora 直连彩排状态脚本时必须给出明确错误。',
|
||||
);
|
||||
if (!existsSync(fixture.maintenanceFile)) {
|
||||
failures.push('部署失败时必须保持维护模式。');
|
||||
}
|
||||
}
|
||||
|
||||
function assertMissingPingoraCutoverStatusSnapshotFails() {
|
||||
const fixture = prepareFixture('missing-pingora-cutover-status-snapshot');
|
||||
rmSync(
|
||||
@@ -1060,6 +1088,10 @@ function prepareFixture(name) {
|
||||
'scripts/ops/pingora-current-release-audit.mjs',
|
||||
path.join(sourceDir, 'scripts/ops/pingora-current-release-audit.mjs'),
|
||||
);
|
||||
copyFile(
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
path.join(sourceDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'),
|
||||
);
|
||||
copyFile(
|
||||
'scripts/ops/pingora-cutover-status-snapshot.mjs',
|
||||
path.join(sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'),
|
||||
|
||||
@@ -155,6 +155,13 @@ function assertApiReleaseContainsPingoraDirectDependencies() {
|
||||
),
|
||||
'API release 必须包含 Pingora current release 自审脚本。',
|
||||
);
|
||||
assertFileExists(
|
||||
path.join(
|
||||
releaseDir,
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
),
|
||||
'API release 必须包含 Pingora 直连彩排状态脚本。',
|
||||
);
|
||||
assertFileExists(
|
||||
path.join(
|
||||
releaseDir,
|
||||
@@ -280,7 +287,7 @@ function assertApiReleaseContainsPingoraDirectDependencies() {
|
||||
);
|
||||
assertIncludes(
|
||||
releaseReadme,
|
||||
'health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审',
|
||||
'health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 直连彩排状态',
|
||||
'API release README 必须说明随包携带 Pingora 运行态复核脚本。',
|
||||
);
|
||||
assertIncludes(
|
||||
|
||||
@@ -66,6 +66,26 @@ const checks = [
|
||||
includes: 'scripts/check-pingora-release-readiness.mjs',
|
||||
reason: 'Pingora current release 自审必须确认 release readiness 聚合门禁脚本已随包发布。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/ops/pingora-current-release-audit.mjs',
|
||||
includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
reason: 'Pingora current release 自审必须确认直连彩排状态脚本已随包发布。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
includes: '该脚本只读采集',
|
||||
reason: 'Pingora 直连彩排状态脚本必须声明只读边界。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
includes: 'ss',
|
||||
reason: 'Pingora 直连彩排状态脚本必须读取端口监听状态。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/check-pingora-direct-rehearsal-status.mjs',
|
||||
includes: 'assertNginxPublicRealpathRehearsalPasses',
|
||||
reason: 'Pingora 直连彩排状态自测必须覆盖 Nginx 公网 + realpath canary 正例。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/ops/pingora-current-release-audit.mjs',
|
||||
includes: 'scripts/check-pingora-canary-live.mjs',
|
||||
@@ -939,6 +959,11 @@ const checks = [
|
||||
includes: 'scripts/ops/pingora-current-release-audit.mjs',
|
||||
reason: '生产 API release 必须携带 Pingora current release 自审脚本。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/build-production-release.sh',
|
||||
includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
reason: '生产 API release 必须携带 Pingora 直连彩排状态脚本。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/build-production-release.sh',
|
||||
includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs',
|
||||
@@ -1114,6 +1139,11 @@ const checks = [
|
||||
includes: 'PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE',
|
||||
reason: 'API deploy 必须从发布产物复制 Pingora current release 自审脚本。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-api-deploy.sh',
|
||||
includes: 'PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE',
|
||||
reason: 'API deploy 必须从发布产物复制 Pingora 直连彩排状态脚本。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-api-deploy.sh',
|
||||
includes: 'PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE',
|
||||
@@ -1139,6 +1169,11 @@ const checks = [
|
||||
includes: 'scripts/ops/pingora-current-release-audit.mjs',
|
||||
reason: 'API release 动态烟测必须断言 Pingora current release 自审脚本随包发布。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/check-production-api-release.mjs',
|
||||
includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
reason: 'API release 动态烟测必须断言 Pingora 直连彩排状态脚本随包发布。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/check-production-api-release.mjs',
|
||||
includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs',
|
||||
@@ -1361,6 +1396,11 @@ const checks = [
|
||||
includes: 'check:pingora-current-release-audit',
|
||||
reason: 'Pingora current release 自审必须保留可重复 npm 自测入口。',
|
||||
},
|
||||
{
|
||||
file: 'package.json',
|
||||
includes: 'check:pingora-direct-rehearsal-status',
|
||||
reason: 'Pingora 直连彩排状态脚本必须保留可重复 npm 自测入口。',
|
||||
},
|
||||
{
|
||||
file: 'package.json',
|
||||
includes: 'check:pingora-cutover-status-snapshot',
|
||||
@@ -1452,6 +1492,11 @@ const checks = [
|
||||
includes: 'check:pingora-current-release-audit',
|
||||
reason: 'Pingora 聚合门禁必须执行 current release 自审烟测。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/check-pingora-release-readiness.mjs',
|
||||
includes: 'check:pingora-direct-rehearsal-status',
|
||||
reason: 'Pingora 聚合门禁必须执行直连彩排状态烟测。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/check-pingora-release-readiness.mjs',
|
||||
includes: 'check:pingora-cutover-status-snapshot',
|
||||
|
||||
@@ -610,6 +610,7 @@ fi
|
||||
BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
|
||||
HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs"
|
||||
PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs"
|
||||
PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
|
||||
PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
|
||||
PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
|
||||
PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
|
||||
@@ -650,6 +651,12 @@ if [[ ! -f "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" ]]; then
|
||||
fi
|
||||
cp "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
|
||||
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
|
||||
if [[ ! -f "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" ]]; then
|
||||
echo "[production-api-deploy] 发布产物缺少 Pingora 直连彩排状态脚本: ${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" >&2
|
||||
exit 1
|
||||
fi
|
||||
cp "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
|
||||
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
|
||||
if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then
|
||||
echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2
|
||||
exit 1
|
||||
|
||||
@@ -11,6 +11,7 @@ const REQUIRED_ARTIFACTS = [
|
||||
{ path: 'scripts/database-backup-to-oss.mjs' },
|
||||
{ path: 'scripts/ops/production-health-patrol.mjs' },
|
||||
{ path: 'scripts/ops/pingora-current-release-audit.mjs' },
|
||||
{ path: 'scripts/ops/pingora-direct-rehearsal-status.mjs' },
|
||||
{ path: 'scripts/ops/pingora-cutover-status-snapshot.mjs' },
|
||||
{ path: 'scripts/ops/pingora-cutover-evidence-bundle.mjs' },
|
||||
{ path: 'scripts/check-production-health-patrol-env.mjs' },
|
||||
|
||||
@@ -21,6 +21,7 @@ const SERVICES = [
|
||||
const RELEASE_ARTIFACTS = [
|
||||
'scripts/ops/pingora-cutover-status-snapshot.mjs',
|
||||
'scripts/ops/pingora-current-release-audit.mjs',
|
||||
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
||||
'scripts/ops/pingora-cutover-evidence-audit.mjs',
|
||||
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
|
||||
'scripts/ops/pingora-cutover-command-evidence.mjs',
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user