补齐流水线维护退出选项与停服公告
Full 发布全程保持维护并在最后按选项退出 API Deploy 支持成功后保留维护并补齐回归门禁 新增停服公告维护页并同步运维文档
This commit is contained in:
@@ -36,6 +36,7 @@ console.log('[check:production-api-deploy] OK');
|
||||
|
||||
function main() {
|
||||
assertDeployCopiesPingoraDirectReleaseDependencies();
|
||||
assertSuccessfulDeployCanKeepMaintenance();
|
||||
assertDeployRestartsActivePingoraWhenArtifactIncluded();
|
||||
assertDeployStartsInactivePingoraWhenArtifactIncluded();
|
||||
assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded();
|
||||
@@ -79,6 +80,22 @@ function main() {
|
||||
assertMissingPingoraCanaryAccessLogParityFails();
|
||||
}
|
||||
|
||||
function assertSuccessfulDeployCanKeepMaintenance() {
|
||||
const fixture = prepareFixture('keep-maintenance-after-success');
|
||||
const result = runDeploy(fixture, { keepMaintenance: true });
|
||||
|
||||
assertStatus(result, 0, '显式保持维护时完整 fixture 应部署成功。');
|
||||
if (result.status !== 0) {
|
||||
return;
|
||||
}
|
||||
assertMaintenanceKept(fixture, '显式要求成功部署后保持维护');
|
||||
assertIncludes(
|
||||
result.stdout,
|
||||
'按参数保持维护模式',
|
||||
'成功部署并保持维护时必须输出明确状态。',
|
||||
);
|
||||
}
|
||||
|
||||
function readOptionalCommandsLog(fixture) {
|
||||
if (!existsSync(fixture.commandsLog)) {
|
||||
return '';
|
||||
@@ -1666,6 +1683,9 @@ function runDeploy(fixture, options = {}) {
|
||||
if (options.requirePingoraGateway) {
|
||||
args.push('--require-pingora-gateway');
|
||||
}
|
||||
if (options.keepMaintenance) {
|
||||
args.push('--keep-maintenance-mode');
|
||||
}
|
||||
return spawnSync(
|
||||
'bash',
|
||||
args,
|
||||
|
||||
@@ -157,6 +157,27 @@ const checks = [
|
||||
includes: '继承已有维护模式;部署失败时不得误退出上游维护窗口',
|
||||
reason: 'API deploy 必须区分自己打开的维护模式与 Stdb gate 继承的维护模式。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-api-deploy.sh',
|
||||
includes: '--keep-maintenance-mode)',
|
||||
reason: 'API deploy 必须允许成功发布后按显式参数保留维护模式。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-api-deploy.sh',
|
||||
includes: 'readiness 通过,按参数保持维护模式',
|
||||
reason: 'API deploy 保留维护模式时必须在 readiness 通过后给出明确状态。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-api-deploy',
|
||||
includes:
|
||||
"booleanParam(name: 'KEEP_MAINTENANCE_MODE', defaultValue: false",
|
||||
reason: 'API Deploy Job 必须向独立发布和 Full 编排暴露成功后保持维护的参数。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-api-deploy',
|
||||
includes: 'maintenance_deploy_args+=(--keep-maintenance-mode)',
|
||||
reason: 'API Deploy Job 必须把保持维护参数传给发布产物内的部署脚本。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes: 'ensure_runtime_bootstrap_secret_file_env',
|
||||
@@ -353,8 +374,32 @@ const checks = [
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
|
||||
includes:
|
||||
"booleanParam(name: 'KEEP_MAINTENANCE_MODE', value: params.STDB_API_ROLLOUT_MODE == 'pause-after-stdb')",
|
||||
reason: 'Full Build 暂停时必须要求 Stdb Publish 保持维护模式和旧运行时停止状态。',
|
||||
"booleanParam(name: 'KEEP_MAINTENANCE_MODE', value: true)",
|
||||
reason: 'Full Build 必须让 Stdb 与 API Deploy 全程保持维护,直到 Web Deploy 结束。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
|
||||
includes:
|
||||
"booleanParam(name: 'EXIT_MAINTENANCE_MODE_AFTER_COMPLETION', defaultValue: true",
|
||||
reason: 'Full Build 必须显式提供完整发布成功后是否退出维护模式的选项。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
|
||||
includes:
|
||||
"stage('Exit Maintenance')",
|
||||
reason: 'Full Build 必须在 Web Deploy 之后用独立阶段决定是否退出维护。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
|
||||
includes:
|
||||
'expression { return params.EXIT_MAINTENANCE_MODE_AFTER_COMPLETION != false }',
|
||||
reason: 'Full Build 只有在显式允许时才执行最终维护退出阶段。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
|
||||
includes:
|
||||
'maintenance_script="/opt/genarrative/current/scripts/deploy/maintenance-off.sh"',
|
||||
reason: 'Full Build 必须使用本次 current release 随包维护脚本退出维护。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
|
||||
@@ -6779,6 +6824,16 @@ if ((forcedBuildOnlyCalls?.length ?? 0) !== 3) {
|
||||
);
|
||||
}
|
||||
|
||||
const fullPipelineMaintenanceHoldCalls = fullPipelineContent.match(
|
||||
/booleanParam\(name: 'KEEP_MAINTENANCE_MODE', value: true\)/gu,
|
||||
);
|
||||
if ((fullPipelineMaintenanceHoldCalls?.length ?? 0) !== 2) {
|
||||
failed = true;
|
||||
console.error(
|
||||
'[check:production-ops] Full Build 必须让 Stdb Publish 与 API Deploy 两个下游阶段都保持维护模式。',
|
||||
);
|
||||
}
|
||||
|
||||
for (const file of nodeEnvFileCommandFiles) {
|
||||
const content = readFileSync(file, 'utf8');
|
||||
const commandText = content.replace(/\\\r?\n\s*/g, ' ');
|
||||
|
||||
@@ -5,13 +5,14 @@ set -euo pipefail
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
|
||||
|
||||
说明:
|
||||
进入维护模式,校验并发布 api-server 单文件,更新 current 链接,重启 systemd 服务并执行 readiness 检查。
|
||||
默认同时重启外部生成 worker controller 和已加载的 worker 实例;未启用 worker 单元时会自动跳过。
|
||||
若传入 --database,会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件,避免服务继续读取旧库。
|
||||
若发布包包含 pingora-gateway,或传入 --require-pingora-gateway,部署脚本会要求 release manifest、二进制与 checksum 一致,再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置,启动或重启 Pingora 影子服务并复核 active。
|
||||
默认在 readiness 通过后退出维护模式;传入 --keep-maintenance-mode 时保留维护文件,供人工验收后再恢复公网。
|
||||
current 链接切换前失败时会退出本次打开的维护模式;current 链接切换后失败时保留维护模式,避免暴露半发布版本。
|
||||
EOF
|
||||
}
|
||||
@@ -687,6 +688,7 @@ SPACETIME_SERVER_URL=""
|
||||
DEPLOY_COMPLETED=0
|
||||
PINGORA_INCLUDED=0
|
||||
REQUIRE_PINGORA_GATEWAY=0
|
||||
KEEP_MAINTENANCE_MODE=0
|
||||
MAINTENANCE_ENABLED_BY_DEPLOY=0
|
||||
MAINTENANCE_FILE="${GENARRATIVE_MAINTENANCE_FILE:-/var/lib/genarrative/maintenance/enabled}"
|
||||
CURRENT_LINK_SWITCHED=0
|
||||
@@ -727,6 +729,10 @@ while [[ $# -gt 0 ]]; do
|
||||
REQUIRE_PINGORA_GATEWAY=1
|
||||
shift
|
||||
;;
|
||||
--keep-maintenance-mode)
|
||||
KEEP_MAINTENANCE_MODE=1
|
||||
shift
|
||||
;;
|
||||
--worker-service-pattern)
|
||||
WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
|
||||
shift 2
|
||||
@@ -1121,7 +1127,11 @@ wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"
|
||||
echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}"
|
||||
for _ in {1..30}; do
|
||||
if curl -fsS "${HEALTH_URL}" >/dev/null; then
|
||||
bash "${SCRIPT_DIR}/maintenance-off.sh"
|
||||
if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
|
||||
echo "[production-api-deploy] readiness 通过,按参数保持维护模式: ${MAINTENANCE_FILE}"
|
||||
else
|
||||
bash "${SCRIPT_DIR}/maintenance-off.sh"
|
||||
fi
|
||||
DEPLOY_COMPLETED=1
|
||||
echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user