自动拉起 Pingora shadow 发布服务

让 API deploy 在发布包包含 Pingora 时先校验 shadow systemd 和 env 安全边界

在 shadow 配置安全时自动启动或重启 genarrative-pingora-gateway.service

拒绝 direct-entry capability、公网监听 env,并在 current 切换前失败

补充 production-api-deploy 动态烟测、生产运维护栏和 Pingora 运维文档

在 dev 服务器真实验证 shadow inactive 场景可由发布脚本自动拉起
This commit is contained in:
2026-06-18 14:53:03 +08:00
parent 10e559701f
commit 0cc2a79c7b
7 changed files with 278 additions and 44 deletions
+141 -20
View File
@@ -37,7 +37,9 @@ console.log('[check:production-api-deploy] OK');
function main() {
assertDeployCopiesPingoraDirectReleaseDependencies();
assertDeployRestartsActivePingoraWhenArtifactIncluded();
assertDeploySkipsInactivePingoraWhenArtifactIncluded();
assertDeployStartsInactivePingoraWhenArtifactIncluded();
assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded();
assertDeployRejectsPingoraPublicListenWhenArtifactIncluded();
assertDeployRejectsPingoraArtifactMissingManifestEntry();
assertMissingReleaseManifestFails();
assertReleaseManifestMissingApiArtifactFails();
@@ -286,22 +288,22 @@ function assertDeployRestartsActivePingoraWhenArtifactIncluded() {
const commandsLog = readFileSync(fixture.commandsLog, 'utf8');
assertIncludes(
commandsLog,
'systemctl is-active --quiet genarrative-pingora-gateway.service',
'部署脚本看到 Pingora 产物后必须先检查 shadow service 是否 active。',
'systemctl cat genarrative-pingora-gateway.service',
'部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。',
);
assertIncludes(
commandsLog,
'systemctl try-restart genarrative-pingora-gateway.service',
'Pingora shadow service 已 active 时必须随 current release 切换 try-restart。',
'systemctl restart genarrative-pingora-gateway.service',
'Pingora shadow service 已 active 时必须随 current release 切换 restart。',
);
assertIncludes(
result.stdout,
'发布包包含 Pingora,重启已运行的影子服务',
'发布包包含 Pingora,启动或重启 shadow 影子服务',
'Pingora shadow service 自动重启时必须输出明确提示。',
);
}
function assertDeploySkipsInactivePingoraWhenArtifactIncluded() {
function assertDeployStartsInactivePingoraWhenArtifactIncluded() {
const fixture = prepareFixture('with-inactive-pingora-artifact');
addPingoraGatewayArtifact(fixture);
const result = runDeploy(fixture, { pingoraActive: false });
@@ -309,7 +311,7 @@ function assertDeploySkipsInactivePingoraWhenArtifactIncluded() {
assertStatus(
result,
0,
'包含 Pingora 但 shadow service inactive 时应部署成功。',
'包含 Pingora 且 shadow service inactive 但仍是安全 shadow 配置时应部署成功。',
);
if (result.status !== 0) {
return;
@@ -318,20 +320,86 @@ function assertDeploySkipsInactivePingoraWhenArtifactIncluded() {
const commandsLog = readFileSync(fixture.commandsLog, 'utf8');
assertIncludes(
commandsLog,
'systemctl is-active --quiet genarrative-pingora-gateway.service',
'部署脚本看到 Pingora 产物后必须检查 shadow service 是否 active。',
'systemctl cat genarrative-pingora-gateway.service',
'部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。',
);
assertIncludes(
commandsLog,
'systemctl restart genarrative-pingora-gateway.service',
'Pingora shadow service inactive 且 shadow 配置安全时必须随 current release 启动。',
);
assertIncludes(
commandsLog,
'systemctl is-active --quiet genarrative-pingora-gateway.service',
'启动或重启 Pingora shadow 后必须复核 active。',
);
if (
commandsLog.includes(
'systemctl try-restart genarrative-pingora-gateway.service',
)
) {
failures.push('Pingora shadow service inactive 时不能主动 try-restart。');
}
assertIncludes(
result.stdout,
'Pingora 影子服务未处于 active,跳过自动重启',
'Pingora shadow service inactive 时必须明确说明不会主动拉起。',
'发布包包含 Pingora,启动或重启 shadow 影子服务',
'Pingora shadow service 自动启动时必须输出明确提示。',
);
}
function assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded() {
const fixture = prepareFixture('with-direct-entry-pingora-artifact');
addPingoraGatewayArtifact(fixture);
const result = runDeploy(fixture, { pingoraDirectEntry: true });
if (result.status === 0) {
failures.push('包含 Pingora 但 systemd 已启用 direct-entry capability 时部署必须失败。');
}
assertIncludes(
result.stderr,
'Pingora systemd 已包含 CAP_NET_BIND_SERVICE',
'direct-entry capability 存在时必须给出明确错误。',
);
const commandsLog = readFileSync(fixture.commandsLog, 'utf8');
if (
commandsLog.includes(
'systemctl restart genarrative-pingora-gateway.service',
)
) {
failures.push('direct-entry capability 存在时不能自动 restart Pingora。');
}
if (!existsSync(fixture.maintenanceFile)) {
failures.push('direct-entry capability 导致部署失败时必须保持维护模式。');
}
assertNoReleasePromoted(
fixture,
'direct-entry capability 导致部署失败时不能提升正式 release。',
);
}
function assertDeployRejectsPingoraPublicListenWhenArtifactIncluded() {
const fixture = prepareFixture('with-public-listen-pingora-artifact');
addPingoraGatewayArtifact(fixture);
writePingoraEnv(fixture, {
listen: '0.0.0.0:443',
});
const result = runDeploy(fixture);
if (result.status === 0) {
failures.push('包含 Pingora 但 env 已配置公网监听时部署必须失败。');
}
assertIncludes(
result.stderr,
'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081',
'公网监听 env 存在时必须给出明确错误。',
);
const commandsLog = readFileSync(fixture.commandsLog, 'utf8');
if (
commandsLog.includes(
'systemctl restart genarrative-pingora-gateway.service',
)
) {
failures.push('公网监听 env 存在时不能自动 restart Pingora。');
}
if (!existsSync(fixture.maintenanceFile)) {
failures.push('公网监听 env 导致部署失败时必须保持维护模式。');
}
assertNoReleasePromoted(
fixture,
'公网监听 env 导致部署失败时不能提升正式 release。',
);
}
@@ -1024,10 +1092,12 @@ function prepareFixture(name) {
const releaseRoot = path.join(root, 'releases');
const currentLink = path.join(root, 'current');
const apiEnvFile = path.join(root, 'etc', 'api-server.env');
const pingoraEnvFile = path.join(root, 'etc', 'pingora-gateway.env');
const maintenanceFile = path.join(root, 'maintenance', 'enabled');
const fakeBin = path.join(root, 'bin');
const commandsLog = path.join(root, 'commands.log');
const workerStateFile = path.join(root, 'worker-service-enabled');
const pingoraStateFile = path.join(root, 'pingora-service-active');
const version = `20260614-${name}`;
mkdirSync(sourceDir, { recursive: true });
@@ -1054,6 +1124,7 @@ function prepareFixture(name) {
].join('\n'),
'utf8',
);
writePingoraEnv({ pingoraEnvFile });
chmodExecutable(path.join(sourceDir, 'api-server'));
writeSha256(sourceDir, 'api-server');
writeFileSync(
@@ -1226,6 +1297,17 @@ function prepareFixture(name) {
'set -euo pipefail',
`printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`,
'worker_state_file="${FAKE_WORKER_STATE_FILE}"',
'pingora_state_file="${FAKE_PINGORA_STATE_FILE}"',
'if [[ "$1" == "cat" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then',
' printf "[Service]\\n"',
' printf "EnvironmentFile=%s\\n" "${FAKE_PINGORA_ENV_FILE}"',
' printf "ExecStart=/opt/genarrative/current/pingora-gateway\\n"',
' if [[ "${FAKE_PINGORA_DIRECT_ENTRY:-false}" == "true" ]]; then',
' printf "AmbientCapabilities=CAP_NET_BIND_SERVICE\\n"',
' printf "CapabilityBoundingSet=CAP_NET_BIND_SERVICE\\n"',
' fi',
' exit 0',
'fi',
'if [[ "$1" == "list-units" ]]; then',
' pattern="${@: -1}"',
' if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" && -f "${worker_state_file}" ]]; then',
@@ -1237,9 +1319,16 @@ function prepareFixture(name) {
' printf "enabled\\n" > "${worker_state_file}"',
' exit 0',
'fi',
'if [[ "$1 $2 ${3:-}" == "is-active --quiet genarrative-pingora-gateway.service" && "${FAKE_PINGORA_ACTIVE:-true}" == "false" ]]; then',
'if [[ "$1 $2 ${3:-}" == "is-active --quiet genarrative-pingora-gateway.service" ]]; then',
' if [[ "${FAKE_PINGORA_ACTIVE:-true}" == "true" || -f "${pingora_state_file}" ]]; then',
' exit 0',
' fi',
' exit 3',
'fi',
'if [[ "$1" == "restart" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then',
' printf "active\\n" > "${pingora_state_file}"',
' exit 0',
'fi',
'exit 0',
'',
].join('\n'),
@@ -1283,14 +1372,32 @@ function prepareFixture(name) {
releaseRoot,
currentLink,
apiEnvFile,
pingoraEnvFile,
maintenanceFile,
fakeBin,
commandsLog,
workerStateFile,
pingoraStateFile,
version,
};
}
function writePingoraEnv(fixture, options = {}) {
const filePath = fixture.pingoraEnvFile;
const lines = [
`GENARRATIVE_PINGORA_GATEWAY_LISTEN=${options.listen ?? '127.0.0.1:18081'}`,
];
if (options.tlsListen) {
lines.push(`GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${options.tlsListen}`);
}
if (options.redirectListen) {
lines.push(
`GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${options.redirectListen}`,
);
}
writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8');
}
function addPingoraGatewayArtifact(fixture, options = {}) {
writeFileSync(
path.join(fixture.sourceDir, 'pingora-gateway'),
@@ -1351,6 +1458,10 @@ function runDeploy(fixture, options = {}) {
PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`,
GENARRATIVE_MAINTENANCE_FILE: fixture.maintenanceFile,
FAKE_PINGORA_ACTIVE: options.pingoraActive === false ? 'false' : 'true',
FAKE_PINGORA_DIRECT_ENTRY:
options.pingoraDirectEntry === true ? 'true' : 'false',
FAKE_PINGORA_ENV_FILE: fixture.pingoraEnvFile,
FAKE_PINGORA_STATE_FILE: fixture.pingoraStateFile,
FAKE_CREATE_RELEASE_DURING_COPY:
options.createReleaseDuringCopy === true ? 'true' : 'false',
FAKE_RELEASE_ROOT: fixture.releaseRoot,
@@ -1397,6 +1508,16 @@ function assertFileExists(filePath, reason) {
}
}
function assertNoReleasePromoted(fixture, reason) {
const releaseDir = path.join(fixture.releaseRoot, fixture.version);
if (existsSync(releaseDir)) {
failures.push(`${reason} 已存在: ${releaseDir}`);
}
if (existsSync(fixture.currentLink)) {
failures.push(`${reason} 不应切换 current: ${fixture.currentLink}`);
}
}
function readDirNames(directory) {
return spawnSync('find', [directory, '-maxdepth', '1', '-mindepth', '1', '-printf', '%f\n'], {
cwd: process.cwd(),
+32 -8
View File
@@ -5317,21 +5317,33 @@ const checks = [
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'systemctl is-active --quiet "${service_name}"',
includes: 'ensure_pingora_shadow_service',
reason:
'API deploy 看到 Pingora 影子网关产物时必须先确认 shadow service 已 active。',
'API deploy 看到 Pingora 影子网关产物时必须先进入受控 shadow service 启动流程。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'systemctl try-restart "${service_name}"',
includes: 'systemctl cat "${service_name}"',
reason:
'API deploy 看到 Pingora 影子网关产物且 shadow service 已 active 时必须随 current release 切换重启。',
'API deploy 自动启动或重启 Pingora 前必须读取 systemd 最终配置。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'Pingora 影子服务未处于 active,跳过自动重启',
includes: 'CAP_NET_BIND_SERVICE',
reason:
'API deploy 不能在 Pingora shadow service 未运行时主动拉起新公网入口候选服务。',
'API deploy 自动启动或重启 Pingora 前必须拒绝疑似 direct-entry 低端口 capability。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081',
reason:
'API deploy 自动启动 Pingora 时必须限制为本机 shadow 高端口,避免误拉起公网直连入口。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'systemctl restart "${service_name}"',
reason:
'API deploy 看到 Pingora 影子网关产物且 shadow 配置安全时必须随 current release 启动或重启。',
},
{
file: 'scripts/check-production-api-deploy.mjs',
@@ -5341,9 +5353,21 @@ const checks = [
},
{
file: 'scripts/check-production-api-deploy.mjs',
includes: 'assertDeploySkipsInactivePingoraWhenArtifactIncluded',
includes: 'assertDeployStartsInactivePingoraWhenArtifactIncluded',
reason:
'API deploy 动态烟测必须覆盖 Pingora shadow service inactive 时不会被主动拉起。',
'API deploy 动态烟测必须覆盖 Pingora shadow service inactive 但配置安全时会被主动拉起。',
},
{
file: 'scripts/check-production-api-deploy.mjs',
includes: 'assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded',
reason:
'API deploy 动态烟测必须覆盖 direct-entry capability 下不会自动拉起 Pingora。',
},
{
file: 'scripts/check-production-api-deploy.mjs',
includes: 'assertDeployRejectsPingoraPublicListenWhenArtifactIncluded',
reason:
'API deploy 动态烟测必须覆盖公网监听 env 下不会自动拉起 Pingora。',
},
{
file: 'scripts/check-production-api-release.mjs',
+99 -10
View File
@@ -11,7 +11,7 @@ usage() {
进入维护模式,校验并发布 api-server 单文件,更新 current 链接,重启 systemd 服务并执行 readiness 检查。
默认同时重启外部生成 worker controller 和已加载的 worker 实例;未启用 worker 单元时会自动跳过。
若传入 --database,会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件,避免服务继续读取旧库。
若发布包包含 pingora-gateway,部署脚本会在 current 链接切换后仅对已 active 的 Pingora 影子服务执行 try-restart 并复核 active;未运行时不会主动拉起。
若发布包包含 pingora-gateway,部署脚本会在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置,再启动或重启 Pingora 影子服务并复核 active。
失败时保留维护模式。
EOF
}
@@ -235,18 +235,101 @@ ensure_runtime_env_and_dirs() {
fi
}
restart_pingora_if_active() {
extract_pingora_env_files_from_unit() {
local service_name="$1"
local unit_content
if ! systemctl is-active --quiet "${service_name}"; then
echo "[production-api-deploy] Pingora 影子服务未处于 active,跳过自动重启: ${service_name}"
return
if ! unit_content="$(systemctl cat "${service_name}")"; then
echo "[production-api-deploy] 无法读取 Pingora systemd 最终配置: ${service_name}" >&2
exit 1
fi
echo "[production-api-deploy] 发布包包含 Pingora,重启已运行的影子服务: ${service_name}"
systemctl try-restart "${service_name}"
if printf "%s\n" "${unit_content}" | grep -Eq '^[[:space:]]*(AmbientCapabilities|CapabilityBoundingSet)=.*CAP_NET_BIND_SERVICE'; then
echo "[production-api-deploy] Pingora systemd 已包含 CAP_NET_BIND_SERVICE,疑似直连入口配置;API deploy 不会自动启动或重启直连服务: ${service_name}" >&2
exit 1
fi
printf "%s\n" "${unit_content}" | while IFS= read -r raw_line; do
local line value token env_file
line="${raw_line#"${raw_line%%[![:space:]]*}"}"
[[ "${line}" == EnvironmentFile=* ]] || continue
value="${line#EnvironmentFile=}"
for token in ${value}; do
env_file="${token#-}"
env_file="${env_file%\"}"
env_file="${env_file#\"}"
env_file="${env_file%\'}"
env_file="${env_file#\'}"
[[ -n "${env_file}" ]] && printf "%s\n" "${env_file}"
done
done
}
find_pingora_gateway_env_file() {
local service_name="$1"
local env_file listen
while IFS= read -r env_file; do
if [[ "${env_file}" != /* ]]; then
echo "[production-api-deploy] Pingora EnvironmentFile 必须使用绝对路径: ${env_file}" >&2
exit 1
fi
listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")"
if [[ -n "${listen}" ]]; then
printf "%s\n" "${env_file}"
return
fi
done < <(extract_pingora_env_files_from_unit "${service_name}")
echo "[production-api-deploy] Pingora systemd 配置缺少包含 GENARRATIVE_PINGORA_GATEWAY_LISTEN 的 EnvironmentFile: ${service_name}" >&2
exit 1
}
require_pingora_shadow_env() {
local env_file="$1"
local listen tls_listen redirect_listen
listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")"
tls_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN")"
redirect_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN")"
if [[ "${listen}" != "127.0.0.1:18081" ]]; then
echo "[production-api-deploy] Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081,当前 GENARRATIVE_PINGORA_GATEWAY_LISTEN=${listen:-<empty>}" >&2
exit 1
fi
if [[ -n "${tls_listen}" ]]; then
echo "[production-api-deploy] Pingora 自动启动不允许启用 TLS_LISTEN,当前 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${tls_listen}" >&2
exit 1
fi
if [[ -n "${redirect_listen}" ]]; then
echo "[production-api-deploy] Pingora 自动启动不允许启用 HTTP_REDIRECT_LISTEN,当前 GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${redirect_listen}" >&2
exit 1
fi
}
check_pingora_shadow_service_config() {
local service_name="$1"
local env_file
env_file="$(find_pingora_gateway_env_file "${service_name}")"
require_pingora_shadow_env "${env_file}"
printf "%s\n" "${env_file}"
}
ensure_pingora_shadow_service() {
local service_name="$1"
local env_file="${2:-}"
if [[ -z "${env_file}" ]]; then
env_file="$(check_pingora_shadow_service_config "${service_name}")"
else
require_pingora_shadow_env "${env_file}"
fi
echo "[production-api-deploy] 发布包包含 Pingora,启动或重启 shadow 影子服务: ${service_name} (${env_file})"
systemctl restart "${service_name}"
if ! systemctl is-active --quiet "${service_name}"; then
echo "[production-api-deploy] Pingora 影子服务重启后不是 active: ${service_name}" >&2
echo "[production-api-deploy] Pingora shadow 影子服务启动或重启后不是 active: ${service_name}" >&2
exit 1
fi
}
@@ -604,7 +687,7 @@ if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then
cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway"
cp "${SOURCE_DIR}/pingora-gateway.sha256" "${RELEASE_CONTENT_DIR}/pingora-gateway.sha256"
chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway"
echo "[production-api-deploy] 已复制 Pingora 影子网关;current 链接切换后将复核并重启已运行的 ${PINGORA_SERVICE_NAME}"
echo "[production-api-deploy] 已复制 Pingora 影子网关;current 链接切换后将复核 shadow 配置并启动或重启 ${PINGORA_SERVICE_NAME}"
fi
BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
@@ -786,6 +869,8 @@ cp -R "${ENV_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/env"
cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_CONTENT_DIR}/release-manifest.api-server.json"
PINGORA_SHADOW_ENV_FILE=""
if [[ -n "${DATABASE}" ]]; then
echo "[production-api-deploy] 写入 api-server SpacetimeDB database: ${DATABASE} -> ${API_ENV_FILE}"
write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_DATABASE" "${DATABASE}"
@@ -798,6 +883,10 @@ fi
ensure_runtime_env_and_dirs "${API_ENV_FILE}"
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")"
fi
mkdir -p "$(dirname "${CURRENT_LINK}")"
if [[ -e "${RELEASE_DIR}" ]]; then
echo "[production-api-deploy] 目标 release 在发布过程中出现,拒绝合并 staging: ${RELEASE_DIR}" >&2
@@ -808,7 +897,7 @@ STAGING_RELEASE_DIR=""
ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}"
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
restart_pingora_if_active "${PINGORA_SERVICE_NAME}"
ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"
fi
echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"