kdletters
e3258f6f1d
新增 Pingora shadow env 回切脚本与对应检查。 补齐直连证据包时间线和 cutoverRunId 审计门禁。 支持 Gitea Host 透传并更新直连多域名文档。 修复百分号编码静态图标路径并补 smoke 覆盖。 更新生产发布与运维护栏对 Pingora 发布包的校验。
569 lines
17 KiB
JavaScript
569 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { spawnSync } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import {
|
|
chmodSync,
|
|
copyFileSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
|
|
const STATUS_SCRIPT = 'scripts/ops/pingora-direct-rehearsal-status.mjs';
|
|
const failures = [];
|
|
const tmpRoot = mkdtempSync(
|
|
path.join(tmpdir(), 'genarrative-pingora-direct-rehearsal-'),
|
|
);
|
|
|
|
try {
|
|
main();
|
|
} finally {
|
|
rmSync(tmpRoot, { recursive: true, force: true });
|
|
}
|
|
|
|
if (failures.length > 0) {
|
|
console.error('[check:pingora-direct-rehearsal-status] FAILED');
|
|
for (const failure of failures) {
|
|
console.error(`- ${failure}`);
|
|
}
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log('[check:pingora-direct-rehearsal-status] OK');
|
|
|
|
function main() {
|
|
assertScriptShape();
|
|
assertNginxPublicRealpathRehearsalPasses();
|
|
assertFailsWhenPublicPortsAlreadyOwnedByPingora();
|
|
assertFailsWhenRealpathCanaryMissing();
|
|
assertRejectsUnsafeArgs();
|
|
assertRejectsInvalidBoolEnv();
|
|
}
|
|
|
|
function assertScriptShape() {
|
|
const content = readFileSync(STATUS_SCRIPT, 'utf8');
|
|
assertIncludes(
|
|
content,
|
|
'该脚本只读采集',
|
|
'rehearsal 状态脚本 usage 必须说明只读边界。',
|
|
);
|
|
assertIncludes(
|
|
content,
|
|
'systemctl',
|
|
'rehearsal 状态脚本必须读取 systemd 状态。',
|
|
);
|
|
assertIncludes(
|
|
content,
|
|
'ss',
|
|
'rehearsal 状态脚本必须读取端口监听状态。',
|
|
);
|
|
assertIncludes(
|
|
content,
|
|
'pingora-current-release-audit.mjs',
|
|
'rehearsal 状态脚本必须复用 current release 自审。',
|
|
);
|
|
if (
|
|
content.includes('writeFile') ||
|
|
content.includes('rmSync(') ||
|
|
content.includes('systemctl reload') ||
|
|
content.includes('daemon-reload')
|
|
) {
|
|
failures.push('rehearsal 状态脚本不应写文件、删除文件或 reload systemd。');
|
|
}
|
|
}
|
|
|
|
function assertNginxPublicRealpathRehearsalPasses() {
|
|
const fixture = prepareFixture('nginx-public-realpath', {
|
|
ssMode: 'nginx-public-realpath',
|
|
});
|
|
const result = runStatus(fixture, [
|
|
'--expect-public-gateway',
|
|
'nginx',
|
|
'--require-realpath-canary',
|
|
'--require-pingora-shadow',
|
|
'--require-current-release-gateway',
|
|
'--fail-on-critical',
|
|
]);
|
|
|
|
assertStatus(result, 0, 'Nginx 接公网 + Pingora shadow + realpath canary 应通过。');
|
|
if (result.status !== 0) {
|
|
return;
|
|
}
|
|
const payload = parseJson(result.stdout, 'rehearsal 状态输出');
|
|
assertEqual(payload.summary.status, 'OK', 'rehearsal summary 应为 OK。');
|
|
assertEqual(
|
|
payload.publicBoundary.status,
|
|
'OK',
|
|
'Nginx 接公网时 public boundary 应通过。',
|
|
);
|
|
assertEqual(
|
|
payload.ports.byPort['80'].hasNginx,
|
|
true,
|
|
'80 端口必须识别为 Nginx。',
|
|
);
|
|
assertEqual(
|
|
payload.ports.byPort['443'].hasNginx,
|
|
true,
|
|
'443 端口必须识别为 Nginx。',
|
|
);
|
|
assertEqual(
|
|
payload.ports.byPort['18081'].hasPingora,
|
|
true,
|
|
'18081 必须识别为 Pingora shadow。',
|
|
);
|
|
assertEqual(
|
|
payload.ports.byPort['18083'].hasNginx,
|
|
true,
|
|
'18083 必须识别为 Nginx realpath canary。',
|
|
);
|
|
assertEqual(
|
|
payload.realpathCanary.templateLooksValid,
|
|
true,
|
|
'realpath canary 配置必须通过关键片段检查。',
|
|
);
|
|
assertEqual(
|
|
payload.systemd.hasAmbientCapability,
|
|
false,
|
|
'Nginx 接公网彩排阶段不得有 Pingora 低端口 capability。',
|
|
);
|
|
assertEqual(
|
|
payload.healthPatrolEnv.values.gatewayMode,
|
|
'nginx',
|
|
'Nginx 接公网彩排阶段 health patrol gateway mode 应为 nginx。',
|
|
);
|
|
const commandsLog = readFileSync(fixture.commandsLog, 'utf8');
|
|
assertIncludes(
|
|
commandsLog,
|
|
'systemctl is-active nginx.service',
|
|
'rehearsal 状态必须读取 Nginx service 状态。',
|
|
);
|
|
assertIncludes(
|
|
commandsLog,
|
|
'ss -H -ltnp',
|
|
'rehearsal 状态必须读取端口监听。',
|
|
);
|
|
}
|
|
|
|
function assertFailsWhenPublicPortsAlreadyOwnedByPingora() {
|
|
const fixture = prepareFixture('pingora-public', {
|
|
ssMode: 'pingora-public',
|
|
});
|
|
const result = runStatus(fixture, [
|
|
'--expect-public-gateway',
|
|
'nginx',
|
|
'--require-pingora-shadow',
|
|
'--fail-on-critical',
|
|
]);
|
|
|
|
assertStatus(result, 1, '期望 Nginx 接公网但 80/443 由 Pingora 监听时必须失败。');
|
|
const payload = parseJson(result.stdout, 'Pingora 抢占公网端口状态输出');
|
|
assertEqual(
|
|
payload.publicBoundary.status,
|
|
'CRITICAL',
|
|
'公网端口归属漂移时 public boundary 必须 CRITICAL。',
|
|
);
|
|
assertIncludes(
|
|
payload.publicBoundary.diagnostics.join('\n'),
|
|
'已被 Pingora 监听',
|
|
'公网端口归属漂移必须给出明确诊断。',
|
|
);
|
|
}
|
|
|
|
function assertFailsWhenRealpathCanaryMissing() {
|
|
const fixture = prepareFixture('missing-realpath', {
|
|
ssMode: 'nginx-public-shadow-only',
|
|
writeRealpathConfig: false,
|
|
});
|
|
const result = runStatus(fixture, [
|
|
'--expect-public-gateway',
|
|
'nginx',
|
|
'--require-realpath-canary',
|
|
'--fail-on-critical',
|
|
]);
|
|
|
|
assertStatus(result, 1, '要求 realpath canary 但配置或端口缺失时必须失败。');
|
|
const payload = parseJson(result.stdout, '缺少 realpath canary 状态输出');
|
|
assertEqual(
|
|
payload.realpathCanary.status,
|
|
'CRITICAL',
|
|
'缺少 realpath canary 时必须 CRITICAL。',
|
|
);
|
|
assertIncludes(
|
|
payload.realpathCanary.diagnostics.join('\n'),
|
|
'realpath canary 配置不存在',
|
|
'缺少 realpath canary 配置时必须给出明确诊断。',
|
|
);
|
|
}
|
|
|
|
function assertRejectsUnsafeArgs() {
|
|
const fixture = prepareFixture('unsafe');
|
|
const cases = [
|
|
{
|
|
args: ['--release-root', 'relative'],
|
|
expected: '--release-root 必须是绝对路径',
|
|
reason: 'release root 相对路径必须失败。',
|
|
},
|
|
{
|
|
args: ['--release-root', '/'],
|
|
expected: '--release-root 不能是文件系统根目录',
|
|
reason: 'release root 为文件系统根目录必须失败。',
|
|
},
|
|
{
|
|
args: ['--expect-public-gateway', 'apache'],
|
|
expected: '--expect-public-gateway 只支持 none / nginx / pingora-direct',
|
|
reason: '未知 public gateway 必须失败。',
|
|
},
|
|
{
|
|
args: ['--timeout-ms', '0'],
|
|
expected: '--timeout-ms 必须是正整数',
|
|
reason: 'timeout 非正整数必须失败。',
|
|
},
|
|
];
|
|
|
|
for (const testCase of cases) {
|
|
const result = runStatus(fixture, testCase.args);
|
|
if ((result.status ?? 0) === 0) {
|
|
failures.push(testCase.reason);
|
|
}
|
|
assertIncludes(
|
|
`${result.stdout}\n${result.stderr}`,
|
|
testCase.expected,
|
|
`${testCase.reason} 必须输出明确错误。`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function assertRejectsInvalidBoolEnv() {
|
|
const fixture = prepareFixture('invalid-bool');
|
|
const result = runStatus(fixture, [], {
|
|
GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_REALPATH_CANARY: 'maybe',
|
|
});
|
|
if ((result.status ?? 0) === 0) {
|
|
failures.push('非法布尔 env 必须失败。');
|
|
}
|
|
assertIncludes(
|
|
`${result.stdout}\n${result.stderr}`,
|
|
'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_REALPATH_CANARY 必须是布尔值',
|
|
'非法布尔 env 必须输出明确错误。',
|
|
);
|
|
}
|
|
|
|
function prepareFixture(name, options = {}) {
|
|
const root = path.join(tmpRoot, name);
|
|
const releaseRoot = path.join(root, 'current');
|
|
const fakeBin = path.join(root, 'bin');
|
|
const commandsLog = path.join(root, 'commands.log');
|
|
const healthPatrolEnvFile = path.join(root, 'etc', 'health-patrol.env');
|
|
const pingoraEnvFile = path.join(root, 'etc', 'pingora-gateway.env');
|
|
const realpathConfigFile = path.join(
|
|
root,
|
|
'nginx',
|
|
'zz-genarrative-pingora-realpath-canary.conf',
|
|
);
|
|
|
|
mkdirSync(releaseRoot, { recursive: true });
|
|
mkdirSync(fakeBin, { recursive: true });
|
|
mkdirSync(path.dirname(healthPatrolEnvFile), { recursive: true });
|
|
mkdirSync(path.dirname(realpathConfigFile), { recursive: true });
|
|
|
|
prepareReleaseRoot(releaseRoot);
|
|
writeFileSync(
|
|
healthPatrolEnvFile,
|
|
[
|
|
'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx',
|
|
'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1',
|
|
'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=',
|
|
'GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN=health-secret-token',
|
|
'',
|
|
].join('\n'),
|
|
'utf8',
|
|
);
|
|
writeFileSync(
|
|
pingoraEnvFile,
|
|
[
|
|
'GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081',
|
|
'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=pingora-secret-token',
|
|
'GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE=/var/log/genarrative/pingora-gateway.access.log',
|
|
'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip',
|
|
'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false',
|
|
'',
|
|
].join('\n'),
|
|
'utf8',
|
|
);
|
|
if (options.writeRealpathConfig !== false) {
|
|
writeFileSync(
|
|
realpathConfigFile,
|
|
[
|
|
'server {',
|
|
' listen 127.0.0.1:18083;',
|
|
' access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;',
|
|
' add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;',
|
|
'}',
|
|
'',
|
|
].join('\n'),
|
|
'utf8',
|
|
);
|
|
}
|
|
|
|
writeFileSync(
|
|
path.join(fakeBin, 'systemctl'),
|
|
[
|
|
'#!/usr/bin/env bash',
|
|
'set -euo pipefail',
|
|
`printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`,
|
|
'if [[ "$1" == "is-active" ]]; then',
|
|
' printf "active\\n"',
|
|
' exit 0',
|
|
'fi',
|
|
'if [[ "$1" == "cat" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then',
|
|
' cat <<UNIT',
|
|
'[Service]',
|
|
`EnvironmentFile=${pingoraEnvFile}`,
|
|
'ExecStart=/opt/genarrative/current/pingora-gateway',
|
|
'UNIT',
|
|
' exit 0',
|
|
'fi',
|
|
'if [[ "$1" == "show" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then',
|
|
' cat <<SHOW',
|
|
'FragmentPath=/etc/systemd/system/genarrative-pingora-gateway.service',
|
|
'DropInPaths=',
|
|
'User=genarrative',
|
|
`ExecStart={ path=${releaseRoot}/pingora-gateway ; argv[]=${releaseRoot}/pingora-gateway ; }`,
|
|
'SHOW',
|
|
' exit 0',
|
|
'fi',
|
|
'exit 0',
|
|
'',
|
|
].join('\n'),
|
|
'utf8',
|
|
);
|
|
writeFileSync(
|
|
path.join(fakeBin, 'ss'),
|
|
[
|
|
'#!/usr/bin/env bash',
|
|
'set -euo pipefail',
|
|
`printf 'ss %s\\n' "$*" >> ${shellQuote(commandsLog)}`,
|
|
`mode="${options.ssMode || 'nginx-public-realpath'}"`,
|
|
'case "${mode}" in',
|
|
' nginx-public-realpath)',
|
|
' cat <<SS',
|
|
'LISTEN 0 511 0.0.0.0:80 0.0.0.0:* users:(("nginx",pid=11,fd=7))',
|
|
'LISTEN 0 511 0.0.0.0:443 0.0.0.0:* users:(("nginx",pid=11,fd=8))',
|
|
'LISTEN 0 128 127.0.0.1:18081 0.0.0.0:* users:(("pingora-gateway",pid=22,fd=9))',
|
|
'LISTEN 0 128 127.0.0.1:18083 0.0.0.0:* users:(("nginx",pid=11,fd=10))',
|
|
'SS',
|
|
' ;;',
|
|
' nginx-public-shadow-only)',
|
|
' cat <<SS',
|
|
'LISTEN 0 511 0.0.0.0:80 0.0.0.0:* users:(("nginx",pid=11,fd=7))',
|
|
'LISTEN 0 511 0.0.0.0:443 0.0.0.0:* users:(("nginx",pid=11,fd=8))',
|
|
'LISTEN 0 128 127.0.0.1:18081 0.0.0.0:* users:(("pingora-gateway",pid=22,fd=9))',
|
|
'SS',
|
|
' ;;',
|
|
' pingora-public)',
|
|
' cat <<SS',
|
|
'LISTEN 0 511 0.0.0.0:80 0.0.0.0:* users:(("pingora-gateway",pid=22,fd=7))',
|
|
'LISTEN 0 511 0.0.0.0:443 0.0.0.0:* users:(("pingora-gateway",pid=22,fd=8))',
|
|
'LISTEN 0 128 127.0.0.1:18081 0.0.0.0:* users:(("pingora-gateway",pid=22,fd=9))',
|
|
'SS',
|
|
' ;;',
|
|
'esac',
|
|
'',
|
|
].join('\n'),
|
|
'utf8',
|
|
);
|
|
chmodSync(path.join(fakeBin, 'systemctl'), 0o755);
|
|
chmodSync(path.join(fakeBin, 'ss'), 0o755);
|
|
|
|
return {
|
|
root,
|
|
releaseRoot,
|
|
fakeBin,
|
|
commandsLog,
|
|
healthPatrolEnvFile,
|
|
pingoraEnvFile,
|
|
realpathConfigFile,
|
|
};
|
|
}
|
|
|
|
function prepareReleaseRoot(releaseRoot) {
|
|
const dirs = [
|
|
'scripts/ops',
|
|
'scripts/deploy',
|
|
'scripts',
|
|
'deploy/systemd',
|
|
'deploy/nginx/snippets',
|
|
'deploy/env',
|
|
'deploy/pingora',
|
|
];
|
|
for (const dir of dirs) {
|
|
mkdirSync(path.join(releaseRoot, dir), { recursive: true });
|
|
}
|
|
writeExecutable(path.join(releaseRoot, 'api-server'));
|
|
writeExecutable(path.join(releaseRoot, 'pingora-gateway'));
|
|
writeSha256(releaseRoot, 'api-server');
|
|
writeSha256(releaseRoot, 'pingora-gateway');
|
|
const files = [
|
|
['scripts/database-backup-to-oss.mjs', 'console.log("backup");\n'],
|
|
['scripts/ops/production-health-patrol.mjs', 'console.log("patrol");\n'],
|
|
['scripts/check-production-health-patrol-env.mjs', 'console.log("env");\n'],
|
|
['scripts/check-pingora-release-readiness.mjs', 'console.log("ready");\n'],
|
|
['scripts/check-pingora-direct-preflight.mjs', 'console.log("preflight");\n'],
|
|
['scripts/check-pingora-direct-live.mjs', 'console.log("live");\n'],
|
|
['scripts/check-pingora-canary-live.mjs', 'console.log("canary");\n'],
|
|
[
|
|
'scripts/check-pingora-canary-access-log-parity.mjs',
|
|
'console.log("parity");\n',
|
|
],
|
|
['deploy/systemd/genarrative-pingora-gateway.service', '[Service]\n'],
|
|
['deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', '[Service]\n'],
|
|
[
|
|
'deploy/nginx/snippets/genarrative-pingora-canary.conf',
|
|
'# canary\n',
|
|
],
|
|
[
|
|
'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf',
|
|
'# realpath\n',
|
|
],
|
|
['deploy/env/health-patrol.env.example', '\n'],
|
|
['deploy/env/pingora-direct-live.env.example', '\n'],
|
|
['deploy/env/pingora-canary-live.env.example', '\n'],
|
|
['deploy/pingora/pingora-gateway.env.example', '\n'],
|
|
['deploy/pingora/nginx-route-parity.matrix.json', '[]\n'],
|
|
];
|
|
for (const [relativePath, content] of files) {
|
|
writeFileSync(path.join(releaseRoot, relativePath), content, 'utf8');
|
|
}
|
|
const copyScripts = [
|
|
'scripts/ops/pingora-current-release-audit.mjs',
|
|
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
|
'scripts/ops/pingora-cutover-status-snapshot.mjs',
|
|
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
|
|
'scripts/ops/pingora-cutover-command-evidence.mjs',
|
|
'scripts/ops/pingora-cutover-evidence-verify.mjs',
|
|
'scripts/ops/pingora-cutover-evidence-audit.mjs',
|
|
'scripts/deploy/pingora-direct-enable.sh',
|
|
'scripts/deploy/pingora-direct-rollback.sh',
|
|
'scripts/deploy/pingora-realpath-canary-enable.sh',
|
|
'scripts/deploy/pingora-realpath-canary-disable.sh',
|
|
'scripts/deploy/pingora-health-patrol-env-switch.mjs',
|
|
'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
|
|
'scripts/deploy/pingora-tls-cert-sync.mjs',
|
|
];
|
|
for (const relativePath of copyScripts) {
|
|
copyFileSync(relativePath, path.join(releaseRoot, relativePath));
|
|
}
|
|
for (const relativePath of [
|
|
'scripts/deploy/pingora-direct-enable.sh',
|
|
'scripts/deploy/pingora-direct-rollback.sh',
|
|
'scripts/deploy/pingora-realpath-canary-enable.sh',
|
|
'scripts/deploy/pingora-realpath-canary-disable.sh',
|
|
'scripts/deploy/pingora-health-patrol-env-switch.mjs',
|
|
'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
|
|
'scripts/deploy/pingora-tls-cert-sync.mjs',
|
|
]) {
|
|
chmodSync(path.join(releaseRoot, relativePath), 0o755);
|
|
}
|
|
writeFileSync(
|
|
path.join(releaseRoot, 'release-manifest.json'),
|
|
`${JSON.stringify(
|
|
{
|
|
version: 'test',
|
|
component_type: 'api-server',
|
|
artifacts: [
|
|
{
|
|
component: 'api-server',
|
|
path: 'api-server',
|
|
checksum_path: 'api-server.sha256',
|
|
},
|
|
{
|
|
component: 'pingora-gateway',
|
|
path: 'pingora-gateway',
|
|
checksum_path: 'pingora-gateway.sha256',
|
|
},
|
|
],
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
'utf8',
|
|
);
|
|
}
|
|
|
|
function writeExecutable(filePath) {
|
|
writeFileSync(filePath, '#!/usr/bin/env bash\n', 'utf8');
|
|
chmodSync(filePath, 0o755);
|
|
}
|
|
|
|
function writeSha256(directory, fileName) {
|
|
const hash = createHash('sha256')
|
|
.update(readFileSync(path.join(directory, fileName)))
|
|
.digest('hex');
|
|
writeFileSync(path.join(directory, `${fileName}.sha256`), `${hash} ${fileName}\n`);
|
|
}
|
|
|
|
function runStatus(fixture, args, extraEnv = {}) {
|
|
return spawnSync(
|
|
'node',
|
|
[
|
|
STATUS_SCRIPT,
|
|
'--release-root',
|
|
fixture.releaseRoot,
|
|
'--health-patrol-env-file',
|
|
fixture.healthPatrolEnvFile,
|
|
'--pingora-env-file',
|
|
fixture.pingoraEnvFile,
|
|
'--realpath-config-file',
|
|
fixture.realpathConfigFile,
|
|
...args,
|
|
],
|
|
{
|
|
cwd: process.cwd(),
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
...extraEnv,
|
|
PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`,
|
|
},
|
|
},
|
|
);
|
|
}
|
|
|
|
function parseJson(value, label) {
|
|
try {
|
|
return JSON.parse(value);
|
|
} catch (error) {
|
|
failures.push(`${label} 不是合法 JSON: ${error.message}`);
|
|
return {};
|
|
}
|
|
}
|
|
|
|
function assertStatus(result, expected, reason) {
|
|
if (result.status !== expected) {
|
|
failures.push(
|
|
`${reason} 退出码 ${result.status}，预期 ${expected}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function assertEqual(actual, expected, reason) {
|
|
if (actual !== expected) {
|
|
failures.push(`${reason} 实际 ${actual}，预期 ${expected}`);
|
|
}
|
|
}
|
|
|
|
function assertIncludes(content, needle, reason) {
|
|
if (!String(content || '').includes(needle)) {
|
|
failures.push(`${reason} 缺少: ${needle}`);
|
|
}
|
|
}
|
|
|
|
function shellQuote(value) {
|
|
return `'${String(value).replace(/'/g, "'\\''")}'`;
|
|
}
|