Genarrative/scripts/check-pingora-gateway-smoke.mjs

#!/usr/bin/env node

import { spawn, spawnSync } from 'node:child_process';
import { createHash, randomBytes } from 'node:crypto';
import { existsSync } from 'node:fs';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import http from 'node:http';
import https from 'node:https';
import net from 'node:net';
import os from 'node:os';
import path from 'node:path';
import { gunzipSync } from 'node:zlib';

const repoRoot = process.cwd();
const failures = [];
const childProcesses = [];
const servers = [];
const sockets = new Set();
const tempDirs = [];
const SPACETIME_WEBSOCKET_PROTOCOL = 'v2.bsatn.spacetimedb';
const SECRET_VALUE_FLAGS = new Set(['--probe-token']);
const PNG_MAGIC_BYTES = Buffer.from([
  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
]);

const config = parseArgs(process.argv.slice(2));

try {
  await main();
} catch (error) {
  failures.push(error instanceof Error ? error.message : String(error));
} finally {
  await cleanup();
}

if (failures.length > 0) {
  console.error('\n[pingora-gateway-smoke] 未通过：');
  for (const failure of failures) {
    console.error(`- ${failure}`);
  }
  process.exit(1);
}

console.log('\n[pingora-gateway-smoke] 通过');

async function main() {
  const tempRoot = await mkdtemp(
    path.join(os.tmpdir(), 'genarrative-pingora-smoke-'),
  );
  tempDirs.push(tempRoot);
  const webRoot = path.join(tempRoot, 'web');
  const acmeRoot = path.join(tempRoot, 'acme');
  const maintenanceFile = path.join(tempRoot, 'maintenance', 'enabled');
  const maintenancePageFile = path.join(tempRoot, 'maintenance', 'page.html');
  const accessLogFile = path.join(tempRoot, 'logs', 'pingora.access.log');

  await prepareStaticRoots(webRoot, acmeRoot);

  const api = await startApiMock();
  const spacetime = await startSpacetimeMock();
  const gitea = await startGiteaMock();
  const listenPort = await getFreePort();
  const tlsListenPort = await getFreePort();
  const redirectListenPort = await getFreePort();
  const tlsFiles = await createSelfSignedCertificate(tempRoot);
  const probeToken = `probe-${randomBytes(8).toString('hex')}`;

  if (!config.skipBuild) {
    runCommand('cargo', [
      'build',
      '-p',
      'pingora-gateway',
      '--manifest-path',
      'server-rs/Cargo.toml',
    ]);
  }

  const gatewayBinary = resolveGatewayBinary();
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN:
        '__GENARRATIVE_PINGORA_PROBE_TOKEN__',
    },
    'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN',
    'probe token 占位值',
  );
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL: '10',
    },
    'GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL',
    'gzip 等级越界',
  );
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS: 'br',
    },
    'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS',
    '未验收压缩算法',
  );
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS: '0',
    },
    'GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS',
    '上游读取超时为 0',
  );
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN: `127.0.0.1:${tlsListenPort}`,
    },
    'GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE',
    'TLS 证书未配置',
  );
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN: `127.0.0.1:${redirectListenPort}`,
    },
    'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN',
    'HTTP 重定向未配置 TLS 入口',
  );
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS: 'git.genarrative.world',
    },
    'GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM',
    'Gitea Host 缺少上游',
  );
  expectInvalidConfigRejected(
    gatewayBinary,
    {
      GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM: '127.0.0.1:3000',
    },
    'GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS',
    'Gitea 上游缺少 Host',
  );
  const gateway = spawn(gatewayBinary, [], {
    cwd: repoRoot,
    env: {
      ...smokeEnv(),
      GENARRATIVE_PINGORA_GATEWAY_LISTEN: `127.0.0.1:${listenPort}`,
      GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN: `127.0.0.1:${tlsListenPort}`,
      GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE: tlsFiles.certFile,
      GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE: tlsFiles.keyFile,
      GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN: `127.0.0.1:${redirectListenPort}`,
      GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME: 'https',
      GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM: `127.0.0.1:${api.port}`,
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM: `127.0.0.1:${spacetime.port}`,
      GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS:
        'git.genarrative.world, Git-Alias.Genarrative.World:443',
      GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM: `127.0.0.1:${gitea.port}`,
      GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT: webRoot,
      GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT: acmeRoot,
      GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE: maintenanceFile,
      GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_PAGE_FILE: maintenancePageFile,
      GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE: accessLogFile,
      GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN: probeToken,
      GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO: 'https',
      GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS: 'gzip',
      GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED: 'true',
      GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL: '5',
      GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_CONNECT_TIMEOUT_MS: '1000',
      GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS: '2',
      GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS: '1',
      GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS: '60',
      GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_WRITE_TIMEOUT_SECONDS: '2',
      GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR: 'true',
      GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED: 'true',
      GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES: '12',
      GENARRATIVE_PINGORA_GATEWAY_API_MAX_CONCURRENT: '1',
      GENARRATIVE_PINGORA_GATEWAY_API_RATE_PER_SECOND: '1',
      GENARRATIVE_PINGORA_GATEWAY_API_BURST: '0',
      GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_MAX_CONCURRENT: '1',
      GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_RATE_PER_SECOND: '0',
      GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_BURST: '0',
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_MAX_CONCURRENT: '16',
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_RATE_PER_SECOND: '100',
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_BURST: '100',
      GENARRATIVE_PINGORA_GATEWAY_LOG: config.verbose
        ? 'info,pingora=info,pingora_gateway=info'
        : 'warn,pingora=warn,pingora_gateway=warn',
    },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  childProcesses.push({ child: gateway, name: 'pingora-gateway' });
  collectProcessLogs(gateway, 'pingora-gateway');

  const baseUrl = `http://127.0.0.1:${listenPort}`;
  const tlsBaseUrl = `https://127.0.0.1:${tlsListenPort}`;
  const redirectBaseUrl = `http://127.0.0.1:${redirectListenPort}`;
  await waitForHttp(`${baseUrl}/`, 200);

  await runDirectLiveSmoke(
    tlsBaseUrl,
    redirectBaseUrl,
    probeToken,
    accessLogFile,
  );
  await runSmokeCases(
    baseUrl,
    tlsBaseUrl,
    redirectBaseUrl,
    probeToken,
    maintenanceFile,
    maintenancePageFile,
    accessLogFile,
    api,
    spacetime,
    gitea,
  );
}

function expectInvalidConfigRejected(
  gatewayBinary,
  env,
  expectedNeedle,
  label,
) {
  console.log(`[pingora-gateway-smoke] 配置错误启动失败: ${label}`);
  const result = spawnSync(gatewayBinary, [], {
    cwd: repoRoot,
    env: {
      ...smokeEnv(),
      GENARRATIVE_PINGORA_GATEWAY_LISTEN: '127.0.0.1:0',
      ...env,
    },
    shell: false,
    encoding: 'utf8',
    timeout: 3000,
  });

  if (result.error?.code === 'ETIMEDOUT') {
    failures.push(`配置错误启动失败: ${label} 未被拒绝，进程超时后已终止`);
  } else if (result.status === 0) {
    failures.push(`配置错误启动失败: ${label} 未被拒绝`);
  }
  const output = `${result.stdout || ''}\n${result.stderr || ''}`;
  if (!output.includes(expectedNeedle)) {
    failures.push(`配置错误启动失败: ${label} 错误输出缺少 ${expectedNeedle}`);
  }
}

function parseArgs(argv) {
  const result = {
    skipBuild: false,
    verbose: false,
  };

  for (const arg of argv) {
    switch (arg) {
      case '--skip-build':
        result.skipBuild = true;
        break;
      case '--verbose':
        result.verbose = true;
        break;
      case '-h':
      case '--help':
        console.log(`Usage:
  node scripts/check-pingora-gateway-smoke.mjs [--skip-build] [--verbose]

本地启动 mock api-server、mock SpacetimeDB 和 pingora-gateway，
覆盖静态路由、API 代理、body limit、429 接流保护、维护模式和 WebSocket Upgrade。
`);
        process.exit(0);
        break;
      default:
        throw new Error(`未知参数: ${arg}`);
    }
  }

  return result;
}

async function prepareStaticRoots(webRoot, acmeRoot) {
  await mkdir(path.join(webRoot, 'admin', 'assets'), { recursive: true });
  await mkdir(path.join(webRoot, 'assets'), { recursive: true });
  await mkdir(
    path.join(
      webRoot,
      'Icons',
      "Admurin's Pixel Items",
      "Admurin's Pixel Items",
      'General',
      'Singles',
    ),
    { recursive: true },
  );
  await mkdir(path.join(acmeRoot, '.well-known', 'acme-challenge'), {
    recursive: true,
  });

  await writeFile(
    path.join(webRoot, 'index.html'),
    [
      '<main>site-shell</main>',
      '<script type="module" src="/assets/app.js"></script>',
      '<script type="module" src="/assets/index-B4dmVw0r.js"></script>',
    ].join(''),
  );
  await writeFile(
    path.join(webRoot, 'admin', 'index.html'),
    '<main>admin-shell</main>',
  );
  await writeFile(
    path.join(webRoot, 'assets', 'app.js'),
    'console.log("site asset");\n',
  );
  await writeFile(
    path.join(webRoot, 'assets', 'index-B4dmVw0r.js'),
    'console.log("fingerprinted site asset");\n',
  );
  await writeFile(
    path.join(webRoot, 'assets', 'large-app.js'),
    `console.log("site asset large");\n${'/* asset filler */\n'.repeat(120)}`,
  );
  await writeFile(
    path.join(webRoot, 'assets', 'large-image.webp'),
    Buffer.alloc(4096, 0x52),
  );
  await writeFile(
    path.join(
      webRoot,
      'Icons',
      "Admurin's Pixel Items",
      "Admurin's Pixel Items",
      'General',
      'Singles',
      '499_Iron_Gear.png',
    ),
    Buffer.concat([
      PNG_MAGIC_BYTES,
      Buffer.from([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52]),
    ]),
  );
  await writeFile(
    path.join(webRoot, 'admin', 'assets', 'admin.js'),
    'console.log("admin asset");',
  );
  await writeFile(
    path.join(webRoot, 'admin', 'assets', 'admin-Ca8f3012.css'),
    'body{color:#123456;}',
  );
  await writeFile(
    path.join(webRoot, 'maintenance.html'),
    '<main>default-maintenance</main>',
  );
  await mkdir(path.join(webRoot, 'branding'), { recursive: true });
  for (const fileName of [
    'taonier-maintenance-page.png',
    'taonier-product-ip.png',
  ]) {
    await writeFile(
      path.join(webRoot, 'branding', fileName),
      Buffer.concat([
        PNG_MAGIC_BYTES,
        Buffer.from([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52]),
      ]),
    );
  }
  await writeFile(
    path.join(webRoot, '404.html'),
    '<main>not-found-page</main>',
  );
  await writeFile(
    path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'),
    'acme-token',
  );
}

async function createSelfSignedCertificate(tempRoot) {
  const tlsDir = path.join(tempRoot, 'tls');
  await mkdir(tlsDir, { recursive: true });
  const certFile = path.join(tlsDir, 'cert.pem');
  const keyFile = path.join(tlsDir, 'key.pem');
  runCommand('openssl', [
    'req',
    '-x509',
    '-newkey',
    'rsa:2048',
    '-nodes',
    '-keyout',
    keyFile,
    '-out',
    certFile,
    '-subj',
    '/CN=localhost',
    '-addext',
    'subjectAltName=DNS:localhost,IP:127.0.0.1',
    '-days',
    '1',
  ]);
  return { certFile, keyFile };
}

async function startApiMock() {
  const state = {
    requests: [],
    releaseHold: undefined,
  };
  const server = http.createServer(async (request, response) => {
    let body;
    try {
      body = await readRequestBody(request);
    } catch (error) {
      state.requests.push({
        method: request.method || '',
        url: request.url || '',
        headers: request.headers,
        body: Buffer.alloc(0),
        aborted: true,
        error: error instanceof Error ? error.message : String(error),
      });
      return;
    }

    state.requests.push({
      method: request.method || '',
      url: request.url || '',
      headers: request.headers,
      body,
    });

    if (request.url?.endsWith('/hold')) {
      await new Promise((resolve) => {
        state.releaseHold = resolve;
      });
    } else if (request.url?.endsWith('/upstream-close')) {
      request.socket.destroy();
      return;
    } else if (request.url?.endsWith('/slow')) {
      await delay(1500);
    }

    response.writeHead(200, {
      'Content-Type': 'application/json; charset=utf-8',
      'X-Upstream': 'api',
    });
    response.end(
      JSON.stringify({
        ok: true,
        upstream: 'api',
        method: request.method,
        url: request.url,
        requestId: request.headers['x-request-id'] || '',
        forwardedProto: request.headers['x-forwarded-proto'] || '',
        forwardedHost: request.headers['x-forwarded-host'] || '',
        forwardedFor: request.headers['x-forwarded-for'] || '',
        realIp: request.headers['x-real-ip'] || '',
        host: request.headers.host || '',
        bodyBytes: body.length,
      }),
    );
  });

  const port = await listen(server);
  return {
    port,
    state,
  };
}

async function startSpacetimeMock() {
  const state = {
    websocketMessages: [],
    requests: [],
  };
  const server = http.createServer(async (request, response) => {
    state.requests.push({
      method: request.method || '',
      url: request.url || '',
      headers: request.headers,
    });
    response.writeHead(200, {
      'Content-Type': 'application/json; charset=utf-8',
    });
    response.end(
      JSON.stringify({ ok: true, upstream: 'spacetime', url: request.url }),
    );
  });

  server.on('upgrade', (request, socket) => {
    socket.on('error', () => {});
    state.requests.push({
      method: request.method || '',
      url: request.url || '',
      headers: request.headers,
    });
    const key = request.headers['sec-websocket-key'];
    if (!key) {
      socket.destroy();
      return;
    }

    const accept = createHash('sha1')
      .update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
      .digest('base64');
    socket.write(
      [
        'HTTP/1.1 101 Switching Protocols',
        'Upgrade: websocket',
        'Connection: Upgrade',
        `Sec-WebSocket-Protocol: ${SPACETIME_WEBSOCKET_PROTOCOL}`,
        `Sec-WebSocket-Accept: ${accept}`,
        '',
        '',
      ].join('\r\n'),
    );
    socket.write(encodeWebSocketTextFrame('spacetime-upgrade-ok'));
    socket.on('data', (chunk) => {
      state.websocketMessages.push(chunk);
    });
  });

  const port = await listen(server);
  return {
    port,
    state,
  };
}

async function startGiteaMock() {
  const state = {
    requests: [],
  };
  const server = http.createServer(async (request, response) => {
    const body = await readRequestBody(request);
    state.requests.push({
      method: request.method || '',
      url: request.url || '',
      headers: request.headers,
      body,
    });

    response.writeHead(200, {
      'Content-Type': 'application/json; charset=utf-8',
      'X-Upstream': 'gitea',
    });
    response.end(
      JSON.stringify({
        ok: true,
        upstream: 'gitea',
        method: request.method,
        url: request.url,
        forwardedProto: request.headers['x-forwarded-proto'] || '',
        forwardedHost: request.headers['x-forwarded-host'] || '',
        host: request.headers.host || '',
      }),
    );
  });

  const port = await listen(server);
  return {
    port,
    state,
  };
}

async function runSmokeCases(
  baseUrl,
  tlsBaseUrl,
  redirectBaseUrl,
  probeToken,
  maintenanceFile,
  maintenancePageFile,
  accessLogFile,
  api,
  spacetime,
  gitea,
) {
  await expectHttp(
    baseUrl,
    '/',
    200,
    'site-shell',
    '主站根路径返回 index.html',
    {
      validate: (response) => response.headers['cache-control'] === 'no-cache',
    },
  );
  await expectHttp(
    baseUrl,
    '/creation',
    200,
    'site-shell',
    '新创作主页深链回退 index.html',
    {
      validate: (response) => response.headers['cache-control'] === 'no-cache',
    },
  );
  await expectHttp(
    baseUrl,
    '/project',
    200,
    'site-shell',
    '主站 allowlist 深链回退 index.html',
    {
      validate: (response) => response.headers['cache-control'] === 'no-cache',
    },
  );
  await expectHttp(
    baseUrl,
    '/profile',
    200,
    'site-shell',
    '个人页深链回退 index.html',
    {
      validate: (response) => response.headers['cache-control'] === 'no-cache',
    },
  );
  await expectHttp(
    baseUrl,
    '/PROJECT/',
    200,
    'site-shell',
    '主站 allowlist 允许大小写差异和尾部斜杠',
  );
  for (const unknownPath of [
    '/some/deep/link',
    '/creation/not-exist',
    '/runtime/not-exist',
    '/puzzle/not-exist',
  ]) {
    await expectHttp(
      baseUrl,
      unknownPath,
      404,
      '',
      `主站未知路径返回真实 404: ${unknownPath}`,
    );
  }
  await expectHttp(
    baseUrl,
    '/some/browser/navigation',
    404,
    'not-found-page',
    '浏览器导航未知路径返回品牌 404 页面',
    {
      headers: {
        Accept: 'text/html,application/xhtml+xml',
      },
      validate: (response) =>
        response.headers['cache-control'] === 'no-store' &&
        response.headers['content-type']?.startsWith('text/html'),
    },
  );
  await expectHttp(baseUrl, '/admin', 301, '', '/admin 301 到 /admin/', {
    validate: (response) => response.headers.location === '/admin/',
  });
  await expectHttp(
    baseUrl,
    '/admin/settings',
    200,
    'admin-shell',
    '后台深链回退 admin/index.html',
  );
  const staticAssetResponse = await expectHttp(
    baseUrl,
    '/assets/app.js',
    200,
    'site asset',
    '主站静态资源精确读取',
    {
      validate: (response) => response.headers['cache-control'] === 'no-cache',
    },
  );
  assertStaticValidatorHeaders(staticAssetResponse, '主站静态资源精确读取');
  ensure(
    staticAssetResponse.headers['accept-ranges'] === 'bytes',
    `主站静态资源精确读取: 缺少 Accept-Ranges=bytes，实际 ${staticAssetResponse.headers['accept-ranges'] || '-'}`,
  );
  await expectHttp(
    baseUrl,
    '/assets/app.js',
    200,
    '',
    'HEAD 静态资源只返回响应头',
    {
      method: 'HEAD',
      validate: (response) =>
        response.body === '' &&
        response.headers.etag === staticAssetResponse.headers.etag &&
        response.headers['last-modified'] ===
          staticAssetResponse.headers['last-modified'] &&
        Number(response.headers['content-length']) ===
          Buffer.byteLength('console.log("site asset");\n'),
    },
  );
  await expectStaticNotModifiedByEtag(
    baseUrl,
    '/assets/app.js',
    staticAssetResponse,
    {
      headers: {
        'X-Request-Id': 'static-etag-304-request-id',
      },
    },
  );
  await expectStaticNotModifiedByLastModified(
    baseUrl,
    '/assets/app.js',
    staticAssetResponse,
    {
      headers: {
        'X-Request-Id': 'static-last-modified-304-request-id',
      },
    },
  );
  await expectHttp(
    baseUrl,
    '/assets/app.js',
    405,
    '',
    '静态资源非读取方法返回 405',
    {
      method: 'POST',
      headers: {
        'X-Request-Id': 'static-method-405-request-id',
      },
      body: 'ignored',
      validate: (response) =>
        response.body === '' &&
        response.headers.allow === 'GET, HEAD' &&
        response.headers['x-genarrative-gateway'] === 'pingora-shadow',
    },
  );
  await expectStaticRange(
    baseUrl,
    '/assets/app.js',
    'bytes=0-6',
    206,
    'console',
    'bytes 0-6/27',
    '静态资源 Range 返回 206',
    {
      headers: {
        'X-Request-Id': 'static-range-206-request-id',
      },
    },
  );
  await expectStaticRange(
    baseUrl,
    '/assets/app.js',
    'bytes=0-6',
    206,
    'console',
    'bytes 0-6/27',
    '静态资源 If-Range 日期匹配返回 206',
    {
      headers: {
        'If-Range': staticAssetResponse.headers['last-modified'],
      },
    },
  );
  await expectStaticRangeFallback(
    baseUrl,
    '/assets/app.js',
    'bytes=0-6',
    'Tue, 14 Nov 2023 22:13:12 GMT',
    '静态资源 If-Range 旧日期回完整文件',
  );
  await expectStaticRangeFallback(
    baseUrl,
    '/assets/app.js',
    'bytes=0-6',
    staticAssetResponse.headers.etag,
    '静态资源 If-Range 弱 ETag 回完整文件',
  );
  await expectStaticRange(
    baseUrl,
    '/assets/app.js',
    'bytes=-8',
    206,
    'sset");\n',
    'bytes 19-26/27',
    '静态资源 suffix Range 返回 206',
  );
  await expectStaticRange(
    baseUrl,
    '/assets/app.js',
    'bytes=999-1000',
    416,
    '',
    'bytes */27',
    '静态资源越界 Range 返回 416',
    {
      headers: {
        'X-Request-Id': 'static-range-416-request-id',
      },
    },
  );
  await expectStaticRange(
    baseUrl,
    '/assets/app.js',
    'bytes=0-6',
    206,
    '',
    'bytes 0-6/27',
    'HEAD 静态资源 Range 只返回响应头',
    {
      method: 'HEAD',
      expectedContentLength: 7,
      headers: {
        'X-Request-Id': 'static-head-range-206-request-id',
      },
    },
  );
  await expectHttp(
    baseUrl,
    '/assets/index-B4dmVw0r.js',
    200,
    'fingerprinted site asset',
    '主站指纹静态资源长缓存',
    {
      validate: (response) =>
        response.headers['cache-control'] ===
        'public, max-age=31536000, immutable',
    },
  );
  await expectNotCompressedResponse(
    baseUrl,
    '/assets/app.js',
    'site asset',
    'gzip 最小长度不压缩小响应',
  );
  await expectGzipResponse(baseUrl, '/assets/large-app.js', 'site asset large');
  await expectGzipResponse(
    baseUrl,
    '/assets/large-app.js',
    'site asset large',
    {
      acceptEncoding: 'br, gzip',
      label: 'gzip-only 压缩算法白名单',
    },
  );
  await expectNotCompressedResponse(
    baseUrl,
    '/assets/large-image.webp',
    undefined,
    'gzip types 不压缩图片资源',
  );
  await expectStaticPng(
    baseUrl,
    '/Icons/Admurin%27s%20Pixel%20Items/Admurin%27s%20Pixel%20Items/General/Singles/499_Iron_Gear.png',
    '百分号编码图标路径返回 PNG',
  );
  await expectHttp(
    baseUrl,
    '/assets%2ffavicon.svg',
    404,
    '',
    '静态路径拒绝编码斜杠',
  );
  await expectRawPathHttp(
    baseUrl,
    '/assets/%2e%2e/favicon.svg',
    404,
    '',
    '静态路径拒绝编码上级目录',
  );
  await expectHttp(
    baseUrl,
    '/assets/%GG',
    404,
    '',
    '静态路径拒绝非法百分号编码',
  );
  await expectHttp(
    baseUrl,
    '/admin/assets/admin.js',
    200,
    'admin asset',
    '后台静态资源精确读取',
    {
      validate: (response) => response.headers['cache-control'] === 'no-cache',
    },
  );
  await expectHttp(
    baseUrl,
    '/admin/assets/admin-Ca8f3012.css',
    200,
    'body{color',
    '后台指纹静态资源长缓存',
    {
      validate: (response) =>
        response.headers['cache-control'] ===
        'public, max-age=31536000, immutable',
    },
  );
  await expectHttp(
    baseUrl,
    '/.well-known/acme-challenge/token',
    200,
    'acme-token',
    'ACME challenge 读取',
    {
      validate: (response) => response.headers['cache-control'] === 'no-cache',
    },
  );
  await expectHttp(
    tlsBaseUrl,
    '/',
    200,
    'site-shell',
    'TLS 直连接口返回 index.html',
    { insecureTls: true },
  );
  await expectHttp(
    redirectBaseUrl,
    '/api/assets/history?kind=character_visual&from=smoke',
    301,
    '',
    'HTTP 入口 301 到 HTTPS',
    {
      headers: { Host: 'example.test' },
      validate: (response) =>
        response.headers.location ===
        'https://example.test/api/assets/history?kind=character_visual&from=smoke',
    },
  );
  await expectHttp(
    redirectBaseUrl,
    '/.well-known/acme-challenge/token',
    200,
    'acme-token',
    'HTTP 重定向入口保留 ACME challenge',
  );
  await expectHttp(
    baseUrl,
    '/generated-puzzle-assets/a.png',
    404,
    '',
    'generated 私有资产公网拒绝',
  );
  await expectHttp(baseUrl, '/healthz', 404, '', '公网 healthz 拒绝');
  await expectHttp(
    baseUrl,
    '/v1/ping',
    404,
    '',
    '非白名单 SpacetimeDB 路由拒绝',
  );

  await expectHttp(
    baseUrl,
    '/__genarrative_pingora/healthz',
    404,
    '',
    'shadow probe 无 token 返回 404',
  );
  await expectHttp(
    baseUrl,
    '/__genarrative_pingora/healthz',
    200,
    '"gateway":"pingora-shadow"',
    'shadow probe token 通过',
    {
      headers: { 'X-Genarrative-Pingora-Probe': probeToken },
    },
  );

  const apiResponse = await expectHttp(
    baseUrl,
    '/api/assets/history',
    200,
    '"upstream":"api"',
    '通用 API 转发',
    {
      headers: {
        'X-Request-Id': 'smoke-request-id',
        'X-Forwarded-For': '203.0.113.10',
        Host: 'example.test',
      },
    },
  );
  ensure(
    apiResponse.headers['x-genarrative-gateway'] === 'pingora-shadow',
    'API 响应缺少网关标记',
  );
  ensure(
    apiResponse.headers['x-accel-buffering'] === 'no',
    'API 响应缺少 X-Accel-Buffering=no',
  );
  const apiPayload = JSON.parse(apiResponse.body);
  ensure(
    apiPayload.requestId === 'smoke-request-id',
    'API 上游未收到 X-Request-Id',
  );
  ensure(
    apiPayload.forwardedProto === 'https',
    'API 上游未收到配置的 X-Forwarded-Proto',
  );
  ensure(apiPayload.host === 'example.test', 'API 上游 Host 未透传');
  ensure(
    apiPayload.forwardedHost === 'example.test',
    'API 上游未收到 X-Forwarded-Host',
  );
  ensure(
    apiPayload.forwardedFor.endsWith(', 127.0.0.1'),
    `API 上游 X-Forwarded-For 未追加 TCP 对端 IP：${apiPayload.forwardedFor}`,
  );
  ensure(
    apiPayload.realIp === '127.0.0.1',
    `API 上游 X-Real-IP 未使用 TCP 对端 IP：${apiPayload.realIp}`,
  );

  await expectHttp(
    baseUrl,
    '/api/upload',
    413,
    'PAYLOAD_TOO_LARGE',
    'Content-Length body limit',
    {
      method: 'POST',
      body: '0123456789abcdef',
    },
  );
  await expectChunkedLimit(baseUrl, '/api/upload');

  await expectConcurrencyLimit(baseUrl, api);

  const rateLimitHeaders = { 'X-Forwarded-For': '203.0.113.13' };
  const beforeRateLimitRequestCount = api.state.requests.length;
  await expectHttp(
    baseUrl,
    '/api/rate-limited',
    200,
    '"upstream":"api"',
    'API RPS 首次请求放行',
    {
      headers: rateLimitHeaders,
    },
  );
  await expectHttp(
    baseUrl,
    '/api/rate-limited',
    429,
    'GATEWAY_RATE_LIMITED',
    'API RPS 保护',
    {
      headers: rateLimitHeaders,
    },
  );
  ensure(
    api.state.requests.length === beforeRateLimitRequestCount + 1,
    'API RPS 保护的 429 请求不应打到上游',
  );
  await expectHttp(
    baseUrl,
    '/api/upstream-close',
    502,
    'GATEWAY_UPSTREAM_ERROR',
    'API 上游断连返回稳定 JSON',
    {
      headers: { 'X-Forwarded-For': '203.0.113.14' },
    },
  );
  await expectHttp(
    baseUrl,
    '/api/slow',
    504,
    'GATEWAY_UPSTREAM_TIMEOUT',
    'API 上游读取超时返回稳定 JSON',
    {
      headers: { 'X-Forwarded-For': '203.0.113.15' },
      timeoutMs: 5000,
    },
  );

  await expectHttp(
    baseUrl,
    '/v1/identity',
    200,
    '"upstream":"spacetime"',
    'SpacetimeDB identity 转发',
  );
  await expectWebSocketUpgrade(
    baseUrl,
    '/v1/database/genarrative/subscribe',
    spacetime,
  );

  const giteaBeforeRequests = gitea.state.requests.length;
  const giteaResponse = await expectHttp(
    baseUrl,
    '/api/v1/repos/GenarrativeAI/Genarrative',
    200,
    '"upstream":"gitea"',
    'Gitea Host 路由转发',
    {
      headers: {
        Host: 'git.genarrative.world:443',
        'X-Request-Id': 'gitea-host-request-id',
      },
    },
  );
  const giteaPayload = JSON.parse(giteaResponse.body || '{}');
  ensure(
    giteaPayload.host === 'git.genarrative.world:443',
    `Gitea 上游 Host 未透传：${giteaPayload.host || '-'}`,
  );
  ensure(
    giteaPayload.forwardedHost === 'git.genarrative.world:443',
    `Gitea 上游 X-Forwarded-Host 未透传：${giteaPayload.forwardedHost || '-'}`,
  );
  ensure(
    giteaPayload.forwardedProto === 'https',
    `Gitea 上游 X-Forwarded-Proto 未对齐：${giteaPayload.forwardedProto || '-'}`,
  );
  await expectHttp(
    baseUrl,
    '/api/v1/repos/GenarrativeAI/Genarrative',
    200,
    '"upstream":"gitea"',
    'Gitea Host 别名路由转发',
    {
      headers: {
        Host: 'git-alias.genarrative.world',
      },
    },
  );
  ensure(
    gitea.state.requests.length === giteaBeforeRequests + 2,
    `Gitea Host 路由没有稳定打到 Gitea mock：${gitea.state.requests.length - giteaBeforeRequests}`,
  );

  await expectAccessLog(accessLogFile);
  await expectAccessLogEntries(accessLogFile, [
    {
      requestId: 'static-etag-304-request-id',
      method: 'GET',
      path: '/assets/app.js',
      status: 304,
    },
    {
      requestId: 'static-last-modified-304-request-id',
      method: 'GET',
      path: '/assets/app.js',
      status: 304,
    },
    {
      requestId: 'static-method-405-request-id',
      method: 'POST',
      path: '/assets/app.js',
      status: 405,
    },
    {
      requestId: 'static-range-206-request-id',
      method: 'GET',
      path: '/assets/app.js',
      status: 206,
    },
    {
      requestId: 'static-range-416-request-id',
      method: 'GET',
      path: '/assets/app.js',
      status: 416,
    },
    {
      requestId: 'static-head-range-206-request-id',
      method: 'HEAD',
      path: '/assets/app.js',
      status: 206,
    },
    {
      requestId: 'gitea-host-request-id',
      method: 'GET',
      path: '/api/v1/repos/GenarrativeAI/Genarrative',
      status: 200,
      proxyTarget: 'Gitea',
    },
  ]);

  await mkdir(path.dirname(maintenanceFile), { recursive: true });
  await writeFile(maintenancePageFile, '<main>runtime-maintenance</main>');
  await writeFile(maintenanceFile, 'enabled');
  const publicClientHeaders = {
    'X-Real-IP': '203.0.113.50',
    'X-Forwarded-For': '192.168.35.50',
  };
  const internalClientHeaders = { 'X-Real-IP': '192.168.35.50' };
  const giteaRequestsBeforeMaintenance = gitea.state.requests.length;
  await expectHttp(
    baseUrl,
    '/user/login',
    200,
    '"upstream":"gitea"',
    '维护模式不拦截 Gitea Host',
    {
      headers: {
        Host: 'git.genarrative.world',
        ...publicClientHeaders,
      },
    },
  );
  ensure(
    gitea.state.requests.length === giteaRequestsBeforeMaintenance + 1,
    '维护模式 Gitea Host 请求没有打到 Gitea mock',
  );
  for (const [path, bodyNeedle, label] of [
    ['/', 'runtime-maintenance', '公网主站页面'],
    ['/api/assets/history', 'MAINTENANCE', '公网普通 API'],
    ['/v1/identity', 'runtime-maintenance', '公网 SpacetimeDB 路由'],
    ['/admin/settings', 'runtime-maintenance', '公网后台页面'],
    ['/admin/assets/admin.js', 'runtime-maintenance', '公网后台静态资源'],
    ['/admin/api/users', 'MAINTENANCE', '公网后台 API'],
  ]) {
    await expectHttp(
      baseUrl,
      path,
      503,
      bodyNeedle,
      `维护模式继续拦截${label}`,
      {
        headers: publicClientHeaders,
      },
    );
  }
  for (const path of [
    '/branding/taonier-maintenance-page.png',
    '/branding/taonier-product-ip.png',
  ]) {
    await expectStaticPng(baseUrl, path, `维护模式放行品牌图片 ${path}`, {
      headers: publicClientHeaders,
    });
  }
  await rm(maintenancePageFile, { force: true });
  await expectHttp(
    baseUrl,
    '/',
    503,
    'default-maintenance',
    '运行态公告缺失时回退 Web 制品默认维护页',
    { headers: publicClientHeaders },
  );
  await expectHttp(
    baseUrl,
    '/',
    200,
    'site-shell',
    '维护模式允许内网主站页面',
    { headers: internalClientHeaders },
  );
  await expectHttp(
    baseUrl,
    '/api/assets/history',
    200,
    '"upstream":"api"',
    '维护模式允许内网普通 API',
    { headers: internalClientHeaders },
  );
  await expectHttp(
    baseUrl,
    '/v1/identity',
    200,
    '"upstream":"spacetime"',
    '维护模式允许内网 SpacetimeDB 路由',
    { headers: internalClientHeaders },
  );
  await expectHttp(
    baseUrl,
    '/admin/settings',
    200,
    'admin-shell',
    '维护模式允许内网后台页面',
    { headers: internalClientHeaders },
  );
  await expectHttp(
    baseUrl,
    '/admin/assets/admin.js',
    200,
    'admin asset',
    '维护模式允许内网后台静态资源',
    { headers: internalClientHeaders },
  );
  await expectHttp(
    baseUrl,
    '/admin/api/users',
    200,
    '"upstream":"api"',
    '维护模式允许内网后台 API',
    { headers: internalClientHeaders },
  );
  await expectAccessLogContains(accessLogFile, [
    'status=503',
    'path=/api/assets/history',
  ]);
}

async function expectAccessLog(accessLogFile) {
  console.log('[pingora-gateway-smoke] access log 落盘');
  await waitForCondition(async () => {
    try {
      const content = await readFile(accessLogFile, 'utf8');
      return (
        content.includes('request_id=smoke-request-id') &&
        content.includes('method=GET') &&
        content.includes('path=/api/assets/history') &&
        content.includes('status=200') &&
        content.includes('proxy_target=Api') &&
        content.includes('protection_class=api') &&
        content.includes('status=429') &&
        content.includes('status=502') &&
        content.includes('status=504')
      );
    } catch {
      return false;
    }
  });
}

function assertStaticValidatorHeaders(response, label) {
  ensure(Boolean(response.headers.etag), `${label}: 缺少 ETag`);
  ensure(
    /^W\/"[0-9a-f]+-[0-9a-f]+"$/iu.test(String(response.headers.etag || '')),
    `${label}: ETag 格式不符合静态弱校验器口径：${response.headers.etag || '-'}`,
  );
  ensure(
    Boolean(response.headers['last-modified']),
    `${label}: 缺少 Last-Modified`,
  );
  ensure(
    Number.isFinite(Date.parse(response.headers['last-modified'] || '')),
    `${label}: Last-Modified 不是有效 HTTP 日期：${response.headers['last-modified'] || '-'}`,
  );
}

async function expectStaticNotModifiedByEtag(
  baseUrl,
  route,
  sourceResponse,
  options = {},
) {
  await expectHttp(baseUrl, route, 304, '', '静态资源 If-None-Match 返回 304', {
    headers: {
      'If-None-Match': sourceResponse.headers.etag,
      ...(options.headers || {}),
    },
    validate: (response) =>
      response.body === '' &&
      response.headers.etag === sourceResponse.headers.etag &&
      response.headers['cache-control'] ===
        sourceResponse.headers['cache-control'],
  });
}

async function expectStaticNotModifiedByLastModified(
  baseUrl,
  route,
  sourceResponse,
  options = {},
) {
  await expectHttp(
    baseUrl,
    route,
    304,
    '',
    '静态资源 If-Modified-Since 返回 304',
    {
      headers: {
        'If-Modified-Since': sourceResponse.headers['last-modified'],
        ...(options.headers || {}),
      },
      validate: (response) =>
        response.body === '' &&
        response.headers['last-modified'] ===
          sourceResponse.headers['last-modified'] &&
        response.headers['cache-control'] ===
          sourceResponse.headers['cache-control'],
    },
  );
}

async function expectStaticRange(
  baseUrl,
  route,
  range,
  status,
  bodyNeedle,
  contentRange,
  label,
  options = {},
) {
  const headers = {
    Range: range,
    'Accept-Encoding': 'gzip',
    ...(options.headers || {}),
  };
  await expectHttp(baseUrl, route, status, bodyNeedle, label, {
    method: options.method,
    headers,
    validate: (response) =>
      response.headers['accept-ranges'] === 'bytes' &&
      response.headers['content-range'] === contentRange &&
      !response.headers['content-encoding'] &&
      (options.method === 'HEAD' ? response.body === '' : true) &&
      (status === 206
        ? Number(response.headers['content-length']) ===
          (options.expectedContentLength ?? Buffer.byteLength(bodyNeedle))
        : Number(response.headers['content-length']) === 0),
  });
}

async function expectStaticRangeFallback(
  baseUrl,
  route,
  range,
  ifRange,
  label,
) {
  await expectHttp(baseUrl, route, 200, 'site asset', label, {
    headers: {
      Range: range,
      'If-Range': ifRange,
      'Accept-Encoding': 'gzip',
    },
    validate: (response) =>
      response.headers['accept-ranges'] === 'bytes' &&
      !response.headers['content-range'] &&
      Number(response.headers['content-length']) ===
        Buffer.byteLength('console.log("site asset");\n'),
  });
}

async function expectGzipResponse(baseUrl, route, bodyNeedle, options = {}) {
  return expectCompressedResponse(baseUrl, route, bodyNeedle, {
    acceptEncoding: options.acceptEncoding || 'gzip',
    contentEncoding: 'gzip',
    decode: (bodyBuffer) => gunzipSync(bodyBuffer).toString('utf8'),
    label: options.label || 'gzip 压缩响应',
  });
}

async function expectNotCompressedResponse(baseUrl, route, bodyNeedle, label) {
  console.log(`[pingora-gateway-smoke] ${label}`);
  const response = await requestHttp(`${baseUrl}${route}`, {
    headers: {
      'Accept-Encoding': 'gzip',
    },
    rawBody: true,
  });

  if (response.status !== 200) {
    failures.push(`${label}: 期望 HTTP 200，实际 ${response.status}`);
    return;
  }
  if (response.headers['content-encoding']) {
    failures.push(
      `${label}: 不应返回 Content-Encoding，实际 ${response.headers['content-encoding']}`,
    );
  }
  const body = response.bodyBuffer.toString('utf8');
  if (bodyNeedle && !body.includes(bodyNeedle)) {
    failures.push(`${label}: 响应体缺少 ${bodyNeedle}`);
  }
}

async function expectStaticPng(baseUrl, route, label, options = {}) {
  console.log(`[pingora-gateway-smoke] ${label}`);
  const response = await requestHttp(`${baseUrl}${route}`, {
    headers: options.headers,
    rawBody: true,
  });

  if (response.status !== 200) {
    failures.push(`${label}: 期望 HTTP 200，实际 ${response.status}`);
    return;
  }
  if (!String(response.headers['content-type'] || '').startsWith('image/png')) {
    failures.push(
      `${label}: 期望 Content-Type=image/png，实际 ${response.headers['content-type'] || '-'}`,
    );
  }
  if (!response.bodyBuffer.subarray(0, 8).equals(PNG_MAGIC_BYTES)) {
    failures.push(`${label}: 响应体不是 PNG magic bytes`);
  }
  if (response.body.includes('<!doctype html>')) {
    failures.push(`${label}: 命中了 SPA fallback HTML`);
  }
}

async function expectRawPathHttp(baseUrl, route, status, bodyNeedle, label) {
  console.log(`[pingora-gateway-smoke] ${label}`);
  let response;
  try {
    response = await rawHttpPathRequest(baseUrl, route);
  } catch (error) {
    failures.push(
      `${label}: ${error instanceof Error ? error.message : String(error)}`,
    );
    return;
  }

  if (response.status !== status) {
    failures.push(`${label}: 期望 HTTP ${status}，实际 ${response.status}`);
  }
  if (bodyNeedle && !response.body.includes(bodyNeedle)) {
    failures.push(`${label}: 响应体缺少 ${bodyNeedle}`);
  }
}

async function expectCompressedResponse(baseUrl, route, bodyNeedle, options) {
  const label = options.label;
  console.log(`[pingora-gateway-smoke] ${label}`);
  const response = await requestHttp(`${baseUrl}${route}`, {
    headers: {
      'Accept-Encoding': options.acceptEncoding,
    },
    rawBody: true,
  });

  if (response.status !== 200) {
    failures.push(`${label}: 期望 HTTP 200，实际 ${response.status}`);
    return;
  }
  if (response.headers['content-encoding'] !== options.contentEncoding) {
    failures.push(
      `${label}: 缺少 Content-Encoding=${options.contentEncoding}，实际 ${response.headers['content-encoding'] || '-'}`,
    );
  }
  if (
    !String(response.headers.vary || '')
      .toLowerCase()
      .includes('accept-encoding')
  ) {
    failures.push(
      `${label}: 缺少 Vary: Accept-Encoding，实际 ${response.headers.vary || '-'}`,
    );
  }

  let decoded = '';
  try {
    decoded = options.decode(response.bodyBuffer);
  } catch (error) {
    failures.push(
      `${label}: 解压失败：${error instanceof Error ? error.message : String(error)}`,
    );
    return;
  }
  if (!decoded.includes(bodyNeedle)) {
    failures.push(`${label}: 解压后内容缺少 ${bodyNeedle}`);
  }
}

async function expectAccessLogContains(accessLogFile, needles) {
  await waitForCondition(async () => {
    try {
      const content = await readFile(accessLogFile, 'utf8');
      return needles.every((needle) => content.includes(needle));
    } catch {
      return false;
    }
  });
}

async function expectAccessLogEntries(accessLogFile, expectedEntries) {
  console.log('[pingora-gateway-smoke] access log 静态边界状态对账');
  await waitForCondition(async () => {
    try {
      const content = await readFile(accessLogFile, 'utf8');
      const entries = content
        .split(/\r?\n/u)
        .filter(Boolean)
        .map(parseAccessLogLine);
      return expectedEntries.every((expected) => {
        const entry = entries.find(
          (item) => item.request_id === expected.requestId,
        );
        return (
          entry &&
          entry.method === expected.method &&
          entry.path === expected.path &&
          Number.parseInt(entry.status || '', 10) === expected.status &&
          entry.proxy_target === (expected.proxyTarget || 'Local')
        );
      });
    } catch {
      return false;
    }
  });
}

function parseAccessLogLine(line) {
  const fields = {};
  for (const part of line.split('\t')) {
    const separator = part.indexOf('=');
    if (separator <= 0) {
      continue;
    }
    fields[part.slice(0, separator)] = unescapeAccessLogValue(
      part.slice(separator + 1),
    );
  }
  return fields;
}

function unescapeAccessLogValue(value) {
  return value
    .replace(/\\\\/gu, '\\')
    .replace(/\\t/gu, '\t')
    .replace(/\\n/gu, '\n')
    .replace(/\\r/gu, '\r');
}

async function expectHttp(
  baseUrl,
  route,
  status,
  bodyNeedle,
  label,
  options = {},
) {
  console.log(`[pingora-gateway-smoke] ${label}`);
  const pending = options.beforeRequest
    ? await options.beforeRequest()
    : undefined;
  let response;
  try {
    response = await requestHttp(`${baseUrl}${route}`, {
      method: options.method,
      headers: options.headers,
      body: options.body,
      insecureTls: options.insecureTls,
    });
  } catch (error) {
    failures.push(
      `${label}: ${error instanceof Error ? error.message : String(error)}`,
    );
  } finally {
    if (options.afterRequest) {
      try {
        await options.afterRequest(pending);
      } catch (error) {
        failures.push(error instanceof Error ? error.message : String(error));
      }
    }
  }

  if (!response) {
    return { status: 0, headers: {}, body: '' };
  }

  if (config.verbose) {
    console.log(`[pingora-gateway-smoke] ${label} -> HTTP ${response.status}`);
  }

  if (response.status !== status) {
    failures.push(`${label}: 期望 HTTP ${status}，实际 ${response.status}`);
  }

  if (bodyNeedle && !response.body.includes(bodyNeedle)) {
    failures.push(`${label}: 响应体缺少 ${bodyNeedle}`);
  }

  if (options.validate && !options.validate(response)) {
    failures.push(`${label}: 自定义响应校验未通过`);
  }

  return response;
}

async function expectChunkedLimit(baseUrl, route) {
  console.log('[pingora-gateway-smoke] chunked body limit');
  const response = await new Promise((resolve, reject) => {
    const request = http.request(
      `${baseUrl}${route}`,
      {
        method: 'POST',
        headers: {
          'Transfer-Encoding': 'chunked',
          'X-Forwarded-For': '203.0.113.11',
        },
      },
      (incoming) => {
        collectIncoming(incoming).then(resolve, reject);
      },
    );
    request.on('error', reject);
    request.write('012345');
    request.write('6789abcdef');
    request.end();
  });

  if (response.status !== 413 || !response.body.includes('PAYLOAD_TOO_LARGE')) {
    failures.push(
      `chunked body limit: 期望 413 PAYLOAD_TOO_LARGE，实际 ${response.status} ${response.body}`,
    );
  }
}

async function expectConcurrencyLimit(baseUrl, api) {
  console.log('[pingora-gateway-smoke] API 并发保护');
  const hold = await openRawHttpRequest(`${baseUrl}/admin/api/hold`, {
    'X-Forwarded-For': '203.0.113.12',
  });

  try {
    await waitForCondition(() => typeof api.state.releaseHold === 'function');
    const beforeLimitedRequestCount = api.state.requests.length;
    const response = await rawHttpRequest(`${baseUrl}/admin/api/users`, {
      'X-Forwarded-For': '203.0.113.12',
    });

    if (response.status !== 429) {
      failures.push(`API 并发保护: 期望 HTTP 429，实际 ${response.status}`);
    }
    if (!response.body.includes('GATEWAY_CONCURRENCY_LIMITED')) {
      failures.push('API 并发保护: 响应体缺少 GATEWAY_CONCURRENCY_LIMITED');
    }
    ensure(
      api.state.requests.length === beforeLimitedRequestCount,
      `API 并发保护的 429 请求不应打到上游，上游请求为 ${describeRequests(api.state.requests.slice(beforeLimitedRequestCount))}`,
    );
  } finally {
    api.state.releaseHold?.();
    hold.socket.end();
    const holdResponse = await hold.done.catch((error) => ({ error }));
    if (holdResponse?.error) {
      failures.push(
        `API 并发保护: hold 请求失败：${holdResponse.error.message}`,
      );
    }
  }
}

async function expectWebSocketUpgrade(baseUrl, route, spacetime) {
  console.log('[pingora-gateway-smoke] SpacetimeDB WebSocket Upgrade');
  const url = new URL(route, baseUrl);
  const key = randomBytes(16).toString('base64');
  const response = await new Promise((resolve, reject) => {
    const socket = net.connect(
      { host: url.hostname, port: Number(url.port) },
      () => {
        socket.write(
          [
            `GET ${url.pathname} HTTP/1.1`,
            `Host: ${url.host}`,
            'Upgrade: websocket',
            'Connection: Upgrade',
            `Sec-WebSocket-Key: ${key}`,
            'Sec-WebSocket-Version: 13',
            '',
            '',
          ].join('\r\n'),
        );
      },
    );

    let raw = Buffer.alloc(0);
    let settled = false;
    const timeout = setTimeout(() => {
      settled = true;
      socket.destroy();
      reject(new Error('WebSocket Upgrade 超时'));
    }, 5000);

    socket.on('data', (chunk) => {
      raw = Buffer.concat([raw, chunk]);
      const text = raw.toString('latin1');
      if (text.includes('\r\n\r\n') && text.includes('spacetime-upgrade-ok')) {
        settled = true;
        clearTimeout(timeout);
        socket.destroy();
        resolve(raw);
      }
    });
    socket.on('error', (error) => {
      settled = true;
      clearTimeout(timeout);
      reject(error);
    });
    socket.on('close', () => {
      clearTimeout(timeout);
      if (!settled) {
        reject(new Error('WebSocket Upgrade 连接提前关闭'));
      }
    });
  });

  const text = response.toString('latin1');
  ensure(
    text.startsWith('HTTP/1.1 101'),
    `WebSocket Upgrade 未返回 101：${text.split('\r\n')[0] || text}`,
  );
  ensure(
    text.includes('spacetime-upgrade-ok'),
    'WebSocket tunnel 未收到上游 welcome frame',
  );
  const upgradeRequest = spacetime.state.requests.find(
    (request) =>
      request.url === route && request.headers.upgrade === 'websocket',
  );
  ensure(
    Boolean(upgradeRequest),
    'SpacetimeDB mock 未收到 WebSocket Upgrade 请求',
  );
}

function requestHttp(url, options = {}) {
  return new Promise((resolve, reject) => {
    const headers = { ...(options.headers || {}) };
    if (
      options.body !== undefined &&
      !hasHeader(headers, 'content-length') &&
      !hasHeader(headers, 'transfer-encoding')
    ) {
      headers['Content-Length'] = Buffer.byteLength(String(options.body));
    }
    const client = url.startsWith('https:') ? https : http;
    const request = client.request(
      url,
      {
        method: options.method || 'GET',
        headers,
        agent: false,
        rejectUnauthorized: options.insecureTls ? false : undefined,
      },
      (incoming) => {
        collectIncoming(incoming).then(resolve, reject);
      },
    );
    const timeout = setTimeout(() => {
      request.destroy(new Error(`HTTP 请求超时：${url}`));
    }, options.timeoutMs || 5000);
    request.on('error', reject);
    request.on('close', () => {
      clearTimeout(timeout);
    });
    if (options.body !== undefined) {
      request.write(options.body);
    }
    request.end();
  });
}

function openRawHttpRequest(url, headers = {}) {
  return new Promise((resolve, reject) => {
    const target = new URL(url);
    const socket = net.connect(
      { host: target.hostname, port: Number(target.port) },
      () => {
        socket.write(buildRawHttpRequest(target, headers));
        resolve({
          socket,
          done: collectRawHttpResponse(socket, url, 35000),
        });
      },
    );
    socket.on('error', reject);
  });
}

async function rawHttpRequest(url, headers = {}) {
  const request = await openRawHttpRequest(url, headers);
  return request.done.finally(() => {
    request.socket.destroy();
  });
}

function rawHttpPathRequest(baseUrl, route, headers = {}) {
  return new Promise((resolve, reject) => {
    const target = new URL(baseUrl);
    const socket = net.connect(
      { host: target.hostname, port: Number(target.port) },
      () => {
        socket.write(buildRawHttpPathRequest(target, route, headers));
        collectRawHttpResponse(socket, `${baseUrl}${route}`, 35000)
          .then(resolve, reject)
          .finally(() => socket.destroy());
      },
    );
    socket.on('error', reject);
  });
}

function buildRawHttpRequest(target, headers) {
  const mergedHeaders = {
    Host: target.host,
    Connection: 'close',
    ...headers,
  };
  const headerLines = Object.entries(mergedHeaders).map(
    ([name, value]) => `${name}: ${value}`,
  );
  return [
    `GET ${target.pathname}${target.search} HTTP/1.1`,
    ...headerLines,
    '',
    '',
  ].join('\r\n');
}

function buildRawHttpPathRequest(target, route, headers) {
  const mergedHeaders = {
    Host: target.host,
    Connection: 'close',
    ...headers,
  };
  const headerLines = Object.entries(mergedHeaders).map(
    ([name, value]) => `${name}: ${value}`,
  );
  return [`GET ${route} HTTP/1.1`, ...headerLines, '', ''].join('\r\n');
}

function collectRawHttpResponse(socket, url, timeoutMs) {
  return new Promise((resolve, reject) => {
    let raw = Buffer.alloc(0);
    let settled = false;
    const timer = setTimeout(() => {
      settled = true;
      socket.destroy();
      reject(new Error(`HTTP 请求超时：${url}`));
    }, timeoutMs);

    socket.on('data', (chunk) => {
      raw = Buffer.concat([raw, chunk]);
      let parsed;
      try {
        parsed = tryParseRawHttpResponse(raw);
      } catch (error) {
        settled = true;
        clearTimeout(timer);
        socket.destroy();
        reject(error);
        return;
      }
      if (parsed) {
        settled = true;
        clearTimeout(timer);
        resolve(parsed);
      }
    });
    socket.on('error', (error) => {
      if (!settled) {
        settled = true;
        clearTimeout(timer);
        reject(error);
      }
    });
    socket.on('close', () => {
      if (!settled) {
        settled = true;
        clearTimeout(timer);
        try {
          const parsed = tryParseRawHttpResponse(raw, { final: true });
          if (parsed) {
            resolve(parsed);
          } else {
            reject(new Error(`HTTP 响应提前关闭：${url}`));
          }
        } catch (error) {
          reject(error);
        }
      }
    });
  });
}

function tryParseRawHttpResponse(raw, options = {}) {
  const headerEnd = raw.indexOf('\r\n\r\n');
  if (headerEnd < 0) {
    return null;
  }

  const headerText = raw.slice(0, headerEnd).toString('latin1');
  const [statusLine, ...headerLines] = headerText.split('\r\n');
  const status = Number.parseInt(statusLine.split(' ')[1] || '0', 10);
  const headers = {};
  for (const line of headerLines) {
    const separator = line.indexOf(':');
    if (separator <= 0) {
      continue;
    }
    headers[line.slice(0, separator).trim().toLowerCase()] = line
      .slice(separator + 1)
      .trim();
  }

  const bodyStart = headerEnd + 4;
  const body = raw.slice(bodyStart);
  if (hasToken(headers['transfer-encoding'], 'chunked')) {
    const decoded = decodeChunkedBody(body);
    if (!decoded.complete) {
      return null;
    }

    return {
      status,
      headers,
      body: decoded.body.toString('utf8'),
    };
  }

  const contentLength = Number.parseInt(headers['content-length'] || '', 10);
  if (Number.isFinite(contentLength) && body.length < contentLength) {
    return null;
  }
  if (Number.isFinite(contentLength)) {
    return {
      status,
      headers,
      body: body.slice(0, contentLength).toString('utf8'),
    };
  }

  if (!options.final) {
    return null;
  }

  return {
    status,
    headers,
    body: body.toString('utf8'),
  };
}

function hasToken(value, expected) {
  if (!value) {
    return false;
  }
  const normalized = expected.toLowerCase();
  return value
    .split(',')
    .map((token) => token.trim().toLowerCase())
    .includes(normalized);
}

function decodeChunkedBody(body) {
  let offset = 0;
  const chunks = [];

  while (offset < body.length) {
    const lineEnd = body.indexOf('\r\n', offset);
    if (lineEnd < 0) {
      return { complete: false, body: Buffer.alloc(0) };
    }

    const sizeLine = body.slice(offset, lineEnd).toString('latin1');
    const sizeText = sizeLine.split(';')[0]?.trim() || '';
    const size = Number.parseInt(sizeText, 16);
    if (!Number.isFinite(size)) {
      throw new Error(`无法解析 chunked 响应大小：${sizeLine}`);
    }

    offset = lineEnd + 2;
    if (size === 0) {
      const trailerEnd = body.indexOf('\r\n\r\n', offset);
      if (trailerEnd >= 0) {
        return { complete: true, body: Buffer.concat(chunks) };
      }
      if (body.slice(offset, offset + 2).toString('latin1') === '\r\n') {
        return { complete: true, body: Buffer.concat(chunks) };
      }
      return { complete: false, body: Buffer.alloc(0) };
    }

    const chunkStart = offset;
    const chunkEnd = chunkStart + size;
    const nextChunkStart = chunkEnd + 2;
    if (body.length < nextChunkStart) {
      return { complete: false, body: Buffer.alloc(0) };
    }
    if (body.slice(chunkEnd, nextChunkStart).toString('latin1') !== '\r\n') {
      throw new Error('chunked 响应缺少 chunk 结束换行');
    }

    chunks.push(body.slice(chunkStart, chunkEnd));
    offset = nextChunkStart;
  }

  return { complete: false, body: Buffer.alloc(0) };
}

function hasHeader(headers, name) {
  const expected = name.toLowerCase();
  return Object.keys(headers).some((key) => key.toLowerCase() === expected);
}

function collectIncoming(incoming, options = {}) {
  return new Promise((resolve, reject) => {
    const chunks = [];
    incoming.on('data', (chunk) => chunks.push(chunk));
    incoming.on('error', reject);
    incoming.on('end', () => {
      const bodyBuffer = Buffer.concat(chunks);
      resolve({
        status: incoming.statusCode || 0,
        headers: incoming.headers,
        body: options.rawBody ? '' : bodyBuffer.toString('utf8'),
        bodyBuffer,
      });
    });
  });
}

function readRequestBody(request) {
  return new Promise((resolve, reject) => {
    const chunks = [];
    request.on('data', (chunk) => chunks.push(chunk));
    request.on('error', reject);
    request.on('end', () => resolve(Buffer.concat(chunks)));
  });
}

function listen(server) {
  return new Promise((resolve, reject) => {
    server.on('error', reject);
    server.on('connection', (socket) => {
      sockets.add(socket);
      socket.on('close', () => {
        sockets.delete(socket);
      });
    });
    server.listen(0, '127.0.0.1', () => {
      servers.push(server);
      const address = server.address();
      if (!address || typeof address === 'string') {
        reject(new Error('无法读取 mock server 端口'));
        return;
      }
      resolve(address.port);
    });
  });
}

function getFreePort() {
  return new Promise((resolve, reject) => {
    const server = net.createServer();
    server.on('error', reject);
    server.listen(0, '127.0.0.1', () => {
      const address = server.address();
      if (!address || typeof address === 'string') {
        server.close();
        reject(new Error('无法分配临时端口'));
        return;
      }
      const { port } = address;
      server.close(() => resolve(port));
    });
  });
}

async function waitForHttp(url, expectedStatus) {
  const startedAt = Date.now();
  let lastError = '';
  while (Date.now() - startedAt < 10000) {
    try {
      const response = await requestHttp(url);
      if (response.status === expectedStatus) {
        return;
      }
      lastError = `HTTP ${response.status}`;
    } catch (error) {
      lastError = error instanceof Error ? error.message : String(error);
    }
    await delay(100);
  }
  throw new Error(`等待 Pingora 网关就绪超时：${lastError}`);
}

function runCommand(command, args) {
  console.log(
    `[pingora-gateway-smoke] ${command} ${redactSecretArgs(args).join(' ')}`,
  );
  const result = spawnSync(command, args, {
    cwd: repoRoot,
    env: smokeEnv(),
    shell: false,
    stdio: 'inherit',
  });

  if (result.error) {
    throw new Error(`${command} 启动失败：${result.error.message}`);
  }
  if (result.signal) {
    throw new Error(`${command} 被信号终止：${result.signal}`);
  }
  if ((result.status ?? 0) !== 0) {
    throw new Error(
      `${command} ${redactSecretArgs(args).join(' ')} 退出码 ${result.status}`,
    );
  }
}

async function runDirectLiveSmoke(
  tlsBaseUrl,
  redirectBaseUrl,
  probeToken,
  accessLogFile,
) {
  const output = await runCommandAsync('node', [
    'scripts/check-pingora-direct-live.mjs',
    '--https-base-url',
    tlsBaseUrl,
    '--http-base-url',
    redirectBaseUrl,
    '--host',
    new URL(tlsBaseUrl).host,
    '--probe-token',
    probeToken,
    '--pingora-access-log',
    accessLogFile,
    '--redirect-host',
    new URL(tlsBaseUrl).host,
    '--spacetime-database',
    'genarrative',
    '--require-wss-upgrade',
    '--insecure-tls',
    '--json',
  ]);
  assertDirectLiveAccessLogEvidence(output.stdout);
}

function runCommandAsync(command, args) {
  console.log(
    `[pingora-gateway-smoke] ${command} ${redactSecretArgs(args).join(' ')}`,
  );
  return new Promise((resolve, reject) => {
    let stdout = '';
    let stderr = '';
    const child = spawn(command, args, {
      cwd: repoRoot,
      env: smokeEnv(),
      shell: false,
      stdio: ['ignore', 'pipe', 'pipe'],
    });
    child.stdout?.on('data', (chunk) => {
      const text = String(chunk);
      stdout += text;
      process.stdout.write(text);
    });
    child.stderr?.on('data', (chunk) => {
      const text = String(chunk);
      stderr += text;
      process.stderr.write(text);
    });
    child.on('error', (error) => {
      reject(new Error(`${command} 启动失败：${error.message}`));
    });
    child.on('exit', (status, signal) => {
      if (signal) {
        reject(new Error(`${command} 被信号终止：${signal}`));
        return;
      }
      if ((status ?? 0) !== 0) {
        reject(
          new Error(
            `${command} ${redactSecretArgs(args).join(' ')} 退出码 ${status}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
          ),
        );
        return;
      }
      resolve({ stdout, stderr });
    });
  });
}

function assertDirectLiveAccessLogEvidence(stdout) {
  const payload = extractJsonObject(stdout, stdout.indexOf('{'));
  if (!payload) {
    failures.push('direct live JSON 输出缺失。');
    return;
  }
  let result;
  try {
    result = JSON.parse(payload);
  } catch (error) {
    failures.push(
      `direct live JSON 输出无法解析：${error instanceof Error ? error.message : String(error)}`,
    );
    return;
  }
  const accessLogCheck = result.results?.find(
    (item) => item.name === 'direct-access-log',
  );
  const staticAssetCheck = result.results?.find(
    (item) => item.name === 'https-static-asset',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.statusCode === 200,
    'direct live 必须验证首页引用的指纹静态资源 GET 200。',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.head?.statusCode === 200,
    'direct live 必须验证首页引用的指纹静态资源 HEAD 200。',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.range?.statusCode === 206,
    'direct live 必须验证首页引用的指纹静态资源 Range 206。',
  );
  ensure(
    staticAssetCheck?.headers?.['cache-control'] === 'no-cache',
    'direct live JSON 必须保留普通静态资源 Cache-Control 证据。',
  );
  ensure(
    Boolean(staticAssetCheck?.headers?.etag),
    'direct live JSON 必须保留普通静态资源 ETag 证据。',
  );
  ensure(
    staticAssetCheck?.range?.headers?.['content-range']?.startsWith(
      'bytes 0-0/',
    ),
    'direct live JSON 必须保留普通静态资源 Content-Range 证据。',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.headers?.['cache-control'] ===
      'public, max-age=31536000, immutable',
    'direct live JSON 必须保留指纹静态资源 immutable Cache-Control 证据。',
  );
  ensure(
    Boolean(staticAssetCheck?.fingerprinted?.headers?.etag),
    'direct live JSON 必须保留指纹静态资源 ETag 证据。',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.range?.headers?.[
      'content-range'
    ]?.startsWith('bytes 0-0/'),
    'direct live JSON 必须保留指纹静态资源 Content-Range 证据。',
  );
  ensure(
    staticAssetCheck?.notModified?.etag?.statusCode === 304,
    'direct live 必须验证首页引用的静态资源 ETag 304。',
  );
  ensure(
    staticAssetCheck?.notModified?.lastModified?.statusCode === 304,
    'direct live 必须验证首页引用的静态资源 Last-Modified 304。',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.notModified?.etag?.statusCode === 304,
    'direct live 必须验证首页引用的指纹静态资源 ETag 304。',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.notModified?.lastModified?.statusCode ===
      304,
    'direct live 必须验证首页引用的指纹静态资源 Last-Modified 304。',
  );
  ensure(
    staticAssetCheck?.notModified?.etag?.headers?.['cache-control'] ===
      staticAssetCheck?.headers?.['cache-control'],
    'direct live JSON 必须保留普通静态 304 Cache-Control 证据。',
  );
  ensure(
    staticAssetCheck?.fingerprinted?.notModified?.etag?.headers?.[
      'cache-control'
    ] === staticAssetCheck?.fingerprinted?.headers?.['cache-control'],
    'direct live JSON 必须保留指纹静态 304 Cache-Control 证据。',
  );
  if (!accessLogCheck) {
    failures.push('direct live JSON 缺少 direct-access-log 检查结果。');
    return;
  }
  ensure(
    accessLogCheck.checked >= 10,
    'direct live access log 检查请求数不足。',
  );
  ensure(
    accessLogCheck.matchedCount === accessLogCheck.checked,
    'direct live access log matchedCount 必须等于 checked。',
  );
  ensure(
    accessLogCheck.missingCount === 0,
    'direct live access log 不应缺少 request_id。',
  );
  ensure(
    accessLogCheck.mismatchCount === 0,
    'direct live access log 不应出现 method/path/status 漂移。',
  );
  ensure(
    Array.isArray(accessLogCheck.matched) &&
      accessLogCheck.matched.some(
        (item) =>
          item.name === 'wss-spacetime-subscribe' &&
          item.expectedMethod === 'GET' &&
          item.actualMethod === 'GET' &&
          item.expectedStatusCode === 101 &&
          item.actualStatusCode === 101,
      ),
    'direct live access log 必须保留 WSS 101 的 method/path/status 明细。',
  );
  ensure(
    Array.isArray(accessLogCheck.matched) &&
      accessLogCheck.matched.some(
        (item) =>
          item.name === 'https-static-asset-head' &&
          item.expectedMethod === 'HEAD' &&
          item.actualMethod === 'HEAD' &&
          item.expectedStatusCode === 200 &&
          item.actualStatusCode === 200,
      ),
    'direct live access log 必须保留静态 HEAD 的 method/path/status 明细。',
  );
  ensure(
    Array.isArray(accessLogCheck.matched) &&
      accessLogCheck.matched.some(
        (item) =>
          item.name === 'https-static-fingerprinted-asset' &&
          item.expectedMethod === 'GET' &&
          item.actualMethod === 'GET' &&
          item.expectedStatusCode === 200 &&
          item.actualStatusCode === 200,
      ),
    'direct live access log 必须保留指纹静态 GET 的 method/path/status 明细。',
  );
  ensure(
    Array.isArray(accessLogCheck.matched) &&
      accessLogCheck.matched.some(
        (item) =>
          item.name === 'https-static-fingerprinted-asset-head' &&
          item.expectedMethod === 'HEAD' &&
          item.actualMethod === 'HEAD' &&
          item.expectedStatusCode === 200 &&
          item.actualStatusCode === 200,
      ),
    'direct live access log 必须保留指纹静态 HEAD 的 method/path/status 明细。',
  );
  ensure(
    Array.isArray(accessLogCheck.matched) &&
      accessLogCheck.matched.some(
        (item) =>
          item.name === 'https-static-fingerprinted-asset-range' &&
          item.expectedMethod === 'GET' &&
          item.actualMethod === 'GET' &&
          item.expectedStatusCode === 206 &&
          item.actualStatusCode === 206,
      ),
    'direct live access log 必须保留指纹静态 Range 的 method/path/status 明细。',
  );
  ensure(
    Array.isArray(accessLogCheck.matched) &&
      accessLogCheck.matched.some(
        (item) =>
          item.name === 'https-static-asset-etag-304' &&
          item.expectedMethod === 'GET' &&
          item.actualMethod === 'GET' &&
          item.expectedStatusCode === 304 &&
          item.actualStatusCode === 304,
      ),
    'direct live access log 必须保留静态 ETag 304 的 method/path/status 明细。',
  );
  ensure(
    Array.isArray(accessLogCheck.matched) &&
      accessLogCheck.matched.some(
        (item) =>
          item.name === 'https-static-fingerprinted-asset-etag-304' &&
          item.expectedMethod === 'GET' &&
          item.actualMethod === 'GET' &&
          item.expectedStatusCode === 304 &&
          item.actualStatusCode === 304,
      ),
    'direct live access log 必须保留指纹静态 ETag 304 的 method/path/status 明细。',
  );
  ensure(
    Array.isArray(accessLogCheck.missing) &&
      accessLogCheck.missing.length === 0,
    'direct live access log missing 明细必须为空数组。',
  );
  ensure(
    Array.isArray(accessLogCheck.mismatches) &&
      accessLogCheck.mismatches.length === 0,
    'direct live access log mismatches 明细必须为空数组。',
  );
}

function extractJsonObject(text, startIndex) {
  if (startIndex < 0) {
    return '';
  }
  let depth = 0;
  let inString = false;
  let escaped = false;

  for (let index = startIndex; index < text.length; index += 1) {
    const char = text[index];
    if (inString) {
      if (escaped) {
        escaped = false;
      } else if (char === '\\') {
        escaped = true;
      } else if (char === '"') {
        inString = false;
      }
      continue;
    }
    if (char === '"') {
      inString = true;
      continue;
    }
    if (char === '{') {
      depth += 1;
      continue;
    }
    if (char === '}') {
      depth -= 1;
      if (depth === 0) {
        return text.slice(startIndex, index + 1);
      }
    }
  }

  return '';
}

function redactSecretArgs(args) {
  return args.map((arg, index) =>
    index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) ? '<redacted>' : arg,
  );
}

function resolveGatewayBinary() {
  const explicit = process.env.GENARRATIVE_PINGORA_GATEWAY_BINARY;
  if (explicit) {
    return explicit;
  }

  const candidates = [
    path.join(repoRoot, 'server-rs', 'target', 'debug', 'pingora-gateway'),
    path.join(repoRoot, 'target', 'debug', 'pingora-gateway'),
  ];
  const found = candidates.find((candidate) => existsSync(candidate));
  if (!found) {
    throw new Error(
      `未找到 pingora-gateway debug 二进制，请先构建或设置 GENARRATIVE_PINGORA_GATEWAY_BINARY。候选：${candidates.join(', ')}`,
    );
  }
  return found;
}

function smokeEnv() {
  return {
    ...process.env,
    PATH: `${path.join(os.homedir(), '.local', 'bin')}:${process.env.PATH || ''}`,
  };
}

function collectProcessLogs(child, name) {
  child.stdout?.on('data', (chunk) => {
    if (config.verbose) {
      process.stdout.write(`[${name}] ${chunk}`);
    }
  });
  child.stderr?.on('data', (chunk) => {
    if (config.verbose) {
      process.stderr.write(`[${name}] ${chunk}`);
    }
  });
}

function encodeWebSocketTextFrame(text) {
  const payload = Buffer.from(text, 'utf8');
  if (payload.length > 125) {
    throw new Error('smoke WebSocket frame payload too large');
  }
  return Buffer.concat([Buffer.from([0x81, payload.length]), payload]);
}

function delay(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function waitForCondition(predicate, timeoutMs = 2000) {
  const startedAt = Date.now();
  while (Date.now() - startedAt < timeoutMs) {
    if (await predicate()) {
      return;
    }
    await delay(20);
  }
  throw new Error('等待 smoke 条件达成超时');
}

function ensure(condition, message) {
  if (!condition) {
    failures.push(message);
  }
}

function describeRequests(requests) {
  return JSON.stringify(
    requests.map((request) => ({
      method: request.method,
      url: request.url,
      xForwardedFor: request.headers?.['x-forwarded-for'],
      aborted: Boolean(request.aborted),
    })),
  );
}

async function cleanup() {
  for (const { child, name } of childProcesses.reverse()) {
    await stopChild(child, name);
  }

  for (const socket of sockets) {
    socket.destroy();
  }
  sockets.clear();

  await Promise.all(
    servers.reverse().map(
      (server) =>
        new Promise((resolve) => {
          const timer = setTimeout(resolve, 1000);
          server.close(() => {
            clearTimeout(timer);
            resolve();
          });
        }),
    ),
  );

  await Promise.all(
    tempDirs.reverse().map((dir) => rm(dir, { recursive: true, force: true })),
  );
}

async function stopChild(child, name) {
  if (child.exitCode !== null || child.signalCode !== null) {
    return;
  }

  child.kill('SIGTERM');
  const exited = await waitForExit(child, 3000);
  if (!exited) {
    child.kill('SIGKILL');
    const killed = await waitForExit(child, 3000);
    if (!killed) {
      failures.push(`${name} 未能退出`);
    }
  }
}

function waitForExit(child, timeoutMs) {
  return new Promise((resolve) => {
    if (child.exitCode !== null || child.signalCode !== null) {
      resolve(true);
      return;
    }

    const timer = setTimeout(() => {
      child.off('exit', onExit);
      resolve(false);
    }, timeoutMs);

    function onExit() {
      clearTimeout(timer);
      resolve(true);
    }

    child.once('exit', onExit);
  });
}