Genarrative/scripts/check-pingora-canary-docker.mjs

#!/usr/bin/env node

import { spawn, spawnSync } from 'node:child_process';
import { randomBytes } from 'node:crypto';
import { existsSync } from 'node:fs';
import {
  chmod,
  mkdir,
  mkdtemp,
  readFile,
  rm,
  writeFile,
} from 'node:fs/promises';
import net from 'node:net';
import os from 'node:os';
import path from 'node:path';

const SNIPPET_PATH = 'deploy/nginx/snippets/genarrative-pingora-canary.conf';
const REALPATH_SNIPPET_PATH =
  'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf';
const PROBE_TOKEN_PLACEHOLDER = '__GENARRATIVE_PINGORA_PROBE_TOKEN__';
const REALPATH_CANARY_ACCESS_LOG =
  '/var/log/nginx/genarrative-pingora-realpath-canary.access.log';
const REALPATH_LISTEN = 'listen 127.0.0.1:18083;';
const DEFAULT_IMAGE = 'nginx:1.27-alpine';

const repoRoot = process.cwd();
const failures = [];
const childProcesses = [];
const containers = [];
const servers = [];
const sockets = new Set();
const tempDirs = [];
let skipped = false;

const config = parseArgs(process.argv.slice(2));

try {
  await main();
} catch (error) {
  failures.push(error instanceof Error ? error.message : String(error));
} finally {
  await cleanup();
}

if (failures.length > 0) {
  console.error('\n[pingora-canary-docker] 未通过：');
  for (const failure of failures) {
    console.error(`- ${failure}`);
  }
  process.exit(1);
}

console.log(
  skipped ? '\n[pingora-canary-docker] 跳过' : '\n[pingora-canary-docker] 通过',
);

async function main() {
  if (!ensureDockerReady()) {
    return;
  }
  if (!ensureDockerImage()) {
    return;
  }

  const tempRoot = await mkdtemp(
    path.join(os.tmpdir(), 'genarrative-pingora-canary-docker-'),
  );
  tempDirs.push(tempRoot);

  const webRoot = path.join(tempRoot, 'web');
  const acmeRoot = path.join(tempRoot, 'acme');
  const logsRoot = path.join(tempRoot, 'logs');
  const nginxAccessLogFile = path.join(logsRoot, 'nginx.access.log');
  const realpathNginxAccessLogFile = path.join(
    logsRoot,
    'nginx.realpath.access.log',
  );
  const accessLogFile = path.join(logsRoot, 'pingora.access.log');
  await mkdir(logsRoot, { recursive: true });
  await chmod(logsRoot, 0o777);

  const gatewayWebRoot = config.webRoot || webRoot;
  const gatewayAcmeRoot = config.acmeRoot || acmeRoot;
  if (!config.webRoot) {
    await prepareStaticWebRoot(webRoot);
  }
  if (!config.acmeRoot) {
    await prepareAcmeRoot(acmeRoot);
  }

  const realUpstreams =
    config.realUpstreams || config.apiUpstream || config.spacetimeUpstream;
  if (realUpstreams && (!config.apiUpstream || !config.spacetimeUpstream)) {
    throw new Error(
      '真实上游模式必须同时提供 --api-upstream 与 --spacetime-upstream。',
    );
  }

  const api = realUpstreams
    ? { upstream: config.apiUpstream, state: null }
    : await startApiMock();
  const spacetime = realUpstreams
    ? { upstream: config.spacetimeUpstream, state: null }
    : await startSpacetimeMock();
  if (realUpstreams) {
    console.log(
      `[pingora-canary-docker] 使用真实上游 api=${api.upstream} spacetime=${spacetime.upstream} webRoot=${gatewayWebRoot}`,
    );
    await assertRealUpstreamsReady(api.upstream, spacetime.upstream);
  }
  const pingoraPort = await getFreePort();
  const nginxPort = await getFreePort();
  const realpathNginxPort = await getFreePort();
  const probeToken = `docker-${randomBytes(12).toString('hex')}`;

  if (!config.skipBuild) {
    runCommand('cargo', [
      'build',
      '-p',
      'pingora-gateway',
      '--manifest-path',
      'server-rs/Cargo.toml',
    ]);
  }

  const gatewayBinary = resolveGatewayBinary();
  const gateway = spawn(gatewayBinary, [], {
    cwd: repoRoot,
    env: {
      ...smokeEnv(),
      GENARRATIVE_PINGORA_GATEWAY_LISTEN: `0.0.0.0:${pingoraPort}`,
      GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM: api.upstream,
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM: spacetime.upstream,
      GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT: gatewayWebRoot,
      GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT: gatewayAcmeRoot,
      GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE: path.join(
        tempRoot,
        'maintenance',
        'enabled',
      ),
      GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE: accessLogFile,
      GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN: probeToken,
      GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO: 'https',
      GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES: '1048576',
      GENARRATIVE_PINGORA_GATEWAY_API_MAX_CONCURRENT: '16',
      GENARRATIVE_PINGORA_GATEWAY_API_RATE_PER_SECOND: '100',
      GENARRATIVE_PINGORA_GATEWAY_API_BURST: '100',
      GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_MAX_CONCURRENT: '16',
      GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_RATE_PER_SECOND: '100',
      GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_BURST: '100',
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_MAX_CONCURRENT: '16',
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_RATE_PER_SECOND: '100',
      GENARRATIVE_PINGORA_GATEWAY_SPACETIME_BURST: '100',
      GENARRATIVE_PINGORA_GATEWAY_LOG: config.verbose
        ? 'info,pingora=info,pingora_gateway=info'
        : 'warn,pingora=warn,pingora_gateway=warn',
    },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  childProcesses.push({ child: gateway, name: 'pingora-gateway' });
  collectProcessLogs(gateway, 'pingora-gateway');

  await waitForHttp(`http://127.0.0.1:${pingoraPort}/`, 200, {
    label: '等待 Pingora 网关就绪',
  });

  const renderedSnippetPath = path.join(
    tempRoot,
    'genarrative-pingora-canary.conf',
  );
  const nginxConfigPath = path.join(tempRoot, 'nginx.conf');
  await renderNginxConfig({
    renderedSnippetPath,
    nginxConfigPath,
    nginxAccessLogFile,
    realpathNginxAccessLogFile,
    pingoraPort,
    nginxPort,
    realpathNginxPort,
    probeToken,
  });

  const containerName = `genarrative-pingora-canary-${process.pid}-${randomBytes(4).toString('hex')}`;
  containers.push(containerName);
  const docker = spawn(
    'docker',
    [
      'run',
      '--rm',
      '--name',
      containerName,
      '--network',
      'host',
      '-v',
      `${tempRoot}:${tempRoot}`,
      config.image,
      'nginx',
      '-c',
      nginxConfigPath,
      '-g',
      'daemon off;',
    ],
    {
      cwd: repoRoot,
      env: smokeEnv(),
      stdio: ['ignore', 'pipe', 'pipe'],
    },
  );
  childProcesses.push({ child: docker, name: 'docker-nginx' });
  collectProcessLogs(docker, 'docker-nginx');

  const baseUrl = `http://127.0.0.1:${nginxPort}`;
  const realpathBaseUrl = `http://127.0.0.1:${realpathNginxPort}`;
  await waitForHttp(`${baseUrl}/__genarrative_pingora_canary/healthz`, 200, {
    headers: { Host: 'example.test' },
    label: '等待 Docker Nginx canary 就绪',
    validate: (response) =>
      response.headers['x-genarrative-nginx-handoff'] === 'pingora-canary' &&
      response.body.includes('"gateway":"pingora-shadow"'),
  });
  await waitForHttp(
    `${realpathBaseUrl}/__genarrative_pingora_realpath_canary/healthz`,
    200,
    {
      headers: { Host: 'example.test' },
      label: '等待 Docker Nginx realpath canary 就绪',
      validate: (response) =>
        response.headers['x-genarrative-nginx-handoff'] ===
          'pingora-realpath-canary' &&
        response.body.includes('"gateway":"pingora-shadow"'),
    },
  );

  await runCommandAsync(process.execPath, [
    'scripts/check-pingora-canary-live.mjs',
    '--base-url',
    baseUrl,
    '--host',
    'example.test',
    '--timeout-ms',
    '5000',
  ]);
  await runCommandAsync(process.execPath, [
    'scripts/check-pingora-canary-live.mjs',
    '--realpath',
    '--base-url',
    realpathBaseUrl,
    '--host',
    'example.test',
    '--timeout-ms',
    '5000',
  ]);

  await expectAccessLogContains(nginxAccessLogFile, [
    '/__genarrative_pingora_canary/healthz',
    '/__genarrative_pingora_canary/api/assets/history',
    '/__genarrative_pingora_canary/v1/identity',
    '/__genarrative_pingora_canary/assets/app.js',
  ]);
  await expectAccessLogContains(realpathNginxAccessLogFile, [
    '/__genarrative_pingora_realpath_canary/healthz',
    '/api/assets/history',
    '/v1/identity',
    '/assets/app.js',
  ]);
  await expectAccessLogContains(accessLogFile, [
    'path=/__genarrative_pingora/healthz',
    'path=/api/assets/history',
    'path=/v1/identity',
    'path=/assets/app.js',
  ]);

  await runCommandAsync(process.execPath, [
    'scripts/check-pingora-canary-access-log-parity.mjs',
    '--nginx-log-file',
    nginxAccessLogFile,
    '--pingora-log-file',
    accessLogFile,
    '--since-lines',
    '2000',
    '--path',
    '/__genarrative_pingora_canary/healthz',
    '--path',
    '/__genarrative_pingora_canary/api/assets/history',
    '--path',
    '/__genarrative_pingora_canary/v1/identity',
    '--path',
    '/__genarrative_pingora_canary/assets/app.js',
  ]);
  await runCommandAsync(process.execPath, [
    'scripts/check-pingora-canary-access-log-parity.mjs',
    '--realpath',
    '--nginx-log-file',
    realpathNginxAccessLogFile,
    '--pingora-log-file',
    accessLogFile,
    '--since-lines',
    '2000',
    '--path',
    '/__genarrative_pingora_realpath_canary/healthz',
    '--path',
    '/api/assets/history',
    '--path',
    '/v1/identity',
    '--path',
    '/assets/app.js',
  ]);

  if (!realUpstreams) {
    ensure(
      api.state.requests.some(
        (request) => request.url === '/api/assets/history',
      ),
      'Docker Nginx canary 未把 API 代表路径交给 mock api-server',
    );
    ensure(
      spacetime.state.requests.some(
        (request) => request.url === '/v1/identity',
      ),
      'Docker Nginx canary 未把 SpacetimeDB identity 代表路径交给 mock SpacetimeDB',
    );
    ensure(
      api.state.requests.filter(
        (request) => request.url === '/api/assets/history',
      ).length >= 2,
      'Docker Nginx realpath canary 未把真实 API 代表路径交给 mock api-server',
    );
    ensure(
      spacetime.state.requests.filter(
        (request) => request.url === '/v1/identity',
      ).length >= 2,
      'Docker Nginx realpath canary 未把真实 SpacetimeDB identity 路径交给 mock SpacetimeDB',
    );
  }
}

function parseArgs(argv) {
  const result = {
    image: process.env.GENARRATIVE_PINGORA_CANARY_DOCKER_IMAGE || DEFAULT_IMAGE,
    pull: false,
    requireDocker: false,
    skipBuild: false,
    verbose: false,
    realUpstreams: false,
    apiUpstream: normalizeOptionalHostPort(
      process.env.GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM,
      'GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM',
    ),
    spacetimeUpstream: normalizeOptionalHostPort(
      process.env.GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM,
      'GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM',
    ),
    webRoot: normalizeOptionalDirectory(
      process.env.GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT,
      'GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT',
    ),
    acmeRoot: normalizeOptionalDirectory(
      process.env.GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT,
      'GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT',
    ),
  };

  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    switch (arg) {
      case '-h':
      case '--help':
        console.log(`Usage:
  node scripts/check-pingora-canary-docker.mjs [options]

Options:
  --image <image>       Nginx image, default ${DEFAULT_IMAGE}.
  --pull                Pull the image when it is not present locally.
  --require-docker      Treat missing Docker/image/host-network as failure instead of skip.
  --skip-build          Reuse an existing pingora-gateway debug binary.
  --real-upstreams      Require real api-server and SpacetimeDB upstream arguments.
  --api-upstream <host:port>
                        Use a real api-server upstream instead of the built-in mock.
  --spacetime-upstream <host:port>
                        Use a real SpacetimeDB upstream instead of the built-in mock.
  --web-root <path>     Serve an existing web root, for example dist/.
  --acme-root <path>    Serve an existing ACME root; temp root is used by default.
  --verbose             Print Pingora and Docker Nginx process logs.

Environment aliases:
  GENARRATIVE_PINGORA_CANARY_DOCKER_IMAGE
  GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM
  GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM
  GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT
  GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT
  GENARRATIVE_PINGORA_GATEWAY_BINARY

默认不会拉取镜像；本机或 CI 要做强验收时建议：
  node scripts/check-pingora-canary-docker.mjs --require-docker --pull
`);
        process.exit(0);
        break;
      case '--image':
        result.image = requireValue(argv, ++index, arg);
        break;
      case '--pull':
        result.pull = true;
        break;
      case '--require-docker':
        result.requireDocker = true;
        break;
      case '--skip-build':
        result.skipBuild = true;
        break;
      case '--real-upstreams':
        result.realUpstreams = true;
        break;
      case '--api-upstream':
        result.apiUpstream = normalizeHostPort(
          requireValue(argv, ++index, arg),
          arg,
        );
        break;
      case '--spacetime-upstream':
        result.spacetimeUpstream = normalizeHostPort(
          requireValue(argv, ++index, arg),
          arg,
        );
        break;
      case '--web-root':
        result.webRoot = normalizeDirectory(
          requireValue(argv, ++index, arg),
          arg,
        );
        break;
      case '--acme-root':
        result.acmeRoot = normalizeDirectory(
          requireValue(argv, ++index, arg),
          arg,
        );
        break;
      case '--verbose':
        result.verbose = true;
        break;
      default:
        throw new Error(`未知参数: ${arg}`);
    }
  }

  return result;
}

function requireValue(argv, index, flag) {
  const value = argv[index];
  if (!value || value.startsWith('--')) {
    throw new Error(`${flag} 缺少参数值`);
  }
  return value;
}

function normalizeOptionalHostPort(value, label) {
  if (!value) {
    return '';
  }
  return normalizeHostPort(value, label);
}

function normalizeHostPort(value, label) {
  validateNoControlCharacters(value, label);
  const raw = String(value || '').trim();
  if (!raw) {
    throw new Error(`${label} 不能为空。`);
  }
  if (raw.includes('://') || /[\s/?#@]/u.test(raw)) {
    throw new Error(
      `${label} 必须是 host:port，不能包含 scheme、路径、查询、片段或空白字符。`,
    );
  }
  try {
    const parsed = new URL(`http://${raw}`);
    const port = Number.parseInt(parsed.port, 10);
    if (
      !parsed.hostname ||
      !Number.isInteger(port) ||
      port <= 0 ||
      port > 65535
    ) {
      throw new Error('invalid host:port');
    }
  } catch {
    throw new Error(`${label} 必须是合法 host:port。`);
  }
  return raw;
}

function normalizeOptionalDirectory(value, label) {
  if (!value) {
    return '';
  }
  return normalizeDirectory(value, label);
}

function normalizeDirectory(value, label) {
  validateNoControlCharacters(value, label);
  const resolved = path.resolve(repoRoot, String(value || '').trim());
  if (!existsSync(resolved)) {
    throw new Error(`${label} 不存在：${resolved}`);
  }
  return resolved;
}

function validateNoControlCharacters(value, label) {
  if (/[\0\r\n]/u.test(String(value ?? ''))) {
    throw new Error(`${label} 不能包含换行或 NUL 字符。`);
  }
}

function ensureDockerReady() {
  const result = spawnSync(
    'docker',
    ['version', '--format', '{{.Server.Version}}'],
    {
      cwd: repoRoot,
      env: smokeEnv(),
      encoding: 'utf8',
      shell: false,
      stdio: 'pipe',
    },
  );

  if (result.status === 0) {
    console.log(
      `[pingora-canary-docker] Docker daemon ${result.stdout.trim() || 'ready'}`,
    );
    return true;
  }

  return skipOrFail(
    `Docker daemon 不可用；需要强制验证时先启动 Docker。${formatSpawnFailure(result)}`,
  );
}

function ensureDockerImage() {
  const inspect = spawnSync('docker', ['image', 'inspect', config.image], {
    cwd: repoRoot,
    env: smokeEnv(),
    encoding: 'utf8',
    shell: false,
    stdio: 'pipe',
  });

  if (inspect.status === 0) {
    return true;
  }

  if (!config.pull) {
    return skipOrFail(
      `本机没有 ${config.image} 镜像；可追加 --pull 拉取，或预先 docker pull ${config.image}。`,
    );
  }

  runCommand('docker', ['pull', config.image]);
  return true;
}

function skipOrFail(message) {
  if (config.requireDocker) {
    failures.push(message);
  } else {
    skipped = true;
    console.log(`[pingora-canary-docker] SKIP: ${message}`);
  }
  return false;
}

async function prepareStaticWebRoot(webRoot) {
  await mkdir(path.join(webRoot, 'admin', 'assets'), { recursive: true });
  await mkdir(path.join(webRoot, 'assets'), { recursive: true });

  await writeFile(path.join(webRoot, 'index.html'), '<main>site-shell</main>');
  await writeFile(
    path.join(webRoot, 'admin', 'index.html'),
    '<main>admin-shell</main>',
  );
  await writeFile(
    path.join(webRoot, 'assets', 'app.js'),
    'console.log("site asset");',
  );
  await writeFile(
    path.join(webRoot, 'admin', 'assets', 'admin.js'),
    'console.log("admin asset");',
  );
}

async function prepareAcmeRoot(acmeRoot) {
  await mkdir(path.join(acmeRoot, '.well-known', 'acme-challenge'), {
    recursive: true,
  });
  await writeFile(
    path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'),
    'acme-token',
  );
}

async function startApiMock() {
  const state = {
    requests: [],
  };
  const server = net.createServer((socket) => {
    sockets.add(socket);
    socket.on('close', () => {
      sockets.delete(socket);
    });
    if (config.verbose) {
      console.log('[api-mock] connection');
    }

    let raw = Buffer.alloc(0);
    socket.on('data', (chunk) => {
      raw = Buffer.concat([raw, chunk]);
      const headerEnd = raw.indexOf('\r\n\r\n');
      if (headerEnd < 0 || socket.writableEnded) {
        return;
      }

      const request = parseRawHttpRequest(raw.slice(0, headerEnd + 4));
      if (config.verbose) {
        console.log(
          `[api-mock] ${request.method} ${request.url} headers=${JSON.stringify(request.headers)}`,
        );
      }
      state.requests.push(request);
      const body = JSON.stringify({
        ok: true,
        upstream: 'api',
        method: request.method,
        url: request.url,
        host: request.headers.host || '',
      });
      socket.end(
        [
          'HTTP/1.1 200 OK',
          'Content-Type: application/json; charset=utf-8',
          'X-Upstream: api',
          `Content-Length: ${Buffer.byteLength(body)}`,
          'Connection: close',
          '',
          body,
        ].join('\r\n'),
      );
    });
  });

  const port = await listen(server);
  return { upstream: `127.0.0.1:${port}`, state };
}

async function startSpacetimeMock() {
  const state = {
    requests: [],
  };
  const server = net.createServer((socket) => {
    sockets.add(socket);
    socket.on('close', () => {
      sockets.delete(socket);
    });
    let raw = Buffer.alloc(0);
    socket.on('data', (chunk) => {
      raw = Buffer.concat([raw, chunk]);
      const headerEnd = raw.indexOf('\r\n\r\n');
      if (headerEnd < 0 || socket.writableEnded) {
        return;
      }

      const request = parseRawHttpRequest(raw.slice(0, headerEnd + 4));
      state.requests.push(request);
      const body = JSON.stringify({
        ok: true,
        upstream: 'spacetime',
        url: request.url,
      });
      socket.end(
        [
          'HTTP/1.1 200 OK',
          'Content-Type: application/json; charset=utf-8',
          `Content-Length: ${Buffer.byteLength(body)}`,
          'Connection: close',
          '',
          body,
        ].join('\r\n'),
      );
    });
  });

  const port = await listen(server);
  return { upstream: `127.0.0.1:${port}`, state };
}

async function assertRealUpstreamsReady(apiUpstream, spacetimeUpstream) {
  await waitForHttp(`http://${apiUpstream}/healthz`, 200, {
    label: '等待真实 api-server 就绪',
  });
  await waitForHttp(`http://${spacetimeUpstream}/v1/ping`, 200, {
    label: '等待真实 SpacetimeDB 就绪',
  });
}

async function renderNginxConfig({
  renderedSnippetPath,
  nginxConfigPath,
  nginxAccessLogFile,
  realpathNginxAccessLogFile,
  pingoraPort,
  nginxPort,
  realpathNginxPort,
  probeToken,
}) {
  const snippet = await readFile(SNIPPET_PATH, 'utf8');
  const renderedSnippet = snippet
    .replaceAll(PROBE_TOKEN_PLACEHOLDER, probeToken)
    .replaceAll('http://127.0.0.1:18081', `http://127.0.0.1:${pingoraPort}`);
  const renderedRealpathSnippetPath = path.join(
    path.dirname(renderedSnippetPath),
    'genarrative-pingora-realpath-canary.conf',
  );
  const realpathSnippet = await readFile(REALPATH_SNIPPET_PATH, 'utf8');
  const renderedRealpathSnippet = realpathSnippet
    .replaceAll(PROBE_TOKEN_PLACEHOLDER, probeToken)
    .replaceAll('http://127.0.0.1:18081', `http://127.0.0.1:${pingoraPort}`)
    .replace(REALPATH_LISTEN, `listen 127.0.0.1:${realpathNginxPort};`)
    .replace(REALPATH_CANARY_ACCESS_LOG, realpathNginxAccessLogFile);
  const nginxConfig = `
pid /tmp/genarrative-pingora-canary-docker.pid;
error_log stderr notice;

events {
    worker_connections 128;
}

http {
    log_format genarrative_upstream
        '$remote_addr - $remote_user [$time_local] "$request" '
        '$status $body_bytes_sent "$http_referer" "$http_user_agent" '
        'request_time=$request_time upstream_connect_time=$upstream_connect_time '
        'upstream_header_time=$upstream_header_time upstream_response_time=$upstream_response_time '
        'upstream_status=$upstream_status request_id=$request_id';

    server {
        listen 127.0.0.1:${nginxPort};
        server_name example.test;
        access_log ${nginxAccessLogFile} genarrative_upstream;

        include ${renderedSnippetPath};
    }

    include ${renderedRealpathSnippetPath};
}
`;

  await writeFile(renderedSnippetPath, renderedSnippet, 'utf8');
  await writeFile(renderedRealpathSnippetPath, renderedRealpathSnippet, 'utf8');
  await writeFile(nginxConfigPath, nginxConfig, 'utf8');
}

async function expectAccessLogContains(accessLogFile, needles) {
  await waitForCondition(async () => {
    try {
      const content = await readFile(accessLogFile, 'utf8');
      return needles.every((needle) => content.includes(needle));
    } catch {
      return false;
    }
  }, 5000);
}

function requestHttp(url, options = {}) {
  return new Promise((resolve, reject) => {
    const target = new URL(url);
    const socket = net.connect(
      { host: target.hostname, port: Number(target.port) },
      () => {
        const headers = {
          Host: target.host,
          Connection: 'close',
          ...(options.headers || {}),
        };
        const headerLines = Object.entries(headers).map(
          ([name, value]) => `${name}: ${value}`,
        );
        socket.write(
          [
            `${options.method || 'GET'} ${target.pathname}${target.search} HTTP/1.1`,
            ...headerLines,
            '',
            '',
          ].join('\r\n'),
        );
      },
    );
    let raw = Buffer.alloc(0);
    const timeout = setTimeout(() => {
      socket.destroy(new Error(`HTTP 请求超时：${url}`));
    }, options.timeoutMs || 5000);
    socket.on('data', (chunk) => {
      raw = Buffer.concat([raw, chunk]);
      const response = tryParseRawHttpResponse(raw);
      if (response) {
        clearTimeout(timeout);
        socket.destroy();
        resolve(response);
      }
    });
    socket.on('error', reject);
    socket.on('close', () => {
      clearTimeout(timeout);
      const response = tryParseRawHttpResponse(raw, { final: true });
      if (response) {
        resolve(response);
      }
    });
  });
}

function parseRawHttpRequest(raw) {
  const text = raw.toString('latin1');
  const [requestLine, ...headerLines] = text.trimEnd().split('\r\n');
  const [method = '', url = ''] = requestLine.split(' ');
  const headers = {};
  for (const line of headerLines) {
    const separator = line.indexOf(':');
    if (separator <= 0) {
      continue;
    }
    headers[line.slice(0, separator).trim().toLowerCase()] = line
      .slice(separator + 1)
      .trim();
  }
  return { method, url, headers };
}

function tryParseRawHttpResponse(raw, options = {}) {
  const headerEnd = raw.indexOf('\r\n\r\n');
  if (headerEnd < 0) {
    return null;
  }

  const headerText = raw.slice(0, headerEnd).toString('latin1');
  const [statusLine, ...headerLines] = headerText.split('\r\n');
  const status = Number.parseInt(statusLine.split(' ')[1] || '0', 10);
  const headers = {};
  for (const line of headerLines) {
    const separator = line.indexOf(':');
    if (separator <= 0) {
      continue;
    }
    headers[line.slice(0, separator).trim().toLowerCase()] = line
      .slice(separator + 1)
      .trim();
  }

  const bodyStart = headerEnd + 4;
  const body = raw.slice(bodyStart);
  const contentLength = Number.parseInt(headers['content-length'] || '', 10);
  if (Number.isFinite(contentLength)) {
    if (body.length < contentLength) {
      return null;
    }
    return {
      status: status,
      headers,
      body: body.slice(0, contentLength).toString('utf8'),
    };
  }

  if (!options.final) {
    return null;
  }
  return {
    status,
    headers,
    body: body.toString('utf8'),
  };
}

function listen(server) {
  return new Promise((resolve, reject) => {
    server.on('error', reject);
    server.on('connection', (socket) => {
      sockets.add(socket);
      socket.on('close', () => {
        sockets.delete(socket);
      });
    });
    server.listen(0, '127.0.0.1', () => {
      servers.push(server);
      const address = server.address();
      if (!address || typeof address === 'string') {
        reject(new Error('无法读取 mock server 端口'));
        return;
      }
      resolve(address.port);
    });
  });
}

function getFreePort() {
  return new Promise((resolve, reject) => {
    const server = net.createServer();
    server.on('error', reject);
    server.listen(0, '127.0.0.1', () => {
      const address = server.address();
      if (!address || typeof address === 'string') {
        server.close();
        reject(new Error('无法分配临时端口'));
        return;
      }
      const { port } = address;
      server.close(() => resolve(port));
    });
  });
}

async function waitForHttp(url, expectedStatus, options = {}) {
  let lastError = '';
  await waitForCondition(async () => {
    try {
      const response = await requestHttp(url, {
        headers: options.headers,
        timeoutMs: 2000,
      });
      if (
        response.status === expectedStatus &&
        (!options.validate || options.validate(response))
      ) {
        return true;
      }
      lastError = `HTTP ${response.status}`;
      return false;
    } catch (error) {
      lastError = error instanceof Error ? error.message : String(error);
      return false;
    }
  }, 15000).catch(() => {
    throw new Error(`${options.label || '等待 HTTP 就绪'}超时：${lastError}`);
  });
}

function runCommand(command, args) {
  console.log(`[pingora-canary-docker] ${command} ${args.join(' ')}`);
  const result = spawnSync(command, args, {
    cwd: repoRoot,
    env: smokeEnv(),
    shell: false,
    stdio: 'inherit',
  });

  if (result.error) {
    throw new Error(`${command} 启动失败：${result.error.message}`);
  }
  if (result.signal) {
    throw new Error(`${command} 被信号终止：${result.signal}`);
  }
  if ((result.status ?? 0) !== 0) {
    throw new Error(`${command} ${args.join(' ')} 退出码 ${result.status}`);
  }
}

function runCommandAsync(command, args) {
  console.log(`[pingora-canary-docker] ${command} ${args.join(' ')}`);
  return new Promise((resolve, reject) => {
    const child = spawn(command, args, {
      cwd: repoRoot,
      env: smokeEnv(),
      shell: false,
      stdio: 'inherit',
    });
    child.on('error', (error) => {
      reject(new Error(`${command} 启动失败：${error.message}`));
    });
    child.on('exit', (status, signal) => {
      if (signal) {
        reject(new Error(`${command} 被信号终止：${signal}`));
        return;
      }
      if ((status ?? 0) !== 0) {
        reject(new Error(`${command} ${args.join(' ')} 退出码 ${status}`));
        return;
      }
      resolve();
    });
  });
}

function resolveGatewayBinary() {
  const explicit = process.env.GENARRATIVE_PINGORA_GATEWAY_BINARY;
  if (explicit) {
    return explicit;
  }

  const candidates = [
    path.join(repoRoot, 'server-rs', 'target', 'debug', 'pingora-gateway'),
    path.join(repoRoot, 'target', 'debug', 'pingora-gateway'),
  ];
  const found = candidates.find((candidate) => existsSync(candidate));
  if (!found) {
    throw new Error(
      `未找到 pingora-gateway debug 二进制，请先构建或设置 GENARRATIVE_PINGORA_GATEWAY_BINARY。候选：${candidates.join(', ')}`,
    );
  }
  return found;
}

function smokeEnv() {
  return {
    ...process.env,
    PATH: `${path.join(os.homedir(), '.local', 'bin')}:${process.env.PATH || ''}`,
  };
}

function collectProcessLogs(child, name) {
  child.stdout?.on('data', (chunk) => {
    if (config.verbose) {
      process.stdout.write(`[${name}] ${chunk}`);
    }
  });
  child.stderr?.on('data', (chunk) => {
    if (config.verbose) {
      process.stderr.write(`[${name}] ${chunk}`);
    }
  });
}

function formatSpawnFailure(result) {
  const chunks = [];
  if (result.error) {
    chunks.push(result.error.message);
  }
  if (result.stderr) {
    chunks.push(result.stderr.trim());
  }
  if (result.stdout) {
    chunks.push(result.stdout.trim());
  }
  return chunks.length > 0 ? ` ${chunks.join(' ')}` : '';
}

function delay(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function waitForCondition(predicate, timeoutMs = 2000) {
  const startedAt = Date.now();
  while (Date.now() - startedAt < timeoutMs) {
    if (await predicate()) {
      return;
    }
    await delay(50);
  }
  throw new Error('等待 smoke 条件达成超时');
}

function ensure(condition, message) {
  if (!condition) {
    failures.push(message);
  }
}

async function cleanup() {
  for (const container of containers.reverse()) {
    spawnSync('docker', ['rm', '-f', container], {
      cwd: repoRoot,
      env: smokeEnv(),
      encoding: 'utf8',
      shell: false,
      stdio: config.verbose ? 'inherit' : 'pipe',
    });
  }

  for (const { child, name } of childProcesses.reverse()) {
    await stopChild(child, name);
  }

  for (const socket of sockets) {
    socket.destroy();
  }
  sockets.clear();

  await Promise.all(
    servers.reverse().map(
      (server) =>
        new Promise((resolve) => {
          const timer = setTimeout(resolve, 1000);
          server.close(() => {
            clearTimeout(timer);
            resolve();
          });
        }),
    ),
  );

  await Promise.all(
    tempDirs.reverse().map((dir) => rm(dir, { recursive: true, force: true })),
  );
}

async function stopChild(child, name) {
  if (child.exitCode !== null || child.signalCode !== null) {
    return;
  }

  child.kill('SIGTERM');
  const exited = await waitForExit(child, 3000);
  if (!exited) {
    child.kill('SIGKILL');
    const killed = await waitForExit(child, 3000);
    if (!killed) {
      failures.push(`${name} 未能退出`);
    }
  }
}

function waitForExit(child, timeoutMs) {
  return new Promise((resolve) => {
    if (child.exitCode !== null || child.signalCode !== null) {
      resolve(true);
      return;
    }

    const timer = setTimeout(() => {
      child.off('exit', onExit);
      resolve(false);
    }, timeoutMs);

    function onExit() {
      clearTimeout(timer);
      resolve(true);
    }

    child.once('exit', onExit);
  });
}