合并 master 到 Pingora 网关分支

保留 Pingora 网关发布与巡检能力

合入外部生成 worker 与 controller 运维链路

对齐 SpacetimeDB 2.5.0 依赖锁定

更新项目共享记忆与生产运维文档

scripts/ops/production-health-patrol.mjs

@@ -20,14 +20,17 @@ const DEFAULT_PUBLIC_PATHS = [
 
 const DEFAULT_SERVICES = [
   'genarrative-api.service',
+  'genarrative-external-generation-controller.service',
   'spacetimedb.service',
   'nginx.service',
 ];
 const PINGORA_DIRECT_SERVICES = [
   'genarrative-api.service',
+  'genarrative-external-generation-controller.service',
   'spacetimedb.service',
   'genarrative-pingora-gateway.service',
 ];
+const WORKER_SERVICE_PATTERN = 'genarrative-external-generation-worker@*.service';
 const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']);
 
 function usage() {
@@ -287,6 +290,61 @@ async function checkService(serviceName, timeoutMs) {
   );
 }
 
+async function checkActiveWorkerInstances(config) {
+  const result = await runCommand(
+    'systemctl',
+    [
+      'list-units',
+      WORKER_SERVICE_PATTERN,
+      '--type=service',
+      '--state=active',
+      '--no-legend',
+      '--plain',
+      '--no-pager',
+    ],
+    config.timeoutMs,
+  );
+  if (result.code !== 0) {
+    return checkResult(
+      'service:external-generation-workers',
+      'CRITICAL',
+      '无法枚举外部生成 worker 实例',
+      {
+        command: result.command,
+        stderr: result.stderr.trim() || result.error,
+      },
+    );
+  }
+
+  const services = result.stdout
+    .split('\n')
+    .map((line) => line.trim().split(/\s+/u)[0])
+    .filter((service) =>
+      /^genarrative-external-generation-worker@.+\.service$/u.test(service),
+    );
+
+  if (services.length === 0) {
+    return checkResult(
+      'service:external-generation-workers',
+      'CRITICAL',
+      '没有 active 的外部生成 worker 实例',
+      {
+        command: result.command,
+      },
+    );
+  }
+
+  return checkResult(
+    'service:external-generation-workers',
+    'OK',
+    `${services.length} 个 worker active`,
+    {
+      command: result.command,
+      services,
+    },
+  );
+}
+
 function requestUrl(url, timeoutMs, headers = {}) {
   return new Promise((resolve) => {
     const startedAt = Date.now();
@@ -493,7 +551,9 @@ function gatewayServiceNames(config) {
 }
 
 async function checkRecentJournal(config) {
-  const serviceNames = gatewayServiceNames(config);
+  const serviceNames = [
+    ...new Set([...gatewayServiceNames(config), WORKER_SERVICE_PATTERN]),
+  ];
   const args = [
     ...serviceNames.flatMap((serviceName) => ['-u', serviceName]),
     '--since',
@@ -619,6 +679,7 @@ async function main() {
   for (const serviceName of gatewayServiceNames(config)) {
     checks.push(await checkService(serviceName, config.timeoutMs));
   }
+  checks.push(await checkActiveWorkerInstances(config));
 
   checks.push(
     await checkHttp(