清理本地遗留生成Worker

启动本地 all 角色前清理同库旧 external-generation-worker

补充 dev 调度脚本的 worker 匹配测试

记录旧 worker 抢队列导致 procedure 超时的排障口径

scripts/dev.mjs

@@ -5,6 +5,7 @@ import {
   existsSync,
   readdirSync,
   readFileSync,
+  readlinkSync,
   realpathSync,
   statSync,
   watch,
@@ -817,6 +818,182 @@ async function stopProcess(child, label) {
   });
 }
 
+function parseProcessEnvBlock(rawEnv) {
+  return String(rawEnv ?? '')
+    .split('\0')
+    .filter(Boolean)
+    .reduce((env, entry) => {
+      const separatorIndex = entry.indexOf('=');
+      if (separatorIndex <= 0) {
+        return env;
+      }
+      env[entry.slice(0, separatorIndex)] = entry.slice(separatorIndex + 1);
+      return env;
+    }, {});
+}
+
+function normalizeProcessLinkTarget(path) {
+  return String(path ?? '').replace(/ \(deleted\)$/u, '');
+}
+
+function isSamePathForDevProcess(left, right) {
+  const normalizedLeft = normalizePath(normalizeProcessLinkTarget(left));
+  const normalizedRight = normalizePath(normalizeProcessLinkTarget(right));
+  return normalizedLeft === normalizedRight;
+}
+
+function isStaleExternalGenerationWorkerProcess({
+  cwd,
+  env,
+  exe,
+  expectedDatabase,
+  expectedExePath,
+  expectedRepoRoot,
+  expectedSpacetimeServer,
+  pid,
+}) {
+  if (!Number.isInteger(pid) || pid === process.pid) {
+    return false;
+  }
+  if (env.GENARRATIVE_PROCESS_ROLE !== 'external-generation-worker') {
+    return false;
+  }
+  if (String(env.GENARRATIVE_SPACETIME_SERVER_URL ?? '') !== expectedSpacetimeServer) {
+    return false;
+  }
+  if (String(env.GENARRATIVE_SPACETIME_DATABASE ?? '') !== expectedDatabase) {
+    return false;
+  }
+  if (!isSamePathForDevProcess(cwd, expectedRepoRoot)) {
+    return false;
+  }
+  return isSamePathForDevProcess(exe, expectedExePath);
+}
+
+function readLinuxApiServerProcessSnapshot(pid) {
+  try {
+    return {
+      cwd: readlinkSync(`/proc/${pid}/cwd`),
+      env: parseProcessEnvBlock(readFileSync(`/proc/${pid}/environ`, 'utf8')),
+      exe: readlinkSync(`/proc/${pid}/exe`),
+      pid,
+    };
+  } catch (error) {
+    if (
+      error?.code === 'ENOENT' ||
+      error?.code === 'EACCES' ||
+      error?.code === 'EPERM'
+    ) {
+      return null;
+    }
+    throw error;
+  }
+}
+
+function listLinuxProcessIds(procDir = '/proc') {
+  try {
+    return readdirSync(procDir)
+      .map((name) => Number.parseInt(name, 10))
+      .filter(Number.isInteger);
+  } catch {
+    return [];
+  }
+}
+
+function isLinuxProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function waitForLinuxProcessExit(pid, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (!isLinuxProcessAlive(pid)) {
+      return true;
+    }
+    await sleep(100);
+  }
+  return !isLinuxProcessAlive(pid);
+}
+
+async function stopLinuxProcessId(pid, label) {
+  try {
+    process.kill(pid, 'SIGTERM');
+  } catch (error) {
+    if (error?.code === 'ESRCH') {
+      return true;
+    }
+    console.warn(`[dev:${label}] 停止旧进程失败 pid=${pid}: ${error.message}`);
+    return false;
+  }
+
+  if (await waitForLinuxProcessExit(pid, 5000)) {
+    return true;
+  }
+
+  try {
+    process.kill(pid, 'SIGKILL');
+  } catch (error) {
+    if (error?.code === 'ESRCH') {
+      return true;
+    }
+    console.warn(`[dev:${label}] 强制停止旧进程失败 pid=${pid}: ${error.message}`);
+    return false;
+  }
+  return waitForLinuxProcessExit(pid, 1000);
+}
+
+async function stopStaleLocalExternalGenerationWorkers({
+  database,
+  logStream = null,
+  repoRootPath = repoRoot,
+  processRole,
+  spacetimeServer,
+}) {
+  if (process.platform !== 'linux') {
+    return [];
+  }
+  if (processRole !== 'all') {
+    return [];
+  }
+
+  const expectedExePath = resolve(repoRootPath, 'server-rs/target/debug/api-server');
+  const stopped = [];
+  for (const pid of listLinuxProcessIds()) {
+    const snapshot = readLinuxApiServerProcessSnapshot(pid);
+    if (
+      !snapshot ||
+      !isStaleExternalGenerationWorkerProcess({
+        ...snapshot,
+        expectedDatabase: database,
+        expectedExePath,
+        expectedRepoRoot: repoRootPath,
+        expectedSpacetimeServer: spacetimeServer,
+      })
+    ) {
+      continue;
+    }
+
+    const label = snapshot.env.GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID
+      ? `${pid}(${snapshot.env.GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID})`
+      : String(pid);
+    if (await stopLinuxProcessId(pid, 'api-server')) {
+      stopped.push(label);
+    }
+  }
+
+  if (stopped.length > 0) {
+    const line = `[dev:api-server] 已停止同库旧 external-generation-worker 进程: ${stopped.join(', ')}\n`;
+    process.stdout.write(line);
+    logStream?.write(line);
+  }
+  return stopped;
+}
+
 function stopWindowsProcessTree(pid) {
   if (!pid) {
     return;
@@ -1436,6 +1613,12 @@ class DevRunner {
     mergedEnv.GENARRATIVE_API_SERVER_LOG_FILE = logFile;
 
     stopExistingWindowsApiServer(logStream);
+    await stopStaleLocalExternalGenerationWorkers({
+      database: this.options.database,
+      logStream,
+      processRole: mergedEnv.GENARRATIVE_PROCESS_ROLE,
+      spacetimeServer: this.state.spacetimeServer,
+    });
 
     console.log(`[dev:api-server] log: ${logFile}`);
     console.log(
@@ -2278,7 +2461,9 @@ export {
   DevRunner,
   isDirectModuleExecution,
   isSpacetimePublishPermissionError,
+  isStaleExternalGenerationWorkerProcess,
   normalizeCargoVersionRequirement,
+  parseProcessEnvBlock,
   parseArgs,
   parseSpacetimeToolVersion,
   resolveDevStackStatePath,