import { assert, codedError, hashValue, sleep, throwIfShutdownRequested, } from '../assertions/core.mjs'; import { fs, path, spawn, withLoopbackNoProxy } from '../dependencies.mjs'; import { activeCommandChildren, appRoot, commandOutputLimit, manifestPath, state, } from '../runtime-state.mjs'; import { isSupervisorSwarmTransientRetrySuite, recordSupervisorSwarmChatSessionFailureDiagnostic, } from '../suites/supervisor-swarm.mjs'; import { isIsolatedRunnerSuite } from './reporting.mjs'; export const activeInteractiveCliSessions = new Set(); export async function prepareCliBinary() { const cargo = process.platform === 'win32' ? 'cargo.exe' : 'cargo'; await runProcess( cargo, ['build', '--quiet', '--manifest-path', manifestPath], { cwd: appRoot, timeoutMs: 15 * 60 * 1000, }, ); const metadata = await runProcess( cargo, [ 'metadata', '--format-version', '1', '--no-deps', '--manifest-path', manifestPath, ], { cwd: appRoot, timeoutMs: 120_000 }, ); const parsed = JSON.parse(metadata.stdout); const executable = path.join( parsed.target_directory, 'debug', `genarrative-ai-game-creator-shell${process.platform === 'win32' ? '.exe' : ''}`, ); const binary = await fs.stat(executable).catch(() => null); assert(binary?.isFile(), 'cli-binary-missing'); return executable; } export async function runCli(args, options = {}) { assert(Boolean(state.cliBinary), 'cli-binary-not-ready'); assert(Boolean(state.runtimeConfigDir), 'runtime-config-dir-not-ready'); if ( isIsolatedRunnerSuite() && state.options?.configDir && path.resolve(state.runtimeConfigDir) === path.resolve(state.options.configDir) ) { state.isolatedRunner.sourceConfigCliCallCount += 1; } return runProcess( state.cliBinary, [...args, '--config-dir', state.runtimeConfigDir], { cwd: appRoot, timeoutMs: options.timeoutMs ?? 60_000, stdin: options.stdin, allowNonZero: options.allowNonZero ?? false, env: buildCliChildEnvironment(), }, ); } export function buildCliChildEnvironment() { const environment = { ...process.env, NO_COLOR: '1', RUST_BACKTRACE: '0', }; // The deterministic playable suite copies its account fixture into the // sibling isolated AppData directory. Set the path explicitly here so // every CLI and the Runner it launches use the isolated copy, even if the // parent harness environment was restored or changed after setup. if (state.isolatedRunner.platformSessionFixturePath) { environment.GENARRATIVE_AGC_PLATFORM_SESSION_FIXTURE = state.isolatedRunner.platformSessionFixturePath; } return isSupervisorSwarmTransientRetrySuite() ? withLoopbackNoProxy(environment) : environment; } export function safeProcessFailureDiagnostic({ code = null, signal = null, error = null, stdout = '', stderr = '', } = {}) { const stdoutText = Buffer.isBuffer(stdout) ? stdout.toString('utf8') : String(stdout); const stderrText = Buffer.isBuffer(stderr) ? stderr.toString('utf8') : String(stderr); const combined = `${error?.message ?? ''}\n${stdoutText}\n${stderrText}`; const processErrorCode = /^[A-Z0-9_]+$/u.test(error?.code ?? '') ? error.code : 'none'; const closeSignal = /^[A-Z0-9]+$/u.test(signal ?? '') ? signal : 'none'; let failureKind = 'closed'; if ( processErrorCode === 'ENOSPC' || /(?:\bENOSPC\b|No space left on device|os error 28)/iu.test(combined) ) { failureKind = 'enospc'; } else if (processErrorCode !== 'none') { failureKind = 'process-error'; } else if (closeSignal !== 'none') { failureKind = 'signal'; } else if (Number.isInteger(code) && code !== 0) { failureKind = 'nonzero-exit'; } return { failureKind, exitCode: Number.isInteger(code) ? String(code) : 'none', signal: closeSignal, processErrorCode, stderrChars: [...stderrText].length, stderrSha256: hashValue(stderrText) ?? 'none', }; } export function codedProcessError(code, details) { const error = codedError(code, details?.error); error.safeFailureDiagnostic = safeProcessFailureDiagnostic(details); return error; } export function startInteractiveCli(args) { assert(Boolean(state.cliBinary), 'interactive-cli-binary-not-ready'); assert(Boolean(state.runtimeConfigDir), 'interactive-config-dir-not-ready'); if ( isIsolatedRunnerSuite() && state.options?.configDir && path.resolve(state.runtimeConfigDir) === path.resolve(state.options.configDir) ) { state.isolatedRunner.sourceConfigCliCallCount += 1; } const child = spawn( state.cliBinary, [...args, '--config-dir', state.runtimeConfigDir], { cwd: appRoot, env: buildCliChildEnvironment(), stdio: ['pipe', 'pipe', 'pipe'], }, ); return createInteractiveCliSession(child); } export function createInteractiveCliSession(child) { activeCommandChildren.add(child); const session = { child, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0), exited: false, exitInfo: null, exitPromise: null, closed: false, closeInfo: null, closePromise: null, stdioClosed: false, stdioCloseInfo: null, spawnError: null, stdinError: null, }; activeInteractiveCliSessions.add(session); session.exitPromise = new Promise((resolve) => { const settle = (result) => { if (session.exited) return; activeCommandChildren.delete(child); session.exited = true; session.closed = true; session.exitInfo = result; session.closeInfo = result; resolve(result); }; child.once('error', (error) => { session.spawnError = error; settle({ code: null, signal: null, error }); }); child.once('exit', (code, signal) => { settle({ code, signal, error: null }); }); }); session.closePromise = new Promise((resolve) => { child.once('close', (code, signal) => { activeCommandChildren.delete(child); activeInteractiveCliSessions.delete(session); session.stdioClosed = true; session.stdioCloseInfo = { code, signal, error: session.spawnError, }; resolve(session.stdioCloseInfo); }); }); child.stdin?.on('error', (error) => { session.stdinError ??= error; }); child.stdout.on('data', (chunk) => { state.transcriptScanner?.scan('interactive-stdout', chunk); state.projectPathTranscriptScanner?.scan('interactive-stdout', chunk); state.formalConfigPathTranscriptScanner?.scan('interactive-stdout', chunk); session.stdout = appendBounded(session.stdout, chunk, commandOutputLimit); }); child.stderr.on('data', (chunk) => { state.transcriptScanner?.scan('interactive-stderr', chunk); state.projectPathTranscriptScanner?.scan('interactive-stderr', chunk); state.formalConfigPathTranscriptScanner?.scan('interactive-stderr', chunk); session.stderr = appendBounded(session.stderr, chunk, commandOutputLimit); }); return session; } export function interactiveCliOutput(session) { return `${session.stdout.toString('utf8')}\n${session.stderr.toString('utf8')}`; } export function writeInteractiveCliLine(session, line) { assert(!session.closed, 'interactive-cli-already-closed'); assert(session.child.stdin.writable, 'interactive-cli-stdin-not-writable'); session.child.stdin.write(`${line}\n`); } export async function waitForInteractiveCliOutput( session, predicate, code, timeoutMs, { allowAfterProcessExit = false } = {}, ) { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const output = interactiveCliOutput(session); if (predicate(output)) return output; if (session.exited && !allowAfterProcessExit) { if (session === state.supervisorSwarmCliSession) { recordSupervisorSwarmChatSessionFailureDiagnostic(session); } throw codedError(`${code}-cli-exited`); } if (session.stdioClosed) { if (session === state.supervisorSwarmCliSession) { recordSupervisorSwarmChatSessionFailureDiagnostic(session); } throw codedError(`${code}-cli-stdio-closed`); } await sleep(50); } throw codedError(code); } export async function waitForInteractiveCliExit(session, timeoutMs) { const result = await Promise.race([ session.exitPromise, sleep(timeoutMs).then(() => null), ]); if (!result) throw codedError('interactive-cli-exit-timeout'); if (result.error) throw codedError('interactive-cli-process-error'); assert( result.code === 0 && result.signal === null, 'interactive-cli-exit-invalid', ); return result; } export async function closeInteractiveCli(session) { if (!session) return null; if (session.exited) return session.exitInfo; if ( session.child.stdin.writable && !session.child.stdin.writableEnded && !session.child.stdin.destroyed ) { session.child.stdin.write('/quit\n'); } let result = await Promise.race([ session.exitPromise, sleep(3_000).then(() => null), ]); if (!result && !session.exited) { session.child.kill('SIGTERM'); result = await Promise.race([ session.exitPromise, sleep(2_000).then(() => null), ]); } if (!result && !session.exited) { session.child.kill('SIGKILL'); result = await Promise.race([ session.exitPromise, sleep(5_000).then(() => null), ]); } assert(Boolean(result), 'interactive-cli-cleanup-timeout'); return result; } export async function waitForInteractiveCliStdioClose(session, timeoutMs) { if (!session || session.stdioClosed) return session?.stdioCloseInfo ?? null; const result = await Promise.race([ session.closePromise, sleep(timeoutMs).then(() => null), ]); if (!result) throw codedError('interactive-cli-stdio-close-timeout'); return result; } export function destroyInteractiveCliOutputStreams(session) { if (!session) return; for (const stream of [session.child.stdout, session.child.stderr]) { if (stream && !stream.destroyed) stream.destroy(); } } export async function runProcess( program, args, { cwd, timeoutMs, stdin, allowNonZero = false, env = null }, ) { throwIfShutdownRequested(); return new Promise((resolve, reject) => { const child = spawn(program, args, { cwd, env: env ?? { ...process.env, NO_COLOR: '1', RUST_BACKTRACE: '0' }, stdio: [stdin === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'], }); activeCommandChildren.add(child); let stdout = Buffer.alloc(0); let stderr = Buffer.alloc(0); let timedOut = false; const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, timeoutMs); child.stdout.on('data', (chunk) => { state.transcriptScanner?.scan('stdout', chunk); state.projectPathTranscriptScanner?.scan('stdout', chunk); state.formalConfigPathTranscriptScanner?.scan('stdout', chunk); stdout = appendBounded(stdout, chunk, commandOutputLimit); }); child.stderr.on('data', (chunk) => { state.transcriptScanner?.scan('stderr', chunk); state.projectPathTranscriptScanner?.scan('stderr', chunk); state.formalConfigPathTranscriptScanner?.scan('stderr', chunk); stderr = appendBounded(stderr, chunk, commandOutputLimit); }); child.on('error', (error) => { clearTimeout(timer); activeCommandChildren.delete(child); reject( codedProcessError('process-spawn-failed', { error, stdout, stderr, }), ); }); child.on('close', (code, signal) => { clearTimeout(timer); activeCommandChildren.delete(child); const result = { stdout: stdout.toString('utf8'), stderr: stderr.toString('utf8'), code, signal, }; if (timedOut) { reject(codedProcessError('process-timeout', result)); } else if (state.shutdownSignal && !state.cleanupInProgress) { reject( codedProcessError( `interrupted-${state.shutdownSignal.toLowerCase()}`, result, ), ); } else if (code !== 0 && !allowNonZero) { reject(codedProcessError('cli-command-failed', result)); } else { resolve(result); } }); if (stdin !== undefined) child.stdin.end(stdin); }); } export async function listSystemProcessIdentities() { if (process.platform === 'win32') { const systemRoot = process.env.SystemRoot ?? process.env.SYSTEMROOT; assert( typeof systemRoot === 'string' && path.isAbsolute(systemRoot), 'owned-process-snapshot-system-root-invalid', ); const powershell = path.join( systemRoot, 'System32/WindowsPowerShell/v1.0/powershell.exe', ); const metadata = await fs.lstat(powershell); assert( metadata.isFile() && !metadata.isSymbolicLink(), 'owned-process-snapshot-powershell-invalid', ); const result = await runProcess( powershell, [ '-NoProfile', '-NonInteractive', '-Command', '$processes = @(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CreationDate,Name); $processes | ConvertTo-Json -Compress', ], { cwd: appRoot, timeoutMs: 30_000, env: { ...process.env, NO_COLOR: '1', RUST_BACKTRACE: '0' }, }, ); const parsed = JSON.parse(result.stdout); return (Array.isArray(parsed) ? parsed : [parsed]) .map((record) => ({ pid: Number(record?.ProcessId), parentPid: Number(record?.ParentProcessId), startedAt: String(record?.CreationDate ?? ''), name: String(record?.Name ?? ''), })) .filter(validSystemProcessIdentity); } assert( process.platform === 'linux' || process.platform === 'darwin', 'owned-process-snapshot-platform-unsupported', ); const result = await runProcess( 'ps', ['-A', '-o', 'pid=', '-o', 'ppid=', '-o', 'lstart=', '-o', 'comm='], { cwd: appRoot, timeoutMs: 30_000 }, ); return result.stdout .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .map((line) => { const fields = line.split(/\s+/u); return { pid: Number(fields[0]), parentPid: Number(fields[1]), startedAt: fields.slice(2, 7).join(' '), name: fields.slice(7).join(' '), }; }) .filter(validSystemProcessIdentity); } function validSystemProcessIdentity(record) { return ( Number.isSafeInteger(record?.pid) && record.pid > 0 && Number.isSafeInteger(record.parentPid) && record.parentPid >= 0 && typeof record.startedAt === 'string' && record.startedAt.length > 0 && typeof record.name === 'string' && record.name.length > 0 ); } export function buildOwnedProcessCleanupSnapshot( processRecords, { rootPids = [], runnerPid = null, helperPids = [] } = {}, ) { assert( Array.isArray(processRecords) && Array.isArray(rootPids) && Array.isArray(helperPids), 'owned-process-snapshot-input-invalid', ); const records = processRecords.filter(validSystemProcessIdentity); const byPid = new Map(records.map((record) => [record.pid, record])); const childrenByParent = new Map(); for (const record of records) { const children = childrenByParent.get(record.parentPid) ?? []; children.push(record.pid); childrenByParent.set(record.parentPid, children); } const normalizedRunnerPid = Number.isSafeInteger(runnerPid) ? runnerPid : null; const helperPidSet = new Set( helperPids.filter((pid) => Number.isSafeInteger(pid) && pid > 0), ); const roots = [ ...new Set( [...rootPids, normalizedRunnerPid, ...helperPidSet].filter( (pid) => Number.isSafeInteger(pid) && pid > 0, ), ), ]; assert(roots.length > 0, 'owned-process-snapshot-root-missing'); const ownedPids = new Set(); const queue = [...roots]; while (queue.length > 0) { const pid = queue.shift(); if (ownedPids.has(pid)) continue; ownedPids.add(pid); queue.push(...(childrenByParent.get(pid) ?? [])); } const identities = [...ownedPids] .map((pid) => byPid.get(pid)) .filter(Boolean) .map((record) => ({ pid: record.pid, startedAt: record.startedAt, name: record.name, kind: ownedProcessKind(record, normalizedRunnerPid, helperPidSet), })) .sort((left, right) => left.pid - right.pid); return { identities, observedCounts: countOwnedProcessKinds(identities), }; } function ownedProcessKind(record, runnerPid, helperPids) { if (record.pid === runnerPid) return 'runner'; if (helperPids.has(record.pid)) return 'helper'; const name = path.basename(record.name).toLowerCase(); if (/^node(?:\.exe)?$/u.test(name)) return 'node'; if (/^(?:chrome|chromium|msedge|google-chrome)(?:\.exe)?$/u.test(name)) { return 'browser'; } return 'command'; } function countOwnedProcessKinds(identities) { const counts = { runner: 0, helper: 0, node: 0, browser: 0, command: 0, total: identities.length, }; for (const identity of identities) counts[identity.kind] += 1; return counts; } export function inspectOwnedProcessCleanupResiduals( snapshot, processRecords, { activeCommandChildCount = 0, activeInteractiveCliSessionCount = 0 } = {}, ) { assert( Array.isArray(snapshot?.identities) && Array.isArray(processRecords), 'owned-process-residual-input-invalid', ); const currentByPid = new Map( processRecords .filter(validSystemProcessIdentity) .map((record) => [record.pid, record]), ); const residualIdentities = snapshot.identities.filter((identity) => { const current = currentByPid.get(identity.pid); return ( current?.startedAt === identity.startedAt && current?.name === identity.name ); }); return { residualCounts: countOwnedProcessKinds(residualIdentities), activeCommandChildCount, activeInteractiveCliSessionCount, clean: residualIdentities.length === 0 && activeCommandChildCount === 0 && activeInteractiveCliSessionCount === 0, }; } export async function captureOwnedProcessCleanupSnapshot(options) { return buildOwnedProcessCleanupSnapshot( await listSystemProcessIdentities(), options, ); } export async function verifyOwnedProcessCleanupSnapshot(snapshot) { return inspectOwnedProcessCleanupResiduals( snapshot, await listSystemProcessIdentities(), { activeCommandChildCount: activeCommandChildren.size, activeInteractiveCliSessionCount: activeInteractiveCliSessions.size, }, ); } export function appendBounded(current, chunk, limit) { const combined = Buffer.concat([current, chunk]); return combined.length <= limit ? combined : combined.subarray(combined.length - limit); }