diff --git a/apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs b/apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs index 9a9a9bbd8..80c96fda2 100644 --- a/apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs +++ b/apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs @@ -371,6 +371,26 @@ async function runWithConcurrency(shards, runner) { return results; } +function extractFailingTestNames(result) { + const names = new Set(); + for (const line of result.failures) { + const normalized = line + .trim() + .replace(/^\[rust-shards\]\s*/, '') + .replace(/^----\s*/, '') + .replace(/\s*stdout\s*----$/, '') + .replace(/\s*\(\d+\)\s*$/, '') + .trim(); + const match = normalized.match( + /^(process_session::tests::[A-Za-z0-9_:]+|tests::[A-Za-z0-9_:]+)$/, + ); + if (match) { + names.add(match[1]); + } + } + return [...names]; +} + async function main() { const executable = await resolveTestExecutable(); const testNames = await listTestNames(executable); @@ -410,15 +430,55 @@ async function main() { const results = await runWithConcurrency( selectedShards, - ({ index, shardTestNames }) => - runShard(executable, index, shards.length, shardTestNames), + async ({ index, shardTestNames }) => { + const result = await runShard( + executable, + index, + shards.length, + shardTestNames, + ); + if (result.ok) { + return result; + } + // 片内串行的时序型用例在高负载 CI 上会偶发假红。只对失败用例做一次 + // 有界复核:复核通过按 flaky 记录,复核失败才判红,避免把真实回归洗掉。 + const failingTestNames = extractFailingTestNames(result).filter((name) => + shardTestNames.includes(name), + ); + if (failingTestNames.length === 0) { + return result; + } + const retry = await runShard( + executable, + index, + shards.length, + failingTestNames, + ); + if (!retry.ok) { + return { + ...result, + failures: [ + `re-run of ${failingTestNames.length} failing test(s) also failed`, + ...retry.failures, + ], + }; + } + return { + ...result, + ok: true, + retriedTestNames: failingTestNames, + }; + }, ); let failed = false; for (const result of results) { if (result.ok) { + const retrySuffix = result.retriedTestNames + ? ` (flaky: re-ran ${result.retriedTestNames.length} failing test(s) and passed: ${result.retriedTestNames.join(', ')})` + : ''; console.log( - `[rust-shards] ${result.label} ok: ${result.testCount} test(s) in ${formatDuration(result.durationMs)}`, + `[rust-shards] ${result.label} ok: ${result.testCount} test(s) in ${formatDuration(result.durationMs)}${retrySuffix}`, ); continue; }