统一生成失败的可重放结算
Project CI / Repository checks (pull_request) Failing after 9s
Project CI / Backend tests (pull_request) Failing after 11s
Project CI / Frontend tests (pull_request) Successful in 3m29s
Project CI / Native shell tests (pull_request) Successful in 16m46s

生成流程内部错误统一先写 pending ledger,再更新公开草稿、staging revision 与终态账本。
增加 pending 写入后故障注入恢复回归。
补齐公开草稿、staging 与终态账本之间三个持久化切点测试。
同步更新失败结算技术合同与共享决策记录。
This commit is contained in:
2026-08-23 19:10:08 +08:00
parent 155052752f
commit fb12ef27c6
3 changed files with 242 additions and 15 deletions
@@ -2805,22 +2805,52 @@ fn mark_generation_error(
reconciliation: bool,
error_code: &str,
emit: &mut (dyn FnMut(AssetCanvasGenerationProgressEvent) + Send),
) -> Result<(), String> {
mark_generation_error_with_pending_hook(root, ledger, reconciliation, error_code, emit, || {
Ok(())
})
}
fn mark_generation_error_with_pending_hook(
root: &Path,
ledger: &mut AssetCanvasGenerationLedger,
reconciliation: bool,
error_code: &str,
emit: &mut (dyn FnMut(AssetCanvasGenerationProgressEvent) + Send),
after_pending: impl FnOnce() -> Result<(), String>,
) -> Result<(), String> {
// Cancellation is authoritative and must not be overwritten by a late
// network/commit error. The private ledger keeps its last durable phase as
// evidence while all public projection and candidate persistence stop.
ensure_generation_draft_active(root, ledger)?;
set_private_phase(
root,
ledger,
if reconciliation {
GenerationLedgerPhase::ReconciliationRequired
let (_, record) = {
let _draft_guard = acquire_asset_canvas_draft_lock(root)?;
if pending_failure_settlement_target(&ledger.phase).is_some() {
finish_generation_failure_settlement_pending_locked(root, ledger)?
} else {
GenerationLedgerPhase::Failed
},
Some(error_code),
)?;
publish_public_phase(root, ledger, emit).map(|_| ())
let draft = read_asset_canvas_draft_locked(root, &ledger.project_id, &ledger.draft_id)?
.ok_or_else(|| "素材画布草稿不存在".to_string())?;
validate_generation_draft_active(ledger, &draft)?;
let reconciliation =
reconciliation || ledger.phase == GenerationLedgerPhase::ReconciliationRequired;
ledger.phase = if reconciliation {
GenerationLedgerPhase::ReconciliationSettlementPending
} else {
GenerationLedgerPhase::FailureSettlementPending
};
ledger.error_code = Some(error_code.to_string());
ledger.current_draft_revision = Some(draft.revision);
write_generation_ledger(root, ledger)?;
after_pending()?;
finish_generation_failure_settlement_pending_locked(root, ledger)?
}
};
emit(generation_progress_event(
ledger,
public_phase_name(&record.phase),
public_progress_value(&record.phase),
record.error_code.clone(),
));
Ok(())
}
fn pending_failure_settlement_target(
@@ -5417,7 +5447,87 @@ mod tests {
}
#[tokio::test]
async fn pending_generation_failure_settlement_recovers_across_both_durable_write_cuts() {
async fn internal_generation_error_recovers_after_pending_ledger_cut() {
let project_id = "asset-canvas-internal-error-pending";
let (directory, draft) = create_generation_fixture(project_id, "内部失败 pending 恢复测试");
let input = generation_input(
directory.path(),
project_id,
&draft,
"恢复生成流程内部失败的 pending 结算",
);
let mut ledger =
validate_and_prepare_ledger(directory.path(), &input).expect("prepare generation");
set_private_phase(
directory.path(),
&mut ledger,
GenerationLedgerPhase::Running,
None,
)
.expect("persist running phase");
publish_public_phase(directory.path(), &mut ledger, &mut |_| {})
.expect("publish running phase");
let running_draft =
read_asset_canvas_draft_locked(directory.path(), project_id, &draft.draft_id)
.expect("read running draft")
.expect("running draft exists");
let error = mark_generation_error_with_pending_hook(
directory.path(),
&mut ledger,
false,
"generation-rejected",
&mut |_| {},
|| Err("fault-injected:failure-settlement-pending".to_string()),
)
.expect_err("fault after pending ledger must interrupt settlement");
assert_eq!(error, "fault-injected:failure-settlement-pending");
let interrupted_ledger = read_generation_ledger(directory.path(), &input.generation_id)
.expect("read interrupted internal failure ledger")
.expect("interrupted internal failure ledger exists");
assert_eq!(
interrupted_ledger.phase,
GenerationLedgerPhase::FailureSettlementPending
);
let interrupted_draft =
read_asset_canvas_draft_locked(directory.path(), project_id, &draft.draft_id)
.expect("read interrupted internal failure draft")
.expect("interrupted internal failure draft exists");
assert_eq!(interrupted_draft.revision, running_draft.revision);
assert!(interrupted_draft.generations.iter().any(|record| {
record.generation_id == input.generation_id
&& record.phase == AssetCanvasGenerationStatus::GenerationRunning
}));
recover_asset_canvas_generations_at(
directory.path(),
&RecoverAssetCanvasGenerationsInput {
project_path: directory.path().to_string_lossy().into_owned(),
expected_project_id: project_id.to_string(),
draft_id: draft.draft_id.clone(),
},
|_| {},
)
.await
.expect("recover internal generation error settlement");
let recovered_ledger = read_generation_ledger(directory.path(), &input.generation_id)
.expect("read recovered internal failure ledger")
.expect("recovered internal failure ledger exists");
assert_eq!(recovered_ledger.phase, GenerationLedgerPhase::Failed);
let recovered_draft =
read_asset_canvas_draft_locked(directory.path(), project_id, &draft.draft_id)
.expect("read recovered internal failure draft")
.expect("recovered internal failure draft exists");
assert_eq!(recovered_draft.revision, running_draft.revision + 1);
assert!(recovered_draft.generations.iter().any(|record| {
record.generation_id == input.generation_id
&& record.phase == AssetCanvasGenerationStatus::Failed
&& record.error_code.as_deref() == Some("generation-rejected")
}));
}
#[tokio::test]
async fn pending_generation_failure_settlement_recovers_across_all_durable_write_cuts() {
let project_id = "asset-canvas-failure-pending-before-public";
let (directory, draft) = create_generation_fixture(project_id, "失败结算 pending 恢复测试");
let input = generation_input(
@@ -5500,9 +5610,24 @@ mod tests {
read_asset_canvas_draft_locked(directory.path(), project_id, &draft.draft_id)
.expect("read running draft")
.expect("running draft exists");
let staged = stage_asset_canvas_image_with_token_at(
directory.path(),
&StageAssetCanvasImageInput {
project_path: directory.path().to_string_lossy().into_owned(),
expected_project_id: project_id.to_string(),
draft_id: draft.draft_id.clone(),
expected_draft_revision: running_draft.revision,
media_type: "image/png".to_string(),
bytes: test_png(),
},
Some(&ledger.staged_image_token),
)
.expect("stage reconciliation candidate");
assert_eq!(staged.status, "staged");
ledger.phase = GenerationLedgerPhase::ReconciliationSettlementPending;
ledger.error_code = Some("poll-result-unknown".to_string());
ledger.current_draft_revision = Some(running_draft.revision);
ledger.staged_draft_revision = Some(running_draft.revision);
write_generation_ledger(directory.path(), &mut ledger)
.expect("persist reconciliation settlement journal");
let mut projection = ledger.clone();
@@ -5510,6 +5635,13 @@ mod tests {
let (_, projected_revision) =
upsert_public_generation_record(directory.path(), &projection)
.expect("simulate public projection before terminal ledger write");
let (staged_before_recovery, _) =
read_staged_image_locked(directory.path(), &ledger.staged_image_token)
.expect("read staging before reconciliation recovery");
assert_eq!(
staged_before_recovery.draft_revision,
running_draft.revision
);
let mut progress = Vec::new();
recover_asset_canvas_generations_at(
@@ -5535,6 +5667,10 @@ mod tests {
.expect("read recovered reconciliation draft")
.expect("recovered reconciliation draft exists");
assert_eq!(recovered_draft.revision, projected_revision);
let (recovered_staging, _) =
read_staged_image_locked(directory.path(), &ledger.staged_image_token)
.expect("read rebound staging after reconciliation recovery");
assert_eq!(recovered_staging.draft_revision, projected_revision);
assert!(recovered_draft.generations.iter().any(|record| {
record.generation_id == input.generation_id
&& record.phase == AssetCanvasGenerationStatus::ReconciliationRequired
@@ -5542,6 +5678,97 @@ mod tests {
}));
assert_eq!(progress.len(), 1);
assert_eq!(progress[0].phase, "reconciliation-required");
let project_id = "asset-canvas-failure-pending-after-staging";
let (directory, draft) =
create_generation_fixture(project_id, "失败结算 staging 后恢复测试");
let input = generation_input(
directory.path(),
project_id,
&draft,
"恢复已推进 staging 但未写终态账本的失败结算",
);
let mut ledger =
validate_and_prepare_ledger(directory.path(), &input).expect("prepare generation");
set_private_phase(
directory.path(),
&mut ledger,
GenerationLedgerPhase::Running,
None,
)
.expect("persist running phase");
publish_public_phase(directory.path(), &mut ledger, &mut |_| {})
.expect("publish running phase");
let running_draft =
read_asset_canvas_draft_locked(directory.path(), project_id, &draft.draft_id)
.expect("read running draft")
.expect("running draft exists");
let staged = stage_asset_canvas_image_with_token_at(
directory.path(),
&StageAssetCanvasImageInput {
project_path: directory.path().to_string_lossy().into_owned(),
expected_project_id: project_id.to_string(),
draft_id: draft.draft_id.clone(),
expected_draft_revision: running_draft.revision,
media_type: "image/png".to_string(),
bytes: test_png_with_color([22, 44, 66, 255]),
},
Some(&ledger.staged_image_token),
)
.expect("stage failure candidate");
assert_eq!(staged.status, "staged");
ledger.phase = GenerationLedgerPhase::FailureSettlementPending;
ledger.error_code = Some("generation-rejected".to_string());
ledger.current_draft_revision = Some(running_draft.revision);
ledger.staged_draft_revision = Some(running_draft.revision);
write_generation_ledger(directory.path(), &mut ledger)
.expect("persist failure settlement journal");
let mut projection = ledger.clone();
projection.phase = GenerationLedgerPhase::Failed;
let (_, projected_revision) =
upsert_public_generation_record(directory.path(), &projection)
.expect("simulate failure public projection before staging revision");
rebind_asset_canvas_staged_image_revision_at(
directory.path(),
&ledger.staged_image_token,
project_id,
&draft.draft_id,
running_draft.revision,
projected_revision,
)
.expect("simulate staging revision before terminal ledger write");
let mut progress = Vec::new();
recover_asset_canvas_generations_at(
directory.path(),
&RecoverAssetCanvasGenerationsInput {
project_path: directory.path().to_string_lossy().into_owned(),
expected_project_id: project_id.to_string(),
draft_id: draft.draft_id.clone(),
},
|event| progress.push(event),
)
.await
.expect("recover pending failure after staging revision");
let recovered_ledger = read_generation_ledger(directory.path(), &input.generation_id)
.expect("read recovered failure ledger")
.expect("recovered failure ledger exists");
assert_eq!(recovered_ledger.phase, GenerationLedgerPhase::Failed);
assert_eq!(
recovered_ledger.staged_draft_revision,
Some(projected_revision)
);
let recovered_draft =
read_asset_canvas_draft_locked(directory.path(), project_id, &draft.draft_id)
.expect("read recovered failure draft")
.expect("recovered failure draft exists");
assert_eq!(recovered_draft.revision, projected_revision);
let (recovered_staging, _) =
read_staged_image_locked(directory.path(), &ledger.staged_image_token)
.expect("read staging after terminal ledger recovery");
assert_eq!(recovered_staging.draft_revision, projected_revision);
assert_eq!(progress.len(), 1);
assert_eq!(progress[0].phase, "failed");
}
#[tokio::test]
@@ -14395,6 +14395,6 @@
## 2026-08-23 AGC 素材画布失败结算使用可重放中间态
- 失败结算的 `expectedDraftRevision` CAS、结算意图和公开投影必须在同一草稿锁边界内串行化;CAS 成功后先持久化 `failure-settlement-pending``reconciliation-settlement-pending`,不能直接跨文件发布终态。
- 失败结算的 `expectedDraftRevision` CAS、结算意图和公开投影必须在同一草稿锁边界内串行化;显式失败结算和生成流程内部错误都先持久化 `failure-settlement-pending``reconciliation-settlement-pending`,不能直接跨文件发布终态。
- pending 恢复不依赖平台登录或 External API Key。恢复按当前权威草稿幂等补齐 generation 投影和 staging revision,再把私有 ledger 发布为 `failed``reconciliation-required`;公开投影已经存在时只完成账本,不重复增加草稿 revision。
- 回归必须覆盖 pending ledger 写入后、公开草稿写入前,以及公开草稿写入后、终态 ledger 写入前种重启切点。
- 回归必须覆盖 pending ledger 写入后、公开草稿写入前,公开草稿写入后、staging revision 写入前,以及 staging revision 写入后、终态 ledger 写入前种重启切点。
@@ -92,7 +92,7 @@
- 生成失败时,公开 generation 记录必须在同一草稿 revision 链路中落为 `failed``reconciliation-required`,不得留下持久化的 `generation-running` 幽灵任务。
- 客户端处理生成失败时必须调用宿主的权威失败结算接口并携带 `expectedDraftRevision`;结算返回 revision 冲突或失败时保持失败/对账阻断态,不得把画布静默恢复为 clean editing。无私有账本的早期失败只允许由后端原子移除公开占位并返回 `not-started`,已进入候选或正式提交终态的结果返回 `already-terminal`,等待重新打开对账。
- 权威失败结算先将私有 ledger 写为 `failure-settlement-pending``reconciliation-settlement-pending`,再幂等更新公开草稿与 staging revision,最后发布 `failed``reconciliation-required`。任一持久化边界中断后,启动恢复必须在解析生成服务凭据前完成 pending 结算;公开草稿已写入时不得重复推进 revision
- 权威失败结算与生成流程内部错误统一先将私有 ledger 写为 `failure-settlement-pending``reconciliation-settlement-pending`,再幂等更新公开草稿与 staging revision,最后发布 `failed``reconciliation-required`,任何路径都不得先发布终态 ledger。任一持久化边界中断后,启动恢复必须在解析生成服务凭据前完成 pending 结算;公开草稿或 staging revision 已写入时不得重复推进。
- 候选成功回包不得整体 hydrate 覆盖生成期间的本地未 autosave 状态;必须以当前 runtime 的图层、viewport、背景、selection 和 history 为基线,只合并权威候选图层、预览地址与 generation 记录,并用一次带 CAS 的草稿保存确认候选。
- 候选生成必须先幂等落盘候选媒体、候选图层和公开记录,回读成功后才能推进私有 ledger 的 `candidate-ready` 终态;两次写入之间中断时,恢复必须可重放。
- 失败归档使用 `archive-pending` 中间态:先记录归档意图,再删除草稿公开记录,最后发布 `archived`;任一边界中断都必须在重启后收敛且保持幂等。