新增external_generation_job的provider任务checkpoint

- provider_kind 与 provider_task_id 追加在表末尾,带显式默认值,旧行迁移时补 NULL
- 新增 set_external_generation_job_provider_checkpoint_and_return procedure,复用 running 租约栅栏
- 已有 checkpoint 时禁止覆盖并留下 provider_checkpoint 事件,作为 at-most-once submit 的数据层保证
- 快照与兼容快照同步投影两个新字段,轻量摘要按无 checkpoint 返回
- 增加 checkpoint 不可覆盖的不变量单测
This commit is contained in:
2026-09-21 13:02:52 +08:00
parent 36036eed7b
commit 09ea0a5e76
2 changed files with 123 additions and 0 deletions
@@ -15,6 +15,7 @@ const EXTERNAL_GENERATION_EVENT_LEASE_RENEWED: &str = "lease_renewed";
const EXTERNAL_GENERATION_EVENT_COMPLETED: &str = "completed";
const EXTERNAL_GENERATION_EVENT_FAILED: &str = "failed";
const EXTERNAL_GENERATION_EVENT_ACKNOWLEDGED: &str = "acknowledged";
const EXTERNAL_GENERATION_EVENT_PROVIDER_CHECKPOINT: &str = "provider_checkpoint";
const EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE: &str = "editor-canvas";
const EXTERNAL_GENERATION_FINAL_ATTEMPT_LEASE_EXPIRED_MESSAGE: &str =
"worker 最终执行次数的 lease 已过期,任务已终止";
@@ -90,6 +91,12 @@ pub struct ExternalGenerationJob {
pub(crate) notification_acknowledged_at: Option<Timestamp>,
#[default(None::<String>)]
pub(crate) phase: Option<String>,
// 中文注释:provider 任务 checkpoint 晚于主表加入。没有 provider_task_id 才允许 submit
// 有 checkpoint 的 attempt 只允许续跑查询与落库,避免崩溃重试造成二次提交与二次扣费。
#[default(None::<String>)]
pub(crate) provider_kind: Option<String>,
#[default(None::<String>)]
pub(crate) provider_task_id: Option<String>,
}
#[spacetimedb::table(
@@ -193,6 +200,15 @@ pub struct ExternalGenerationJobPhaseUpdateInput {
pub phase: String,
}
#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)]
pub struct ExternalGenerationJobProviderCheckpointInput {
pub job_id: String,
pub worker_id: String,
pub lease_token: String,
pub provider_kind: String,
pub provider_task_id: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)]
pub enum ExternalGenerationJobPhaseUpdateFailureKind {
LeaseFencingRejected,
@@ -283,6 +299,8 @@ pub struct ExternalGenerationJobSnapshot {
pub refund_ledger_id: Option<String>,
pub notification_acknowledged_at_micros: Option<i64>,
pub phase: Option<String>,
pub provider_kind: Option<String>,
pub provider_task_id: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)]
@@ -504,6 +522,23 @@ pub fn renew_external_generation_job_lease_and_return(
}
}
#[spacetimedb::procedure]
pub fn set_external_generation_job_provider_checkpoint_and_return(
ctx: &mut ProcedureContext,
input: ExternalGenerationJobProviderCheckpointInput,
) -> ExternalGenerationJobProcedureResult {
let caller = ctx.sender();
match ctx.try_with_tx(|tx| {
crate::editor_project_storage::require_editor_generation_runtime_service_identity(
tx, caller,
)?;
set_external_generation_job_provider_checkpoint_tx(tx, input.clone())
}) {
Ok(job) => single_external_generation_job_result(job),
Err(message) => failed_external_generation_job_result(message),
}
}
#[spacetimedb::procedure]
pub fn update_external_generation_job_phase_and_return(
ctx: &mut ProcedureContext,
@@ -819,6 +854,8 @@ fn enqueue_external_generation_job_tx(
refund_ledger_id: None,
notification_acknowledged_at: None,
phase: None,
provider_kind: None,
provider_task_id: None,
};
persist_external_generation_job_row(ctx, row.clone());
insert_external_generation_job_event(
@@ -1578,6 +1615,45 @@ fn update_external_generation_job_phase_tx(
Ok(map_external_generation_job_row(row))
}
/// 写入 provider 任务 checkpoint。
///
/// 只有持有当前租约的 worker 能写;checkpoint 一旦存在就不再允许覆盖,
/// 这是 at-most-once submit 的数据层保证:崩溃后重新 claim 的 attempt 只能读到既有
/// provider_task_id 并继续查询,不会再去 submit 一次。
fn set_external_generation_job_provider_checkpoint_tx(
ctx: &ReducerContext,
input: ExternalGenerationJobProviderCheckpointInput,
) -> Result<ExternalGenerationJobSnapshot, String> {
let provider_kind = normalize_optional_text(&input.provider_kind)
.ok_or_else(|| "external_generation_job.provider_kind 不能为空".to_string())?;
let provider_task_id = normalize_optional_text(&input.provider_task_id)
.ok_or_else(|| "external_generation_job.provider_task_id 不能为空".to_string())?;
let mut row = get_worker_owned_external_generation_job(
ctx,
&input.job_id,
&input.worker_id,
&input.lease_token,
)?;
ensure_external_generation_job_provider_checkpoint_absent(&row)?;
row.provider_kind = Some(provider_kind);
row.provider_task_id = Some(provider_task_id.clone());
row.updated_at = ctx.timestamp;
persist_external_generation_job_row(ctx, row.clone());
insert_external_generation_job_event(
ctx,
&row,
EXTERNAL_GENERATION_EVENT_PROVIDER_CHECKPOINT,
Some(format!(
"worker 已记录 provider 任务 checkpoint {provider_task_id}"
)),
Some(input.worker_id),
ctx.timestamp,
);
Ok(map_external_generation_job_row(row))
}
fn fail_external_generation_job_tx(
ctx: &ReducerContext,
input: ExternalGenerationJobFailInput,
@@ -1801,6 +1877,19 @@ fn get_worker_owned_external_generation_job(
.map_err(|error| error.message)
}
/// at-most-once submit 的数据层不变量:已有 checkpoint 的 job 不允许再写第二次。
fn ensure_external_generation_job_provider_checkpoint_absent(
row: &ExternalGenerationJob,
) -> Result<(), String> {
if row.provider_task_id.is_some() {
return Err(
"external_generation_job.provider_task_id 已存在,禁止覆盖 provider checkpoint"
.to_string(),
);
}
Ok(())
}
#[derive(Debug)]
struct ExternalGenerationJobPhaseUpdateError {
kind: ExternalGenerationJobPhaseUpdateFailureKind,
@@ -2573,6 +2662,8 @@ fn map_external_generation_job_row(row: ExternalGenerationJob) -> ExternalGenera
refund_ledger_id: row.refund_ledger_id,
notification_acknowledged_at_micros,
phase: row.phase,
provider_kind: row.provider_kind,
provider_task_id: row.provider_task_id,
}
}
@@ -2641,6 +2732,9 @@ fn map_external_generation_job_summary_to_compat_snapshot(
refund_ledger_id: summary.refund_ledger_id,
notification_acknowledged_at_micros: summary.notification_acknowledged_at_micros,
phase: summary.phase,
// 中文注释:轻量摘要不投影 provider checkpoint,兼容快照按无 checkpoint 返回。
provider_kind: None,
provider_task_id: None,
}
}
@@ -3095,6 +3189,22 @@ mod tests {
}
}
#[test]
fn provider_checkpoint_cannot_be_overwritten_after_first_write() {
let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_RUNNING);
assert!(
ensure_external_generation_job_provider_checkpoint_absent(&row).is_ok(),
"首次 checkpoint 必须允许写入"
);
row.provider_kind = Some("tripo".to_string());
row.provider_task_id = Some("task-1".to_string());
assert!(
ensure_external_generation_job_provider_checkpoint_absent(&row).is_err(),
"已有 checkpoint 必须拒绝覆盖,保证 at-most-once submit"
);
}
#[test]
fn external_generation_job_result_failure_is_structured() {
let result = failed_external_generation_job_result("失败".to_string());
@@ -3847,6 +3957,8 @@ mod tests {
refund_ledger_id: None,
notification_acknowledged_at: None,
phase: None,
provider_kind: None,
provider_task_id: None,
}
}
@@ -1593,6 +1593,17 @@ fn normalize_migration_row(table_name: &str, value: &serde_json::Value) -> serde
.or_insert(serde_json::Value::Null);
}
}
if table_name == "external_generation_job" {
if let Some(object) = next_value.as_object_mut() {
// 中文注释:provider 任务 checkpoint 晚于外部生成主表加入,旧迁移包按尚未提交 provider 兼容。
object
.entry("provider_kind".to_string())
.or_insert(serde_json::Value::Null);
object
.entry("provider_task_id".to_string())
.or_insert(serde_json::Value::Null);
}
}
if table_name == "big_fish_creation_session" {
if let Some(object) = next_value.as_object_mut() {
// 中文注释:旧迁移包没有公开游玩次数字段,导入时按新建作品默认 0 兼容。