GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善外部生成Worker动态扩缩容

Browse Source 
新增外部生成controller进程角色与systemd服务

补齐队列统计procedure与spacetime-client绑定

更新生产部署脚本、健康巡检和server provision的worker/controller口径

新增容器worker smoke脚本并同步运维文档与团队记忆
This commit is contained in:
kdletters
2026-06-12 15:21:35 +08:00
parent 69815d918a
commit 4a6c126366
30 changed files with 2030 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
server-rs/crates/spacetime-client/src/external_generation.rs
View File

@@ -126,4 +126,23 @@ impl SpacetimeClient {
)
.await
}
pub async fn get_external_generation_queue_stats(
&self,
) -> Result<ExternalGenerationQueueStatsRecord, SpacetimeClientError> {
self.call_after_connect(
"get_external_generation_queue_stats_and_return",
move |connection, sender| {
connection
.procedures()
.get_external_generation_queue_stats_and_return_then(move |_, result| {
let mapped = result
.map_err(SpacetimeClientError::from_sdk_error)
.and_then(map_external_generation_queue_stats_result);
send_once(&sender, mapped);
});
},
)
.await
}
}

13
server-rs/crates/spacetime-client/src/lib.rs
View File

@@ -33,12 +33,13 @@ pub use mapper::{
CustomWorldWorkSummaryRecord, ExternalGenerationJobClaimRecordInput,
ExternalGenerationJobCompleteRecordInput, ExternalGenerationJobEnqueueRecordInput,
ExternalGenerationJobFailRecordInput, ExternalGenerationJobRecord,
ExternalGenerationJobRenewLeaseRecordInput, JumpHopActionRequest, JumpHopActionResponse,
JumpHopActionType, JumpHopCharacterAsset, JumpHopDifficulty, JumpHopDraftResponse,
JumpHopGalleryCardResponse, JumpHopGalleryDetailResponse, JumpHopGalleryResponse,
JumpHopGenerationStatus, JumpHopJumpRequest, JumpHopJumpResponse, JumpHopJumpResult,
JumpHopLastJump, JumpHopPath, JumpHopPlatform, JumpHopRestartRunRequest, JumpHopRunResponse,
JumpHopRunStatus, JumpHopRuntimeRunSnapshotResponse, JumpHopScoring, JumpHopSessionResponse,
ExternalGenerationJobRenewLeaseRecordInput, ExternalGenerationQueueStatsRecord,
JumpHopActionRequest, JumpHopActionResponse, JumpHopActionType, JumpHopCharacterAsset,
JumpHopDifficulty, JumpHopDraftResponse, JumpHopGalleryCardResponse,
JumpHopGalleryDetailResponse, JumpHopGalleryResponse, JumpHopGenerationStatus,
JumpHopJumpRequest, JumpHopJumpResponse, JumpHopJumpResult, JumpHopLastJump, JumpHopPath,
JumpHopPlatform, JumpHopRestartRunRequest, JumpHopRunResponse, JumpHopRunStatus,
JumpHopRuntimeRunSnapshotResponse, JumpHopScoring, JumpHopSessionResponse,
JumpHopSessionSnapshotResponse, JumpHopStartRunRequest, JumpHopStylePreset, JumpHopTileAsset,
JumpHopTileType, JumpHopWorkDetailResponse, JumpHopWorkMutationResponse,
JumpHopWorkProfileResponse, JumpHopWorkSummaryResponse, JumpHopWorksResponse,

2
server-rs/crates/spacetime-client/src/mapper.rs
View File

@@ -73,6 +73,7 @@ pub use self::external_generation::{
ExternalGenerationJobClaimRecordInput, ExternalGenerationJobCompleteRecordInput,
ExternalGenerationJobEnqueueRecordInput, ExternalGenerationJobFailRecordInput,
ExternalGenerationJobRecord, ExternalGenerationJobRenewLeaseRecordInput,
ExternalGenerationQueueStatsRecord,
};
pub use self::jump_hop::{
JumpHopActionRequest, JumpHopActionResponse, JumpHopActionType, JumpHopCharacterAsset,
@@ -186,6 +187,7 @@ pub(crate) use self::custom_world::{
};
pub(crate) use self::external_generation::{
map_external_generation_job_claim_result, map_external_generation_job_procedure_result,
map_external_generation_queue_stats_result,
};
pub(crate) use self::inventory::{
map_runtime_inventory_state_procedure_result, map_runtime_item_reward_item_snapshot,

37
server-rs/crates/spacetime-client/src/mapper/external_generation.rs
View File

@@ -94,6 +94,30 @@ pub(crate) fn map_external_generation_job_claim_result(
.collect())
}
pub(crate) fn map_external_generation_queue_stats_result(
result: ExternalGenerationQueueStatsProcedureResult,
) -> Result<ExternalGenerationQueueStatsRecord, SpacetimeClientError> {
if !result.ok {
return Err(SpacetimeClientError::procedure_failed(result.error_message));
}
let stats = result.stats.ok_or_else(|| {
SpacetimeClientError::missing_snapshot("external_generation queue stats 快照")
})?;
Ok(ExternalGenerationQueueStatsRecord {
pending_count: stats.pending_count,
delayed_pending_count: stats.delayed_pending_count,
claimable_pending_count: stats.claimable_pending_count,
running_active_count: stats.running_active_count,
expired_running_count: stats.expired_running_count,
terminal_count: stats.terminal_count,
claimable_count: stats.claimable_count,
oldest_claimable_age_micros: stats.oldest_claimable_age_micros,
now_micros: stats.now_micros,
})
}
fn map_external_generation_job_snapshot(
snapshot: ExternalGenerationJobSnapshot,
) -> ExternalGenerationJobRecord {
@@ -199,3 +223,16 @@ pub struct ExternalGenerationJobRecord {
pub updated_at: String,
pub lease_token: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExternalGenerationQueueStatsRecord {
pub pending_count: u32,
pub delayed_pending_count: u32,
pub claimable_pending_count: u32,
pub running_active_count: u32,
pub expired_running_count: u32,
pub terminal_count: u32,
pub claimable_count: u32,
pub oldest_claimable_age_micros: Option<i64>,
pub now_micros: i64,
}

6
server-rs/crates/spacetime-client/src/module_bindings.rs
View File

@@ -360,6 +360,8 @@ pub mod external_generation_job_renew_lease_input_type;
pub mod external_generation_job_snapshot_type;
pub mod external_generation_job_table;
pub mod external_generation_job_type;
pub mod external_generation_queue_stats_procedure_result_type;
pub mod external_generation_queue_stats_snapshot_type;
pub mod fail_ai_task_and_return_procedure;
pub mod fail_external_generation_job_and_return_procedure;
pub mod finalize_big_fish_agent_message_turn_procedure;
@@ -386,6 +388,7 @@ pub mod get_custom_world_agent_session_procedure;
pub mod get_custom_world_gallery_detail_by_code_procedure;
pub mod get_custom_world_gallery_detail_procedure;
pub mod get_custom_world_library_detail_procedure;
pub mod get_external_generation_queue_stats_and_return_procedure;
pub mod get_jump_hop_agent_session_procedure;
pub mod get_jump_hop_leaderboard_procedure;
pub mod get_jump_hop_run_procedure;
@@ -1491,6 +1494,8 @@ pub use external_generation_job_renew_lease_input_type::ExternalGenerationJobRen
pub use external_generation_job_snapshot_type::ExternalGenerationJobSnapshot;
pub use external_generation_job_table::*;
pub use external_generation_job_type::ExternalGenerationJob;
pub use external_generation_queue_stats_procedure_result_type::ExternalGenerationQueueStatsProcedureResult;
pub use external_generation_queue_stats_snapshot_type::ExternalGenerationQueueStatsSnapshot;
pub use fail_ai_task_and_return_procedure::fail_ai_task_and_return;
pub use fail_external_generation_job_and_return_procedure::fail_external_generation_job_and_return;
pub use finalize_big_fish_agent_message_turn_procedure::finalize_big_fish_agent_message_turn;
@@ -1517,6 +1522,7 @@ pub use get_custom_world_agent_session_procedure::get_custom_world_agent_session
pub use get_custom_world_gallery_detail_by_code_procedure::get_custom_world_gallery_detail_by_code;
pub use get_custom_world_gallery_detail_procedure::get_custom_world_gallery_detail;
pub use get_custom_world_library_detail_procedure::get_custom_world_library_detail;
pub use get_external_generation_queue_stats_and_return_procedure::get_external_generation_queue_stats_and_return;
pub use get_jump_hop_agent_session_procedure::get_jump_hop_agent_session;
pub use get_jump_hop_leaderboard_procedure::get_jump_hop_leaderboard;
pub use get_jump_hop_run_procedure::get_jump_hop_run;

19
server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_procedure_result_type.rs Normal file
View File

@@ -0,0 +1,19 @@
// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE
// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD.
#![allow(unused, clippy::all)]
use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws};
use super::external_generation_queue_stats_snapshot_type::ExternalGenerationQueueStatsSnapshot;
#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)]
#[sats(crate = __lib)]
pub struct ExternalGenerationQueueStatsProcedureResult {
pub ok: bool,
pub stats: Option<ExternalGenerationQueueStatsSnapshot>,
pub error_message: Option<String>,
}
impl __sdk::InModule for ExternalGenerationQueueStatsProcedureResult {
type Module = super::RemoteModule;
}

23
server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_snapshot_type.rs Normal file
View File

@@ -0,0 +1,23 @@
// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE
// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD.
#![allow(unused, clippy::all)]
use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws};
#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)]
#[sats(crate = __lib)]
pub struct ExternalGenerationQueueStatsSnapshot {
pub pending_count: u32,
pub delayed_pending_count: u32,
pub claimable_pending_count: u32,
pub running_active_count: u32,
pub expired_running_count: u32,
pub terminal_count: u32,
pub claimable_count: u32,
pub oldest_claimable_age_micros: Option<i64>,
pub now_micros: i64,
}
impl __sdk::InModule for ExternalGenerationQueueStatsSnapshot {
type Module = super::RemoteModule;
}

54
server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_queue_stats_and_return_procedure.rs Normal file
View File

@@ -0,0 +1,54 @@
// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE
// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD.
#![allow(unused, clippy::all)]
use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws};
use super::external_generation_queue_stats_procedure_result_type::ExternalGenerationQueueStatsProcedureResult;
#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)]
#[sats(crate = __lib)]
struct GetExternalGenerationQueueStatsAndReturnArgs {}
impl __sdk::InModule for GetExternalGenerationQueueStatsAndReturnArgs {
type Module = super::RemoteModule;
}
#[allow(non_camel_case_types)]
/// Extension trait for access to the procedure `get_external_generation_queue_stats_and_return`.
///
/// Implemented for [`super::RemoteProcedures`].
pub trait get_external_generation_queue_stats_and_return {
fn get_external_generation_queue_stats_and_return(&self) {
self.get_external_generation_queue_stats_and_return_then(|_, _| {});
}
fn get_external_generation_queue_stats_and_return_then(
&self,
__callback: impl FnOnce(
&super::ProcedureEventContext,
Result<ExternalGenerationQueueStatsProcedureResult, __sdk::InternalError>,
) + Send
+ 'static,
);
}
impl get_external_generation_queue_stats_and_return for super::RemoteProcedures {
fn get_external_generation_queue_stats_and_return_then(
&self,
__callback: impl FnOnce(
&super::ProcedureEventContext,
Result<ExternalGenerationQueueStatsProcedureResult, __sdk::InternalError>,
) + Send
+ 'static,
) {
self.imp
.invoke_procedure_with_callback::<_, ExternalGenerationQueueStatsProcedureResult>(
"get_external_generation_queue_stats_and_return",
GetExternalGenerationQueueStatsAndReturnArgs {},
__callback,
);
}
}