From cf5eac1c63d3e6c618fc6f7a22fd89badf484c76 Mon Sep 17 00:00:00 2001 From: kdletters Date: Sat, 15 Aug 2026 18:38:59 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8DJenkins=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E6=94=B6=E5=B0=BE=E7=8A=B6=E6=80=81=E7=AB=9E=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为队列、构建状态和部署产物查询增加有限重试 避免 Jenkins 瞬态无效响应永久覆盖成功部署状态 增加构建收尾竞态回归测试和技术说明 --- ...Jenkins容器预览部署控制面技术方案-2026-08-15.md | 2 + .../preview-deployer-server/src/jenkins.rs | 23 ++++++-- .../preview-deployer-server/src/tests.rs | 57 ++++++++++++++++++- 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/docs/technical/【开发运维】Jenkins容器预览部署控制面技术方案-2026-08-15.md b/docs/technical/【开发运维】Jenkins容器预览部署控制面技术方案-2026-08-15.md index 3c441be27..de41c048d 100644 --- a/docs/technical/【开发运维】Jenkins容器预览部署控制面技术方案-2026-08-15.md +++ b/docs/technical/【开发运维】Jenkins容器预览部署控制面技术方案-2026-08-15.md @@ -92,6 +92,8 @@ SpacetimeDB 2.7 CLI 发布到受控 Compose 网络地址时固定使用 `--yes=r 页面状态统一为 `queued / building / deploying / running / uninstalling / stopped / failed / cancelled`，健康状态统一为 `pending / healthy / unhealthy / unknown`。 +Jenkins 在构建完成、归档 artifact 和更新 REST 状态之间可能短暂返回不可解析的状态正文。控制服务对队列、构建状态和 artifact 查询执行有限重试；单次瞬态响应不得把已经成功并健康的部署永久写成 `failed`。 + ## 安全边界 - 服务端缺少控制面访问口令或 Jenkins service account 凭据时必须拒绝启动，不允许退化成匿名写接口。 diff --git a/server-rs/crates/preview-deployer-server/src/jenkins.rs b/server-rs/crates/preview-deployer-server/src/jenkins.rs index 840c4a0f0..27f8538c3 100644 --- a/server-rs/crates/preview-deployer-server/src/jenkins.rs +++ b/server-rs/crates/preview-deployer-server/src/jenkins.rs @@ -1,7 +1,7 @@ use std::{future::Future, time::Duration}; use reqwest::{Client, StatusCode, header}; -use serde::Deserialize; +use serde::{Deserialize, de::DeserializeOwned}; use url::Url; use crate::{Config, DeploymentStatus, HealthStatus}; @@ -209,7 +209,7 @@ impl JenkinsClient { .queue_url .join("api/json") .map_err(|error| error.to_string())?; - let item: QueueItem = self.get_json(item_url).await?; + let item: QueueItem = self.get_json_with_retry(item_url).await?; if item.cancelled.unwrap_or(false) { return Ok(JenkinsOutcome { success: false, @@ -229,7 +229,7 @@ impl JenkinsClient { let state_url = build_url .join("api/json") .map_err(|error| error.to_string())?; - let state: BuildState = self.get_json(state_url).await?; + let state: BuildState = self.get_json_with_retry(state_url).await?; if state.building { tokio::time::sleep(self.poll_interval).await; continue; @@ -247,7 +247,7 @@ impl JenkinsClient { let artifact_url = build_url .join("artifact/preview-result.json") .map_err(|error| error.to_string())?; - let result = self.get_json(artifact_url).await?; + let result = self.get_json_with_retry(artifact_url).await?; Ok(JenkinsOutcome { success: true, cancelled: false, @@ -273,6 +273,21 @@ impl JenkinsClient { .map_err(|_| "Jenkins 状态响应格式无效".to_string()) } + async fn get_json_with_retry(&self, url: Url) -> Result { + const MAX_ATTEMPTS: usize = 10; + let mut last_error = None; + for attempt in 1..=MAX_ATTEMPTS { + match self.get_json(url.clone()).await { + Ok(value) => return Ok(value), + Err(error) => last_error = Some(error), + } + if attempt < MAX_ATTEMPTS { + tokio::time::sleep(self.poll_interval).await; + } + } + Err(last_error.unwrap_or_else(|| "Jenkins 状态请求失败".to_string())) + } + fn resolve_trusted_url(&self, value: &str) -> Result { let returned = Url::parse(value) .or_else(|_| self.job_url.join(value)) diff --git a/server-rs/crates/preview-deployer-server/src/tests.rs b/server-rs/crates/preview-deployer-server/src/tests.rs index d78e065e2..03f90e6ed 100644 --- a/server-rs/crates/preview-deployer-server/src/tests.rs +++ b/server-rs/crates/preview-deployer-server/src/tests.rs @@ -1,5 +1,8 @@ use std::{ - sync::{Arc, Mutex}, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, time::Duration, }; @@ -25,6 +28,7 @@ const ORIGIN: &str = "http://preview.internal:8080"; #[derive(Clone, Default)] struct MockJenkinsState { requests: Arc>>, + invalid_build_responses: Arc, } #[derive(Debug)] @@ -99,8 +103,17 @@ async fn mock_queue(axum::extract::Path(id): axum::extract::Path) -> Json Json { - Json(json!({"building": false, "result": "SUCCESS"})) +async fn mock_build(State(state): State) -> impl IntoResponse { + if state + .invalid_build_responses + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return (StatusCode::OK, "Jenkins is finalizing the build").into_response(); + } + Json(json!({"building": false, "result": "SUCCESS"})).into_response() } async fn mock_artifact(axum::extract::Path(build): axum::extract::Path) -> Json { @@ -424,6 +437,44 @@ async fn deploy_and_uninstall_use_fixed_job_and_apply_owned_artifacts() { std::fs::remove_file(state_file).unwrap(); } +#[tokio::test] +async fn transient_invalid_build_status_is_retried_before_reading_artifact() { + let (jenkins_url, mock) = start_mock_jenkins().await; + mock.invalid_build_responses.store(1, Ordering::SeqCst); + let state = AppState::new(test_config(jenkins_url)).unwrap(); + let app = build_router(state.clone()); + let cookie = login_cookie(&app).await; + let id = super::derive_deployment_id("feature/preview-ui"); + let deploy_request = axum::http::Request::builder() + .method("POST") + .uri("/api/preview-deployer/deployments") + .header(header::HOST, HOST) + .header(header::ORIGIN, ORIGIN) + .header(header::COOKIE, &cookie) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(r#"{"branch":"feature/preview-ui"}"#)) + .unwrap(); + assert_eq!( + app.oneshot(deploy_request).await.unwrap().status(), + StatusCode::ACCEPTED + ); + wait_for_status(&state, &id, super::DeploymentStatus::Running) + .await + .expect("transient invalid Jenkins response is retried"); + assert_eq!( + state + .deployments + .read() + .await + .get(&id) + .unwrap() + .public + .health, + super::HealthStatus::Healthy + ); + std::fs::remove_file(&state.config.state_file).unwrap(); +} + #[tokio::test] async fn duplicate_active_branch_is_rejected_without_second_jenkins_trigger() { let (jenkins_url, mock) = start_mock_jenkins().await;