chore: unify production pipelines on linux

This commit is contained in:
2026-06-01 06:05:19 +00:00
parent 42a9f3cce1
commit 6cd554473b
7 changed files with 171 additions and 476 deletions

View File

@@ -10,7 +10,8 @@ pipeline {
}
environment {
GIT_REMOTE_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
GIT_REMOTE_URL = 'http://127.0.0.1:3000/GenarrativeAI/Genarrative.git'
GIT_REMOTE_FALLBACK_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
CARGO_HOME = '/home/dsk/.cache/genarrative-jenkins/api-server/cargo-home'
CARGO_TARGET_DIR = '/home/dsk/.cache/genarrative-jenkins/api-server/cargo-target/prod-release'
CARGO_INCREMENTAL = '0'
@@ -33,23 +34,36 @@ pipeline {
stages {
stage('Checkout') {
steps {
checkout([
$class: 'GitSCM',
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
],
userRemoteConfigs: [[url: "${GIT_REMOTE_URL}", refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
])
script {
def checkoutFromRemote = { String remoteUrl ->
checkout([
$class: 'GitSCM',
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
],
userRemoteConfigs: [[url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
])
}
try {
checkoutFromRemote(env.GIT_REMOTE_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
} catch (error) {
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL},改用备用地址: ${env.GIT_REMOTE_FALLBACK_URL}"
checkoutFromRemote(env.GIT_REMOTE_FALLBACK_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_FALLBACK_URL
}
}
sh '''
bash -lc '
set -euo pipefail
chmod +x scripts/jenkins-checkout-source.sh
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
COMMIT_HASH="${COMMIT_HASH:-}" \
GIT_REMOTE_URL="${GIT_REMOTE_URL}" \
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
GIT_REMOTE_FALLBACK_URL="${GIT_REMOTE_FALLBACK_URL:-}" \
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
scripts/jenkins-checkout-source.sh
'

View File

@@ -1,5 +1,7 @@
pipeline {
agent none
agent {
label 'linux && genarrative-build'
}
options {
disableConcurrentBuilds()

View File

@@ -1,27 +1,3 @@
def runWindowsPowerShell(String scriptName, String scriptBody) {
def scriptPath = ".jenkins-${scriptName}.ps1"
writeFile file: scriptPath, text: scriptBody, encoding: 'UTF-8'
bat label: "PowerShell ${scriptName}", script: """
@echo off
setlocal
set "GENARRATIVE_POWERSHELL=%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"
if not exist "%GENARRATIVE_POWERSHELL%" (
echo [jenkins-powershell] powershell.exe not found: %GENARRATIVE_POWERSHELL%
exit /b 1
)
echo [jenkins-powershell] user:
whoami
echo [jenkins-powershell] workspace: %CD%
echo [jenkins-powershell] exe: %GENARRATIVE_POWERSHELL%
if not exist "%CD%\\${scriptPath}" (
echo [jenkins-powershell] script not found: %CD%\\${scriptPath}
exit /b 1
)
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "try { \$path = Join-Path (Get-Location).ProviderPath '${scriptPath}'; Write-Host '[jenkins-powershell] script:' \$path; \$text = [System.IO.File]::ReadAllText(\$path, [System.Text.Encoding]::UTF8); Write-Host '[jenkins-powershell] loaded bytes:' ([System.IO.File]::ReadAllBytes(\$path).Length); \$scriptBlock = [ScriptBlock]::Create(\$text); & \$scriptBlock; if (\$LASTEXITCODE -is [int] -and \$LASTEXITCODE -ne 0) { exit \$LASTEXITCODE } } catch { Write-Host '[jenkins-powershell] failed:' \$_.Exception.Message; if (\$_.ScriptStackTrace) { Write-Host \$_.ScriptStackTrace }; exit 1 }"
exit /b %ERRORLEVEL%
"""
}
pipeline {
agent none
@@ -46,11 +22,11 @@ pipeline {
string(name: 'COMMIT_HASH', defaultValue: '', description: '部署脚本来源 commit')
string(name: 'SERVER_NAME', defaultValue: 'genarrative.example.com', description: '证书主域名;也作为 Nginx server_name 的第一个域名')
string(name: 'SERVER_ALIASES', defaultValue: '', description: '可选,额外 Nginx server_name多个用空格或逗号分隔例如 www.genarrative.world')
string(name: 'PROVISION_DOWNLOADS_DIR', defaultValue: 'provision-tool-downloads', description: 'Windows 下载阶段暂存 SpacetimeDB/otelcol 安装包的工作区相对目录')
string(name: 'PROVISION_DOWNLOADS_DIR', defaultValue: 'provision-tool-downloads', description: 'Linux 预下载阶段暂存 SpacetimeDB/otelcol 安装包的工作区相对目录')
string(name: 'PROVISION_TOOLS_DIR', defaultValue: 'provision-tools', description: '目标机工作区内由已下载安装包生成的工具包目录')
string(name: 'PROVISION_DOWNLOAD_PROXY', defaultValue: '', description: '可选,Windows 下载 SpacetimeDB 和 otelcol-contrib 时使用的代理地址,例如 http://127.0.0.1:7890留空不设置代理')
string(name: 'SPACETIME_DOWNLOAD_ROOT', defaultValue: 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download', description: 'Windows 下载 SpacetimeDB Linux release tarball 的根地址;目标机不访问该地址')
string(name: 'SPACETIME_TARGET_HOST', defaultValue: 'x86_64-unknown-linux-gnu', description: '目标机 SpacetimeDB 预编译包 host tripledevelopment/release Linux amd64 使用默认值')
string(name: 'PROVISION_DOWNLOAD_PROXY', defaultValue: '', description: '可选,Linux 预下载阶段下载 SpacetimeDB 和 otelcol-contrib 时使用的代理地址,例如 http://127.0.0.1:7890留空不设置代理')
string(name: 'SPACETIME_DOWNLOAD_ROOT', defaultValue: 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download', description: 'Linux 预下载阶段使用的 SpacetimeDB Linux release tarball 地址')
string(name: 'SPACETIME_TARGET_HOST', defaultValue: 'x86_64-unknown-linux-gnu', description: 'SpacetimeDB 预编译包 host tripledevelopment/release Linux amd64 使用默认值')
string(name: 'SPACETIME_ROOT', defaultValue: '/stdb', description: 'SpacetimeDB root-dir')
string(name: 'RELEASE_ROOT', defaultValue: '/opt/genarrative/releases', description: 'release 根目录')
string(name: 'CURRENT_LINK', defaultValue: '/opt/genarrative/current', description: '当前版本软链接')
@@ -66,7 +42,7 @@ pipeline {
stages {
stage('Prepare') {
agent {
label 'windows'
label 'linux && genarrative-build'
}
steps {
script {
@@ -134,258 +110,64 @@ pipeline {
}
}
stage('Download Provision Tool Archives') {
stage('Prepare Provision Tools') {
agent {
label 'windows'
label 'linux && genarrative-build'
}
steps {
script {
runWindowsPowerShell('server-provision-tool-downloads', '''
$ErrorActionPreference = 'Stop'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$downloadsDir = if ($env:PROVISION_DOWNLOADS_DIR) { $env:PROVISION_DOWNLOADS_DIR } else { 'provision-tool-downloads' }
$otelVersion = if ($env:OTELCOL_VERSION) { $env:OTELCOL_VERSION } else { '0.151.0' }
$prepareOtel = if ($env:ENABLE_OTELCOL) { $env:ENABLE_OTELCOL } else { 'true' }
$otelRoot = if ($env:OTELCOL_DOWNLOAD_ROOT) { $env:OTELCOL_DOWNLOAD_ROOT.TrimEnd('/') } else { 'https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download' }
$spacetimeDownloadRoot = if ($env:SPACETIME_DOWNLOAD_ROOT) { $env:SPACETIME_DOWNLOAD_ROOT.TrimEnd('/') } else { 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download' }
$spacetimeTargetHost = if ($env:SPACETIME_TARGET_HOST) { $env:SPACETIME_TARGET_HOST } else { 'x86_64-unknown-linux-gnu' }
$downloadProxy = if ($env:PROVISION_DOWNLOAD_PROXY) { $env:PROVISION_DOWNLOAD_PROXY } else { '' }
$workspace = (Get-Location).ProviderPath
if ([System.IO.Path]::IsPathRooted($downloadsDir)) {
throw "[prepare-provision-downloads] PROVISION_DOWNLOADS_DIR 只能是工作区内相对路径: ${downloadsDir}"
}
$downloadsDir = Join-Path $workspace $downloadsDir
Write-Host "[prepare-provision-downloads] Windows workspace: ${workspace}"
Write-Host "[prepare-provision-downloads] download dir: ${downloadsDir}"
if (Test-Path -LiteralPath $downloadsDir) {
Write-Host "[prepare-provision-downloads] 复用已有下载目录: ${downloadsDir}"
} else {
New-Item -ItemType Directory -Force -Path $downloadsDir | Out-Null
Write-Host "[prepare-provision-downloads] 已创建下载目录: ${downloadsDir}"
}
if ($downloadProxy) {
$env:HTTP_PROXY = $downloadProxy
$env:HTTPS_PROXY = $downloadProxy
$env:ALL_PROXY = $downloadProxy
Write-Host "[prepare-provision-downloads] 已配置 Windows 下载代理: $($downloadProxy -replace '://.*', '://***')"
}
function Get-GithubReleaseAssetDigest {
param(
[Parameter(Mandatory=$true)][string]$Repository,
[Parameter(Mandatory=$true)][string]$ReleaseSelector,
[Parameter(Mandatory=$true)][string]$AssetName
)
$request = @{
Uri = "https://api.github.com/repos/${Repository}/${ReleaseSelector}"
Headers = @{
Accept = 'application/vnd.github+json'
'User-Agent' = 'Genarrative-Server-Provision'
}
ErrorAction = 'Stop'
}
if ($downloadProxy) {
$request.Proxy = $downloadProxy
}
Write-Host "[prepare-provision-downloads] 查询 GitHub digest: repo=${Repository} release=${ReleaseSelector} asset=${AssetName}"
$release = Invoke-RestMethod @request
$asset = $release.assets | Where-Object { $_.name -eq $AssetName } | Select-Object -First 1
if (-not $asset) {
throw "[prepare-provision-downloads] GitHub release 未找到资产: ${Repository}/${AssetName}"
}
if (-not $asset.digest) {
throw "[prepare-provision-downloads] GitHub release 未返回 digest: ${Repository}/${AssetName}"
}
Write-Host "[prepare-provision-downloads] GitHub digest ${AssetName}: $($asset.digest)"
return $asset.digest
}
function Get-FileDigest {
param(
[Parameter(Mandatory=$true)][string]$Path,
[Parameter(Mandatory=$true)][string]$Algorithm
)
$fileHash = Get-FileHash -Algorithm $Algorithm -LiteralPath $Path
return $fileHash.Hash.ToLowerInvariant()
}
function Test-DownloadDigestMatch {
param(
[Parameter(Mandatory=$true)][string]$Path,
[Parameter(Mandatory=$true)][string]$ExpectedDigest
)
$parts = $ExpectedDigest.Split(':', 2)
if ($parts.Length -ne 2) {
throw "[prepare-provision-downloads] 无法解析 GitHub digest: ${ExpectedDigest}"
}
$algorithm = $parts[0].Trim().ToLowerInvariant()
$expectedHash = $parts[1].Trim().ToLowerInvariant()
if ($algorithm -ne 'sha256') {
throw "[prepare-provision-downloads] 暂不支持的 GitHub digest 算法: ${algorithm}"
}
$localHash = Get-FileDigest -Path $Path -Algorithm 'SHA256'
return $localHash -eq $expectedHash
}
function Invoke-ProvisionDownload {
param(
[Parameter(Mandatory=$true)][string]$Label,
[Parameter(Mandatory=$true)][string]$Url,
[Parameter(Mandatory=$true)][string]$Output,
[string]$ExpectedDigest = ''
)
if ($ExpectedDigest) {
if (Test-Path -LiteralPath $Output) {
if (Test-DownloadDigestMatch -Path $Output -ExpectedDigest $ExpectedDigest) {
$existingItem = Get-Item -LiteralPath $Output
Write-Host "[prepare-provision-downloads] 已存在且校验一致,跳过下载: ${Label} bytes=$($existingItem.Length) path=${Output}"
return
}
Write-Host "[prepare-provision-downloads] 已存在但校验不一致,重新下载: ${Label} path=${Output}"
}
}
Write-Host "[prepare-provision-downloads] 下载 ${Label}: ${Url}"
$tempOutput = "${Output}.download"
if (Test-Path -LiteralPath $tempOutput) {
$tempItem = Get-Item -LiteralPath $tempOutput
if ($ExpectedDigest -and $tempItem.Length -gt 0 -and (Test-DownloadDigestMatch -Path $tempOutput -ExpectedDigest $ExpectedDigest)) {
Move-Item -LiteralPath $tempOutput -Destination $Output -Force
$finalItem = Get-Item -LiteralPath $Output
Write-Host "[prepare-provision-downloads] 已复用校验通过的临时下载: ${Label} bytes=$($finalItem.Length) path=${Output}"
return
}
if ($tempItem.Length -gt 0) {
Write-Host "[prepare-provision-downloads] 发现未完成临时文件,后续尝试断点续传: ${Label} bytes=$($tempItem.Length) path=${tempOutput}"
} else {
Remove-Item -LiteralPath $tempOutput -Force
}
}
$curl = Get-Command curl.exe -ErrorAction SilentlyContinue
$maxAttempts = 8
for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) {
$resumeBytes = 0
if (Test-Path -LiteralPath $tempOutput) {
$resumeBytes = (Get-Item -LiteralPath $tempOutput).Length
}
try {
if ($curl) {
$arguments = @('-fL', '--retry', '3', '--retry-delay', '3', '--retry-all-errors', '--connect-timeout', '30', '--speed-time', '60', '--speed-limit', '1024')
if ($resumeBytes -gt 0) {
$arguments += @('-C', '-')
Write-Host "[prepare-provision-downloads] curl 断点续传 ${Label}: attempt=${attempt}/${maxAttempts} resumeBytes=${resumeBytes}"
} else {
Write-Host "[prepare-provision-downloads] curl 下载 ${Label}: attempt=${attempt}/${maxAttempts}"
}
$arguments += @('-o', $tempOutput)
if ($downloadProxy) {
$arguments += @('--proxy', $downloadProxy)
}
$arguments += $Url
& $curl.Source @arguments
$exitCode = $LASTEXITCODE
if ($exitCode -ne 0) {
$currentBytes = if (Test-Path -LiteralPath $tempOutput) { (Get-Item -LiteralPath $tempOutput).Length } else { 0 }
Write-Host "[prepare-provision-downloads] curl 下载未完成: ${Label}, attempt=${attempt}/${maxAttempts}, exit=${exitCode}, tempBytes=${currentBytes}"
if ($attempt -lt $maxAttempts) {
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
continue
}
throw "[prepare-provision-downloads] curl 下载失败: ${Label}, exit=${exitCode}, temp=${tempOutput}"
}
} else {
Write-Host "[prepare-provision-downloads] Invoke-WebRequest 下载 ${Label}: attempt=${attempt}/${maxAttempts}"
if ($resumeBytes -gt 0) {
Write-Host "[prepare-provision-downloads] Invoke-WebRequest 不支持断点续传,删除临时文件后重新下载: ${Label}, bytes=${resumeBytes}"
Remove-Item -LiteralPath $tempOutput -Force
}
$parameters = @{
Uri = $Url
OutFile = $tempOutput
UseBasicParsing = $true
}
if ($downloadProxy) {
$parameters.Proxy = $downloadProxy
}
Invoke-WebRequest @parameters
}
} catch {
$currentBytes = if (Test-Path -LiteralPath $tempOutput) { (Get-Item -LiteralPath $tempOutput).Length } else { 0 }
Write-Host "[prepare-provision-downloads] 下载尝试失败: ${Label}, attempt=${attempt}/${maxAttempts}, tempBytes=${currentBytes}, error=$($_.Exception.Message)"
if ($attempt -lt $maxAttempts) {
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
continue
}
throw
}
if (-not (Test-Path -LiteralPath $tempOutput)) {
throw "[prepare-provision-downloads] 下载未生成临时文件: ${tempOutput}"
}
$item = Get-Item -LiteralPath $tempOutput
if ($item.Length -le 0) {
if ($attempt -lt $maxAttempts) {
Write-Host "[prepare-provision-downloads] 下载结果为空,将重试: ${Label}"
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
continue
}
throw "[prepare-provision-downloads] 下载结果为空: ${tempOutput}"
}
if ($ExpectedDigest) {
if (-not (Test-DownloadDigestMatch -Path $tempOutput -ExpectedDigest $ExpectedDigest)) {
Write-Host "[prepare-provision-downloads] 下载结果校验未通过,将继续重试: ${Label}, attempt=${attempt}/${maxAttempts}, tempBytes=$($item.Length)"
if ($attempt -lt $maxAttempts) {
Remove-Item -LiteralPath $tempOutput -Force
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
continue
}
throw "[prepare-provision-downloads] 下载结果校验失败: ${Label}, temp=${tempOutput}"
}
}
Move-Item -LiteralPath $tempOutput -Destination $Output -Force
$finalItem = Get-Item -LiteralPath $Output
Write-Host "[prepare-provision-downloads] 已下载 ${Label}: bytes=$($finalItem.Length) path=${Output}"
return
}
throw "[prepare-provision-downloads] 下载重试耗尽: ${Label}"
}
$spacetimeArchiveName = "spacetime-${spacetimeTargetHost}.tar.gz"
$spacetimeArchiveUrl = "${spacetimeDownloadRoot}/${spacetimeArchiveName}"
$spacetimeArchiveDigest = Get-GithubReleaseAssetDigest -Repository 'clockworklabs/SpacetimeDB' -ReleaseSelector 'releases/latest' -AssetName $spacetimeArchiveName
Invoke-ProvisionDownload -Label "SpacetimeDB release tarball ${spacetimeTargetHost}" -Url $spacetimeArchiveUrl -Output (Join-Path $downloadsDir $spacetimeArchiveName) -ExpectedDigest $spacetimeArchiveDigest
if ($prepareOtel -eq 'true') {
$otelArchiveName = "otelcol-contrib_${otelVersion}_linux_amd64.tar.gz"
$otelUrl = "${otelRoot}/v${otelVersion}/${otelArchiveName}"
$otelDigest = Get-GithubReleaseAssetDigest -Repository 'open-telemetry/opentelemetry-collector-releases' -ReleaseSelector "releases/tags/v${otelVersion}" -AssetName $otelArchiveName
Invoke-ProvisionDownload -Label "otelcol-contrib ${otelVersion} linux amd64" -Url $otelUrl -Output (Join-Path $downloadsDir $otelArchiveName) -ExpectedDigest $otelDigest
} else {
Write-Host "[prepare-provision-downloads] ENABLE_OTELCOL=${prepareOtel},跳过 otelcol-contrib 下载。"
}
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
$manifest = @(
"spacetime release tarball ${spacetimeArchiveUrl}",
"spacetime target host ${spacetimeTargetHost}",
"otelcol-contrib ${otelVersion} prepare=${prepareOtel}"
)
[System.IO.File]::WriteAllLines((Join-Path $downloadsDir 'DOWNLOADS-MANIFEST.txt'), $manifest, $utf8NoBom)
Get-ChildItem -LiteralPath $downloadsDir | Sort-Object Name | ForEach-Object {
Write-Host "[prepare-provision-downloads] artifact $($_.Length) $($_.Name)"
}
''')
def checkoutFromRemote = { String remoteUrl ->
checkout([
$class: 'GitSCM',
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
],
userRemoteConfigs: [[url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
])
}
try {
checkoutFromRemote(env.GIT_REMOTE_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
} catch (error) {
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL},改用备用地址: ${env.GIT_REMOTE_FALLBACK_URL}"
checkoutFromRemote(env.GIT_REMOTE_FALLBACK_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_FALLBACK_URL
}
}
stash name: 'server-provision-tool-downloads', includes: "${params.PROVISION_DOWNLOADS_DIR}/**", useDefaultExcludes: false
sh '''
bash <<'BASH'
set -euo pipefail
chmod +x scripts/jenkins-checkout-source.sh
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
COMMIT_HASH="${COMMIT_HASH:-}" \
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
GIT_REMOTE_FALLBACK_URL="${GIT_REMOTE_FALLBACK_URL:-}" \
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
scripts/jenkins-checkout-source.sh
BASH
'''
sh '''
bash -lc '
set -euo pipefail
chmod +x scripts/prepare-server-provision-tools.sh
PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" \
PROVISION_DOWNLOADS_DIR="${PROVISION_DOWNLOADS_DIR:-provision-tool-downloads}" \
OTELCOL_VERSION="${OTELCOL_VERSION:-0.151.0}" \
PREPARE_OTELCOL="${ENABLE_OTELCOL:-true}" \
PROVISION_DOWNLOAD_PROXY="${PROVISION_DOWNLOAD_PROXY:-}" \
SPACETIME_DOWNLOAD_ROOT="${SPACETIME_DOWNLOAD_ROOT:-https://github.com/clockworklabs/SpacetimeDB/releases/latest/download}" \
SPACETIME_TARGET_HOST="${SPACETIME_TARGET_HOST:-x86_64-unknown-linux-gnu}" \
scripts/prepare-server-provision-tools.sh
'
'''
script {
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
echo "Provision 工具包源码 commit=${env.SOURCE_COMMIT}"
}
stash name: 'server-provision-tools', includes: "${params.PROVISION_TOOLS_DIR}/**", useDefaultExcludes: false
}
}
@@ -440,20 +222,10 @@ BASH
label "${params.DEPLOY_TARGET == 'development' ? 'linux && genarrative-build' : 'linux && genarrative-release-deploy'}"
}
steps {
unstash 'server-provision-tool-downloads'
unstash 'server-provision-tools'
sh '''
bash <<'BASH'
set -euo pipefail
chmod +x scripts/prepare-server-provision-tools.sh
PROVISION_DOWNLOADS_DIR="${PROVISION_DOWNLOADS_DIR:-provision-tool-downloads}" \
PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" \
OTELCOL_VERSION="${OTELCOL_VERSION:-0.151.0}" \
PREPARE_OTELCOL="${ENABLE_OTELCOL:-true}" \
PROVISION_REQUIRE_LOCAL_DOWNLOADS="true" \
SPACETIME_DOWNLOAD_ROOT="${SPACETIME_DOWNLOAD_ROOT:-https://github.com/clockworklabs/SpacetimeDB/releases/latest/download}" \
SPACETIME_TARGET_HOST="${SPACETIME_TARGET_HOST:-x86_64-unknown-linux-gnu}" \
scripts/prepare-server-provision-tools.sh
if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
chmod +x "${PROVISION_TOOLS_DIR:-provision-tools}/otelcol-contrib"
fi
@@ -504,4 +276,4 @@ BASH
echo "Server provision 完成: target=${params.DEPLOY_TARGET}, dryRun=${params.DRY_RUN}, nginxConfigMode=${params.NGINX_CONFIG_MODE}"
}
}
}
}

View File

@@ -1,27 +1,6 @@
def runWindowsPowerShell(String scriptName, String scriptBody) {
def scriptPath = ".jenkins-${scriptName}.ps1"
writeFile file: scriptPath, text: scriptBody, encoding: 'UTF-8'
bat label: "PowerShell ${scriptName}", script: """
@echo off
setlocal
set "GENARRATIVE_POWERSHELL=%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"
if not exist "%GENARRATIVE_POWERSHELL%" (
echo [jenkins-powershell] powershell.exe not found: %GENARRATIVE_POWERSHELL%
exit /b 1
)
echo [jenkins-powershell] user:
whoami
echo [jenkins-powershell] exe: %GENARRATIVE_POWERSHELL%
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "\$path = '%CD%\\${scriptPath}'; \$text = [System.IO.File]::ReadAllText(\$path, [System.Text.Encoding]::UTF8); \$utf8Bom = New-Object System.Text.UTF8Encoding(\$true); [System.IO.File]::WriteAllText(\$path, \$text, \$utf8Bom)"
if errorlevel 1 exit /b %ERRORLEVEL%
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "%CD%\\${scriptPath}"
exit /b %ERRORLEVEL%
"""
}
pipeline {
agent {
label 'windows'
label 'linux && genarrative-build'
}
options {
@@ -31,12 +10,10 @@ pipeline {
}
environment {
GIT_REMOTE_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
CARGO_HOME = '${env.WORKSPACE_TMP}/cargo-home'
CARGO_TARGET_DIR = '${env.WORKSPACE_TMP}/cargo-target/prod-release'
GIT_REMOTE_URL = 'http://127.0.0.1:3000/GenarrativeAI/Genarrative.git'
GIT_REMOTE_FALLBACK_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
CARGO_INCREMENTAL = '0'
RUSTC_WRAPPER = 'sccache'
SCCACHE_DIR = '${env.USERPROFILE}\\.cache\\sccache-stdb-module'
SCCACHE_CACHE_SIZE = '30G'
}
@@ -56,106 +33,42 @@ pipeline {
stages {
stage('Checkout') {
steps {
checkout([
$class: 'GitSCM',
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
],
userRemoteConfigs: [[url: "${GIT_REMOTE_URL}", refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
])
script {
runWindowsPowerShell('stdb-checkout', '''
$ErrorActionPreference = 'Stop'
$sourceBranch = if ($env:SOURCE_BRANCH) { $env:SOURCE_BRANCH } else { 'master' }
$commitHash = if ($env:COMMIT_HASH) { $env:COMMIT_HASH } else { '' }
$gitRemoteUrl = if ($env:GIT_REMOTE_URL) { $env:GIT_REMOTE_URL } else { 'https://git.genarrative.world/GenarrativeAI/Genarrative.git' }
function Invoke-GitCommand {
param(
[string]$Label,
[string[]]$Arguments
)
Write-Host "[stdb-checkout] $Label"
& git @Arguments
$exitCode = $LASTEXITCODE
if ($exitCode -ne 0) {
throw "[stdb-checkout] $Label failed with exit code $exitCode"
}
}
Write-Host "[stdb-checkout] sourceBranch: $sourceBranch"
Write-Host "[stdb-checkout] remote: $gitRemoteUrl"
$currentCommit = (git rev-parse HEAD).Trim()
if ($LASTEXITCODE -ne 0 -or -not $currentCommit) {
throw '[stdb-checkout] cannot resolve current HEAD'
}
Write-Host "[stdb-checkout] current HEAD: $currentCommit"
if ($commitHash) {
Write-Host "[stdb-checkout] requested commit: $commitHash"
$resolvedCommit = (git rev-parse --verify "${commitHash}^{commit}" 2>$null).Trim()
if ($LASTEXITCODE -eq 0 -and $resolvedCommit -eq $currentCommit) {
Write-Host '[stdb-checkout] requested commit already matches Jenkins GitSCM checkout'
} else {
Invoke-GitCommand -Label 'fetch source branch history' -Arguments @(
'fetch',
'--no-tags',
'--prune',
$gitRemoteUrl,
"+refs/heads/${sourceBranch}:refs/remotes/origin/${sourceBranch}"
)
$isShallowRepository = (git rev-parse --is-shallow-repository 2>$null).Trim()
if ($LASTEXITCODE -ne 0) {
throw '[stdb-checkout] cannot determine whether repository is shallow'
}
if ($isShallowRepository -eq 'true') {
Invoke-GitCommand -Label 'deepen source branch history' -Arguments @(
'fetch',
'--unshallow',
'--no-tags',
$gitRemoteUrl,
"+refs/heads/${sourceBranch}:refs/remotes/origin/${sourceBranch}"
)
}
Invoke-GitCommand -Label 'validate source branch ref' -Arguments @(
'cat-file',
'-e',
"refs/remotes/origin/${sourceBranch}^{commit}"
)
Invoke-GitCommand -Label 'validate requested commit' -Arguments @(
'cat-file',
'-e',
"${commitHash}^{commit}"
)
$resolvedCommit = (git rev-parse --verify "${commitHash}^{commit}").Trim()
if ($LASTEXITCODE -ne 0 -or -not $resolvedCommit) {
throw "[stdb-checkout] cannot resolve requested commit: $commitHash"
}
Invoke-GitCommand -Label 'validate requested commit belongs to branch' -Arguments @(
'merge-base',
'--is-ancestor',
$resolvedCommit,
"refs/remotes/origin/${sourceBranch}"
)
Invoke-GitCommand -Label "checkout commit $resolvedCommit" -Arguments @(
'checkout',
'--force',
$resolvedCommit
)
}
} else {
Write-Host "[stdb-checkout] COMMIT_HASH empty, reusing Jenkins GitSCM checkout result"
}
$resolvedCommit = (git rev-parse HEAD).Trim()
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
[System.IO.File]::WriteAllText((Join-Path (Get-Location) '.jenkins-source-commit'), "$resolvedCommit`n", $utf8NoBom)
''')
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').replace('\uFEFF', '').trim()
def checkoutFromRemote = { String remoteUrl ->
checkout([
$class: 'GitSCM',
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
],
userRemoteConfigs: [[url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
])
}
try {
checkoutFromRemote(env.GIT_REMOTE_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
} catch (error) {
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL},改用备用地址: ${env.GIT_REMOTE_FALLBACK_URL}"
checkoutFromRemote(env.GIT_REMOTE_FALLBACK_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_FALLBACK_URL
}
}
sh '''
bash -lc '
set -euo pipefail
chmod +x scripts/jenkins-checkout-source.sh
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
COMMIT_HASH="${COMMIT_HASH:-}" \
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
GIT_REMOTE_FALLBACK_URL="${GIT_REMOTE_FALLBACK_URL:-}" \
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
scripts/jenkins-checkout-source.sh
'
'''
script {
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
env.EFFECTIVE_BUILD_VERSION = params.BUILD_VERSION?.trim() ? params.BUILD_VERSION.trim() : env.BUILD_NUMBER
}
}
@@ -165,45 +78,27 @@ pipeline {
steps {
script {
def buildStep = {
runWindowsPowerShell('stdb-build', '''
$ErrorActionPreference = 'Stop'
$workspaceTmp = if ($env:WORKSPACE_TMP) { $env:WORKSPACE_TMP } else { "$env:WORKSPACE@tmp" }
$env:CARGO_HOME = "$workspaceTmp/cargo-home"
$env:CARGO_TARGET_DIR = "$workspaceTmp/cargo-target/prod-release"
$env:SCCACHE_DIR = "$env:USERPROFILE/.cache/sccache-stdb-module"
$env:PATH = "$env:CARGO_HOME/bin;$env:PATH"
$gitBash = @(
$env:GENARRATIVE_BASH,
'C:/Program Files/Git/bin/bash.exe',
'C:/Program Files/Git/usr/bin/bash.exe',
'C:/msys64/usr/bin/bash.exe',
'bash'
) | Where-Object { $_ -and (($_ -eq 'bash') -or (Test-Path $_)) } | Select-Object -First 1
if (-not $gitBash) {
throw '[stdb-build] Windows 构建节点缺少 Git Bash无法执行仓库现有生产构建脚本。请先安装 Git for Windows并确保 bash 在 PATH 中。'
}
$env:GENARRATIVE_BASH = $gitBash
if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
throw '[stdb-build] 缺少 cargo。请先在 Windows 构建节点安装 Rust 工具链,并确保 cargo 在 PATH 中。'
}
# sccache 只是可选缓存PATH 中存在但不可执行时必须回退到 rustc。
$sccacheCommand = Get-Command sccache -ErrorAction SilentlyContinue
$sccacheUsable = $false
if ($sccacheCommand) {
try {
& $sccacheCommand.Source --version | Out-Host
$sccacheUsable = $true
} catch {
Write-Host "[stdb-build] sccache 无法执行:$($_.Exception.Message)"
}
}
if (-not $sccacheUsable) {
Write-Host '[stdb-build] 未找到可用 sccache改用 rustc 直接构建。'
Remove-Item Env:RUSTC_WRAPPER -ErrorAction SilentlyContinue
}
npm run build:production-release -- --component spacetime-module --name "$env:EFFECTIVE_BUILD_VERSION"
sh '''
bash -lc '
set -euo pipefail
workspace_tmp="${WORKSPACE_TMP:-${WORKSPACE}@tmp}"
export CARGO_HOME="${workspace_tmp}/cargo-home"
export CARGO_TARGET_DIR="${workspace_tmp}/cargo-target/prod-release"
export CARGO_INCREMENTAL=0
export RUSTC_WRAPPER=sccache
export SCCACHE_DIR="${workspace_tmp}/sccache-stdb-module"
export SCCACHE_CACHE_SIZE=30G
mkdir -p "${CARGO_HOME}" "${CARGO_TARGET_DIR}" "${SCCACHE_DIR}"
chmod +x scripts/jenkins-prepare-cargo-env.sh
source scripts/jenkins-prepare-cargo-env.sh
if ! command -v sccache >/dev/null 2>&1; then
echo "[stdb-build] 未找到可用 sccache改用 rustc 直接构建。"
unset RUSTC_WRAPPER
fi
SOURCE_BRANCH="${SOURCE_BRANCH}" SOURCE_COMMIT="${SOURCE_COMMIT}" \
npm run build:production-release -- --component spacetime-module --name "${EFFECTIVE_BUILD_VERSION}"
'
'''
)
}
if (params.MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID?.trim()) {
withCredentials([