chore: unify production pipelines on linux
This commit is contained in:
@@ -10,7 +10,8 @@ pipeline {
|
||||
}
|
||||
|
||||
environment {
|
||||
GIT_REMOTE_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
|
||||
GIT_REMOTE_URL = 'http://127.0.0.1:3000/GenarrativeAI/Genarrative.git'
|
||||
GIT_REMOTE_FALLBACK_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
|
||||
CARGO_HOME = '/home/dsk/.cache/genarrative-jenkins/api-server/cargo-home'
|
||||
CARGO_TARGET_DIR = '/home/dsk/.cache/genarrative-jenkins/api-server/cargo-target/prod-release'
|
||||
CARGO_INCREMENTAL = '0'
|
||||
@@ -33,23 +34,36 @@ pipeline {
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
checkout([
|
||||
$class: 'GitSCM',
|
||||
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
|
||||
doGenerateSubmoduleConfigurations: false,
|
||||
extensions: [
|
||||
[$class: 'CleanBeforeCheckout'],
|
||||
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
|
||||
],
|
||||
userRemoteConfigs: [[url: "${GIT_REMOTE_URL}", refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
|
||||
])
|
||||
script {
|
||||
def checkoutFromRemote = { String remoteUrl ->
|
||||
checkout([
|
||||
$class: 'GitSCM',
|
||||
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
|
||||
doGenerateSubmoduleConfigurations: false,
|
||||
extensions: [
|
||||
[$class: 'CleanBeforeCheckout'],
|
||||
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
|
||||
],
|
||||
userRemoteConfigs: [[url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
|
||||
])
|
||||
}
|
||||
try {
|
||||
checkoutFromRemote(env.GIT_REMOTE_URL)
|
||||
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
|
||||
} catch (error) {
|
||||
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL},改用备用地址: ${env.GIT_REMOTE_FALLBACK_URL}"
|
||||
checkoutFromRemote(env.GIT_REMOTE_FALLBACK_URL)
|
||||
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_FALLBACK_URL
|
||||
}
|
||||
}
|
||||
sh '''
|
||||
bash -lc '
|
||||
set -euo pipefail
|
||||
chmod +x scripts/jenkins-checkout-source.sh
|
||||
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
|
||||
COMMIT_HASH="${COMMIT_HASH:-}" \
|
||||
GIT_REMOTE_URL="${GIT_REMOTE_URL}" \
|
||||
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
|
||||
GIT_REMOTE_FALLBACK_URL="${GIT_REMOTE_FALLBACK_URL:-}" \
|
||||
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
|
||||
scripts/jenkins-checkout-source.sh
|
||||
'
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
pipeline {
|
||||
agent none
|
||||
agent {
|
||||
label 'linux && genarrative-build'
|
||||
}
|
||||
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
|
||||
@@ -1,27 +1,3 @@
|
||||
def runWindowsPowerShell(String scriptName, String scriptBody) {
|
||||
def scriptPath = ".jenkins-${scriptName}.ps1"
|
||||
writeFile file: scriptPath, text: scriptBody, encoding: 'UTF-8'
|
||||
bat label: "PowerShell ${scriptName}", script: """
|
||||
@echo off
|
||||
setlocal
|
||||
set "GENARRATIVE_POWERSHELL=%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"
|
||||
if not exist "%GENARRATIVE_POWERSHELL%" (
|
||||
echo [jenkins-powershell] powershell.exe not found: %GENARRATIVE_POWERSHELL%
|
||||
exit /b 1
|
||||
)
|
||||
echo [jenkins-powershell] user:
|
||||
whoami
|
||||
echo [jenkins-powershell] workspace: %CD%
|
||||
echo [jenkins-powershell] exe: %GENARRATIVE_POWERSHELL%
|
||||
if not exist "%CD%\\${scriptPath}" (
|
||||
echo [jenkins-powershell] script not found: %CD%\\${scriptPath}
|
||||
exit /b 1
|
||||
)
|
||||
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "try { \$path = Join-Path (Get-Location).ProviderPath '${scriptPath}'; Write-Host '[jenkins-powershell] script:' \$path; \$text = [System.IO.File]::ReadAllText(\$path, [System.Text.Encoding]::UTF8); Write-Host '[jenkins-powershell] loaded bytes:' ([System.IO.File]::ReadAllBytes(\$path).Length); \$scriptBlock = [ScriptBlock]::Create(\$text); & \$scriptBlock; if (\$LASTEXITCODE -is [int] -and \$LASTEXITCODE -ne 0) { exit \$LASTEXITCODE } } catch { Write-Host '[jenkins-powershell] failed:' \$_.Exception.Message; if (\$_.ScriptStackTrace) { Write-Host \$_.ScriptStackTrace }; exit 1 }"
|
||||
exit /b %ERRORLEVEL%
|
||||
"""
|
||||
}
|
||||
|
||||
pipeline {
|
||||
agent none
|
||||
|
||||
@@ -46,11 +22,11 @@ pipeline {
|
||||
string(name: 'COMMIT_HASH', defaultValue: '', description: '部署脚本来源 commit')
|
||||
string(name: 'SERVER_NAME', defaultValue: 'genarrative.example.com', description: '证书主域名;也作为 Nginx server_name 的第一个域名')
|
||||
string(name: 'SERVER_ALIASES', defaultValue: '', description: '可选,额外 Nginx server_name,多个用空格或逗号分隔,例如 www.genarrative.world')
|
||||
string(name: 'PROVISION_DOWNLOADS_DIR', defaultValue: 'provision-tool-downloads', description: 'Windows 下载阶段暂存 SpacetimeDB/otelcol 安装包的工作区相对目录')
|
||||
string(name: 'PROVISION_DOWNLOADS_DIR', defaultValue: 'provision-tool-downloads', description: 'Linux 预下载阶段暂存 SpacetimeDB/otelcol 安装包的工作区相对目录')
|
||||
string(name: 'PROVISION_TOOLS_DIR', defaultValue: 'provision-tools', description: '目标机工作区内由已下载安装包生成的工具包目录')
|
||||
string(name: 'PROVISION_DOWNLOAD_PROXY', defaultValue: '', description: '可选,Windows 下载 SpacetimeDB 和 otelcol-contrib 时使用的代理地址,例如 http://127.0.0.1:7890;留空不设置代理')
|
||||
string(name: 'SPACETIME_DOWNLOAD_ROOT', defaultValue: 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download', description: 'Windows 下载 SpacetimeDB Linux release tarball 的根地址;目标机不访问该地址')
|
||||
string(name: 'SPACETIME_TARGET_HOST', defaultValue: 'x86_64-unknown-linux-gnu', description: '目标机 SpacetimeDB 预编译包 host triple,development/release Linux amd64 使用默认值')
|
||||
string(name: 'PROVISION_DOWNLOAD_PROXY', defaultValue: '', description: '可选,Linux 预下载阶段下载 SpacetimeDB 和 otelcol-contrib 时使用的代理地址,例如 http://127.0.0.1:7890;留空不设置代理')
|
||||
string(name: 'SPACETIME_DOWNLOAD_ROOT', defaultValue: 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download', description: 'Linux 预下载阶段使用的 SpacetimeDB Linux release tarball 根地址')
|
||||
string(name: 'SPACETIME_TARGET_HOST', defaultValue: 'x86_64-unknown-linux-gnu', description: 'SpacetimeDB 预编译包 host triple,development/release Linux amd64 使用默认值')
|
||||
string(name: 'SPACETIME_ROOT', defaultValue: '/stdb', description: 'SpacetimeDB root-dir')
|
||||
string(name: 'RELEASE_ROOT', defaultValue: '/opt/genarrative/releases', description: 'release 根目录')
|
||||
string(name: 'CURRENT_LINK', defaultValue: '/opt/genarrative/current', description: '当前版本软链接')
|
||||
@@ -66,7 +42,7 @@ pipeline {
|
||||
stages {
|
||||
stage('Prepare') {
|
||||
agent {
|
||||
label 'windows'
|
||||
label 'linux && genarrative-build'
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
@@ -134,258 +110,64 @@ pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
stage('Download Provision Tool Archives') {
|
||||
stage('Prepare Provision Tools') {
|
||||
agent {
|
||||
label 'windows'
|
||||
label 'linux && genarrative-build'
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
runWindowsPowerShell('server-provision-tool-downloads', '''
|
||||
$ErrorActionPreference = 'Stop'
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||
|
||||
$downloadsDir = if ($env:PROVISION_DOWNLOADS_DIR) { $env:PROVISION_DOWNLOADS_DIR } else { 'provision-tool-downloads' }
|
||||
$otelVersion = if ($env:OTELCOL_VERSION) { $env:OTELCOL_VERSION } else { '0.151.0' }
|
||||
$prepareOtel = if ($env:ENABLE_OTELCOL) { $env:ENABLE_OTELCOL } else { 'true' }
|
||||
$otelRoot = if ($env:OTELCOL_DOWNLOAD_ROOT) { $env:OTELCOL_DOWNLOAD_ROOT.TrimEnd('/') } else { 'https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download' }
|
||||
$spacetimeDownloadRoot = if ($env:SPACETIME_DOWNLOAD_ROOT) { $env:SPACETIME_DOWNLOAD_ROOT.TrimEnd('/') } else { 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download' }
|
||||
$spacetimeTargetHost = if ($env:SPACETIME_TARGET_HOST) { $env:SPACETIME_TARGET_HOST } else { 'x86_64-unknown-linux-gnu' }
|
||||
$downloadProxy = if ($env:PROVISION_DOWNLOAD_PROXY) { $env:PROVISION_DOWNLOAD_PROXY } else { '' }
|
||||
$workspace = (Get-Location).ProviderPath
|
||||
if ([System.IO.Path]::IsPathRooted($downloadsDir)) {
|
||||
throw "[prepare-provision-downloads] PROVISION_DOWNLOADS_DIR 只能是工作区内相对路径: ${downloadsDir}"
|
||||
}
|
||||
$downloadsDir = Join-Path $workspace $downloadsDir
|
||||
Write-Host "[prepare-provision-downloads] Windows workspace: ${workspace}"
|
||||
Write-Host "[prepare-provision-downloads] download dir: ${downloadsDir}"
|
||||
|
||||
if (Test-Path -LiteralPath $downloadsDir) {
|
||||
Write-Host "[prepare-provision-downloads] 复用已有下载目录: ${downloadsDir}"
|
||||
} else {
|
||||
New-Item -ItemType Directory -Force -Path $downloadsDir | Out-Null
|
||||
Write-Host "[prepare-provision-downloads] 已创建下载目录: ${downloadsDir}"
|
||||
}
|
||||
|
||||
if ($downloadProxy) {
|
||||
$env:HTTP_PROXY = $downloadProxy
|
||||
$env:HTTPS_PROXY = $downloadProxy
|
||||
$env:ALL_PROXY = $downloadProxy
|
||||
Write-Host "[prepare-provision-downloads] 已配置 Windows 下载代理: $($downloadProxy -replace '://.*', '://***')"
|
||||
}
|
||||
|
||||
function Get-GithubReleaseAssetDigest {
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][string]$Repository,
|
||||
[Parameter(Mandatory=$true)][string]$ReleaseSelector,
|
||||
[Parameter(Mandatory=$true)][string]$AssetName
|
||||
)
|
||||
|
||||
$request = @{
|
||||
Uri = "https://api.github.com/repos/${Repository}/${ReleaseSelector}"
|
||||
Headers = @{
|
||||
Accept = 'application/vnd.github+json'
|
||||
'User-Agent' = 'Genarrative-Server-Provision'
|
||||
}
|
||||
ErrorAction = 'Stop'
|
||||
}
|
||||
if ($downloadProxy) {
|
||||
$request.Proxy = $downloadProxy
|
||||
}
|
||||
|
||||
Write-Host "[prepare-provision-downloads] 查询 GitHub digest: repo=${Repository} release=${ReleaseSelector} asset=${AssetName}"
|
||||
$release = Invoke-RestMethod @request
|
||||
$asset = $release.assets | Where-Object { $_.name -eq $AssetName } | Select-Object -First 1
|
||||
if (-not $asset) {
|
||||
throw "[prepare-provision-downloads] GitHub release 未找到资产: ${Repository}/${AssetName}"
|
||||
}
|
||||
if (-not $asset.digest) {
|
||||
throw "[prepare-provision-downloads] GitHub release 未返回 digest: ${Repository}/${AssetName}"
|
||||
}
|
||||
Write-Host "[prepare-provision-downloads] GitHub digest ${AssetName}: $($asset.digest)"
|
||||
return $asset.digest
|
||||
}
|
||||
|
||||
function Get-FileDigest {
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][string]$Path,
|
||||
[Parameter(Mandatory=$true)][string]$Algorithm
|
||||
)
|
||||
|
||||
$fileHash = Get-FileHash -Algorithm $Algorithm -LiteralPath $Path
|
||||
return $fileHash.Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Test-DownloadDigestMatch {
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][string]$Path,
|
||||
[Parameter(Mandatory=$true)][string]$ExpectedDigest
|
||||
)
|
||||
|
||||
$parts = $ExpectedDigest.Split(':', 2)
|
||||
if ($parts.Length -ne 2) {
|
||||
throw "[prepare-provision-downloads] 无法解析 GitHub digest: ${ExpectedDigest}"
|
||||
}
|
||||
$algorithm = $parts[0].Trim().ToLowerInvariant()
|
||||
$expectedHash = $parts[1].Trim().ToLowerInvariant()
|
||||
if ($algorithm -ne 'sha256') {
|
||||
throw "[prepare-provision-downloads] 暂不支持的 GitHub digest 算法: ${algorithm}"
|
||||
}
|
||||
$localHash = Get-FileDigest -Path $Path -Algorithm 'SHA256'
|
||||
return $localHash -eq $expectedHash
|
||||
}
|
||||
|
||||
function Invoke-ProvisionDownload {
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][string]$Label,
|
||||
[Parameter(Mandatory=$true)][string]$Url,
|
||||
[Parameter(Mandatory=$true)][string]$Output,
|
||||
[string]$ExpectedDigest = ''
|
||||
)
|
||||
|
||||
if ($ExpectedDigest) {
|
||||
if (Test-Path -LiteralPath $Output) {
|
||||
if (Test-DownloadDigestMatch -Path $Output -ExpectedDigest $ExpectedDigest) {
|
||||
$existingItem = Get-Item -LiteralPath $Output
|
||||
Write-Host "[prepare-provision-downloads] 已存在且校验一致,跳过下载: ${Label} bytes=$($existingItem.Length) path=${Output}"
|
||||
return
|
||||
}
|
||||
Write-Host "[prepare-provision-downloads] 已存在但校验不一致,重新下载: ${Label} path=${Output}"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[prepare-provision-downloads] 下载 ${Label}: ${Url}"
|
||||
$tempOutput = "${Output}.download"
|
||||
if (Test-Path -LiteralPath $tempOutput) {
|
||||
$tempItem = Get-Item -LiteralPath $tempOutput
|
||||
if ($ExpectedDigest -and $tempItem.Length -gt 0 -and (Test-DownloadDigestMatch -Path $tempOutput -ExpectedDigest $ExpectedDigest)) {
|
||||
Move-Item -LiteralPath $tempOutput -Destination $Output -Force
|
||||
$finalItem = Get-Item -LiteralPath $Output
|
||||
Write-Host "[prepare-provision-downloads] 已复用校验通过的临时下载: ${Label} bytes=$($finalItem.Length) path=${Output}"
|
||||
return
|
||||
}
|
||||
if ($tempItem.Length -gt 0) {
|
||||
Write-Host "[prepare-provision-downloads] 发现未完成临时文件,后续尝试断点续传: ${Label} bytes=$($tempItem.Length) path=${tempOutput}"
|
||||
} else {
|
||||
Remove-Item -LiteralPath $tempOutput -Force
|
||||
}
|
||||
}
|
||||
$curl = Get-Command curl.exe -ErrorAction SilentlyContinue
|
||||
$maxAttempts = 8
|
||||
for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) {
|
||||
$resumeBytes = 0
|
||||
if (Test-Path -LiteralPath $tempOutput) {
|
||||
$resumeBytes = (Get-Item -LiteralPath $tempOutput).Length
|
||||
}
|
||||
try {
|
||||
if ($curl) {
|
||||
$arguments = @('-fL', '--retry', '3', '--retry-delay', '3', '--retry-all-errors', '--connect-timeout', '30', '--speed-time', '60', '--speed-limit', '1024')
|
||||
if ($resumeBytes -gt 0) {
|
||||
$arguments += @('-C', '-')
|
||||
Write-Host "[prepare-provision-downloads] curl 断点续传 ${Label}: attempt=${attempt}/${maxAttempts} resumeBytes=${resumeBytes}"
|
||||
} else {
|
||||
Write-Host "[prepare-provision-downloads] curl 下载 ${Label}: attempt=${attempt}/${maxAttempts}"
|
||||
}
|
||||
$arguments += @('-o', $tempOutput)
|
||||
if ($downloadProxy) {
|
||||
$arguments += @('--proxy', $downloadProxy)
|
||||
}
|
||||
$arguments += $Url
|
||||
& $curl.Source @arguments
|
||||
$exitCode = $LASTEXITCODE
|
||||
if ($exitCode -ne 0) {
|
||||
$currentBytes = if (Test-Path -LiteralPath $tempOutput) { (Get-Item -LiteralPath $tempOutput).Length } else { 0 }
|
||||
Write-Host "[prepare-provision-downloads] curl 下载未完成: ${Label}, attempt=${attempt}/${maxAttempts}, exit=${exitCode}, tempBytes=${currentBytes}"
|
||||
if ($attempt -lt $maxAttempts) {
|
||||
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
|
||||
continue
|
||||
}
|
||||
throw "[prepare-provision-downloads] curl 下载失败: ${Label}, exit=${exitCode}, temp=${tempOutput}"
|
||||
}
|
||||
} else {
|
||||
Write-Host "[prepare-provision-downloads] Invoke-WebRequest 下载 ${Label}: attempt=${attempt}/${maxAttempts}"
|
||||
if ($resumeBytes -gt 0) {
|
||||
Write-Host "[prepare-provision-downloads] Invoke-WebRequest 不支持断点续传,删除临时文件后重新下载: ${Label}, bytes=${resumeBytes}"
|
||||
Remove-Item -LiteralPath $tempOutput -Force
|
||||
}
|
||||
$parameters = @{
|
||||
Uri = $Url
|
||||
OutFile = $tempOutput
|
||||
UseBasicParsing = $true
|
||||
}
|
||||
if ($downloadProxy) {
|
||||
$parameters.Proxy = $downloadProxy
|
||||
}
|
||||
Invoke-WebRequest @parameters
|
||||
}
|
||||
} catch {
|
||||
$currentBytes = if (Test-Path -LiteralPath $tempOutput) { (Get-Item -LiteralPath $tempOutput).Length } else { 0 }
|
||||
Write-Host "[prepare-provision-downloads] 下载尝试失败: ${Label}, attempt=${attempt}/${maxAttempts}, tempBytes=${currentBytes}, error=$($_.Exception.Message)"
|
||||
if ($attempt -lt $maxAttempts) {
|
||||
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
|
||||
continue
|
||||
}
|
||||
throw
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $tempOutput)) {
|
||||
throw "[prepare-provision-downloads] 下载未生成临时文件: ${tempOutput}"
|
||||
}
|
||||
$item = Get-Item -LiteralPath $tempOutput
|
||||
if ($item.Length -le 0) {
|
||||
if ($attempt -lt $maxAttempts) {
|
||||
Write-Host "[prepare-provision-downloads] 下载结果为空,将重试: ${Label}"
|
||||
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
|
||||
continue
|
||||
}
|
||||
throw "[prepare-provision-downloads] 下载结果为空: ${tempOutput}"
|
||||
}
|
||||
if ($ExpectedDigest) {
|
||||
if (-not (Test-DownloadDigestMatch -Path $tempOutput -ExpectedDigest $ExpectedDigest)) {
|
||||
Write-Host "[prepare-provision-downloads] 下载结果校验未通过,将继续重试: ${Label}, attempt=${attempt}/${maxAttempts}, tempBytes=$($item.Length)"
|
||||
if ($attempt -lt $maxAttempts) {
|
||||
Remove-Item -LiteralPath $tempOutput -Force
|
||||
Start-Sleep -Seconds ([Math]::Min(30, 3 * $attempt))
|
||||
continue
|
||||
}
|
||||
throw "[prepare-provision-downloads] 下载结果校验失败: ${Label}, temp=${tempOutput}"
|
||||
}
|
||||
}
|
||||
Move-Item -LiteralPath $tempOutput -Destination $Output -Force
|
||||
$finalItem = Get-Item -LiteralPath $Output
|
||||
Write-Host "[prepare-provision-downloads] 已下载 ${Label}: bytes=$($finalItem.Length) path=${Output}"
|
||||
return
|
||||
}
|
||||
throw "[prepare-provision-downloads] 下载重试耗尽: ${Label}"
|
||||
}
|
||||
|
||||
$spacetimeArchiveName = "spacetime-${spacetimeTargetHost}.tar.gz"
|
||||
$spacetimeArchiveUrl = "${spacetimeDownloadRoot}/${spacetimeArchiveName}"
|
||||
$spacetimeArchiveDigest = Get-GithubReleaseAssetDigest -Repository 'clockworklabs/SpacetimeDB' -ReleaseSelector 'releases/latest' -AssetName $spacetimeArchiveName
|
||||
Invoke-ProvisionDownload -Label "SpacetimeDB release tarball ${spacetimeTargetHost}" -Url $spacetimeArchiveUrl -Output (Join-Path $downloadsDir $spacetimeArchiveName) -ExpectedDigest $spacetimeArchiveDigest
|
||||
|
||||
if ($prepareOtel -eq 'true') {
|
||||
$otelArchiveName = "otelcol-contrib_${otelVersion}_linux_amd64.tar.gz"
|
||||
$otelUrl = "${otelRoot}/v${otelVersion}/${otelArchiveName}"
|
||||
$otelDigest = Get-GithubReleaseAssetDigest -Repository 'open-telemetry/opentelemetry-collector-releases' -ReleaseSelector "releases/tags/v${otelVersion}" -AssetName $otelArchiveName
|
||||
Invoke-ProvisionDownload -Label "otelcol-contrib ${otelVersion} linux amd64" -Url $otelUrl -Output (Join-Path $downloadsDir $otelArchiveName) -ExpectedDigest $otelDigest
|
||||
} else {
|
||||
Write-Host "[prepare-provision-downloads] ENABLE_OTELCOL=${prepareOtel},跳过 otelcol-contrib 下载。"
|
||||
}
|
||||
|
||||
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
|
||||
$manifest = @(
|
||||
"spacetime release tarball ${spacetimeArchiveUrl}",
|
||||
"spacetime target host ${spacetimeTargetHost}",
|
||||
"otelcol-contrib ${otelVersion} prepare=${prepareOtel}"
|
||||
)
|
||||
[System.IO.File]::WriteAllLines((Join-Path $downloadsDir 'DOWNLOADS-MANIFEST.txt'), $manifest, $utf8NoBom)
|
||||
|
||||
Get-ChildItem -LiteralPath $downloadsDir | Sort-Object Name | ForEach-Object {
|
||||
Write-Host "[prepare-provision-downloads] artifact $($_.Length) $($_.Name)"
|
||||
}
|
||||
''')
|
||||
def checkoutFromRemote = { String remoteUrl ->
|
||||
checkout([
|
||||
$class: 'GitSCM',
|
||||
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
|
||||
doGenerateSubmoduleConfigurations: false,
|
||||
extensions: [
|
||||
[$class: 'CleanBeforeCheckout'],
|
||||
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
|
||||
],
|
||||
userRemoteConfigs: [[url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
|
||||
])
|
||||
}
|
||||
try {
|
||||
checkoutFromRemote(env.GIT_REMOTE_URL)
|
||||
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
|
||||
} catch (error) {
|
||||
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL},改用备用地址: ${env.GIT_REMOTE_FALLBACK_URL}"
|
||||
checkoutFromRemote(env.GIT_REMOTE_FALLBACK_URL)
|
||||
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_FALLBACK_URL
|
||||
}
|
||||
}
|
||||
stash name: 'server-provision-tool-downloads', includes: "${params.PROVISION_DOWNLOADS_DIR}/**", useDefaultExcludes: false
|
||||
sh '''
|
||||
bash <<'BASH'
|
||||
set -euo pipefail
|
||||
chmod +x scripts/jenkins-checkout-source.sh
|
||||
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
|
||||
COMMIT_HASH="${COMMIT_HASH:-}" \
|
||||
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
|
||||
GIT_REMOTE_FALLBACK_URL="${GIT_REMOTE_FALLBACK_URL:-}" \
|
||||
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
|
||||
scripts/jenkins-checkout-source.sh
|
||||
BASH
|
||||
'''
|
||||
sh '''
|
||||
bash -lc '
|
||||
set -euo pipefail
|
||||
chmod +x scripts/prepare-server-provision-tools.sh
|
||||
PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" \
|
||||
PROVISION_DOWNLOADS_DIR="${PROVISION_DOWNLOADS_DIR:-provision-tool-downloads}" \
|
||||
OTELCOL_VERSION="${OTELCOL_VERSION:-0.151.0}" \
|
||||
PREPARE_OTELCOL="${ENABLE_OTELCOL:-true}" \
|
||||
PROVISION_DOWNLOAD_PROXY="${PROVISION_DOWNLOAD_PROXY:-}" \
|
||||
SPACETIME_DOWNLOAD_ROOT="${SPACETIME_DOWNLOAD_ROOT:-https://github.com/clockworklabs/SpacetimeDB/releases/latest/download}" \
|
||||
SPACETIME_TARGET_HOST="${SPACETIME_TARGET_HOST:-x86_64-unknown-linux-gnu}" \
|
||||
scripts/prepare-server-provision-tools.sh
|
||||
'
|
||||
'''
|
||||
script {
|
||||
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
|
||||
echo "Provision 工具包源码 commit=${env.SOURCE_COMMIT}"
|
||||
}
|
||||
stash name: 'server-provision-tools', includes: "${params.PROVISION_TOOLS_DIR}/**", useDefaultExcludes: false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,20 +222,10 @@ BASH
|
||||
label "${params.DEPLOY_TARGET == 'development' ? 'linux && genarrative-build' : 'linux && genarrative-release-deploy'}"
|
||||
}
|
||||
steps {
|
||||
unstash 'server-provision-tool-downloads'
|
||||
unstash 'server-provision-tools'
|
||||
sh '''
|
||||
bash <<'BASH'
|
||||
set -euo pipefail
|
||||
chmod +x scripts/prepare-server-provision-tools.sh
|
||||
PROVISION_DOWNLOADS_DIR="${PROVISION_DOWNLOADS_DIR:-provision-tool-downloads}" \
|
||||
PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" \
|
||||
OTELCOL_VERSION="${OTELCOL_VERSION:-0.151.0}" \
|
||||
PREPARE_OTELCOL="${ENABLE_OTELCOL:-true}" \
|
||||
PROVISION_REQUIRE_LOCAL_DOWNLOADS="true" \
|
||||
SPACETIME_DOWNLOAD_ROOT="${SPACETIME_DOWNLOAD_ROOT:-https://github.com/clockworklabs/SpacetimeDB/releases/latest/download}" \
|
||||
SPACETIME_TARGET_HOST="${SPACETIME_TARGET_HOST:-x86_64-unknown-linux-gnu}" \
|
||||
scripts/prepare-server-provision-tools.sh
|
||||
|
||||
if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
|
||||
chmod +x "${PROVISION_TOOLS_DIR:-provision-tools}/otelcol-contrib"
|
||||
fi
|
||||
@@ -504,4 +276,4 @@ BASH
|
||||
echo "Server provision 完成: target=${params.DEPLOY_TARGET}, dryRun=${params.DRY_RUN}, nginxConfigMode=${params.NGINX_CONFIG_MODE}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +1,6 @@
|
||||
def runWindowsPowerShell(String scriptName, String scriptBody) {
|
||||
def scriptPath = ".jenkins-${scriptName}.ps1"
|
||||
writeFile file: scriptPath, text: scriptBody, encoding: 'UTF-8'
|
||||
bat label: "PowerShell ${scriptName}", script: """
|
||||
@echo off
|
||||
setlocal
|
||||
set "GENARRATIVE_POWERSHELL=%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"
|
||||
if not exist "%GENARRATIVE_POWERSHELL%" (
|
||||
echo [jenkins-powershell] powershell.exe not found: %GENARRATIVE_POWERSHELL%
|
||||
exit /b 1
|
||||
)
|
||||
echo [jenkins-powershell] user:
|
||||
whoami
|
||||
echo [jenkins-powershell] exe: %GENARRATIVE_POWERSHELL%
|
||||
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "\$path = '%CD%\\${scriptPath}'; \$text = [System.IO.File]::ReadAllText(\$path, [System.Text.Encoding]::UTF8); \$utf8Bom = New-Object System.Text.UTF8Encoding(\$true); [System.IO.File]::WriteAllText(\$path, \$text, \$utf8Bom)"
|
||||
if errorlevel 1 exit /b %ERRORLEVEL%
|
||||
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "%CD%\\${scriptPath}"
|
||||
exit /b %ERRORLEVEL%
|
||||
"""
|
||||
}
|
||||
|
||||
pipeline {
|
||||
agent {
|
||||
label 'windows'
|
||||
label 'linux && genarrative-build'
|
||||
}
|
||||
|
||||
options {
|
||||
@@ -31,12 +10,10 @@ pipeline {
|
||||
}
|
||||
|
||||
environment {
|
||||
GIT_REMOTE_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
|
||||
CARGO_HOME = '${env.WORKSPACE_TMP}/cargo-home'
|
||||
CARGO_TARGET_DIR = '${env.WORKSPACE_TMP}/cargo-target/prod-release'
|
||||
GIT_REMOTE_URL = 'http://127.0.0.1:3000/GenarrativeAI/Genarrative.git'
|
||||
GIT_REMOTE_FALLBACK_URL = 'https://git.genarrative.world/GenarrativeAI/Genarrative.git'
|
||||
CARGO_INCREMENTAL = '0'
|
||||
RUSTC_WRAPPER = 'sccache'
|
||||
SCCACHE_DIR = '${env.USERPROFILE}\\.cache\\sccache-stdb-module'
|
||||
SCCACHE_CACHE_SIZE = '30G'
|
||||
}
|
||||
|
||||
@@ -56,106 +33,42 @@ pipeline {
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
checkout([
|
||||
$class: 'GitSCM',
|
||||
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
|
||||
doGenerateSubmoduleConfigurations: false,
|
||||
extensions: [
|
||||
[$class: 'CleanBeforeCheckout'],
|
||||
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
|
||||
],
|
||||
userRemoteConfigs: [[url: "${GIT_REMOTE_URL}", refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
|
||||
])
|
||||
script {
|
||||
runWindowsPowerShell('stdb-checkout', '''
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$sourceBranch = if ($env:SOURCE_BRANCH) { $env:SOURCE_BRANCH } else { 'master' }
|
||||
$commitHash = if ($env:COMMIT_HASH) { $env:COMMIT_HASH } else { '' }
|
||||
$gitRemoteUrl = if ($env:GIT_REMOTE_URL) { $env:GIT_REMOTE_URL } else { 'https://git.genarrative.world/GenarrativeAI/Genarrative.git' }
|
||||
|
||||
function Invoke-GitCommand {
|
||||
param(
|
||||
[string]$Label,
|
||||
[string[]]$Arguments
|
||||
)
|
||||
|
||||
Write-Host "[stdb-checkout] $Label"
|
||||
& git @Arguments
|
||||
$exitCode = $LASTEXITCODE
|
||||
if ($exitCode -ne 0) {
|
||||
throw "[stdb-checkout] $Label failed with exit code $exitCode"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[stdb-checkout] sourceBranch: $sourceBranch"
|
||||
Write-Host "[stdb-checkout] remote: $gitRemoteUrl"
|
||||
$currentCommit = (git rev-parse HEAD).Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or -not $currentCommit) {
|
||||
throw '[stdb-checkout] cannot resolve current HEAD'
|
||||
}
|
||||
Write-Host "[stdb-checkout] current HEAD: $currentCommit"
|
||||
|
||||
if ($commitHash) {
|
||||
Write-Host "[stdb-checkout] requested commit: $commitHash"
|
||||
$resolvedCommit = (git rev-parse --verify "${commitHash}^{commit}" 2>$null).Trim()
|
||||
if ($LASTEXITCODE -eq 0 -and $resolvedCommit -eq $currentCommit) {
|
||||
Write-Host '[stdb-checkout] requested commit already matches Jenkins GitSCM checkout'
|
||||
} else {
|
||||
Invoke-GitCommand -Label 'fetch source branch history' -Arguments @(
|
||||
'fetch',
|
||||
'--no-tags',
|
||||
'--prune',
|
||||
$gitRemoteUrl,
|
||||
"+refs/heads/${sourceBranch}:refs/remotes/origin/${sourceBranch}"
|
||||
)
|
||||
$isShallowRepository = (git rev-parse --is-shallow-repository 2>$null).Trim()
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw '[stdb-checkout] cannot determine whether repository is shallow'
|
||||
}
|
||||
if ($isShallowRepository -eq 'true') {
|
||||
Invoke-GitCommand -Label 'deepen source branch history' -Arguments @(
|
||||
'fetch',
|
||||
'--unshallow',
|
||||
'--no-tags',
|
||||
$gitRemoteUrl,
|
||||
"+refs/heads/${sourceBranch}:refs/remotes/origin/${sourceBranch}"
|
||||
)
|
||||
}
|
||||
Invoke-GitCommand -Label 'validate source branch ref' -Arguments @(
|
||||
'cat-file',
|
||||
'-e',
|
||||
"refs/remotes/origin/${sourceBranch}^{commit}"
|
||||
)
|
||||
Invoke-GitCommand -Label 'validate requested commit' -Arguments @(
|
||||
'cat-file',
|
||||
'-e',
|
||||
"${commitHash}^{commit}"
|
||||
)
|
||||
$resolvedCommit = (git rev-parse --verify "${commitHash}^{commit}").Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or -not $resolvedCommit) {
|
||||
throw "[stdb-checkout] cannot resolve requested commit: $commitHash"
|
||||
}
|
||||
Invoke-GitCommand -Label 'validate requested commit belongs to branch' -Arguments @(
|
||||
'merge-base',
|
||||
'--is-ancestor',
|
||||
$resolvedCommit,
|
||||
"refs/remotes/origin/${sourceBranch}"
|
||||
)
|
||||
Invoke-GitCommand -Label "checkout commit $resolvedCommit" -Arguments @(
|
||||
'checkout',
|
||||
'--force',
|
||||
$resolvedCommit
|
||||
)
|
||||
}
|
||||
} else {
|
||||
Write-Host "[stdb-checkout] COMMIT_HASH empty, reusing Jenkins GitSCM checkout result"
|
||||
}
|
||||
|
||||
$resolvedCommit = (git rev-parse HEAD).Trim()
|
||||
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
|
||||
[System.IO.File]::WriteAllText((Join-Path (Get-Location) '.jenkins-source-commit'), "$resolvedCommit`n", $utf8NoBom)
|
||||
''')
|
||||
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').replace('\uFEFF', '').trim()
|
||||
def checkoutFromRemote = { String remoteUrl ->
|
||||
checkout([
|
||||
$class: 'GitSCM',
|
||||
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
|
||||
doGenerateSubmoduleConfigurations: false,
|
||||
extensions: [
|
||||
[$class: 'CleanBeforeCheckout'],
|
||||
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
|
||||
],
|
||||
userRemoteConfigs: [[url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
|
||||
])
|
||||
}
|
||||
try {
|
||||
checkoutFromRemote(env.GIT_REMOTE_URL)
|
||||
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
|
||||
} catch (error) {
|
||||
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL},改用备用地址: ${env.GIT_REMOTE_FALLBACK_URL}"
|
||||
checkoutFromRemote(env.GIT_REMOTE_FALLBACK_URL)
|
||||
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_FALLBACK_URL
|
||||
}
|
||||
}
|
||||
sh '''
|
||||
bash -lc '
|
||||
set -euo pipefail
|
||||
chmod +x scripts/jenkins-checkout-source.sh
|
||||
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
|
||||
COMMIT_HASH="${COMMIT_HASH:-}" \
|
||||
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
|
||||
GIT_REMOTE_FALLBACK_URL="${GIT_REMOTE_FALLBACK_URL:-}" \
|
||||
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
|
||||
scripts/jenkins-checkout-source.sh
|
||||
'
|
||||
'''
|
||||
script {
|
||||
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
|
||||
env.EFFECTIVE_BUILD_VERSION = params.BUILD_VERSION?.trim() ? params.BUILD_VERSION.trim() : env.BUILD_NUMBER
|
||||
}
|
||||
}
|
||||
@@ -165,45 +78,27 @@ pipeline {
|
||||
steps {
|
||||
script {
|
||||
def buildStep = {
|
||||
runWindowsPowerShell('stdb-build', '''
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$workspaceTmp = if ($env:WORKSPACE_TMP) { $env:WORKSPACE_TMP } else { "$env:WORKSPACE@tmp" }
|
||||
$env:CARGO_HOME = "$workspaceTmp/cargo-home"
|
||||
$env:CARGO_TARGET_DIR = "$workspaceTmp/cargo-target/prod-release"
|
||||
$env:SCCACHE_DIR = "$env:USERPROFILE/.cache/sccache-stdb-module"
|
||||
$env:PATH = "$env:CARGO_HOME/bin;$env:PATH"
|
||||
$gitBash = @(
|
||||
$env:GENARRATIVE_BASH,
|
||||
'C:/Program Files/Git/bin/bash.exe',
|
||||
'C:/Program Files/Git/usr/bin/bash.exe',
|
||||
'C:/msys64/usr/bin/bash.exe',
|
||||
'bash'
|
||||
) | Where-Object { $_ -and (($_ -eq 'bash') -or (Test-Path $_)) } | Select-Object -First 1
|
||||
if (-not $gitBash) {
|
||||
throw '[stdb-build] Windows 构建节点缺少 Git Bash,无法执行仓库现有生产构建脚本。请先安装 Git for Windows,并确保 bash 在 PATH 中。'
|
||||
}
|
||||
$env:GENARRATIVE_BASH = $gitBash
|
||||
if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
|
||||
throw '[stdb-build] 缺少 cargo。请先在 Windows 构建节点安装 Rust 工具链,并确保 cargo 在 PATH 中。'
|
||||
}
|
||||
# sccache 只是可选缓存;PATH 中存在但不可执行时必须回退到 rustc。
|
||||
$sccacheCommand = Get-Command sccache -ErrorAction SilentlyContinue
|
||||
$sccacheUsable = $false
|
||||
if ($sccacheCommand) {
|
||||
try {
|
||||
& $sccacheCommand.Source --version | Out-Host
|
||||
$sccacheUsable = $true
|
||||
} catch {
|
||||
Write-Host "[stdb-build] sccache 无法执行:$($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
if (-not $sccacheUsable) {
|
||||
Write-Host '[stdb-build] 未找到可用 sccache,改用 rustc 直接构建。'
|
||||
Remove-Item Env:RUSTC_WRAPPER -ErrorAction SilentlyContinue
|
||||
}
|
||||
npm run build:production-release -- --component spacetime-module --name "$env:EFFECTIVE_BUILD_VERSION"
|
||||
sh '''
|
||||
bash -lc '
|
||||
set -euo pipefail
|
||||
workspace_tmp="${WORKSPACE_TMP:-${WORKSPACE}@tmp}"
|
||||
export CARGO_HOME="${workspace_tmp}/cargo-home"
|
||||
export CARGO_TARGET_DIR="${workspace_tmp}/cargo-target/prod-release"
|
||||
export CARGO_INCREMENTAL=0
|
||||
export RUSTC_WRAPPER=sccache
|
||||
export SCCACHE_DIR="${workspace_tmp}/sccache-stdb-module"
|
||||
export SCCACHE_CACHE_SIZE=30G
|
||||
mkdir -p "${CARGO_HOME}" "${CARGO_TARGET_DIR}" "${SCCACHE_DIR}"
|
||||
chmod +x scripts/jenkins-prepare-cargo-env.sh
|
||||
source scripts/jenkins-prepare-cargo-env.sh
|
||||
if ! command -v sccache >/dev/null 2>&1; then
|
||||
echo "[stdb-build] 未找到可用 sccache,改用 rustc 直接构建。"
|
||||
unset RUSTC_WRAPPER
|
||||
fi
|
||||
SOURCE_BRANCH="${SOURCE_BRANCH}" SOURCE_COMMIT="${SOURCE_COMMIT}" \
|
||||
npm run build:production-release -- --component spacetime-module --name "${EFFECTIVE_BUILD_VERSION}"
|
||||
'
|
||||
'''
|
||||
)
|
||||
}
|
||||
if (params.MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID?.trim()) {
|
||||
withCredentials([
|
||||
|
||||
Reference in New Issue
Block a user