GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(jenkins): download provision tools on windows

Browse Source
This commit is contained in:
kdletters
2026-05-19 17:57:55 +08:00
parent a9c54b0e1a
commit b4a6cb2465
5 changed files with 266 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
jenkins/Jenkinsfile.production-server-provision
View File

@@ -1,3 +1,24 @@
def runWindowsPowerShell(String scriptName, String scriptBody) {
def scriptPath = ".jenkins-${scriptName}.ps1"
writeFile file: scriptPath, text: scriptBody, encoding: 'UTF-8'
bat label: "PowerShell ${scriptName}", script: """
@echo off
setlocal
set "GENARRATIVE_POWERSHELL=%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"
if not exist "%GENARRATIVE_POWERSHELL%" (
echo [jenkins-powershell] powershell.exe not found: %GENARRATIVE_POWERSHELL%
exit /b 1
)
echo [jenkins-powershell] user:
whoami
echo [jenkins-powershell] exe: %GENARRATIVE_POWERSHELL%
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "\$path = '%CD%\\${scriptPath}'; \$text = [System.IO.File]::ReadAllText(\$path, [System.Text.Encoding]::UTF8); \$utf8Bom = New-Object System.Text.UTF8Encoding(\$true); [System.IO.File]::WriteAllText(\$path, \$text, \$utf8Bom)"
if errorlevel 1 exit /b %ERRORLEVEL%
"%GENARRATIVE_POWERSHELL%" -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "%CD%\\${scriptPath}"
exit /b %ERRORLEVEL%
"""
}
pipeline {
agent none
@@ -22,9 +43,12 @@ pipeline {
string(name: 'COMMIT_HASH', defaultValue: '', description: '部署脚本来源 commit')
string(name: 'SERVER_NAME', defaultValue: 'genarrative.example.com', description: '证书主域名；也作为 Nginx server_name 的第一个域名')
string(name: 'SERVER_ALIASES', defaultValue: '', description: '可选，额外 Nginx server_name，多个用空格或逗号分隔，例如 www.genarrative.world')
string(name: 'PROVISION_TOOLS_DIR', defaultValue: 'provision-tools', description: '构建机准备并上传到目标机工作区的工具包目录')
string(name: 'PROVISION_DOWNLOAD_PROXY', defaultValue: '', description: '可选，构建机下载 SpacetimeDB 和 otelcol-contrib 时使用的代理地址，例如 http://proxy-host:7890；留空不设置代理')
string(name: 'SPACETIME_DOWNLOAD_ROOT', defaultValue: 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download', description: '构建机下载 SpacetimeDB 官方安装产物的根地址；目标机不访问该地址')
string(name: 'PROVISION_DOWNLOADS_DIR', defaultValue: 'provision-tool-downloads', description: 'Windows 下载阶段暂存 SpacetimeDB/otelcol 安装包的工作区相对目录')
string(name: 'PROVISION_TOOLS_DIR', defaultValue: 'provision-tools', description: '目标机工作区内由已下载安装包生成的工具包目录')
string(name: 'PROVISION_DOWNLOAD_PROXY', defaultValue: '', description: '可选，Windows 下载 SpacetimeDB 和 otelcol-contrib 时使用的代理地址，例如 http://127.0.0.1:7890；留空不设置代理')
string(name: 'SPACETIME_INSTALLER_URL', defaultValue: 'https://install.spacetimedb.com', description: 'Windows 下载 SpacetimeDB 官方安装脚本的地址；目标机不访问该地址')
string(name: 'SPACETIME_DOWNLOAD_ROOT', defaultValue: 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download', description: 'Windows 下载 SpacetimeDB Linux update installer 的根地址；目标机不访问该地址')
string(name: 'SPACETIME_TARGET_HOST', defaultValue: 'x86_64-unknown-linux-gnu', description: '目标机 SpacetimeDB 预编译包 host triple，development/release Linux amd64 使用默认值')
string(name: 'SPACETIME_ROOT', defaultValue: '/stdb', description: 'SpacetimeDB root-dir')
string(name: 'RELEASE_ROOT', defaultValue: '/opt/genarrative/releases', description: 'release 根目录')
string(name: 'CURRENT_LINK', defaultValue: '/opt/genarrative/current', description: '当前版本软链接')
@@ -40,7 +64,7 @@ pipeline {
stages {
stage('Prepare') {
agent {
label 'linux && genarrative-build'
label 'windows'
}
steps {
script {
@@ -67,9 +91,20 @@ pipeline {
if (!params.PROVISION_TOOLS_DIR?.trim()) {
error('PROVISION_TOOLS_DIR 不能为空。')
}
if (!(params.PROVISION_TOOLS_DIR.trim() ==~ /^[0-9A-Za-z._\/-]+$/) || params.PROVISION_TOOLS_DIR.startsWith('/') || params.PROVISION_TOOLS_DIR.contains('..')) {
if (!(params.PROVISION_TOOLS_DIR.trim() ==~ /^[0-9A-Za-z._\/-]+$/) || params.PROVISION_TOOLS_DIR.startsWith('/') || params.PROVISION_TOOLS_DIR.contains('..') || params.PROVISION_TOOLS_DIR.trim() == '.') {
error("PROVISION_TOOLS_DIR 只能是工作区内的相对目录，不能包含绝对路径或连续点号: ${params.PROVISION_TOOLS_DIR}")
}
if (!params.PROVISION_DOWNLOADS_DIR?.trim()) {
error('PROVISION_DOWNLOADS_DIR 不能为空。')
}
if (!(params.PROVISION_DOWNLOADS_DIR.trim() ==~ /^[0-9A-Za-z._\/-]+$/) || params.PROVISION_DOWNLOADS_DIR.startsWith('/') || params.PROVISION_DOWNLOADS_DIR.contains('..') || params.PROVISION_DOWNLOADS_DIR.trim() == '.') {
error("PROVISION_DOWNLOADS_DIR 只能是工作区内的相对目录，不能包含绝对路径或连续点号: ${params.PROVISION_DOWNLOADS_DIR}")
}
def provisionToolsDir = params.PROVISION_TOOLS_DIR.trim()
def provisionDownloadsDir = params.PROVISION_DOWNLOADS_DIR.trim()
if (provisionToolsDir == provisionDownloadsDir || provisionDownloadsDir.startsWith("${provisionToolsDir}/")) {
error("PROVISION_DOWNLOADS_DIR 不能等于或位于 PROVISION_TOOLS_DIR 内，否则目标机生成工具包时会删除下载缓存: ${provisionDownloadsDir}")
}
def provisionDownloadProxy = params.PROVISION_DOWNLOAD_PROXY?.trim()
if (provisionDownloadProxy && !(provisionDownloadProxy ==~ /^https?:\/\/\S+$/)) {
error("PROVISION_DOWNLOAD_PROXY 只能填写 http:// 或 https:// 开头的代理地址，当前值: ${params.PROVISION_DOWNLOAD_PROXY}")
@@ -77,9 +112,15 @@ pipeline {
if (!(params.OTELCOL_VERSION?.trim() ==~ /^[0-9]+\.[0-9]+\.[0-9]+$/)) {
error("OTELCOL_VERSION 格式应为 x.y.z: ${params.OTELCOL_VERSION}")
}
if (!params.SPACETIME_DOWNLOAD_ROOT?.trim()) {
if (!(params.SPACETIME_INSTALLER_URL?.trim() ==~ /^https?:\/\/\S+$/)) {
error("SPACETIME_INSTALLER_URL 只能填写 http:// 或 https:// 开头的地址: ${params.SPACETIME_INSTALLER_URL}")
}
if (!(params.SPACETIME_DOWNLOAD_ROOT?.trim() ==~ /^https?:\/\/\S+$/)) {
error('SPACETIME_DOWNLOAD_ROOT 不能为空。')
}
if (!(params.SPACETIME_TARGET_HOST?.trim() ==~ /^[0-9A-Za-z._-]+$/)) {
error("SPACETIME_TARGET_HOST 只能包含字母、数字、点号、下划线和短横线: ${params.SPACETIME_TARGET_HOST}")
}
def nginxMode = params.NGINX_CONFIG_MODE?.trim()
if (!(nginxMode in ['none', 'production-https', 'development-http'])) {
error("NGINX_CONFIG_MODE 只能是 none、production-https 或 development-http，当前值: ${params.NGINX_CONFIG_MODE}")
@@ -94,65 +135,106 @@ pipeline {
}
}
stage('Prepare Provision Tools') {
stage('Download Provision Tool Archives') {
agent {
label 'linux && genarrative-build'
label 'windows'
}
steps {
script {
def checkoutFromRemote = { String remoteUrl ->
checkout([
$class: 'GitSCM',
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
],
userRemoteConfigs: [[url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]],
])
}
try {
checkoutFromRemote(env.GIT_REMOTE_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
} catch (error) {
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL}，改用备用地址: ${env.GIT_REMOTE_FALLBACK_URL}"
checkoutFromRemote(env.GIT_REMOTE_FALLBACK_URL)
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_FALLBACK_URL
}
}
sh '''
bash <<'BASH'
set -euo pipefail
chmod +x scripts/jenkins-checkout-source.sh
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
COMMIT_HASH="${COMMIT_HASH:-}" \
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
GIT_REMOTE_FALLBACK_URL="${GIT_REMOTE_FALLBACK_URL:-}" \
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
scripts/jenkins-checkout-source.sh
# jenkins-checkout-source.sh 会 reset/clean 到目标 commit，前面的临时 chmod 可能被 Git mode 还原；
# 直接执行脚本前在二次 checkout 之后再补执行位，避免 Linux agent 报 Permission denied。
chmod +x scripts/prepare-server-provision-tools.sh
BASH
'''
sh '''
bash <<'BASH'
set -euo pipefail
runWindowsPowerShell('server-provision-tool-downloads', '''
$ErrorActionPreference = 'Stop'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" \
OTELCOL_VERSION="${OTELCOL_VERSION:-0.151.0}" \
PREPARE_OTELCOL="${ENABLE_OTELCOL:-true}" \
PROVISION_DOWNLOAD_PROXY="${PROVISION_DOWNLOAD_PROXY:-}" \
SPACETIME_DOWNLOAD_ROOT="${SPACETIME_DOWNLOAD_ROOT:-https://github.com/clockworklabs/SpacetimeDB/releases/latest/download}" \
scripts/prepare-server-provision-tools.sh
BASH
'''
script {
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
echo "Provision 工具包已准备，源码 commit=${env.SOURCE_COMMIT}"
$downloadsDir = if ($env:PROVISION_DOWNLOADS_DIR) { $env:PROVISION_DOWNLOADS_DIR } else { 'provision-tool-downloads' }
$otelVersion = if ($env:OTELCOL_VERSION) { $env:OTELCOL_VERSION } else { '0.151.0' }
$prepareOtel = if ($env:ENABLE_OTELCOL) { $env:ENABLE_OTELCOL } else { 'true' }
$otelRoot = if ($env:OTELCOL_DOWNLOAD_ROOT) { $env:OTELCOL_DOWNLOAD_ROOT.TrimEnd('/') } else { 'https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download' }
$spacetimeInstallerUrl = if ($env:SPACETIME_INSTALLER_URL) { $env:SPACETIME_INSTALLER_URL } else { 'https://install.spacetimedb.com' }
$spacetimeDownloadRoot = if ($env:SPACETIME_DOWNLOAD_ROOT) { $env:SPACETIME_DOWNLOAD_ROOT.TrimEnd('/') } else { 'https://github.com/clockworklabs/SpacetimeDB/releases/latest/download' }
$spacetimeTargetHost = if ($env:SPACETIME_TARGET_HOST) { $env:SPACETIME_TARGET_HOST } else { 'x86_64-unknown-linux-gnu' }
$downloadProxy = if ($env:PROVISION_DOWNLOAD_PROXY) { $env:PROVISION_DOWNLOAD_PROXY } else { '' }
if (Test-Path -LiteralPath $downloadsDir) {
Remove-Item -LiteralPath $downloadsDir -Recurse -Force
}
New-Item -ItemType Directory -Force -Path $downloadsDir | Out-Null
if ($downloadProxy) {
$env:HTTP_PROXY = $downloadProxy
$env:HTTPS_PROXY = $downloadProxy
$env:ALL_PROXY = $downloadProxy
Write-Host "[prepare-provision-downloads] 已配置 Windows 下载代理: $($downloadProxy -replace '://.*', '://***')"
}
function Invoke-ProvisionDownload {
param(
[Parameter(Mandatory=$true)][string]$Label,
[Parameter(Mandatory=$true)][string]$Url,
[Parameter(Mandatory=$true)][string]$Output
)
Write-Host "[prepare-provision-downloads] 下载 ${Label}: ${Url}"
$curl = Get-Command curl.exe -ErrorAction SilentlyContinue
if ($curl) {
$arguments = @('-fL', '--retry', '3', '--retry-delay', '2', '-o', $Output)
if ($downloadProxy) {
$arguments += @('--proxy', $downloadProxy)
}
$arguments += $Url
& $curl.Source @arguments
$exitCode = $LASTEXITCODE
if ($exitCode -ne 0) {
throw "[prepare-provision-downloads] curl 下载失败: ${Label}, exit=${exitCode}"
}
} else {
$parameters = @{
Uri = $Url
OutFile = $Output
UseBasicParsing = $true
}
if ($downloadProxy) {
$parameters.Proxy = $downloadProxy
}
Invoke-WebRequest @parameters
}
$item = Get-Item -LiteralPath $Output
if ($item.Length -le 0) {
throw "[prepare-provision-downloads] 下载结果为空: ${Output}"
}
Write-Host "[prepare-provision-downloads] 已下载 ${Label}: bytes=$($item.Length) path=${Output}"
}
$installerPath = Join-Path $downloadsDir 'spacetime-install.sh'
Invoke-ProvisionDownload -Label 'SpacetimeDB install script' -Url $spacetimeInstallerUrl -Output $installerPath
$spacetimeUpdateName = "spacetimedb-update-${spacetimeTargetHost}"
$spacetimeUpdateUrl = "${spacetimeDownloadRoot}/${spacetimeUpdateName}"
Invoke-ProvisionDownload -Label "SpacetimeDB Linux update installer ${spacetimeTargetHost}" -Url $spacetimeUpdateUrl -Output (Join-Path $downloadsDir $spacetimeUpdateName)
if ($prepareOtel -eq 'true') {
$otelArchiveName = "otelcol-contrib_${otelVersion}_linux_amd64.tar.gz"
$otelUrl = "${otelRoot}/v${otelVersion}/${otelArchiveName}"
Invoke-ProvisionDownload -Label "otelcol-contrib ${otelVersion} linux amd64" -Url $otelUrl -Output (Join-Path $downloadsDir $otelArchiveName)
} else {
Write-Host "[prepare-provision-downloads] ENABLE_OTELCOL=${prepareOtel}，跳过 otelcol-contrib 下载。"
}
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
$manifest = @(
"spacetime installer ${spacetimeInstallerUrl}",
"spacetime update ${spacetimeDownloadRoot}/${spacetimeUpdateName}",
"spacetime target host ${spacetimeTargetHost}",
"otelcol-contrib ${otelVersion} prepare=${prepareOtel}"
)
[System.IO.File]::WriteAllLines((Join-Path $downloadsDir 'DOWNLOADS-MANIFEST.txt'), $manifest, $utf8NoBom)
Get-ChildItem -LiteralPath $downloadsDir | Sort-Object Name | ForEach-Object {
Write-Host "[prepare-provision-downloads] artifact $($_.Length) $($_.Name)"
}
''')
}
stash name: 'server-provision-tools', includes: "${params.PROVISION_TOOLS_DIR}/**", useDefaultExcludes: false
stash name: 'server-provision-tool-downloads', includes: "${params.PROVISION_DOWNLOADS_DIR}/**", useDefaultExcludes: false
}
}
@@ -195,6 +277,10 @@ BASH
scripts/jenkins-checkout-source.sh
BASH
'''
script {
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
echo "Provision 源码 commit=${env.SOURCE_COMMIT}"
}
}
}
@@ -203,10 +289,21 @@ BASH
label "${params.DEPLOY_TARGET == 'development' ? 'linux && genarrative-build' : 'linux && genarrative-release-deploy'}"
}
steps {
unstash 'server-provision-tools'
unstash 'server-provision-tool-downloads'
sh '''
bash <<'BASH'
set -euo pipefail
chmod +x scripts/prepare-server-provision-tools.sh
PROVISION_DOWNLOADS_DIR="${PROVISION_DOWNLOADS_DIR:-provision-tool-downloads}" \
PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" \
OTELCOL_VERSION="${OTELCOL_VERSION:-0.151.0}" \
PREPARE_OTELCOL="${ENABLE_OTELCOL:-true}" \
PROVISION_REQUIRE_LOCAL_DOWNLOADS="true" \
SPACETIME_INSTALLER_URL="${SPACETIME_INSTALLER_URL:-https://install.spacetimedb.com}" \
SPACETIME_DOWNLOAD_ROOT="${SPACETIME_DOWNLOAD_ROOT:-https://github.com/clockworklabs/SpacetimeDB/releases/latest/download}" \
SPACETIME_TARGET_HOST="${SPACETIME_TARGET_HOST:-x86_64-unknown-linux-gnu}" \
scripts/prepare-server-provision-tools.sh
if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
chmod +x "${PROVISION_TOOLS_DIR:-provision-tools}/otelcol-contrib"
fi