kdletters
8965a53fb8
将 Exit Maintenance 改为 agent none 并显式分配部署节点，只执行 current release 维护脚本。 增加生产运维静态门禁，禁止维护退出节点 checkout Git 或挂载 Git 凭据。 同步生产运维说明与 Jenkins 重启恢复场景的排障记录。
322 lines
18 KiB
Plaintext
322 lines
18 KiB
Plaintext
pipeline {
|
|
agent none
|
|
|
|
options {
|
|
disableConcurrentBuilds()
|
|
skipDefaultCheckout(true)
|
|
buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
|
|
}
|
|
|
|
triggers {
|
|
cron('0 4 * * *')
|
|
}
|
|
|
|
environment {
|
|
GIT_REMOTE_URL = 'ssh://git@127.0.0.1:2222/GenarrativeAI/Genarrative.git'
|
|
GIT_REMOTE_CREDENTIAL_ID = 'genarrative-local-gitea-ssh'
|
|
GIT_REMOTE_FALLBACK_URL = ''
|
|
GIT_REMOTE_FALLBACK_CREDENTIAL_ID = ''
|
|
}
|
|
|
|
parameters {
|
|
string(name: 'SOURCE_BRANCH', defaultValue: 'master', description: '源码分支，默认 master 最新提交')
|
|
string(name: 'COMMIT_HASH', defaultValue: '', description: '可选，指定属于 SOURCE_BRANCH 的 Git commit')
|
|
string(name: 'BUILD_VERSION', defaultValue: '', description: '发布版本号，留空则使用 Jenkins BUILD_NUMBER')
|
|
booleanParam(name: 'RUN_NPM_CI', defaultValue: true, description: 'Web 构建前是否执行 npm ci')
|
|
string(name: 'NOTIFICATION_EMAILS', defaultValue: '', description: '本次运行追加通知邮箱；会与 Jenkins Secret Text 凭据 genarrative-notification-emails 合并发送')
|
|
string(name: 'MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID', defaultValue: 'genarrative-spacetime-bootstrap-secret-dev-file', description: '必填：Stdb module 构建与发布共用的 Jenkins Secret File 凭据 ID')
|
|
booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: true, description: 'API release 是否构建、归档并部署 Pingora 影子网关；release 默认必须包含')
|
|
string(name: 'WEB_BUILD_JOB_NAME', defaultValue: 'Genarrative-Web-Build', description: 'Web 构建流水线作业名')
|
|
string(name: 'API_BUILD_JOB_NAME', defaultValue: 'Genarrative-Api-Build', description: 'API 构建流水线作业名')
|
|
string(name: 'STDB_BUILD_JOB_NAME', defaultValue: 'Genarrative-Stdb-Module-Build', description: 'Stdb 构建流水线作业名')
|
|
string(name: 'WEB_DEPLOY_JOB_NAME', defaultValue: 'Genarrative-Web-Deploy', description: 'Web 发布流水线作业名')
|
|
string(name: 'API_DEPLOY_JOB_NAME', defaultValue: 'Genarrative-Api-Deploy', description: 'API 发布流水线作业名')
|
|
string(name: 'STDB_PUBLISH_JOB_NAME', defaultValue: 'Genarrative-Stdb-Module-Publish', description: 'Stdb 发布流水线作业名')
|
|
choice(name: 'DEPLOY_TARGET', choices: ['development', 'release'], description: '逻辑部署目标；development 使用当前 Linux 开发/构建/开发部署 agent')
|
|
booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', defaultValue: false, description: '确认 release 目标已有独立 release 部署 agent；当前 Linux 开发/构建/开发部署 agent 不可冒充 release 部署机')
|
|
string(name: 'DATABASE', defaultValue: 'genarrative-prod', description: '生产 SpacetimeDB database')
|
|
choice(name: 'DATABASE_BACKUP_MODE', choices: ['async', 'sync', 'skip'], description: 'Stdb publish 备份策略；人工 release 必须显式选择，已有验真冷备时才可选 skip')
|
|
string(name: 'SPACETIME_SERVER_URL', defaultValue: 'http://127.0.0.1:3101', description: 'Stdb 发布目标 URL；默认避开本机 Git/Web 使用的 3000 端口')
|
|
string(name: 'SPACETIME_ROOT_DIR', defaultValue: '/stdb', description: 'Stdb 发布使用的 spacetime CLI root-dir')
|
|
string(name: 'SPACETIME_RUN_AS_USER', defaultValue: 'spacetimedb', description: 'Stdb 发布使用的本机用户')
|
|
string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'API 与 Stdb publish 共用的 api-server 环境文件')
|
|
string(name: 'WORKER_ENV_FILE', defaultValue: '/etc/genarrative/external-generation-worker.env', description: 'API 与 Stdb publish 共用的 worker 环境文件')
|
|
choice(name: 'STDB_API_ROLLOUT_MODE', choices: ['normal', 'pause-after-stdb'], description: '定时任务默认 normal 完整发布 dev；人工维护窗口才选择 pause-after-stdb')
|
|
string(name: 'STDB_API_ROLLOUT_APPROVERS', defaultValue: '', description: 'pause-after-stdb 必填：允许放行 API 的 Jenkins 用户/组，多个值用逗号分隔')
|
|
booleanParam(name: 'EXIT_MAINTENANCE_MODE_AFTER_COMPLETION', defaultValue: true, description: '完整发布成功后是否退出维护模式；取消勾选会保留维护页，便于内网验收后人工恢复公网')
|
|
}
|
|
|
|
stages {
|
|
stage('Resolve Source') {
|
|
agent {
|
|
label 'linux && genarrative-build'
|
|
}
|
|
steps {
|
|
script {
|
|
def checkoutFromRemote = { String remoteUrl, String credentialsId ->
|
|
def remoteConfig = [url: remoteUrl, refspec: "+refs/heads/${params.SOURCE_BRANCH}:refs/remotes/origin/${params.SOURCE_BRANCH}"]
|
|
def checkoutCredentialId = credentialsId?.trim()
|
|
if (checkoutCredentialId) {
|
|
remoteConfig.credentialsId = checkoutCredentialId
|
|
}
|
|
checkout([
|
|
$class: 'GitSCM',
|
|
branches: [[name: "*/${params.SOURCE_BRANCH}"]],
|
|
doGenerateSubmoduleConfigurations: false,
|
|
extensions: [
|
|
[$class: 'CleanBeforeCheckout'],
|
|
[$class: 'CloneOption', shallow: true, depth: 1, noTags: true, timeout: 30, honorRefspec: true],
|
|
],
|
|
userRemoteConfigs: [remoteConfig],
|
|
])
|
|
}
|
|
try {
|
|
checkoutFromRemote(env.GIT_REMOTE_URL, env.GIT_REMOTE_CREDENTIAL_ID)
|
|
env.EFFECTIVE_GIT_REMOTE_URL = env.GIT_REMOTE_URL
|
|
} catch (error) {
|
|
def fallbackRemoteUrl = env.GIT_REMOTE_FALLBACK_URL?.trim()
|
|
if (!fallbackRemoteUrl) {
|
|
throw error
|
|
}
|
|
echo "Git 主地址拉取失败: ${env.GIT_REMOTE_URL}，改用备用地址: ${fallbackRemoteUrl}"
|
|
checkoutFromRemote(fallbackRemoteUrl, env.GIT_REMOTE_FALLBACK_CREDENTIAL_ID)
|
|
env.EFFECTIVE_GIT_REMOTE_URL = fallbackRemoteUrl
|
|
}
|
|
}
|
|
withCredentials([sshUserPrivateKey(credentialsId: env.GIT_REMOTE_CREDENTIAL_ID, keyFileVariable: 'GENARRATIVE_GIT_SSH_KEY')]) {
|
|
sh '''
|
|
bash -lc '
|
|
set -euo pipefail
|
|
chmod +x scripts/jenkins-checkout-source.sh
|
|
SOURCE_BRANCH="${SOURCE_BRANCH:-master}" \
|
|
COMMIT_HASH="${COMMIT_HASH:-}" \
|
|
GIT_REMOTE_URL="${EFFECTIVE_GIT_REMOTE_URL:-${GIT_REMOTE_URL}}" \
|
|
GIT_REMOTE_FALLBACK_URL="" \
|
|
SOURCE_COMMIT_FILE=".jenkins-source-commit" \
|
|
GENARRATIVE_JENKINS_REUSE_EXISTING_CHECKOUT="true" \
|
|
GIT_SSH_COMMAND="ssh -i ${GENARRATIVE_GIT_SSH_KEY:?GENARRATIVE_GIT_SSH_KEY 不能为空} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" \
|
|
scripts/jenkins-checkout-source.sh
|
|
'
|
|
'''
|
|
}
|
|
script {
|
|
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
|
|
env.EFFECTIVE_BUILD_VERSION = params.BUILD_VERSION?.trim() ? params.BUILD_VERSION.trim() : env.BUILD_NUMBER
|
|
if (!params.MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID?.trim()) {
|
|
error('MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID 必须引用受保护的 Jenkins Secret File 凭据。')
|
|
}
|
|
if (params.DEPLOY_TARGET == 'release' && !params.CONFIRM_RELEASE_DEPLOY_AGENT) {
|
|
error('release 部署需要先配置独立 release 部署 agent，并勾选 CONFIRM_RELEASE_DEPLOY_AGENT。当前 Linux 开发/构建/开发部署 agent 不能执行 release 部署。')
|
|
}
|
|
if (params.STDB_API_ROLLOUT_MODE == 'pause-after-stdb' && !params.STDB_API_ROLLOUT_APPROVERS?.trim()) {
|
|
error('pause-after-stdb 必须填写 STDB_API_ROLLOUT_APPROVERS。')
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Build Components') {
|
|
// 任一构建分支失败时立即中断其他并行分支，避免失败后继续消耗构建资源。
|
|
failFast true
|
|
parallel {
|
|
stage('Web') {
|
|
steps {
|
|
script {
|
|
def webRun = build job: params.WEB_BUILD_JOB_NAME,
|
|
wait: true,
|
|
propagate: true,
|
|
parameters: [
|
|
string(name: 'SOURCE_BRANCH', value: params.SOURCE_BRANCH),
|
|
string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT),
|
|
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
|
|
string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
|
|
booleanParam(name: 'RUN_NPM_CI', value: params.RUN_NPM_CI),
|
|
booleanParam(name: 'PUBLISH_AFTER_BUILD', value: false),
|
|
]
|
|
env.WEB_BUILD_NUMBER = webRun.number.toString()
|
|
}
|
|
}
|
|
}
|
|
stage('Api') {
|
|
steps {
|
|
script {
|
|
def apiRun = build job: params.API_BUILD_JOB_NAME,
|
|
wait: true,
|
|
propagate: true,
|
|
parameters: [
|
|
string(name: 'SOURCE_BRANCH', value: params.SOURCE_BRANCH),
|
|
string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT),
|
|
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
|
|
string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
|
|
booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', value: params.INCLUDE_PINGORA_GATEWAY),
|
|
booleanParam(name: 'PUBLISH_AFTER_BUILD', value: false),
|
|
]
|
|
env.API_BUILD_NUMBER = apiRun.number.toString()
|
|
}
|
|
}
|
|
}
|
|
stage('Stdb') {
|
|
steps {
|
|
script {
|
|
def stdbRun = build job: params.STDB_BUILD_JOB_NAME,
|
|
wait: true,
|
|
propagate: true,
|
|
parameters: [
|
|
string(name: 'SOURCE_BRANCH', value: params.SOURCE_BRANCH),
|
|
string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT),
|
|
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
|
|
string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
|
|
string(name: 'MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID', value: params.MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID ?: ''),
|
|
string(name: 'DATABASE', value: params.DATABASE),
|
|
booleanParam(name: 'PUBLISH_AFTER_BUILD', value: false),
|
|
]
|
|
env.STDB_BUILD_NUMBER = stdbRun.number.toString()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Deploy Stdb') {
|
|
steps {
|
|
build job: params.STDB_PUBLISH_JOB_NAME,
|
|
wait: true,
|
|
propagate: true,
|
|
parameters: [
|
|
string(name: 'SOURCE_BRANCH', value: params.SOURCE_BRANCH),
|
|
string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT),
|
|
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
|
|
string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
|
|
string(name: 'MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID', value: params.MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID),
|
|
string(name: 'DATABASE', value: params.DATABASE),
|
|
string(name: 'DATABASE_BACKUP_MODE', value: params.DATABASE_BACKUP_MODE),
|
|
string(name: 'SPACETIME_SERVER_URL', value: params.SPACETIME_SERVER_URL ?: ''),
|
|
string(name: 'SPACETIME_ROOT_DIR', value: params.SPACETIME_ROOT_DIR ?: '/stdb'),
|
|
string(name: 'SPACETIME_RUN_AS_USER', value: params.SPACETIME_RUN_AS_USER ?: 'spacetimedb'),
|
|
string(name: 'API_ENV_FILE', value: params.API_ENV_FILE ?: '/etc/genarrative/api-server.env'),
|
|
string(name: 'WORKER_ENV_FILE', value: params.WORKER_ENV_FILE ?: '/etc/genarrative/external-generation-worker.env'),
|
|
booleanParam(name: 'KEEP_MAINTENANCE_MODE', value: true),
|
|
string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET),
|
|
booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: params.CONFIRM_RELEASE_DEPLOY_AGENT),
|
|
string(name: 'BUILD_JOB_NAME', value: params.STDB_BUILD_JOB_NAME),
|
|
string(name: 'BUILD_NUMBER_TO_DEPLOY', value: env.STDB_BUILD_NUMBER),
|
|
]
|
|
}
|
|
}
|
|
|
|
stage('Stdb / Api Rollout Gate') {
|
|
when {
|
|
expression { return params.STDB_API_ROLLOUT_MODE == 'pause-after-stdb' }
|
|
}
|
|
steps {
|
|
timeout(time: 4, unit: 'HOURS') {
|
|
input message: 'SpacetimeDB module 已发布，站点和外部生成服务保持维护态。请使用与 api-server GENARRATIVE_SPACETIME_TOKEN 不同的 migration operator identity，在目标部署 agent 用本次 Stdb artifact 中的维护脚本完成 dry-run、分批 apply 与摘要回填；确认无 has_more，并确认 API runtime identity 未被授权为 migration operator 后再继续部署 API。', ok: '确认维护完成，继续部署 API', submitter: params.STDB_API_ROLLOUT_APPROVERS.trim(), submitterParameter: 'STDB_API_ROLLOUT_APPROVED_BY'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Deploy Api') {
|
|
steps {
|
|
build job: params.API_DEPLOY_JOB_NAME,
|
|
wait: true,
|
|
propagate: true,
|
|
parameters: [
|
|
string(name: 'SOURCE_BRANCH', value: params.SOURCE_BRANCH),
|
|
string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT),
|
|
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
|
|
string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
|
|
string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET),
|
|
booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: params.CONFIRM_RELEASE_DEPLOY_AGENT),
|
|
string(name: 'BUILD_JOB_NAME', value: params.API_BUILD_JOB_NAME),
|
|
string(name: 'BUILD_NUMBER_TO_DEPLOY', value: env.API_BUILD_NUMBER),
|
|
booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', value: params.INCLUDE_PINGORA_GATEWAY),
|
|
booleanParam(name: 'KEEP_MAINTENANCE_MODE', value: true),
|
|
string(name: 'API_ENV_FILE', value: params.API_ENV_FILE ?: '/etc/genarrative/api-server.env'),
|
|
string(name: 'WORKER_ENV_FILE', value: params.WORKER_ENV_FILE ?: '/etc/genarrative/external-generation-worker.env'),
|
|
string(name: 'DATABASE', value: params.DATABASE),
|
|
string(name: 'SPACETIME_SERVER_URL', value: params.SPACETIME_SERVER_URL ?: ''),
|
|
]
|
|
}
|
|
}
|
|
|
|
stage('Deploy Web') {
|
|
steps {
|
|
build job: params.WEB_DEPLOY_JOB_NAME,
|
|
wait: true,
|
|
propagate: true,
|
|
parameters: [
|
|
string(name: 'SOURCE_BRANCH', value: params.SOURCE_BRANCH),
|
|
string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT),
|
|
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
|
|
string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
|
|
string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET),
|
|
booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: params.CONFIRM_RELEASE_DEPLOY_AGENT),
|
|
string(name: 'BUILD_JOB_NAME', value: params.WEB_BUILD_JOB_NAME),
|
|
string(name: 'BUILD_NUMBER_TO_DEPLOY', value: env.WEB_BUILD_NUMBER),
|
|
]
|
|
}
|
|
}
|
|
|
|
stage('Exit Maintenance') {
|
|
when {
|
|
expression { return params.EXIT_MAINTENANCE_MODE_AFTER_COMPLETION != false }
|
|
}
|
|
agent none
|
|
steps {
|
|
script {
|
|
def deployLabel = params.DEPLOY_TARGET == 'development' ? 'linux && genarrative-dev-deploy' : 'linux && genarrative-release-deploy'
|
|
node(deployLabel) {
|
|
sh '''
|
|
bash -lc '
|
|
set -euo pipefail
|
|
maintenance_script="/opt/genarrative/current/scripts/deploy/maintenance-off.sh"
|
|
if [[ ! -f "${maintenance_script}" ]]; then
|
|
echo "Full 发布完成但 current release 缺少维护退出脚本: ${maintenance_script}" >&2
|
|
exit 1
|
|
fi
|
|
bash "${maintenance_script}"
|
|
'
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
post {
|
|
always {
|
|
script {
|
|
def notificationParameters = [
|
|
string(name: 'SOURCE_JOB_NAME', value: env.JOB_NAME),
|
|
string(name: 'SOURCE_BUILD_NUMBER', value: env.BUILD_NUMBER),
|
|
string(name: 'SOURCE_BUILD_URL', value: env.BUILD_URL ?: ''),
|
|
string(name: 'SOURCE_RESULT', value: currentBuild.currentResult ?: 'UNKNOWN'),
|
|
string(name: 'SOURCE_BRANCH', value: params.SOURCE_BRANCH ?: ''),
|
|
string(name: 'SOURCE_COMMIT', value: env.SOURCE_COMMIT ?: (params.COMMIT_HASH ?: '')),
|
|
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION ?: (params.BUILD_VERSION ?: '')),
|
|
string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET ?: ''),
|
|
string(name: 'DATABASE', value: params.DATABASE ?: ''),
|
|
string(name: 'INCLUDE_PINGORA_GATEWAY', value: String.valueOf(params.INCLUDE_PINGORA_GATEWAY)),
|
|
string(name: 'EXIT_MAINTENANCE_MODE_AFTER_COMPLETION', value: String.valueOf(params.EXIT_MAINTENANCE_MODE_AFTER_COMPLETION)),
|
|
string(name: 'SUMMARY', value: '全量构建发布编排结束'),
|
|
]
|
|
def notificationRecipients = params.NOTIFICATION_EMAILS?.trim()
|
|
if (notificationRecipients) {
|
|
notificationParameters.add(string(name: 'EMAIL_RECIPIENTS', value: notificationRecipients))
|
|
}
|
|
try {
|
|
build job: 'Genarrative-Notify-Email',
|
|
wait: false,
|
|
propagate: false,
|
|
parameters: notificationParameters
|
|
} catch (error) {
|
|
echo "邮件通知触发失败: ${error.message}"
|
|
}
|
|
}
|
|
}
|
|
success {
|
|
echo "Full Build-And-Deploy 完成: version=${env.EFFECTIVE_BUILD_VERSION}, commit=${env.SOURCE_COMMIT}"
|
|
}
|
|
}
|
|
}
|