master #14

Merged
kdletters merged 226 commits from master into release 2026-05-13 13:23:09 +08:00
6 changed files with 94 additions and 56 deletions
Showing only changes of commit e390b72a0c - Show all commits

View File

@@ -225,6 +225,14 @@
- 验证:发布链路使用当前 `deploy/systemd``deploy/nginx``scripts/deploy``jenkins/Jenkinsfile.production-*`
- 关联:`docs/technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md`
## Jenkins 可选参数在 set -u 下不能裸读
- 现象:数据库导入或导出流水线报 `INCLUDE_TABLES: unbound variable`,或其它可选参数在 Bash 中未定义即退出。
- 原因Jenkins string/boolean 参数留空时不一定会导出同名环境变量,而生产数据库导入导出脚本块启用了 `set -u`
- 处理:进入 Bash 执行块后先使用 `${VAR:-}``${VAR:-默认值}` 收敛成本地变量;必填项使用 `${VAR:?中文错误}` 明确失败原因。
- 验证:扫描 `jenkins/Jenkinsfile.production-database-export``jenkins/Jenkinsfile.production-database-import`,确认 `INCLUDE_TABLES``CHUNK_SIZE``SERVER_BACKUP_DIRECTORY``SMOKE_HEALTH_URL` 等可选参数不再裸读。
- 关联:`docs/technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md``jenkins/Jenkinsfile.production-database-export``jenkins/Jenkinsfile.production-database-import`
## 个人任务 scope 不得扩成 work/site/module
- 现象:个人任务配置为 `work` / `site` / `module` 后进度串桶或静默按 0 处理。

View File

@@ -430,6 +430,7 @@ WASM_SOURCE="${CARGO_TARGET_DIR}/wasm32-unknown-unknown/release/spacetime_module
- `COMMIT_HASH` 非空时,先解析到完整 commit再用 `git merge-base --is-ancestor <commit> refs/remotes/origin/<SOURCE_BRANCH>` 校验该提交属于指定分支,校验通过后 detached checkout。
- 流水线日志必须输出最终 `SOURCE_BRANCH` 与实际 `SOURCE_COMMIT`
- 构建产物必须写入 `release-manifest.json`,至少包含 `version``source_branch``source_commit``built_at` 和组件类型,供发布、回滚和审计使用。
- Windows 构建 Job 写入 `.jenkins-source-commit` 时必须使用 UTF-8 无 BOM部署脚本在校验 `COMMIT_HASH` 前也会剥离 UTF-8 BOM 和 CRLF避免上游 PowerShell 5.1 `Set-Content -Encoding UTF8` 产生的不可见 BOM 让下游发布误判 commit hash 非法。
构建流水线使用上述参数决定实际构建源码。发布流水线也暴露同名参数,但只用于选择本次发布使用的部署脚本、配置模板和 smoke test 逻辑;被发布的应用文件仍必须来自 Jenkins 归档产物或指定 release 包,不允许在发布流水线中重新构建。
@@ -550,6 +551,7 @@ WASM_SOURCE="${CARGO_TARGET_DIR}/wasm32-unknown-unknown/release/spacetime_module
- 从目标机器本机 SpacetimeDB 导出指定数据库数据,默认连接 `SPACETIME_SERVER_URL=http://127.0.0.1:3101`,自托管 `root-dir` 默认 `/stdb`
- 产物归档到 Jenkins并可额外保存到 `SERVER_BACKUP_DIRECTORY`
- 敏感 token 与 bootstrap secret 只通过 Jenkins Secret Text 凭据 ID 注入,不作为明文 Job 参数。
- 导出和导入流水线的 Bash 执行块启用 `set -u`;所有可选 Jenkins 参数必须先通过 `${VAR:-}` 收敛成本地默认值,再传给 Node 迁移脚本,避免空参数没有导出时触发 `unbound variable`
- 成功后解除维护模式。
- 失败时保留维护模式并邮件通知。

View File

@@ -116,8 +116,15 @@ pipeline {
chmod +x scripts/deploy/maintenance-on.sh scripts/deploy/maintenance-off.sh
export_dir="${WORKSPACE_EXPORT_DIRECTORY}"
output_path="${export_dir}/${EFFECTIVE_EXPORT_NAME}"
database="${DATABASE:?DATABASE 不能为空}"
spacetime_server_url="${SPACETIME_SERVER_URL:-}"
spacetime_server="${SPACETIME_SERVER:-}"
spacetime_root_dir="${EFFECTIVE_SPACETIME_ROOT_DIR:-}"
include_tables="${INCLUDE_TABLES:-}"
server_backup_directory="${EFFECTIVE_SERVER_BACKUP_DIRECTORY:-}"
export_dir="${WORKSPACE_EXPORT_DIRECTORY:-database-exports}"
export_name="${EFFECTIVE_EXPORT_NAME:-spacetime-migration-${BUILD_NUMBER:-manual}.json}"
output_path="${export_dir}/${export_name}"
mkdir -p "${export_dir}"
maintenance_entered=0
@@ -132,20 +139,20 @@ pipeline {
}
trap on_exit EXIT
scripts/deploy/maintenance-on.sh "database export ${DATABASE}"
scripts/deploy/maintenance-on.sh "database export ${database}"
maintenance_entered=1
args=(scripts/spacetime-export-migration-json.mjs --out "${output_path}" --database "${DATABASE}")
if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
args+=(--server-url "${SPACETIME_SERVER_URL}")
elif [[ -n "${SPACETIME_SERVER}" ]]; then
args+=(--server "${SPACETIME_SERVER}")
args=(scripts/spacetime-export-migration-json.mjs --out "${output_path}" --database "${database}")
if [[ -n "${spacetime_server_url}" ]]; then
args+=(--server-url "${spacetime_server_url}")
elif [[ -n "${spacetime_server}" ]]; then
args+=(--server "${spacetime_server}")
fi
if [[ -n "${EFFECTIVE_SPACETIME_ROOT_DIR}" ]]; then
args+=(--root-dir "${EFFECTIVE_SPACETIME_ROOT_DIR}")
if [[ -n "${spacetime_root_dir}" ]]; then
args+=(--root-dir "${spacetime_root_dir}")
fi
if [[ -n "${INCLUDE_TABLES}" ]]; then
args+=(--include "${INCLUDE_TABLES}")
if [[ -n "${include_tables}" ]]; then
args+=(--include "${include_tables}")
fi
args+=(--note "jenkins database export ${BUILD_TAG}")
@@ -153,10 +160,10 @@ pipeline {
test -s "${output_path}"
sha256sum "${output_path}" >"${output_path}.sha256"
if [[ -n "${EFFECTIVE_SERVER_BACKUP_DIRECTORY}" ]]; then
mkdir -p "${EFFECTIVE_SERVER_BACKUP_DIRECTORY}"
install -m 0640 "${output_path}" "${EFFECTIVE_SERVER_BACKUP_DIRECTORY}/${EFFECTIVE_EXPORT_NAME}"
install -m 0640 "${output_path}.sha256" "${EFFECTIVE_SERVER_BACKUP_DIRECTORY}/${EFFECTIVE_EXPORT_NAME}.sha256"
if [[ -n "${server_backup_directory}" ]]; then
mkdir -p "${server_backup_directory}"
install -m 0640 "${output_path}" "${server_backup_directory}/${export_name}"
install -m 0640 "${output_path}.sha256" "${server_backup_directory}/${export_name}.sha256"
fi
echo "[database-export] 完成: ${output_path}, source_commit=$(cat .jenkins-source-commit)"

View File

@@ -181,11 +181,12 @@ pipeline {
bash -lc '
set -euo pipefail
manual_filename="${MANUAL_INPUT_FILE_FILENAME:-}"
confirm_input_file="${CONFIRM_INPUT_FILE:-}"
if [[ -z "${manual_filename}" ]]; then
echo "[database-import] 无法读取 MANUAL_INPUT_FILE_FILENAME不能确认手动上传文件名。" >&2
exit 1
fi
if [[ "${CONFIRM_INPUT_FILE}" != "${manual_filename}" ]]; then
if [[ "${confirm_input_file}" != "${manual_filename}" ]]; then
echo "[database-import] CONFIRM_INPUT_FILE 必须与手动上传文件原始文件名一致: ${manual_filename}" >&2
exit 1
fi
@@ -209,7 +210,20 @@ pipeline {
chmod +x scripts/deploy/maintenance-on.sh scripts/deploy/maintenance-off.sh
input_path="${EFFECTIVE_INPUT_FILE}"
database="${DATABASE:?DATABASE 不能为空}"
spacetime_server_url="${SPACETIME_SERVER_URL:-}"
spacetime_server="${SPACETIME_SERVER:-}"
spacetime_root_dir="${SPACETIME_ROOT_DIR:-}"
server_backup_directory="${SERVER_BACKUP_DIRECTORY:-}"
include_tables="${INCLUDE_TABLES:-}"
chunk_size="${CHUNK_SIZE:-}"
dry_run="${DRY_RUN:-true}"
incremental="${INCREMENTAL:-true}"
replace_existing="${REPLACE_EXISTING:-false}"
run_smoke_test="${RUN_SMOKE_TEST:-true}"
smoke_health_url="${SMOKE_HEALTH_URL:-}"
input_path="${EFFECTIVE_INPUT_FILE:?EFFECTIVE_INPUT_FILE 不能为空}"
if [[ "${input_path}" != /* ]]; then
input_path="${WORKSPACE}/${input_path}"
fi
@@ -218,8 +232,9 @@ pipeline {
exit 1
fi
backup_dir="${PRE_IMPORT_BACKUP_DIRECTORY}"
backup_path="${backup_dir}/${EFFECTIVE_PRE_IMPORT_BACKUP_NAME}"
backup_dir="${PRE_IMPORT_BACKUP_DIRECTORY:-database-pre-import-backups}"
backup_name="${EFFECTIVE_PRE_IMPORT_BACKUP_NAME:-pre-import-${BUILD_NUMBER:-manual}.json}"
backup_path="${backup_dir}/${backup_name}"
mkdir -p "${backup_dir}"
completed=0
@@ -232,20 +247,20 @@ pipeline {
}
trap on_exit EXIT
scripts/deploy/maintenance-on.sh "database import ${DATABASE}"
scripts/deploy/maintenance-on.sh "database import ${database}"
backup_args=(scripts/spacetime-export-migration-json.mjs --out "${backup_path}" --database "${DATABASE}")
import_args=(scripts/spacetime-import-migration-json.mjs --in "${input_path}" --database "${DATABASE}")
backup_args=(scripts/spacetime-export-migration-json.mjs --out "${backup_path}" --database "${database}")
import_args=(scripts/spacetime-import-migration-json.mjs --in "${input_path}" --database "${database}")
for args_name in backup_args import_args; do
declare -n current_args="${args_name}"
# server-url 明确指向目标实例时,不再同时透传默认 alias避免 CLI 授权与 HTTP 导入落到不同目标。
if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
current_args+=(--server-url "${SPACETIME_SERVER_URL}")
elif [[ -n "${SPACETIME_SERVER}" ]]; then
current_args+=(--server "${SPACETIME_SERVER}")
if [[ -n "${spacetime_server_url}" ]]; then
current_args+=(--server-url "${spacetime_server_url}")
elif [[ -n "${spacetime_server}" ]]; then
current_args+=(--server "${spacetime_server}")
fi
if [[ -n "${SPACETIME_ROOT_DIR}" ]]; then
current_args+=(--root-dir "${SPACETIME_ROOT_DIR}")
if [[ -n "${spacetime_root_dir}" ]]; then
current_args+=(--root-dir "${spacetime_root_dir}")
fi
done
@@ -254,25 +269,25 @@ pipeline {
test -s "${backup_path}"
sha256sum "${backup_path}" >"${backup_path}.sha256"
if [[ -n "${SERVER_BACKUP_DIRECTORY}" ]]; then
mkdir -p "${SERVER_BACKUP_DIRECTORY}"
install -m 0640 "${backup_path}" "${SERVER_BACKUP_DIRECTORY}/${EFFECTIVE_PRE_IMPORT_BACKUP_NAME}"
install -m 0640 "${backup_path}.sha256" "${SERVER_BACKUP_DIRECTORY}/${EFFECTIVE_PRE_IMPORT_BACKUP_NAME}.sha256"
if [[ -n "${server_backup_directory}" ]]; then
mkdir -p "${server_backup_directory}"
install -m 0640 "${backup_path}" "${server_backup_directory}/${backup_name}"
install -m 0640 "${backup_path}.sha256" "${server_backup_directory}/${backup_name}.sha256"
fi
if [[ -n "${INCLUDE_TABLES}" ]]; then
import_args+=(--include "${INCLUDE_TABLES}")
if [[ -n "${include_tables}" ]]; then
import_args+=(--include "${include_tables}")
fi
if [[ -n "${CHUNK_SIZE}" ]]; then
import_args+=(--chunk-size "${CHUNK_SIZE}")
if [[ -n "${chunk_size}" ]]; then
import_args+=(--chunk-size "${chunk_size}")
fi
if [[ "${DRY_RUN}" == "true" ]]; then
if [[ "${dry_run}" == "true" ]]; then
import_args+=(--dry-run)
fi
if [[ "${INCREMENTAL}" == "true" ]]; then
if [[ "${incremental}" == "true" ]]; then
import_args+=(--incremental)
fi
if [[ "${REPLACE_EXISTING}" == "true" ]]; then
if [[ "${replace_existing}" == "true" ]]; then
import_args+=(--replace-existing)
fi
import_args+=(--note "jenkins database import ${BUILD_TAG}")
@@ -280,13 +295,13 @@ pipeline {
node "${import_args[@]}"
# 导入成功后只做本机健康检查;业务级数据核验仍以迁移脚本的表级统计为准。
if [[ "${RUN_SMOKE_TEST}" == "true" && -n "${SMOKE_HEALTH_URL}" ]]; then
curl -fsS --max-time 10 "${SMOKE_HEALTH_URL}" >/dev/null
if [[ "${run_smoke_test}" == "true" && -n "${smoke_health_url}" ]]; then
curl -fsS --max-time 10 "${smoke_health_url}" >/dev/null
fi
scripts/deploy/maintenance-off.sh
completed=1
echo "[database-import] 完成: dry_run=${DRY_RUN}, database=${DATABASE}, source_commit=$(cat .jenkins-source-commit)"
echo "[database-import] 完成: dry_run=${dry_run}, database=${database}, source_commit=$(cat .jenkins-source-commit)"
'
'''
}

View File

@@ -57,10 +57,12 @@ pipeline {
git checkout --force "origin/$sourceBranch"
}
git clean -ffdx
git rev-parse HEAD | Set-Content -Encoding UTF8 .jenkins-source-commit
$resolvedCommit = (git rev-parse HEAD).Trim()
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
[System.IO.File]::WriteAllText((Join-Path (Get-Location) '.jenkins-source-commit'), "$resolvedCommit`n", $utf8NoBom)
'''
script {
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').trim()
env.SOURCE_COMMIT = readFile('.jenkins-source-commit').replace('\uFEFF', '').trim()
env.EFFECTIVE_BUILD_VERSION = params.BUILD_VERSION?.trim() ? params.BUILD_VERSION.trim() : env.BUILD_NUMBER
}
}

View File

@@ -7,6 +7,10 @@ COMMIT_HASH="${COMMIT_HASH:-}"
GIT_REMOTE_URL="${GIT_REMOTE_URL:-}"
SOURCE_COMMIT_FILE="${SOURCE_COMMIT_FILE:-.jenkins-source-commit}"
# Windows PowerShell 5.1 的 UTF-8 输出可能带 BOM下游参数校验前先剥离不可见字节。
SOURCE_BRANCH="$(printf "%s" "${SOURCE_BRANCH}" | sed $'s/^\xef\xbb\xbf//' | tr -d '\r\n')"
COMMIT_HASH="$(printf "%s" "${COMMIT_HASH}" | sed $'s/^\xef\xbb\xbf//' | tr -d '\r\n')"
if [[ ! "${SOURCE_BRANCH}" =~ ^[0-9A-Za-z._/-]+$ ]]; then
echo "[jenkins-checkout-source] SOURCE_BRANCH 只能包含数字、字母、点、下划线、短横线和斜杠: ${SOURCE_BRANCH}" >&2
exit 1