Add resilient Jenkins inbound agent setup

2026-05-03 14:01:19 +08:00
parent 8e6d1971ea
commit 01b302d7eb
6 changed files with 417 additions and 3 deletions
--- a/scripts/build-production-release.sh
+++ b/scripts/build-production-release.sh
@@ -370,10 +370,15 @@ mkdir -p "${TARGET_DIR}/scripts" "${TARGET_DIR}/deploy"
 cp "${SCRIPT_DIR}/deploy/maintenance-on.sh" "${TARGET_DIR}/scripts/maintenance-on.sh"
 cp "${SCRIPT_DIR}/deploy/maintenance-off.sh" "${TARGET_DIR}/scripts/maintenance-off.sh"
 cp "${SCRIPT_DIR}/deploy/maintenance-status.sh" "${TARGET_DIR}/scripts/maintenance-status.sh"
+cp "${SCRIPT_DIR}/deploy/jenkins-inbound-agent-start.sh" "${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh"
+cp "${SCRIPT_DIR}/deploy/install-jenkins-inbound-agent.sh" "${TARGET_DIR}/scripts/install-jenkins-inbound-agent.sh"
+cp "${SCRIPT_DIR}/deploy/jenkins-agent-reverse-tunnel.ps1" "${TARGET_DIR}/scripts/jenkins-agent-reverse-tunnel.ps1"
 chmod +x \
  "${TARGET_DIR}/scripts/maintenance-on.sh" \
  "${TARGET_DIR}/scripts/maintenance-off.sh" \
-  "${TARGET_DIR}/scripts/maintenance-status.sh"
+  "${TARGET_DIR}/scripts/maintenance-status.sh" \
+  "${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh" \
+  "${TARGET_DIR}/scripts/install-jenkins-inbound-agent.sh"

 copy_required_file "${SCRIPT_DIR}/spacetime-export-migration-json.mjs" "${TARGET_DIR}/scripts/database-export.mjs" "数据库导出脚本"
 copy_required_file "${SCRIPT_DIR}/spacetime-import-migration-json.mjs" "${TARGET_DIR}/scripts/database-import.mjs" "数据库导入脚本"
@@ -398,7 +403,7 @@ cat >"${TARGET_DIR}/README.md" <<EOF
 - \`spacetime_module.wasm\`：SpacetimeDB 模块 wasm。
 - \`*.sha256\`：发布产物 checksum，用于部署前校验。
 - \`release-manifest.json\`：发布版本、源码 commit 与产物清单。
- \`scripts/\`：维护模式脚本、数据库导入导出脚本和迁移授权脚本。
+- \`scripts/\`：维护模式脚本、数据库导入导出脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
 - \`deploy/\`：systemd、Nginx 和生产环境变量示例；\`deploy/nginx/genarrative-dev-http.conf\` 仅供无域名开发服初始化使用。

 ## 生产部署口径
--- a/scripts/deploy/install-jenkins-inbound-agent.sh
+++ b/scripts/deploy/install-jenkins-inbound-agent.sh
@@ -0,0 +1,218 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+用法:
+  sudo scripts/deploy/install-jenkins-inbound-agent.sh \
+    --agent-name genarrative-release-deploy-01 \
+    --jenkins-url http://<jenkins-controller>:8080/ \
+    --secret-file /path/to/inbound-agent.secret
+
+可选参数:
+  --run-user <user>       systemd 运行用户，默认 root；当前生产流水线仍需要特权操作。
+  --run-group <group>     systemd 运行用户组，默认跟随 --run-user。
+  --workdir <path>        agent 工作目录，默认 /var/lib/jenkins/agent/<agent-name>。
+  --jar-path <path>       agent.jar 落盘路径，默认 /opt/jenkins-agent/agent.jar。
+  --java-bin <path>       Java 命令路径，默认 java；需要固定 JDK 时传绝对路径。
+  --no-websocket          不使用 WebSocket inbound 连接。
+  --no-enable             只安装 unit，不执行 systemctl enable。
+  --no-start              只安装 unit，不立即启动服务。
+  --dry-run               只打印操作，不写入系统。
+
+密钥来源:
+  优先使用 --secret-file；如果未传入，则读取环境变量 JENKINS_AGENT_SECRET；
+  如果目标机已存在 /etc/jenkins-agent/<agent-name>.secret，则保留原密钥。
+EOF
+}
+
+AGENT_NAME=""
+JENKINS_URL_VALUE=""
+SECRET_FILE=""
+RUN_USER="root"
+RUN_GROUP=""
+WORKDIR=""
+JAR_PATH="/opt/jenkins-agent/agent.jar"
+JAVA_BIN="java"
+USE_WEBSOCKET="true"
+ENABLE_SERVICE="true"
+START_SERVICE="true"
+DRY_RUN="false"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --agent-name)
+      AGENT_NAME="${2:?缺少 --agent-name 的值}"
+      shift 2
+      ;;
+    --jenkins-url)
+      JENKINS_URL_VALUE="${2:?缺少 --jenkins-url 的值}"
+      shift 2
+      ;;
+    --secret-file)
+      SECRET_FILE="${2:?缺少 --secret-file 的值}"
+      shift 2
+      ;;
+    --run-user)
+      RUN_USER="${2:?缺少 --run-user 的值}"
+      shift 2
+      ;;
+    --run-group)
+      RUN_GROUP="${2:?缺少 --run-group 的值}"
+      shift 2
+      ;;
+    --workdir)
+      WORKDIR="${2:?缺少 --workdir 的值}"
+      shift 2
+      ;;
+    --jar-path)
+      JAR_PATH="${2:?缺少 --jar-path 的值}"
+      shift 2
+      ;;
+    --java-bin)
+      JAVA_BIN="${2:?缺少 --java-bin 的值}"
+      shift 2
+      ;;
+    --no-websocket)
+      USE_WEBSOCKET="false"
+      shift
+      ;;
+    --no-enable)
+      ENABLE_SERVICE="false"
+      shift
+      ;;
+    --no-start)
+      START_SERVICE="false"
+      shift
+      ;;
+    --dry-run)
+      DRY_RUN="true"
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "[jenkins-agent-install] 未知参数: $1" >&2
+      usage
+      exit 2
+      ;;
+  esac
+done
+
+if [[ -z "${AGENT_NAME}" || -z "${JENKINS_URL_VALUE}" ]]; then
+  usage
+  exit 2
+fi
+
+if [[ -z "${RUN_GROUP}" ]]; then
+  RUN_GROUP="${RUN_USER}"
+fi
+
+if [[ -z "${WORKDIR}" ]]; then
+  WORKDIR="/var/lib/jenkins/agent/${AGENT_NAME}"
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+START_SOURCE="${SCRIPT_DIR}/jenkins-inbound-agent-start.sh"
+UNIT_SOURCE="${REPO_ROOT}/deploy/systemd/jenkins-agent@.service"
+CONFIG_DIR="/etc/jenkins-agent"
+CONFIG_FILE="${CONFIG_DIR}/${AGENT_NAME}.env"
+SECRET_TARGET="${CONFIG_DIR}/${AGENT_NAME}.secret"
+SERVICE_NAME="jenkins-agent@${AGENT_NAME}.service"
+
+run_cmd() {
+  echo "+ $*"
+  if [[ "${DRY_RUN}" != "true" ]]; then
+    "$@"
+  fi
+}
+
+write_file() {
+  local target="$1"
+  local mode="$2"
+  local owner="$3"
+  local group="$4"
+  local temp_file
+
+  temp_file="$(mktemp)"
+  cat >"${temp_file}"
+  echo "+ install -m ${mode} ${temp_file} ${target}"
+  if [[ "${DRY_RUN}" != "true" ]]; then
+    install -m "${mode}" -o "${owner}" -g "${group}" "${temp_file}" "${target}"
+  fi
+  rm -f "${temp_file}"
+}
+
+if [[ ! -f "${START_SOURCE}" ]]; then
+  echo "[jenkins-agent-install] 缺少启动脚本: ${START_SOURCE}" >&2
+  exit 1
+fi
+
+if [[ ! -f "${UNIT_SOURCE}" ]]; then
+  echo "[jenkins-agent-install] 缺少 systemd 模板: ${UNIT_SOURCE}" >&2
+  exit 1
+fi
+
+if [[ "${RUN_USER}" != "root" ]] && ! id "${RUN_USER}" >/dev/null 2>&1; then
+  run_cmd useradd --system --create-home --home-dir "/var/lib/${RUN_USER}" --shell /bin/bash "${RUN_USER}"
+fi
+
+run_cmd mkdir -p "${CONFIG_DIR}" "$(dirname "${JAR_PATH}")" "${WORKDIR}"
+run_cmd chmod 0755 "${CONFIG_DIR}" "$(dirname "${JAR_PATH}")"
+
+if [[ "${DRY_RUN}" != "true" ]]; then
+  chown -R "${RUN_USER}:${RUN_GROUP}" "$(dirname "${JAR_PATH}")" "${WORKDIR}"
+fi
+
+run_cmd install -m 0755 "${START_SOURCE}" /usr/local/bin/jenkins-inbound-agent-start
+
+UNIT_TMP="$(mktemp)"
+sed \
+  -e "s|^User=.*|User=${RUN_USER}|" \
+  -e "s|^Group=.*|Group=${RUN_GROUP}|" \
+  "${UNIT_SOURCE}" >"${UNIT_TMP}"
+run_cmd install -m 0644 "${UNIT_TMP}" /etc/systemd/system/jenkins-agent@.service
+rm -f "${UNIT_TMP}"
+
+write_file "${CONFIG_FILE}" 0644 root root <<EOF
+JENKINS_URL='${JENKINS_URL_VALUE}'
+JENKINS_AGENT_NAME='${AGENT_NAME}'
+JENKINS_AGENT_WORKDIR='${WORKDIR}'
+JENKINS_AGENT_JAR='${JAR_PATH}'
+JENKINS_AGENT_SECRET_FILE='${SECRET_TARGET}'
+JENKINS_AGENT_USE_WEBSOCKET='${USE_WEBSOCKET}'
+JENKINS_AGENT_JAVA_BIN='${JAVA_BIN}'
+EOF
+
+if [[ -n "${SECRET_FILE}" ]]; then
+  if [[ ! -r "${SECRET_FILE}" ]]; then
+    echo "[jenkins-agent-install] 密钥文件不可读: ${SECRET_FILE}" >&2
+    exit 1
+  fi
+  run_cmd install -m 0600 -o "${RUN_USER}" -g "${RUN_GROUP}" "${SECRET_FILE}" "${SECRET_TARGET}"
+elif [[ -n "${JENKINS_AGENT_SECRET:-}" ]]; then
+  write_file "${SECRET_TARGET}" 0600 "${RUN_USER}" "${RUN_GROUP}" <<EOF
+${JENKINS_AGENT_SECRET}
+EOF
+elif [[ -f "${SECRET_TARGET}" ]]; then
+  echo "[jenkins-agent-install] 已存在密钥文件，保留不覆盖: ${SECRET_TARGET}"
+else
+  echo "[jenkins-agent-install] 缺少 inbound agent secret。请传 --secret-file，或设置 JENKINS_AGENT_SECRET。" >&2
+  exit 1
+fi
+
+run_cmd systemctl daemon-reload
+
+if [[ "${ENABLE_SERVICE}" == "true" ]]; then
+  run_cmd systemctl enable "${SERVICE_NAME}"
+fi
+
+if [[ "${START_SERVICE}" == "true" ]]; then
+  run_cmd systemctl restart "${SERVICE_NAME}"
+  run_cmd systemctl status "${SERVICE_NAME}" --no-pager -l
+fi
+
+echo "[jenkins-agent-install] 完成: ${SERVICE_NAME}"
--- a/scripts/deploy/jenkins-agent-reverse-tunnel.ps1
+++ b/scripts/deploy/jenkins-agent-reverse-tunnel.ps1
@@ -0,0 +1,50 @@
+param(
+  [string]$RemoteHost = "",
+  [string]$RemoteUser = "root",
+  [string]$SshKeyPath = "$env:USERPROFILE\.ssh\dsk.pem",
+  [string]$LocalJenkinsHost = "127.0.0.1",
+  [int]$LocalJenkinsPort = 8080,
+  [int]$LocalAgentPort = 50000,
+  [int]$RemoteJenkinsPort = 18080,
+  [int]$RemoteAgentPort = 50000,
+  [int]$RestartDelaySeconds = 10
+)
+
+$ErrorActionPreference = "Stop"
+
+function Write-Log {
+  param([string]$Message)
+  $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+  Write-Output "[$timestamp] $Message"
+}
+
+$ssh = (Get-Command ssh.exe -ErrorAction Stop).Source
+$remote = "$RemoteUser@$RemoteHost"
+
+if (-not $RemoteHost) {
+  throw "RemoteHost is required."
+}
+
+if (-not (Test-Path -LiteralPath $SshKeyPath)) {
+  throw "SSH key not found: $SshKeyPath"
+}
+
+while ($true) {
+  $args = @(
+    "-i", $SshKeyPath,
+    "-o", "StrictHostKeyChecking=accept-new",
+    "-o", "ExitOnForwardFailure=yes",
+    "-o", "ServerAliveInterval=30",
+    "-o", "ServerAliveCountMax=3",
+    "-N",
+    "-R", "127.0.0.1:${RemoteJenkinsPort}:${LocalJenkinsHost}:${LocalJenkinsPort}",
+    "-R", "127.0.0.1:${RemoteAgentPort}:${LocalJenkinsHost}:${LocalAgentPort}",
+    $remote
+  )
+
+  Write-Log "Starting Jenkins agent reverse tunnel: $remote"
+  & $ssh @args
+  $exitCode = $LASTEXITCODE
+  Write-Log "Reverse tunnel exited, exitCode=$exitCode; retrying in ${RestartDelaySeconds}s."
+  Start-Sleep -Seconds $RestartDelaySeconds
+}
--- a/scripts/deploy/jenkins-inbound-agent-start.sh
+++ b/scripts/deploy/jenkins-inbound-agent-start.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+用法:
+  jenkins-inbound-agent-start <agent-name>
+
+说明:
+  该脚本由 systemd 调用，读取 /etc/jenkins-agent/<agent-name>.env，
+  下载 Jenkins agent.jar，并通过 inbound WebSocket 连接 Jenkins controller。
+EOF
+}
+
+AGENT_INSTANCE="${1:-}"
+if [[ -z "${AGENT_INSTANCE}" ]]; then
+  usage
+  exit 2
+fi
+
+CONFIG_FILE="${JENKINS_AGENT_CONFIG_FILE:-/etc/jenkins-agent/${AGENT_INSTANCE}.env}"
+if [[ ! -r "${CONFIG_FILE}" ]]; then
+  echo "[jenkins-agent] 配置文件不可读: ${CONFIG_FILE}" >&2
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1090
+source "${CONFIG_FILE}"
+set +a
+
+JENKINS_AGENT_NAME="${JENKINS_AGENT_NAME:-${AGENT_INSTANCE}}"
+JENKINS_AGENT_WORKDIR="${JENKINS_AGENT_WORKDIR:-/var/lib/jenkins/agent/${JENKINS_AGENT_NAME}}"
+JENKINS_AGENT_JAR="${JENKINS_AGENT_JAR:-/opt/jenkins-agent/agent.jar}"
+JENKINS_AGENT_SECRET_FILE="${JENKINS_AGENT_SECRET_FILE:-/etc/jenkins-agent/${JENKINS_AGENT_NAME}.secret}"
+JENKINS_AGENT_USE_WEBSOCKET="${JENKINS_AGENT_USE_WEBSOCKET:-true}"
+JENKINS_AGENT_JAVA_BIN="${JENKINS_AGENT_JAVA_BIN:-java}"
+
+if [[ -z "${JENKINS_URL:-}" ]]; then
+  echo "[jenkins-agent] JENKINS_URL 不能为空。" >&2
+  exit 1
+fi
+
+if [[ -z "${JENKINS_AGENT_SECRET:-}" ]]; then
+  if [[ ! -r "${JENKINS_AGENT_SECRET_FILE}" ]]; then
+    echo "[jenkins-agent] 未提供 JENKINS_AGENT_SECRET，且密钥文件不可读: ${JENKINS_AGENT_SECRET_FILE}" >&2
+    exit 1
+  fi
+  JENKINS_AGENT_SECRET="$(tr -d '\r\n' <"${JENKINS_AGENT_SECRET_FILE}")"
+fi
+
+if [[ -z "${JENKINS_AGENT_SECRET}" ]]; then
+  echo "[jenkins-agent] Jenkins inbound agent secret 不能为空。" >&2
+  exit 1
+fi
+
+mkdir -p "$(dirname "${JENKINS_AGENT_JAR}")" "${JENKINS_AGENT_WORKDIR}"
+
+AGENT_JAR_URL="${JENKINS_URL%/}/jnlpJars/agent.jar"
+AGENT_JAR_TMP="${JENKINS_AGENT_JAR}.tmp"
+
+echo "[jenkins-agent] 下载 agent.jar: ${AGENT_JAR_URL}"
+curl -fsSL --retry 5 --retry-delay 5 "${AGENT_JAR_URL}" -o "${AGENT_JAR_TMP}"
+mv "${AGENT_JAR_TMP}" "${JENKINS_AGENT_JAR}"
+
+agent_args=(
+  "${JENKINS_AGENT_JAVA_BIN}"
+  -jar "${JENKINS_AGENT_JAR}"
+  -url "${JENKINS_URL}"
+  -secret "${JENKINS_AGENT_SECRET}"
+  -name "${JENKINS_AGENT_NAME}"
+  -workDir "${JENKINS_AGENT_WORKDIR}"
+)
+
+if [[ "${JENKINS_AGENT_USE_WEBSOCKET}" == "true" ]]; then
+  agent_args+=(-webSocket)
+fi
+
+echo "[jenkins-agent] 启动 inbound agent: ${JENKINS_AGENT_NAME}"
+exec "${agent_args[@]}"