fix(deploy): ensure release tracking outbox path

2026-05-21 15:27:19 +08:00
parent 8d716e84ad
commit 487efff9c4
4 changed files with 167 additions and 0 deletions
--- a/.hermes/shared-memory/pitfalls.md
+++ b/.hermes/shared-memory/pitfalls.md
@@ -46,6 +46,14 @@
 - 验证：普通 route 请求在 SpacetimeDB 不可用时仍能返回，恢复后 sealed 文件会继续被清理。
 - 关联：`server-rs/crates/api-server/src/tracking_outbox.rs`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。

+## release tracking outbox 权限错误先查 env 缺失
+
+- 现象：release 机器 `journalctl -u genarrative-api.service` 每秒刷 `tracking outbox 定时封存 active 文件失败 error=Permission denied (os error 13)` 和 `tracking outbox 批量写入 SpacetimeDB 失败`。
+- 原因：旧 `/etc/genarrative/api-server.env` 没有 `GENARRATIVE_TRACKING_OUTBOX_DIR` 时，api-server 会回退到本地开发默认相对路径 `server-rs/.data/tracking-outbox`；systemd 工作目录是只读发布目录 `/opt/genarrative/releases/<version>`，`genarrative` 用户不能在其中创建 `server-rs`。
+- 处理：补齐 `GENARRATIVE_TRACKING_OUTBOX_DIR=/var/lib/genarrative/tracking-outbox` 及 batch/flush/max 配置，创建并授权 `/var/lib/genarrative/tracking-outbox` 给 `genarrative:genarrative`，再重启 `genarrative-api.service`。Server-Provision 与 API-Deploy 会保留旧 env 但自动补缺这些运行态路径。
+- 验证：`tr '\0' '\n' < /proc/$(systemctl show genarrative-api.service -p MainPID --value)/environ | grep GENARRATIVE_TRACKING_OUTBOX_DIR` 应指向 `/var/lib/genarrative/tracking-outbox`；重启后当前 PID 不再出现 `Permission denied (os error 13)`。
+- 关联：`scripts/deploy/production-api-deploy.sh`、`scripts/jenkins-server-provision.sh`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。
+
 ## 汪汪声浪入口不要再回到独立配置阶段

 - 现象：汪汪声浪入口如果继续切换到独立配置阶段，会和拼图、抓大鹅的创作页内嵌结构不一致，用户会感觉入口跳页。
--- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md
+++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md
@@ -260,6 +260,17 @@ GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES=268435456

 outbox 采用 NDJSON 文件保存原始事件。达到 `BATCH_SIZE` 时会立刻把当前 active 文件原子封存为 sealed 文件，并马上切到新的 active 继续写入；后台 worker 异步 flush sealed 文件，HTTP 请求线程不等待 SpacetimeDB。`FLUSH_INTERVAL_MS` 只负责兜底封存长时间未满批的 active 文件。SpacetimeDB 批量 procedure 返回成功后删除 sealed 文件，失败则保留文件并重试。`MAX_BYTES` 是磁盘保护阈值，不是 flush 阈值；超过后低价值 route tracking 可以被丢弃并记录日志 / 指标，关键同步事件不进入该丢弃路径。sealed 文件若出现无法解析的坏行，会重命名为 `corrupt-*` 隔离并记录 `genarrative.tracking_outbox.files.corrupt` 指标，避免一个坏文件阻塞后续批量入库。该机制提供至少一次投递语义，依赖 `tracking_event.event_id` 幂等跳过重复事件。

+release 机器如果日志每秒刷 `tracking outbox ... Permission denied (os error 13)`，先检查 `/etc/genarrative/api-server.env` 是否缺少 `GENARRATIVE_TRACKING_OUTBOX_DIR`。缺少时 `api-server` 会回退到本地开发默认相对路径 `server-rs/.data/tracking-outbox`，而 systemd 的工作目录是只读发布目录 `/opt/genarrative/releases/<version>`，`genarrative` 用户无法在其中创建 `server-rs`。修复顺序：
+
+```bash
+install -d -o genarrative -g genarrative -m 0750 /var/lib/genarrative/tracking-outbox
+grep -n '^GENARRATIVE_TRACKING_OUTBOX' /etc/genarrative/api-server.env
+systemctl restart genarrative-api.service
+journalctl -u genarrative-api.service --since '30 seconds ago' --no-pager | grep -E 'tracking outbox|Permission denied|os error 13'
+```
+
+`Genarrative-Server-Provision` 和 `Genarrative-Api-Deploy` 会在保留旧 `/etc/genarrative/api-server.env` 的前提下补齐缺失的 tracking outbox 与 auth-store 运行态路径，并确保 `/var/lib/genarrative/tracking-outbox`、`/var/lib/genarrative/auth` 归属 `genarrative:genarrative`。
+
 常用检查思路：

 ```sql
--- a/scripts/deploy/production-api-deploy.sh
+++ b/scripts/deploy/production-api-deploy.sh
@@ -120,6 +120,115 @@ PY
  fi
 }

+read_env_value() {
+  local file_path="$1"
+  local key="$2"
+
+  if [[ ! -f "${file_path}" ]]; then
+    return 0
+  fi
+
+  local python_script='
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+key = sys.argv[2]
+if not path.exists():
+    raise SystemExit(0)
+for raw_line in path.read_text(encoding="utf-8").splitlines():
+    line = raw_line.strip()
+    if not line or line.startswith("#") or "=" not in line:
+        continue
+    current_key, value = line.split("=", 1)
+    if current_key == key:
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", "'\''"):
+            value = value[1:-1]
+        print(value)
+        raise SystemExit(0)
+'
+
+  if [[ -r "${file_path}" ]]; then
+    python3 -c "${python_script}" "${file_path}" "${key}"
+  else
+    if ! sudo -n true >/dev/null 2>&1; then
+      echo "[production-api-deploy] 当前用户无权读取 ${file_path}，且 sudo -n 不可用；无法检查运行态环境变量。" >&2
+      exit 1
+    fi
+    sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
+  fi
+}
+
+ensure_env_value() {
+  local file_path="$1"
+  local key="$2"
+  local default_value="$3"
+  local current_value
+
+  current_value="$(read_env_value "${file_path}" "${key}")"
+  if [[ -n "${current_value}" ]]; then
+    return
+  fi
+
+  echo "[production-api-deploy] 补齐 api-server 环境变量: ${key} -> ${file_path}"
+  write_env_value "${file_path}" "${key}" "${default_value}"
+}
+
+run_privileged() {
+  if [[ "$(id -u)" -eq 0 ]]; then
+    "$@"
+    return
+  fi
+  if ! sudo -n true >/dev/null 2>&1; then
+    echo "[production-api-deploy] 当前用户不是 root，且 sudo -n 不可用；无法执行: $*" >&2
+    exit 1
+  fi
+  sudo -n "$@"
+}
+
+ensure_runtime_dir() {
+  local path="$1"
+  local mode="$2"
+
+  if [[ -z "${path}" ]]; then
+    return
+  fi
+  if [[ "${path}" != /* ]]; then
+    echo "[production-api-deploy] 运行态目录必须使用绝对路径，避免写入只读发布目录: ${path}" >&2
+    exit 1
+  fi
+
+  echo "[production-api-deploy] 确保运行态目录可写: ${path}"
+  run_privileged install -d -o genarrative -g genarrative -m "${mode}" "${path}"
+}
+
+ensure_runtime_env_and_dirs() {
+  local api_env_file="$1"
+  local tracking_enabled tracking_outbox_dir auth_store_path auth_store_dir
+
+  # 旧生产环境文件会被 server-provision 保留，不一定包含新增的运行态写入路径。
+  # 发布前只补缺省值，不覆盖线上已经定制过的目录或开关。
+  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED" "true"
+  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR" "/var/lib/genarrative/tracking-outbox"
+  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE" "500"
+  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS" "1000"
+  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES" "268435456"
+  ensure_env_value "${api_env_file}" "GENARRATIVE_AUTH_STORE_PATH" "/var/lib/genarrative/auth/auth-store.json"
+
+  tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")"
+  tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")"
+  if [[ "$(printf "%s" "${tracking_enabled}" | tr '[:upper:]' '[:lower:]')" != "false" ]]; then
+    ensure_runtime_dir "${tracking_outbox_dir}" "0750"
+  fi
+
+  auth_store_path="$(read_env_value "${api_env_file}" "GENARRATIVE_AUTH_STORE_PATH")"
+  if [[ -n "${auth_store_path}" ]]; then
+    auth_store_dir="$(dirname "${auth_store_path}")"
+    ensure_runtime_dir "${auth_store_dir}" "0750"
+  fi
+}
+
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_DIR=""
 VERSION=""
@@ -243,6 +352,8 @@ if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
  write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_SERVER_URL" "${SPACETIME_SERVER_URL}"
 fi

+ensure_runtime_env_and_dirs "${API_ENV_FILE}"
+
 mkdir -p "$(dirname "${CURRENT_LINK}")"
 ln -sfn "${RELEASE_DIR}" "${CURRENT_LINK}"

--- a/scripts/jenkins-server-provision.sh
+++ b/scripts/jenkins-server-provision.sh
@@ -292,6 +292,42 @@ write_env_value() {
  chown root:root "${file}"
 }

+ensure_env_value() {
+  local file="$1"
+  local key="$2"
+  local default_value="$3"
+  local current_value
+
+  current_value="$(read_env_value "${file}" "${key}")"
+  if [[ -n "${current_value}" ]]; then
+    return
+  fi
+
+  echo "[server-provision] 补齐 api-server 环境变量: ${key} -> ${file}"
+  if [[ "${DRY_RUN}" != "true" ]]; then
+    write_env_value "${file}" "${key}" "${default_value}"
+  fi
+}
+
+ensure_api_runtime_env_defaults() {
+  if [[ "${DRY_RUN}" == "true" ]]; then
+    echo "+ ensure api-server runtime env defaults in ${API_ENV_FILE}"
+    return
+  fi
+  if [[ ! -f "${API_ENV_FILE}" ]]; then
+    echo "[server-provision] 环境文件不存在，无法补齐 api-server 运行态目录变量: ${API_ENV_FILE}" >&2
+    exit 1
+  fi
+
+  # 已存在的生产 env 会被保留，不会整文件覆盖；这里仅补后续版本新增的运行态写入路径。
+  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED" "true"
+  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_DIR" "/var/lib/genarrative/tracking-outbox"
+  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE" "500"
+  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS" "1000"
+  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES" "268435456"
+  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_AUTH_STORE_PATH" "/var/lib/genarrative/auth/auth-store.json"
+}
+
 parse_json_string_field() {
  local json="$1"
  local key="$2"
@@ -673,6 +709,7 @@ if [[ ! -f "${API_ENV_FILE}" ]]; then
 else
  echo "[server-provision] 已存在环境文件，保留不覆盖: ${API_ENV_FILE}"
 fi
+ensure_api_runtime_env_defaults

 if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
  sync_otelcol_install