diff --git a/.hermes/shared-memory/pitfalls.md b/.hermes/shared-memory/pitfalls.md index a85427e1..ac4d1258 100644 --- a/.hermes/shared-memory/pitfalls.md +++ b/.hermes/shared-memory/pitfalls.md @@ -1735,6 +1735,12 @@ - 现象:release 机器调用 VectorEngine `gpt-image-2` 的 `/v1/images/generations` 或 `/v1/images/edits` 偶发 `client error (SendRequest) -> connection error -> Connection timed out (os error 110)`,应用层表现为 504;本地通常正常。 - 原因:本地 DNS 可能走代理 / 加速出口,而腾讯云 release 直接解析到 VectorEngine 真实边缘节点。实测同一张约 2.37MB PNG、同一 edits 请求,`curl` 5/5 成功,但 `reqwest/hyper` 会间歇性超时;固定 `40.160.33.47` 也只能改善,不能根治。 - 处理:不要优先关闭 multipart,也不要直接把 `SendRequest` 解释成上游业务拒绝。VectorEngine 图片 `generations` / `edits` 上游 POST 单独使用 `libcurl`;参考图下载和响应图片 URL 下载仍用 `reqwest`。send 阶段 timeout / connect error 在 `platform-image` 内最多重试 5 次,使用指数退避和短抖动;日志字段 `attempt`、`max_attempts`、`retry_delay_ms`、`reference_image_bytes_total`、`request_params` 是定位依据。 + +### api-server libcurl / OpenSSL 3.2 runtime + +- 症状:release 部署新 `api-server` 后服务反复 `exit-code`,`LD_TRACE_LOADED_OBJECTS=1 /opt/genarrative/current/api-server` 或 `ldd` 报 `/lib/x86_64-linux-gnu/libssl.so.3: version 'OPENSSL_3.2.0' not found`。 +- 根因:`platform-image` 使用 `libcurl` 后,Linux release 构建产物可能直接要求 `OPENSSL_3.2.0` 符号;Ubuntu 24.04 apt 默认 OpenSSL 仍是 `3.0.13`,不能满足该符号版本。 +- 处理:`Genarrative-Server-Provision` 独立安装 OpenSSL `3.2.0` 到 `/opt/genarrative/openssl-3.2.0`,并只通过 `genarrative-api.service` 的 `LD_LIBRARY_PATH=/opt/genarrative/openssl-3.2.0/lib64:/opt/genarrative/openssl-3.2.0/lib` 给 api-server 使用,避免替换系统 OpenSSL。 - 验证:release 上先看 `journalctl -u genarrative-api.service` 中 `VectorEngine 图片请求发送失败,准备重试` 与最终 `HTTP 返回`;若仍失败,再用同一图片分别跑 curl 与最小 reqwest 探针对照。 - 关联:`server-rs/crates/platform-image/src/vector_engine/client.rs`、`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`。 diff --git a/deploy/systemd/genarrative-api.service b/deploy/systemd/genarrative-api.service index 82ddd339..c061d024 100644 --- a/deploy/systemd/genarrative-api.service +++ b/deploy/systemd/genarrative-api.service @@ -10,6 +10,7 @@ User=genarrative Group=genarrative WorkingDirectory=/opt/genarrative/current EnvironmentFile=/etc/genarrative/api-server.env +Environment="LD_LIBRARY_PATH=/opt/genarrative/openssl-3.2.0/lib64:/opt/genarrative/openssl-3.2.0/lib" ExecStart=/opt/genarrative/current/api-server Restart=always RestartSec=5 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 68666bb1..e2e05bd8 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -255,7 +255,7 @@ Jenkins 按 web / api / Spacetime module / build / deploy / publish 拆分 生产环境变量模板:`deploy/env/api-server.env.example`。真实密钥只放服务器,不提交 Git,不写入文档示例。 -`Genarrative-Server-Provision` 会安装 systemd 模板和 Nginx 站点模板,不再安装 clang / lld / pkg-config / OpenSSL headers / sccache 等构建链依赖。Ubuntu / apt 目标机会额外安装 `libnginx-mod-http-brotli-filter` 与 `libnginx-mod-http-brotli-static`,随后由 `scripts/jenkins-server-provision.sh` 通过临时 `nginx -t` 配置探测 Brotli 指令是否可用;该临时配置必须先 `include /etc/nginx/modules-enabled/*.conf`,因为 apt 安装的 Brotli 是动态模块,不会出现在普通 `nginx -V` 编译参数里。探测成功才在渲染后的 `deploy/nginx/genarrative.conf` / `genarrative-dev-http.conf` 中启用 Brotli,避免未安装模块的机器直接写入无效配置。Provision 写入 Genarrative Nginx 站点时会把 `/etc/nginx/sites-enabled/default*` 移到 `/etc/nginx/sites-disabled/`,避免 Debian / Certbot 默认站点继续占用 `genarrative.world` / `www.genarrative.world` 并在 `nginx -T` 中出现 `conflicting server name ... ignored`。如果 `nginx -t` 失败,脚本会恢复写入前的 Genarrative 配置和被移动的默认站点。 +`Genarrative-Server-Provision` 会安装 systemd 模板和 Nginx 站点模板,不再安装 clang / lld / pkg-config / OpenSSL headers / sccache 等通用构建链依赖。因 VectorEngine 图片上游 POST 已改用 `libcurl`,当前 Linux release 构建出的 `api-server` 运行时需要 `OPENSSL_3.2.0` 符号;Ubuntu 24.04 apt 默认只提供 OpenSSL 3.0.x,不能直接满足该符号版本。Provision 会把 OpenSSL `3.2.0` 独立安装到 `/opt/genarrative/openssl-3.2.0`,校验官方 tarball SHA256,并只通过 `genarrative-api.service` 的 `LD_LIBRARY_PATH=/opt/genarrative/openssl-3.2.0/lib64:/opt/genarrative/openssl-3.2.0/lib` 让 api-server 使用,避免替换系统 OpenSSL 或影响 ssh / nginx / apt。Ubuntu / apt 目标机会额外安装 `libnginx-mod-http-brotli-filter` 与 `libnginx-mod-http-brotli-static`,随后由 `scripts/jenkins-server-provision.sh` 通过临时 `nginx -t` 配置探测 Brotli 指令是否可用;该临时配置必须先 `include /etc/nginx/modules-enabled/*.conf`,因为 apt 安装的 Brotli 是动态模块,不会出现在普通 `nginx -V` 编译参数里。探测成功才在渲染后的 `deploy/nginx/genarrative.conf` / `genarrative-dev-http.conf` 中启用 Brotli,避免未安装模块的机器直接写入无效配置。Provision 写入 Genarrative Nginx 站点时会把 `/etc/nginx/sites-enabled/default*` 移到 `/etc/nginx/sites-disabled/`,避免 Debian / Certbot 默认站点继续占用 `genarrative.world` / `www.genarrative.world` 并在 `nginx -T` 中出现 `conflicting server name ... ignored`。如果 `nginx -t` 失败,脚本会恢复写入前的 Genarrative 配置和被移动的默认站点。 50 HTTP req/s 首版压测优化口径: diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 51a6b216..5d3535ed 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -4,6 +4,10 @@ set -euo pipefail PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/spacetime}" OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}" +GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}" +GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}" +GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}" +GENARRATIVE_OPENSSL_SOURCE_SHA256="${GENARRATIVE_OPENSSL_SOURCE_SHA256:-14c826f07c7e433706fb5c69fa9e25dab95684844b4c962a2cf1bf183eb4690e}" require_non_root_relative_path() { local label="$1" @@ -27,6 +31,14 @@ require_path() { fi } +require_cmd() { + local name="$1" + if ! command -v "${name}" >/dev/null 2>&1; then + echo "[server-provision] 缺少命令: ${name}" >&2 + exit 1 + fi +} + normalize_server_aliases() { printf "%s" "${SERVER_ALIASES:-}" | tr ',' ' ' | xargs } @@ -87,6 +99,113 @@ install_nginx_brotli_modules() { fi } +download_file() { + local url="$1" + local output="$2" + + if command -v curl >/dev/null 2>&1; then + curl -fsSL --retry 3 --retry-delay 2 "${url}" -o "${output}" + elif command -v wget >/dev/null 2>&1; then + wget -O "${output}" "${url}" + else + echo "[server-provision] 需要 curl 或 wget 下载: ${url}" >&2 + exit 1 + fi +} + +openssl_lib_dir_candidates() { + printf "%s\n" \ + "${GENARRATIVE_OPENSSL_PREFIX}/lib64" \ + "${GENARRATIVE_OPENSSL_PREFIX}/lib" +} + +find_genarrative_openssl_lib_dir() { + local lib_dir + while IFS= read -r lib_dir; do + if [[ -f "${lib_dir}/libssl.so.3" && -f "${lib_dir}/libcrypto.so.3" ]]; then + printf "%s" "${lib_dir}" + return 0 + fi + done < <(openssl_lib_dir_candidates) + return 1 +} + +genarrative_openssl_has_required_symbol() { + local lib_dir + lib_dir="$(find_genarrative_openssl_lib_dir 2>/dev/null || true)" + if [[ -z "${lib_dir}" ]]; then + return 1 + fi + grep -a -q "OPENSSL_${GENARRATIVE_OPENSSL_VERSION}" "${lib_dir}/libssl.so.3" +} + +verify_genarrative_openssl_install() { + local lib_dir + lib_dir="$(find_genarrative_openssl_lib_dir 2>/dev/null || true)" + if [[ -z "${lib_dir}" ]]; then + echo "[server-provision] OpenSSL ${GENARRATIVE_OPENSSL_VERSION} 安装后缺少 libssl.so.3/libcrypto.so.3: ${GENARRATIVE_OPENSSL_PREFIX}" >&2 + exit 1 + fi + if ! grep -a -q "OPENSSL_${GENARRATIVE_OPENSSL_VERSION}" "${lib_dir}/libssl.so.3"; then + echo "[server-provision] OpenSSL 动态库缺少 OPENSSL_${GENARRATIVE_OPENSSL_VERSION} 符号: ${lib_dir}/libssl.so.3" >&2 + exit 1 + fi + if ! env "LD_LIBRARY_PATH=${lib_dir}" "${GENARRATIVE_OPENSSL_PREFIX}/bin/openssl" version | grep -q "OpenSSL ${GENARRATIVE_OPENSSL_VERSION}"; then + echo "[server-provision] OpenSSL ${GENARRATIVE_OPENSSL_VERSION} 安装后命令验证失败: ${GENARRATIVE_OPENSSL_PREFIX}/bin/openssl" >&2 + exit 1 + fi + echo "[server-provision] OpenSSL ${GENARRATIVE_OPENSSL_VERSION} 已就绪: ${lib_dir}" +} + +install_genarrative_openssl_runtime() { + local tmp_dir archive source_dir jobs lib_dir + + echo "[server-provision] 检查 api-server/libcurl 运行时 OpenSSL ${GENARRATIVE_OPENSSL_VERSION}" + if [[ "${DRY_RUN}" == "true" ]]; then + echo "+ install OpenSSL ${GENARRATIVE_OPENSSL_VERSION} into ${GENARRATIVE_OPENSSL_PREFIX}" + echo "+ verify OPENSSL_${GENARRATIVE_OPENSSL_VERSION} symbol for api-server/libcurl" + return + fi + + if genarrative_openssl_has_required_symbol; then + verify_genarrative_openssl_install + return + fi + + if command -v apt-get >/dev/null 2>&1; then + run_cmd apt-get install -y build-essential ca-certificates curl perl tar + else + echo "[server-provision] 当前系统未使用 apt,无法自动构建 OpenSSL ${GENARRATIVE_OPENSSL_VERSION};请手动安装到 ${GENARRATIVE_OPENSSL_PREFIX}。" >&2 + exit 1 + fi + require_cmd sha256sum + require_cmd tar + + tmp_dir="$(mktemp -d)" + archive="${tmp_dir}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz" + echo "[server-provision] 下载 OpenSSL ${GENARRATIVE_OPENSSL_VERSION}: ${GENARRATIVE_OPENSSL_SOURCE_URL}" + download_file "${GENARRATIVE_OPENSSL_SOURCE_URL}" "${archive}" + printf "%s %s\n" "${GENARRATIVE_OPENSSL_SOURCE_SHA256}" "${archive}" | sha256sum -c - + + tar -xzf "${archive}" -C "${tmp_dir}" + source_dir="${tmp_dir}/openssl-${GENARRATIVE_OPENSSL_VERSION}" + jobs="$(nproc 2>/dev/null || echo 2)" + ( + cd "${source_dir}" + ./config --prefix="${GENARRATIVE_OPENSSL_PREFIX}" --openssldir="${GENARRATIVE_OPENSSL_PREFIX}/ssl" shared + make -j "${jobs}" + make install_sw + ) + rm -rf "${tmp_dir}" + + lib_dir="$(find_genarrative_openssl_lib_dir 2>/dev/null || true)" + if [[ -n "${lib_dir}" ]]; then + chmod 0755 "${GENARRATIVE_OPENSSL_PREFIX}" "${lib_dir}" || true + chmod 0644 "${lib_dir}/libssl.so.3" "${lib_dir}/libcrypto.so.3" || true + fi + verify_genarrative_openssl_install +} + sync_otelcol_install() { local target_bin="/usr/local/bin/otelcol-contrib" local source_bin="${OTELCOL_BIN_SOURCE}" @@ -651,6 +770,7 @@ fi run_cmd chown -R spacetimedb:spacetimedb "${SPACETIME_ROOT}" run_cmd chown -R genarrative:genarrative /opt/genarrative /var/lib/genarrative /srv/genarrative +install_genarrative_openssl_runtime if [[ ! -x "${SPACETIME_BIN_SOURCE}" ]]; then echo "[server-provision] spacetime CLI 不存在或不可执行: ${SPACETIME_BIN_SOURCE}" >&2