GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

合并 master 并保留外部生成 worker 模式

Browse Source 
合入 master 的生产健康巡检、JumpHop 和 SpacetimeDB 更新
保留外部生成 worker、队列/内联模式与 lease guard 口径
合并 Server-Provision 工具复用、health patrol 和外部生成 worker systemd 配置
补齐 SpacetimeDB 生成绑定并通过本地检查
This commit is contained in:
kdletters
2026-06-10 21:26:53 +08:00
parent 4f86c1a75b e29992cf01
commit 31ad55b0cf
93 changed files with 7872 additions and 2244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/build-production-release.sh
View File

@@ -445,7 +445,7 @@ if [[ "${BUILD_SPACETIME}" -eq 1 ]]; then
write_migration_bootstrap_secret_file
fi
mkdir -p "${TARGET_DIR}/scripts" "${TARGET_DIR}/deploy"
mkdir -p "${TARGET_DIR}/scripts" "${TARGET_DIR}/scripts/ops" "${TARGET_DIR}/deploy"
cp "${SCRIPT_DIR}/deploy/maintenance-on.sh" "${TARGET_DIR}/scripts/maintenance-on.sh"
cp "${SCRIPT_DIR}/deploy/maintenance-off.sh" "${TARGET_DIR}/scripts/maintenance-off.sh"
cp "${SCRIPT_DIR}/deploy/maintenance-status.sh" "${TARGET_DIR}/scripts/maintenance-status.sh"
@@ -466,6 +466,7 @@ copy_required_file "${SCRIPT_DIR}/spacetime-migration-common.mjs" "${TARGET_DIR}
copy_required_file "${SCRIPT_DIR}/spacetime-authorize-migration-operator.mjs" "${TARGET_DIR}/scripts/spacetime-authorize-migration-operator.mjs" "数据库迁移授权脚本"
copy_required_file "${SCRIPT_DIR}/spacetime-revoke-migration-operator.mjs" "${TARGET_DIR}/scripts/spacetime-revoke-migration-operator.mjs" "数据库迁移撤权脚本"
copy_required_file "${SCRIPT_DIR}/database-backup-to-oss.mjs" "${TARGET_DIR}/scripts/database-backup-to-oss.mjs" "数据库 OSS 备份脚本"
copy_required_file "${SCRIPT_DIR}/ops/production-health-patrol.mjs" "${TARGET_DIR}/scripts/ops/production-health-patrol.mjs" "生产健康巡检脚本"
copy_required_dir "${REPO_ROOT}/deploy/systemd" "${TARGET_DIR}/deploy/systemd" "systemd 配置"
copy_required_dir "${REPO_ROOT}/deploy/nginx" "${TARGET_DIR}/deploy/nginx" "Nginx 配置"
@@ -485,7 +486,7 @@ cat >"${TARGET_DIR}/README.md" <<EOF
- \`migration-bootstrap-secret.txt\`：构建 \`spacetime_module.wasm\` 时注入的迁移引导密钥，仅用于创建首个迁移操作员；请作为敏感文件保存到 Jenkins Secret Text，授权完成后不要长期留在公开归档中。
- \`*.sha256\`：发布产物 checksum，用于部署前校验。
- \`release-manifest.json\`：发布版本、源码 commit 与产物清单。
- \`scripts/\`：维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
- \`scripts/\`：维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、生产健康巡检脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
- \`deploy/\`：systemd、Nginx 和生产环境变量示例；\`deploy/nginx/genarrative-dev-http.conf\` 仅供无域名开发服初始化使用。
## 生产部署口径

64
scripts/check-production-ops-guardrails.mjs Normal file
View File

@@ -0,0 +1,64 @@
#!/usr/bin/env node
import {readFileSync} from 'node:fs';
const checks = [
{
file: 'deploy/systemd/genarrative-database-backup.service',
includes: '--restart-service-after genarrative-api.service',
reason: '生产冷备份恢复 SpacetimeDB 后必须显式拉起依赖它的 API 服务。',
},
{
file: 'deploy/systemd/genarrative-health-patrol.service',
includes: 'scripts/ops/production-health-patrol.mjs',
reason: '健康巡检 systemd service 必须调用随 API release 发布的巡检脚本。',
},
{
file: 'deploy/systemd/genarrative-health-patrol.timer',
includes: 'genarrative-health-patrol.service',
reason: '健康巡检 timer 必须绑定巡检 service。',
},
{
file: 'scripts/jenkins-server-provision.sh',
includes: 'genarrative-health-patrol.timer',
reason: 'Server-Provision 必须安装并启用健康巡检 timer。',
},
{
file: 'scripts/build-production-release.sh',
includes: 'production-health-patrol.mjs',
reason: '生产 API release 必须携带健康巡检脚本。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'production-health-patrol.mjs',
reason: 'API deploy 必须把健康巡检脚本复制到 current release。',
},
{
file: 'jenkins/Jenkinsfile.production-api-build',
includes: 'scripts/ops/production-health-patrol.mjs',
reason: 'API Build 归档必须包含健康巡检脚本。',
},
{
file: 'jenkins/Jenkinsfile.production-api-deploy',
includes: 'scripts/ops/production-health-patrol.mjs',
reason: 'API Deploy 复制上游产物时必须包含健康巡检脚本。',
},
];
let failed = false;
for (const check of checks) {
const content = readFileSync(check.file, 'utf8');
if (!content.includes(check.includes)) {
failed = true;
console.error(
`[check:production-ops] ${check.file} 缺少 ${check.includes}。${check.reason}`,
);
}
}
if (failed) {
process.exit(1);
}
console.log('[check:production-ops] OK');

86
scripts/check-server-provision-tools.sh Executable file
View File

@@ -0,0 +1,86 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "${TMP_ROOT}"' EXIT
WORK_DIR="${TMP_ROOT}/workspace"
FAKE_BIN_DIR="${TMP_ROOT}/fake-bin"
TARGET_BIN_DIR="${TMP_ROOT}/target-bin"
SPACETIME_ROOT_DIR="${TMP_ROOT}/stdb"
OUTPUT_LOG="${TMP_ROOT}/prepare.log"
mkdir -p \
"${WORK_DIR}" \
"${FAKE_BIN_DIR}" \
"${TARGET_BIN_DIR}" \
"${SPACETIME_ROOT_DIR}/bin/current"
cat >"${FAKE_BIN_DIR}/curl" <<'EOF'
#!/usr/bin/env bash
echo "curl should not be called when target tools are already ready" >&2
exit 97
EOF
cat >"${FAKE_BIN_DIR}/wget" <<'EOF'
#!/usr/bin/env bash
echo "wget should not be called when target tools are already ready" >&2
exit 97
EOF
chmod +x "${FAKE_BIN_DIR}/curl" "${FAKE_BIN_DIR}/wget"
cat >"${TARGET_BIN_DIR}/otelcol-contrib" <<'EOF'
#!/usr/bin/env bash
echo "otelcol-contrib version 0.151.0"
EOF
chmod +x "${TARGET_BIN_DIR}/otelcol-contrib"
cat >"${SPACETIME_ROOT_DIR}/bin/current/spacetimedb-cli" <<'EOF'
#!/usr/bin/env bash
echo "spacetimedb-cli 2.4.1"
EOF
cat >"${SPACETIME_ROOT_DIR}/bin/current/spacetimedb-standalone" <<'EOF'
#!/usr/bin/env bash
echo "spacetimedb-standalone 2.4.1"
EOF
chmod +x \
"${SPACETIME_ROOT_DIR}/bin/current/spacetimedb-cli" \
"${SPACETIME_ROOT_DIR}/bin/current/spacetimedb-standalone"
if ! (
cd "${WORK_DIR}"
PATH="${FAKE_BIN_DIR}:${PATH}" \
WORKSPACE="${WORK_DIR}" \
PROVISION_TOOLS_DIR="provision-tools" \
PROVISION_DOWNLOADS_DIR="downloads" \
PROVISION_TOOLS_TMP_PARENT="${WORK_DIR}/.tmp/server-provision-tools" \
PROVISION_REQUIRE_LOCAL_DOWNLOADS="true" \
OTELCOL_TARGET_BIN="${TARGET_BIN_DIR}/otelcol-contrib" \
OTELCOL_VERSION="0.151.0" \
SPACETIME_ROOT="${SPACETIME_ROOT_DIR}" \
SPACETIME_EXPECTED_VERSION="2.4.1" \
"${REPO_ROOT}/scripts/prepare-server-provision-tools.sh" \
>"${OUTPUT_LOG}" 2>&1
); then
echo "[check-server-provision-tools] prepare-server-provision-tools.sh 执行失败。" >&2
cat "${OUTPUT_LOG}" >&2
exit 1
fi
grep -q "复用目标机已有 otelcol-contrib" "${OUTPUT_LOG}"
grep -q "复用目标机已有 SpacetimeDB 安装" "${OUTPUT_LOG}"
grep -q "otelcol-contrib 0.151.0 target existing" "${WORK_DIR}/provision-tools/MANIFEST.txt"
grep -q "spacetime target existing" "${WORK_DIR}/provision-tools/MANIFEST.txt"
test -x "${WORK_DIR}/provision-tools/otelcol-contrib"
test -x "${WORK_DIR}/provision-tools/spacetime/spacetime"
test -x "${WORK_DIR}/provision-tools/spacetime/bin/current/spacetimedb-cli"
test -x "${WORK_DIR}/provision-tools/spacetime/bin/current/spacetimedb-standalone"
if grep -q "下载 " "${OUTPUT_LOG}"; then
echo "[check-server-provision-tools] 已有目标机工具时不应进入下载分支。" >&2
cat "${OUTPUT_LOG}" >&2
exit 1
fi
echo "[check-server-provision-tools] OK"

17
scripts/deploy/production-api-deploy.sh
View File

@@ -437,7 +437,9 @@ chmod +x "${RELEASE_DIR}/api-server"
BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
WORKSPACE_BACKUP_SCRIPT_SOURCE="$(cd "${SCRIPT_DIR}/../.." && pwd)/scripts/database-backup-to-oss.mjs"
mkdir -p "${RELEASE_DIR}/scripts"
HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs"
WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE="$(cd "${SCRIPT_DIR}/../.." && pwd)/scripts/ops/production-health-patrol.mjs"
mkdir -p "${RELEASE_DIR}/scripts" "${RELEASE_DIR}/scripts/ops"
if [[ ! -f "${BACKUP_SCRIPT_SOURCE}" ]]; then
if [[ -f "${WORKSPACE_BACKUP_SCRIPT_SOURCE}" ]]; then
echo "[production-api-deploy] 发布产物缺少 scripts/database-backup-to-oss.mjs，回退使用部署工作区脚本；请重新触发包含该脚本的 API 构建。" >&2
@@ -449,6 +451,19 @@ if [[ ! -f "${BACKUP_SCRIPT_SOURCE}" ]]; then
fi
cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_DIR}/scripts/database-backup-to-oss.mjs"
chmod 0644 "${RELEASE_DIR}/scripts/database-backup-to-oss.mjs"
if [[ ! -f "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then
if [[ -f "${WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then
echo "[production-api-deploy] 发布产物缺少 scripts/ops/production-health-patrol.mjs，回退使用部署工作区脚本；请重新触发包含该脚本的 API 构建。" >&2
HEALTH_PATROL_SCRIPT_SOURCE="${WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE}"
else
echo "[production-api-deploy] 未找到生产健康巡检脚本，跳过复制；genarrative-health-patrol.service 会因脚本缺失而跳过执行。" >&2
HEALTH_PATROL_SCRIPT_SOURCE=""
fi
fi
if [[ -n "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then
cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_DIR}/scripts/ops/production-health-patrol.mjs"
chmod 0644 "${RELEASE_DIR}/scripts/ops/production-health-patrol.mjs"
fi
if [[ -f "${SOURCE_DIR}/release-manifest.json" ]]; then
cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_DIR}/release-manifest.api-server.json"

20
scripts/jenkins-server-provision.sh
View File

@@ -750,11 +750,21 @@ render_database_backup_service() {
deploy/systemd/genarrative-database-backup.service
}
render_health_patrol_service() {
local current_escaped
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
sed \
-e "s|/opt/genarrative/current|${current_escaped}|g" \
deploy/systemd/genarrative-health-patrol.service
}
require_path deploy/systemd/spacetimedb.service
require_path deploy/systemd/genarrative-api.service
require_path deploy/systemd/genarrative-external-generation-worker@.service
require_path deploy/systemd/genarrative-database-backup.service
require_path deploy/systemd/genarrative-database-backup.timer
require_path deploy/systemd/genarrative-health-patrol.service
require_path deploy/systemd/genarrative-health-patrol.timer
require_path deploy/systemd/otelcol-contrib.service
require_path deploy/otelcol/genarrative-debug.yaml
require_path deploy/nginx/genarrative.conf
@@ -774,7 +784,7 @@ echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_conf
run_cmd id
require_root_for_real_provision
install_nginx_brotli_modules
run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/database-backups
run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol
if ! id spacetimedb >/dev/null 2>&1; then
run_cmd useradd --system --home-dir "${SPACETIME_ROOT}" --shell /usr/sbin/nologin spacetimedb
@@ -807,16 +817,20 @@ spacetimedb_service="$(mktemp)"
api_service="$(mktemp)"
external_generation_worker_service="$(mktemp)"
database_backup_service="$(mktemp)"
health_patrol_service="$(mktemp)"
render_spacetimedb_service >"${spacetimedb_service}"
render_api_service >"${api_service}"
render_external_generation_worker_service >"${external_generation_worker_service}"
render_database_backup_service >"${database_backup_service}"
render_health_patrol_service >"${health_patrol_service}"
install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644
install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644
install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644
install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644
install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644
rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${database_backup_service}"
install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644
install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/genarrative-health-patrol.timer 0644
rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${database_backup_service}" "${health_patrol_service}"
if [[ ! -f "${API_ENV_FILE}" ]]; then
echo "+ create ${API_ENV_FILE} from example"
@@ -862,7 +876,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
run_cmd systemctl enable otelcol-contrib.service
fi
run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service
run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-health-patrol.timer
if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
run_cmd systemctl restart otelcol-contrib.service
fi

477
scripts/ops/production-health-patrol.mjs Normal file
View File

@@ -0,0 +1,477 @@
#!/usr/bin/env node
import {execFile} from 'node:child_process';
import http from 'node:http';
import https from 'node:https';
import {mkdir, writeFile} from 'node:fs/promises';
import {dirname} from 'node:path';
const STATUS_RANK = {
OK: 0,
WARNING: 1,
CRITICAL: 2,
};
const DEFAULT_PUBLIC_PATHS = [
'/api/creation-entry/config',
'/api/runtime/puzzle/gallery',
'/api/runtime/custom-world-gallery',
];
const DEFAULT_SERVICES = [
'genarrative-api.service',
'spacetimedb.service',
'nginx.service',
];
function usage() {
console.log(`Usage:
node scripts/ops/production-health-patrol.mjs [options]
Options:
--api-base-url <url> API direct base URL, default http://127.0.0.1:8082
--spacetime-base-url <url> SpacetimeDB base URL, default http://127.0.0.1:3101
--public-base-url <url> Nginx/public base URL, default http://127.0.0.1
--public-path <path> Public API path to probe; repeatable
--status-file <path> Write the last patrol result as JSON
--timeout-ms <ms> HTTP/command timeout, default 5000
--slow-ms <ms> Mark successful probes slower than this as WARNING, default 3000
--fail-on-warning Exit 1 when the total status is WARNING
--skip-journal Skip recent journal error scan
--json Print JSON instead of text
`);
}
function readBoolEnv(name, fallback = false) {
const value = process.env[name];
if (!value) {
return fallback;
}
return ['1', 'true', 'yes', 'on'].includes(value.trim().toLowerCase());
}
function parsePositiveInt(raw, fallback) {
const value = Number.parseInt(String(raw ?? ''), 10);
return Number.isFinite(value) && value > 0 ? value : fallback;
}
function parseArgs(argv) {
const config = {
apiBaseUrl:
process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL ||
'http://127.0.0.1:8082',
spacetimeBaseUrl:
process.env.GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL ||
'http://127.0.0.1:3101',
publicBaseUrl:
process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||
process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL ||
'http://127.0.0.1:8082',
publicPaths: [],
statusFile: process.env.GENARRATIVE_HEALTH_PATROL_STATUS_FILE || '',
timeoutMs: parsePositiveInt(
process.env.GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS,
5000,
),
slowMs: parsePositiveInt(
process.env.GENARRATIVE_HEALTH_PATROL_SLOW_MS,
3000,
),
failOnWarning: readBoolEnv('GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING'),
skipJournal: readBoolEnv('GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL'),
json: false,
webhookUrl: process.env.GENARRATIVE_HEALTH_PATROL_WEBHOOK_URL || '',
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
switch (arg) {
case '-h':
case '--help':
usage();
process.exit(0);
break;
case '--api-base-url':
config.apiBaseUrl = requireValue(argv, ++index, arg);
break;
case '--spacetime-base-url':
config.spacetimeBaseUrl = requireValue(argv, ++index, arg);
break;
case '--public-base-url':
config.publicBaseUrl = requireValue(argv, ++index, arg);
break;
case '--public-path':
config.publicPaths.push(requireValue(argv, ++index, arg));
break;
case '--status-file':
config.statusFile = requireValue(argv, ++index, arg);
break;
case '--timeout-ms':
config.timeoutMs = parsePositiveInt(requireValue(argv, ++index, arg), 5000);
break;
case '--slow-ms':
config.slowMs = parsePositiveInt(requireValue(argv, ++index, arg), 3000);
break;
case '--fail-on-warning':
config.failOnWarning = true;
break;
case '--skip-journal':
config.skipJournal = true;
break;
case '--json':
config.json = true;
break;
default:
throw new Error(`未知参数: ${arg}`);
}
}
if (config.publicPaths.length === 0) {
config.publicPaths = DEFAULT_PUBLIC_PATHS;
}
return config;
}
function requireValue(argv, index, flag) {
const value = argv[index];
if (!value || value.startsWith('--')) {
throw new Error(`${flag} 缺少参数值`);
}
return value;
}
function joinUrl(baseUrl, path) {
const base = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl;
const suffix = path.startsWith('/') ? path : `/${path}`;
return `${base}${suffix}`;
}
function maxStatus(checks) {
return checks.reduce((current, check) => {
return STATUS_RANK[check.status] > STATUS_RANK[current] ? check.status : current;
}, 'OK');
}
function checkResult(name, status, summary, details = {}) {
return {
name,
status,
summary,
...details,
};
}
function runCommand(command, args, timeoutMs) {
return new Promise((resolve) => {
execFile(
command,
args,
{
timeout: timeoutMs,
windowsHide: true,
maxBuffer: 256 * 1024,
},
(error, stdout, stderr) => {
resolve({
command: [command, ...args].join(' '),
code:
typeof error?.code === 'number'
? error.code
: error
? 1
: 0,
signal: error?.signal || '',
stdout: String(stdout || ''),
stderr: String(stderr || ''),
timedOut: Boolean(error?.killed),
error: error ? error.message : '',
});
},
);
});
}
async function checkService(serviceName, timeoutMs) {
const result = await runCommand(
'systemctl',
['is-active', serviceName],
timeoutMs,
);
const state = result.stdout.trim() || result.stderr.trim() || result.error;
if (result.code === 0 && state === 'active') {
return checkResult(`service:${serviceName}`, 'OK', 'active', {
command: result.command,
});
}
return checkResult(
`service:${serviceName}`,
'CRITICAL',
`服务状态异常: ${state || `exit ${result.code}`}`,
{
command: result.command,
stderr: result.stderr.trim(),
},
);
}
function requestUrl(url, timeoutMs) {
return new Promise((resolve) => {
const startedAt = Date.now();
const parsed = new URL(url);
const client = parsed.protocol === 'https:' ? https : http;
const request = client.request(
parsed,
{
method: 'GET',
timeout: timeoutMs,
headers: {
'User-Agent': 'genarrative-health-patrol/1.0',
Accept: 'application/json,text/plain,*/*',
},
},
(response) => {
let body = '';
response.setEncoding('utf8');
response.on('data', (chunk) => {
if (body.length < 2048) {
body += chunk;
}
});
response.on('end', () => {
resolve({
elapsedMs: Date.now() - startedAt,
statusCode: response.statusCode || 0,
body: body.slice(0, 2048),
});
});
},
);
request.on('timeout', () => {
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
});
request.on('error', (error) => {
resolve({
elapsedMs: Date.now() - startedAt,
error: error.message,
});
});
request.end();
});
}
async function checkHttp(name, url, config) {
const result = await requestUrl(url, config.timeoutMs);
const curlCommand = `curl -fsS --max-time ${Math.ceil(config.timeoutMs / 1000)} ${url}`;
if (result.error) {
return checkResult(name, 'CRITICAL', `请求失败: ${result.error}`, {
command: curlCommand,
elapsedMs: result.elapsedMs,
});
}
const ok = result.statusCode >= 200 && result.statusCode < 300;
if (!ok) {
return checkResult(
name,
'CRITICAL',
`HTTP ${result.statusCode}，耗时 ${result.elapsedMs}ms`,
{
command: curlCommand,
elapsedMs: result.elapsedMs,
body: result.body.trim(),
},
);
}
if (result.elapsedMs > config.slowMs) {
return checkResult(
name,
'WARNING',
`HTTP ${result.statusCode} 但耗时偏高: ${result.elapsedMs}ms`,
{
command: curlCommand,
elapsedMs: result.elapsedMs,
},
);
}
return checkResult(name, 'OK', `HTTP ${result.statusCode} ${result.elapsedMs}ms`, {
command: curlCommand,
elapsedMs: result.elapsedMs,
});
}
async function checkRecentJournal(config) {
const args = [
'-u',
'genarrative-api.service',
'-u',
'spacetimedb.service',
'-u',
'nginx.service',
'--since',
'15 minutes ago',
'-p',
'err..alert',
'--no-pager',
'-o',
'short-iso',
'-n',
'20',
];
const result = await runCommand('journalctl', args, config.timeoutMs);
if (result.code !== 0) {
return checkResult('journal:recent-errors', 'WARNING', '无法读取最近错误日志', {
command: result.command,
stderr: result.stderr.trim() || result.error,
});
}
const lines = result.stdout
.split('\n')
.map((line) => line.trim())
.filter((line) => line && line !== '-- No entries --');
if (lines.length === 0) {
return checkResult('journal:recent-errors', 'OK', '最近 15 分钟无 err..alert 日志', {
command: result.command,
});
}
return checkResult(
'journal:recent-errors',
'WARNING',
`最近 15 分钟有 ${lines.length} 条 err..alert 日志`,
{
command: result.command,
lines,
},
);
}
async function writeStatusFile(statusFile, payload) {
if (!statusFile) {
return;
}
await mkdir(dirname(statusFile), {recursive: true});
await writeFile(statusFile, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
}
async function notifyWebhook(config, payload) {
if (!config.webhookUrl || payload.status === 'OK') {
return;
}
const body = JSON.stringify(payload);
const parsed = new URL(config.webhookUrl);
const client = parsed.protocol === 'https:' ? https : http;
await new Promise((resolve) => {
const request = client.request(
parsed,
{
method: 'POST',
timeout: config.timeoutMs,
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(body),
},
},
(response) => {
response.resume();
response.on('end', resolve);
},
);
request.on('timeout', () => {
request.destroy(new Error(`timeout after ${config.timeoutMs}ms`));
});
request.on('error', (error) => {
console.error(`[health-patrol] webhook notify failed: ${error.message}`);
resolve();
});
request.end(body);
});
}
function printText(payload) {
console.log(`[health-patrol] ${payload.status} ${payload.checkedAt}`);
for (const check of payload.checks) {
console.log(`[${check.status}] ${check.name}: ${check.summary}`);
if (check.command && check.status !== 'OK') {
console.log(` command: ${check.command}`);
}
if (check.stderr) {
console.log(` stderr: ${check.stderr}`);
}
if (check.body) {
console.log(` body: ${check.body}`);
}
if (Array.isArray(check.lines) && check.lines.length > 0) {
for (const line of check.lines) {
console.log(` ${line}`);
}
}
}
}
async function main() {
const config = parseArgs(process.argv.slice(2));
const checks = [];
for (const serviceName of DEFAULT_SERVICES) {
checks.push(await checkService(serviceName, config.timeoutMs));
}
checks.push(await checkHttp('api:/healthz', joinUrl(config.apiBaseUrl, '/healthz'), config));
checks.push(await checkHttp('api:/readyz', joinUrl(config.apiBaseUrl, '/readyz'), config));
checks.push(
await checkHttp(
'spacetimedb:/v1/ping',
joinUrl(config.spacetimeBaseUrl, '/v1/ping'),
config,
),
);
for (const path of config.publicPaths) {
checks.push(
await checkHttp(`public:${path}`, joinUrl(config.publicBaseUrl, path), config),
);
}
if (!config.skipJournal) {
checks.push(await checkRecentJournal(config));
}
const payload = {
status: maxStatus(checks),
checkedAt: new Date().toISOString(),
host: process.env.HOSTNAME || '',
checks,
};
await writeStatusFile(config.statusFile, payload);
await notifyWebhook(config, payload);
if (config.json) {
console.log(JSON.stringify(payload, null, 2));
} else {
printText(payload);
}
if (payload.status === 'CRITICAL') {
process.exit(2);
}
if (payload.status === 'WARNING' && config.failOnWarning) {
process.exit(1);
}
}
main().catch((error) => {
console.error(`[health-patrol] CRITICAL ${error instanceof Error ? error.message : String(error)}`);
process.exit(2);
});

82
scripts/prepare-server-provision-tools.sh
View File

@@ -7,9 +7,12 @@ OTELCOL_VERSION="${OTELCOL_VERSION:-0.151.0}"
PREPARE_OTELCOL="${PREPARE_OTELCOL:-${ENABLE_OTELCOL:-true}}"
OTELCOL_DOWNLOAD_ROOT="${OTELCOL_DOWNLOAD_ROOT:-https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download}"
OTELCOL_ARCHIVE_PATH="${OTELCOL_ARCHIVE_PATH:-}"
OTELCOL_TARGET_BIN="${OTELCOL_TARGET_BIN:-/usr/local/bin/otelcol-contrib}"
SPACETIME_INSTALLER_URL="${SPACETIME_INSTALLER_URL:-https://install.spacetimedb.com}"
SPACETIME_DOWNLOAD_ROOT="${SPACETIME_DOWNLOAD_ROOT:-https://github.com/clockworklabs/SpacetimeDB/releases/download/v2.4.1}"
SPACETIME_TARGET_HOST="${SPACETIME_TARGET_HOST:-x86_64-unknown-linux-gnu}"
SPACETIME_ROOT="${SPACETIME_ROOT:-/stdb}"
SPACETIME_EXPECTED_VERSION="${SPACETIME_EXPECTED_VERSION:-}"
SPACETIME_ARCHIVE_PATH="${SPACETIME_ARCHIVE_PATH:-}"
SPACETIME_INSTALLER_PATH="${SPACETIME_INSTALLER_PATH:-}"
SPACETIME_UPDATE_INSTALLER_PATH="${SPACETIME_UPDATE_INSTALLER_PATH:-}"
@@ -65,6 +68,60 @@ download_file() {
fi
}
resolve_spacetime_expected_version() {
local download_root="${SPACETIME_DOWNLOAD_ROOT%/}"
if [[ -n "${SPACETIME_EXPECTED_VERSION}" ]]; then
printf "%s" "${SPACETIME_EXPECTED_VERSION}"
return
fi
if [[ "${download_root}" =~ /v([0-9]+(\.[0-9]+){1,2})$ ]]; then
printf "%s" "${BASH_REMATCH[1]}"
fi
}
target_otelcol_ready() {
local version_output
echo "[prepare-provision-tools] 检查目标机 otelcol-contrib: ${OTELCOL_TARGET_BIN}"
if [[ ! -x "${OTELCOL_TARGET_BIN}" ]]; then
echo "[prepare-provision-tools] 目标机 otelcol-contrib 不存在或不可执行，将准备交付文件。"
return 1
fi
version_output="$("${OTELCOL_TARGET_BIN}" --version 2>/dev/null || true)"
if [[ -n "${OTELCOL_VERSION}" && "${version_output}" != *"${OTELCOL_VERSION}"* ]]; then
echo "[prepare-provision-tools] 目标机 otelcol-contrib 版本不匹配，期望 ${OTELCOL_VERSION}，当前: ${version_output:-unknown}"
return 1
fi
echo "[prepare-provision-tools] 目标机 otelcol-contrib 已满足要求: ${version_output:-version unknown}"
return 0
}
target_spacetime_ready() {
local target_cli="${SPACETIME_ROOT}/bin/current/spacetimedb-cli"
local target_standalone="${SPACETIME_ROOT}/bin/current/spacetimedb-standalone"
local expected_version version_output
echo "[prepare-provision-tools] 检查目标机 SpacetimeDB: ${SPACETIME_ROOT}/bin/current"
if [[ ! -x "${target_cli}" || ! -x "${target_standalone}" ]]; then
echo "[prepare-provision-tools] 目标机 SpacetimeDB current 目录不完整，将准备交付文件。"
return 1
fi
expected_version="$(resolve_spacetime_expected_version)"
version_output="$("${target_cli}" --version 2>/dev/null || true)"
if [[ -n "${expected_version}" && "${version_output}" != *"${expected_version}"* ]]; then
echo "[prepare-provision-tools] 目标机 SpacetimeDB 版本不匹配，期望 ${expected_version}，当前: ${version_output:-unknown}"
return 1
fi
echo "[prepare-provision-tools] 目标机 SpacetimeDB 已满足要求: ${version_output:-version unknown}"
return 0
}
validate_relative_dir() {
local label="$1"
local path="$2"
@@ -101,13 +158,21 @@ prepare_otelcol() {
require_cmd tar
if target_otelcol_ready; then
echo "[prepare-provision-tools] 复用目标机已有 otelcol-contrib: ${OTELCOL_TARGET_BIN}"
install -m 0755 "${OTELCOL_TARGET_BIN}" "${target}"
"${target}" --version >/dev/null
OTELCOL_SOURCE_DESCRIPTION="target existing ${OTELCOL_TARGET_BIN}"
return
fi
if [[ -n "${OTELCOL_ARCHIVE_PATH}" && -f "${OTELCOL_ARCHIVE_PATH}" ]]; then
source_archive="${OTELCOL_ARCHIVE_PATH}"
elif [[ -n "${PROVISION_DOWNLOADS_DIR}" && -f "${downloaded_archive}" ]]; then
source_archive="${downloaded_archive}"
fi
if [[ "${PROVISION_REQUIRE_LOCAL_DOWNLOADS}" == "true" && -z "${source_archive}" ]]; then
echo "[prepare-provision-tools] 要求使用 Windows 已下载的 otelcol-contrib 包，但未找到: ${downloaded_archive}" >&2
echo "[prepare-provision-tools] 要求使用本地已有的 otelcol-contrib 来源，但目标机未满足且未找到下载包: ${downloaded_archive}" >&2
exit 1
fi
@@ -146,6 +211,17 @@ prepare_spacetime() {
local downloaded_installer="${PROVISION_DOWNLOADS_DIR}/spacetime-install.sh"
local source_installer=""
if target_spacetime_ready; then
echo "[prepare-provision-tools] 复用目标机已有 SpacetimeDB 安装: ${SPACETIME_ROOT}/bin/current"
mkdir -p "${target_dir}"
cp -a "${SPACETIME_ROOT}/bin" "${target_dir}/bin"
chmod 0755 "${target_dir}/bin/current/spacetimedb-cli" "${target_dir}/bin/current/spacetimedb-standalone"
make_spacetime_wrapper "${target_dir}/spacetime"
"${target_dir}/spacetime" --version >/dev/null
SPACETIME_SOURCE_DESCRIPTION="target existing ${SPACETIME_ROOT}/bin/current"
return
fi
mkdir -p "${install_root}"
if [[ -n "${SPACETIME_ARCHIVE_PATH}" && -f "${SPACETIME_ARCHIVE_PATH}" ]]; then
source_archive="${SPACETIME_ARCHIVE_PATH}"
@@ -165,7 +241,7 @@ prepare_spacetime() {
source_update="${downloaded_update}"
fi
if [[ "${PROVISION_REQUIRE_LOCAL_DOWNLOADS}" == "true" && -z "${source_archive}" ]]; then
echo "[prepare-provision-tools] 要求使用 Windows 已下载的 SpacetimeDB release tarball，但未找到: ${downloaded_archive}" >&2
echo "[prepare-provision-tools] 要求使用本地已有的 SpacetimeDB release tarball，但目标机未满足且未找到下载包: ${downloaded_archive}" >&2
exit 1
fi
@@ -185,7 +261,7 @@ prepare_spacetime() {
fi
if [[ "${PROVISION_REQUIRE_LOCAL_DOWNLOADS}" == "true" && -z "${source_installer}" ]]; then
echo "[prepare-provision-tools] 要求使用 Windows 已下载的 SpacetimeDB 官方安装器脚本，但未找到: ${downloaded_installer}" >&2
echo "[prepare-provision-tools] 要求使用本地已有的 SpacetimeDB 官方安装器脚本，但未找到: ${downloaded_installer}" >&2
exit 1
elif [[ -n "${source_installer}" ]]; then
echo "[prepare-provision-tools] 使用已下载的 SpacetimeDB 官方安装器脚本: ${source_installer}"

163
scripts/test-ve-llm.mjs Normal file
View File

@@ -0,0 +1,163 @@
import { readFileSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = dirname(fileURLToPath(import.meta.url));
const root = resolve(__dirname, '..');
function loadEnv(path) {
const content = readFileSync(path, 'utf-8');
const env = {};
for (const line of content.split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eqIndex = trimmed.indexOf('=');
if (eqIndex === -1) continue;
const key = trimmed.slice(0, eqIndex).trim();
let value = trimmed.slice(eqIndex + 1).trim();
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
value = value.slice(1, -1);
}
env[key] = value;
}
return env;
}
const env = loadEnv(resolve(root, '.env.secrets.local'));
const BASE = env.VECTOR_ENGINE_BASE_URL?.replace(/\/+$/, '') || 'https://api.vectorengine.cn';
const KEY = env.VECTOR_ENGINE_API_KEY || '';
if (!KEY) {
console.error('未找到 VECTOR_ENGINE_API_KEY');
process.exit(1);
}
const TIMEOUT_MS = 60_000;
async function test(name, method, path, body = null) {
const url = `${BASE}${path}`;
const start = Date.now();
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
const headers = {
'Authorization': `Bearer ${KEY}`,
'Content-Type': 'application/json',
};
const options = { method, headers, signal: controller.signal };
if (body) options.body = JSON.stringify(body);
const resp = await fetch(url, options);
clearTimeout(timer);
const elapsed = Date.now() - start;
const text = await resp.text();
let json = null;
try { json = JSON.parse(text); } catch {}
if (resp.ok) {
const model = json?.model || json?.data?.[0]?.id || '?';
const summary = json?.choices?.[0] ? `choices[0]: ${json.choices[0].message?.content?.slice(0, 80)}` :
json?.output_text ? `output_text: ${json.output_text.slice(0, 80)}` :
json?.data ? `${json.data.length} models` : JSON.stringify(json).slice(0, 120);
return { ok: true, elapsed, code: resp.status, model, summary };
} else {
const errMsg = json?.error?.message || json?.message || text.slice(0, 200);
return { ok: false, elapsed, code: resp.status, error: errMsg };
}
} catch (e) {
const elapsed = Date.now() - start;
return { ok: false, elapsed, code: 0, error: e.name === 'AbortError' ? `超时(${TIMEOUT_MS / 1000}s)` : e.message };
}
}
console.log(`VectorEngine LLM 能力探测`);
console.log(`目标: ${BASE}\n`);
const tests = [
// 1. 探测 /v1/models - 基础连通性 + 列出可用模型
{ name: 'GET /v1/models (列出可用模型)', method: 'GET', path: '/v1/models' },
// 2. Chat Completions - 最标准协议，项目已有 LlmProvider::OpenAiCompatible 支持
{
name: 'POST /v1/chat/completions (Chat)',
method: 'POST',
path: '/v1/chat/completions',
body: {
model: 'gpt-4o',
messages: [{ role: 'user', content: '回复 ok，不要解释' }],
max_tokens: 10,
},
},
// 3. Responses - Apimart 当前使用的协议
{
name: 'POST /v1/responses (Responses)',
method: 'POST',
path: '/v1/responses',
body: {
model: 'gpt-4o',
input: [
{ role: 'user', content: [{ type: 'input_text', text: '回复 ok，不要解释' }] },
],
},
},
// 4. 测试 gpt-5 (creative_agent 模型)
{
name: 'POST /v1/chat/completions (gpt-5, Chat)',
method: 'POST',
path: '/v1/chat/completions',
body: {
model: 'gpt-5',
messages: [{ role: 'user', content: '回复 ok' }],
max_tokens: 10,
},
},
// 5. 抓大鹅生成需要的 JSON 输出能力验证
{
name: 'POST /v1/chat/completions (JSON 输出: 抓大鹅物品)',
method: 'POST',
path: '/v1/chat/completions',
body: {
model: 'gpt-4o',
messages: [
{ role: 'system', content: '你是抓大鹅游戏编辑，只返回 JSON。' },
{ role: 'user', content: '题材：水果。请生成 JSON：{"gameName":"水果切切乐","items":[{"name":"苹果","itemSize":"中"},{"name":"西瓜","itemSize":"大"}]}' },
],
max_tokens: 200,
},
},
];
let pass = 0;
let fail = 0;
for (let i = 0; i < tests.length; i++) {
const t = tests[i];
console.log(`[${i + 1}/${tests.length}] ${t.name}`);
const result = await test(t.name, t.method, t.path, t.body);
if (result.ok) {
console.log(` ✅ HTTP ${result.code} ${result.elapsed}ms model: ${result.model}`);
console.log(` ${result.summary}`);
pass++;
} else {
const codeStr = result.code === 0 ? 'NET' : `HTTP ${result.code}`;
console.log(` ❌ ${codeStr} ${result.elapsed}ms ${result.error}`);
fail++;
}
console.log();
}
console.log(`=== 结果: ${pass}/${tests.length} 通过, ${fail}/${tests.length} 失败 ===`);
// 结论
if (pass >= 3) {
console.log('\n✅ VectorEngine 支持 LLM 文本调用，可替代 Apimart。');
console.log(' 将 .env.secrets.local 中 APIMART_BASE_URL 改为 VectorEngine 地址即可。');
} else if (pass <= 1) {
console.log('\n❌ VectorEngine 不支持 LLM 文本调用。');
} else {
console.log('\n⚠️ 部分支持，需进一步评估。');
}