Merge branch 'master' of http://82.157.175.59:3000/GenarrativeAI/Genarrative
This commit is contained in:
@@ -71,6 +71,31 @@ normalize_env_file() {
|
||||
cp "${temp_file}" "${env_file}"
|
||||
}
|
||||
|
||||
write_env_override() {
|
||||
local env_file="$1"
|
||||
local key="$2"
|
||||
local value="$3"
|
||||
local temp_file="${env_file}.tmp.$$"
|
||||
|
||||
mkdir -p "$(dirname "${env_file}")"
|
||||
if [[ -f "${env_file}" ]]; then
|
||||
# 发布包参数是本次构建的权威值,必须覆盖从 Jenkins 工作区复制进来的旧 .env.local。
|
||||
awk -v target_key="${key}" '
|
||||
BEGIN {
|
||||
pattern = "^[[:space:]]*(export[[:space:]]+)?" target_key "="
|
||||
}
|
||||
$0 !~ pattern {
|
||||
print
|
||||
}
|
||||
' "${env_file}" >"${temp_file}"
|
||||
else
|
||||
: >"${temp_file}"
|
||||
fi
|
||||
|
||||
printf "%s=%s\n" "${key}" "${value}" >>"${temp_file}"
|
||||
cp "${temp_file}" "${env_file}"
|
||||
}
|
||||
|
||||
copy_optional_file() {
|
||||
local source_path="$1"
|
||||
local target_path_a="$2"
|
||||
@@ -270,11 +295,13 @@ if [[ ! "${BUILD_NAME}" =~ ^[0-9A-Za-z._-]+$ ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${DATABASE}" =~ ^[0-9A-Za-z.-]+$ ]]; then
|
||||
echo "[deploy:rust] --database 只能包含数字、字母、点和短横线,不能包含下划线: ${DATABASE}" >&2
|
||||
if [[ ! "${DATABASE}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
|
||||
echo "[deploy:rust] --database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$,只能使用小写字母、数字,并用单个短横线分隔: ${DATABASE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[deploy:rust] SpacetimeDB 发布数据库: ${DATABASE}"
|
||||
|
||||
if [[ "${SKIP_SPACETIME_BUILD}" -eq 1 && "${MIGRATION_BOOTSTRAP_SECRET_MODE}" != "disabled" ]]; then
|
||||
echo "[deploy:rust] --skip-spacetime-build 无法把迁移引导密钥注入 wasm。" >&2
|
||||
echo "[deploy:rust] 请移除 --skip-spacetime-build,或同时传 --no-migration-bootstrap-secret。" >&2
|
||||
@@ -328,6 +355,8 @@ echo "[deploy:rust] 发布包目录: ${TARGET_DIR}"
|
||||
|
||||
copy_optional_file "${REPO_ROOT}/.env" "${TARGET_DIR}/.env" "${WEB_DIR}/.env" ".env"
|
||||
copy_optional_file "${REPO_ROOT}/.env.local" "${TARGET_DIR}/.env.local" "${WEB_DIR}/.env.local" ".env.local"
|
||||
write_env_override "${TARGET_DIR}/.env.local" "GENARRATIVE_SPACETIME_DATABASE" "${DATABASE}"
|
||||
write_env_override "${WEB_DIR}/.env.local" "GENARRATIVE_SPACETIME_DATABASE" "${DATABASE}"
|
||||
|
||||
if [[ "${SKIP_WEB_BUILD}" -ne 1 ]]; then
|
||||
echo "[deploy:rust] 构建 Vite release -> ${WEB_DIR}"
|
||||
@@ -613,6 +642,8 @@ SPACETIME_DATABASE="${GENARRATIVE_SPACETIME_DATABASE:-__GENARRATIVE_DEFAULT_SPAC
|
||||
SPACETIME_TIMEOUT_SECONDS="${GENARRATIVE_SPACETIME_TIMEOUT_SECONDS:-60}"
|
||||
SPACETIME_MIGRATE_ON_CONFLICT="${GENARRATIVE_SPACETIME_MIGRATE_ON_CONFLICT:-true}"
|
||||
SPACETIME_MIGRATION_DIR="${GENARRATIVE_SPACETIME_MIGRATION_DIR:-}"
|
||||
SPACETIME_MIGRATION_EXPORT_TOKEN="${GENARRATIVE_SPACETIME_MIGRATION_EXPORT_TOKEN:-}"
|
||||
SPACETIME_MIGRATION_IMPORT_TOKEN="${GENARRATIVE_SPACETIME_MIGRATION_IMPORT_TOKEN:-}"
|
||||
API_HOST="${GENARRATIVE_API_HOST:-__GENARRATIVE_DEFAULT_API_HOST__}"
|
||||
API_PORT="${GENARRATIVE_API_PORT:-__GENARRATIVE_DEFAULT_API_PORT__}"
|
||||
API_LOG="${GENARRATIVE_API_LOG:-info,tower_http=info}"
|
||||
@@ -662,8 +693,8 @@ sanitize_path_segment() {
|
||||
validate_spacetime_database_name() {
|
||||
local database="$1"
|
||||
|
||||
if [[ ! "${database}" =~ ^[0-9A-Za-z.-]+$ ]]; then
|
||||
echo "[start] GENARRATIVE_SPACETIME_DATABASE 只能包含数字、字母、点和短横线,不能包含下划线: ${database}" >&2
|
||||
if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
|
||||
echo "[start] GENARRATIVE_SPACETIME_DATABASE 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$,只能使用小写字母、数字,并用单个短横线分隔: ${database}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
@@ -737,13 +768,28 @@ run_publish() {
|
||||
run_conflict_migration_publish() {
|
||||
local export_bootstrap_secret=""
|
||||
local import_bootstrap_secret=""
|
||||
local export_auth_args=()
|
||||
local import_auth_args=()
|
||||
local migration_database_slug=""
|
||||
local migration_root=""
|
||||
local migration_file=""
|
||||
local publish_log=""
|
||||
|
||||
export_bootstrap_secret="$(read_export_migration_bootstrap_secret)"
|
||||
import_bootstrap_secret="$(read_import_migration_bootstrap_secret)"
|
||||
if [[ -n "${SPACETIME_MIGRATION_EXPORT_TOKEN}" ]]; then
|
||||
echo "[start] 使用 GENARRATIVE_SPACETIME_MIGRATION_EXPORT_TOKEN 导出旧库"
|
||||
export_auth_args=(--token "${SPACETIME_MIGRATION_EXPORT_TOKEN}")
|
||||
else
|
||||
export_bootstrap_secret="$(read_export_migration_bootstrap_secret)"
|
||||
export_auth_args=(--bootstrap-secret "${export_bootstrap_secret}")
|
||||
fi
|
||||
|
||||
if [[ -n "${SPACETIME_MIGRATION_IMPORT_TOKEN}" ]]; then
|
||||
echo "[start] 使用 GENARRATIVE_SPACETIME_MIGRATION_IMPORT_TOKEN 导入新库"
|
||||
import_auth_args=(--token "${SPACETIME_MIGRATION_IMPORT_TOKEN}")
|
||||
else
|
||||
import_bootstrap_secret="$(read_import_migration_bootstrap_secret)"
|
||||
import_auth_args=(--bootstrap-secret "${import_bootstrap_secret}")
|
||||
fi
|
||||
require_migration_script "${MIGRATION_EXPORT_SCRIPT}"
|
||||
require_migration_script "${MIGRATION_IMPORT_SCRIPT}"
|
||||
|
||||
@@ -758,7 +804,7 @@ run_conflict_migration_publish() {
|
||||
--server-url "${SPACETIME_SERVER_URL}" \
|
||||
--root-dir "${SPACETIME_ROOT_DIR}" \
|
||||
--database "${SPACETIME_DATABASE}" \
|
||||
--bootstrap-secret "${export_bootstrap_secret}" \
|
||||
"${export_auth_args[@]}" \
|
||||
--out "${migration_file}" \
|
||||
--note "deploy conflict export $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
@@ -783,7 +829,7 @@ run_conflict_migration_publish() {
|
||||
--server-url "${SPACETIME_SERVER_URL}" \
|
||||
--root-dir "${SPACETIME_ROOT_DIR}" \
|
||||
--database "${SPACETIME_DATABASE}" \
|
||||
--bootstrap-secret "${import_bootstrap_secret}" \
|
||||
"${import_auth_args[@]}" \
|
||||
--in "${migration_file}" \
|
||||
--replace-existing \
|
||||
--note "deploy conflict import $(date -u +%Y-%m-%dT%H:%M:%SZ)"; then
|
||||
@@ -985,9 +1031,15 @@ start_process() {
|
||||
echo "$!" >"${pid_file}"
|
||||
}
|
||||
|
||||
validate_spacetime_database_name "${SPACETIME_DATABASE}"
|
||||
|
||||
echo "[start] SpacetimeDB 发布配置:"
|
||||
echo "[start] - database: ${SPACETIME_DATABASE}"
|
||||
echo "[start] - server: ${SPACETIME_SERVER_URL}"
|
||||
echo "[start] - root-dir: ${SPACETIME_ROOT_DIR}"
|
||||
|
||||
require_command node
|
||||
require_command spacetime
|
||||
validate_spacetime_database_name "${SPACETIME_DATABASE}"
|
||||
|
||||
mkdir -p "${PID_DIR}" "${LOG_DIR}" "${SPACETIME_ROOT_DIR}"
|
||||
sync_ubuntu_spacetime_install "${SPACETIME_ROOT_DIR}"
|
||||
@@ -1075,6 +1127,7 @@ echo "[start] 完成"
|
||||
echo "[start] Web: http://${WEB_HOST}:${WEB_PORT}"
|
||||
echo "[start] API: http://${API_HOST}:${API_PORT}"
|
||||
echo "[start] SpacetimeDB: ${SPACETIME_SERVER_URL}"
|
||||
echo "[start] SpacetimeDB database: ${SPACETIME_DATABASE}"
|
||||
START_SCRIPT
|
||||
|
||||
replace_placeholder_in_file "${TARGET_DIR}/start.sh" "__GENARRATIVE_DEFAULT_SPACETIME_HOST__" "${SPACETIME_HOST}"
|
||||
|
||||
@@ -96,12 +96,23 @@ is_spacetime_ready() {
|
||||
local root_dir="$2"
|
||||
local output
|
||||
|
||||
if ! output="$(spacetime --root-dir="${root_dir}" server ping "${server}" 2>&1)"; then
|
||||
return 1
|
||||
if output="$(spacetime --root-dir="${root_dir}" server ping "${server}" 2>&1)" &&
|
||||
[[ "${output}" == *"Server is online:"* ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# SpacetimeDB CLI 2.1.0 在 502 Bad Gateway 时仍可能返回 0,不能只依赖退出码。
|
||||
[[ "${output}" == *"Server is online:"* ]]
|
||||
# SpacetimeDB CLI 2.1.0 在 Windows 下可能对已监听的 standalone 返回 502;
|
||||
# 直接探测 HTTP 健康端点,避免 npm run dev:rust 卡在“等待 SpacetimeDB 就绪”。
|
||||
node -e '
|
||||
const target = new URL("/v1/ping", process.argv[1]);
|
||||
const client = target.protocol === "https:" ? require("https") : require("http");
|
||||
const request = client.get(target, { timeout: 1000 }, (response) => {
|
||||
response.resume();
|
||||
process.exit(response.statusCode >= 200 && response.statusCode < 300 ? 0 : 1);
|
||||
});
|
||||
request.on("timeout", () => request.destroy(new Error("timeout")));
|
||||
request.on("error", () => process.exit(1));
|
||||
' "${server}" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
describe_spacetime_root_owner() {
|
||||
|
||||
@@ -5,7 +5,7 @@ set -euo pipefail
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/jenkins-deploy-release.sh --source-dir /path/to/build/123 --deploy-dir /var/lib/jenkins/deploy/Genarrative --web-port 25001 [--clear-database] [--no-migrate-on-conflict] [--migration-dir /path/to/migrations] [--hook-with-sudo]
|
||||
./scripts/jenkins-deploy-release.sh --source-dir /path/to/build/123 --deploy-dir /var/lib/jenkins/deploy/Genarrative --web-port 25001 [--clear-database] [--no-migrate-on-conflict] [--migration-dir /path/to/migrations] [--migration-export-token <token>] [--migration-import-token <token>] [--hook-with-sudo]
|
||||
|
||||
说明:
|
||||
1. 如果部署目录已有旧版本且存在 stop.sh,则先执行旧版本 stop.sh。
|
||||
@@ -23,6 +23,8 @@ usage() {
|
||||
--migrate-on-conflict 可选,普通发布遇到 schema 冲突时自动迁移,默认启用
|
||||
--no-migrate-on-conflict 可选,禁用 schema 冲突自动迁移
|
||||
--migration-dir <path> 可选,自动迁移 JSON 输出目录,默认部署目录内 database-migrations/<database>
|
||||
--migration-export-token <token> 可选,旧库已授权迁移操作员 token,仅用于 schema 冲突导出
|
||||
--migration-import-token <token> 可选,新库已授权迁移操作员 token,仅用于 schema 冲突导入
|
||||
--hook-with-sudo 可选,仅对 start.sh/stop.sh 使用 sudo -n 执行
|
||||
EOF
|
||||
}
|
||||
@@ -59,6 +61,15 @@ validate_port() {
|
||||
fi
|
||||
}
|
||||
|
||||
validate_spacetime_database_name() {
|
||||
local database="$1"
|
||||
|
||||
if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
|
||||
echo "[jenkins-deploy] GENARRATIVE_SPACETIME_DATABASE 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$,只能使用小写字母、数字,并用单个短横线分隔: ${database}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
normalize_env_file() {
|
||||
local env_file="$1"
|
||||
local temp_file="${env_file}.tmp.$$"
|
||||
@@ -72,6 +83,57 @@ normalize_env_file() {
|
||||
cp "${temp_file}" "${env_file}"
|
||||
}
|
||||
|
||||
read_env_value() {
|
||||
local key="$1"
|
||||
shift
|
||||
local env_file=""
|
||||
local line=""
|
||||
local line_number=0
|
||||
local parsed_key=""
|
||||
local parsed_value=""
|
||||
local value=""
|
||||
local utf8_bom=$'\xef\xbb\xbf'
|
||||
|
||||
for env_file in "$@"; do
|
||||
if [[ ! -f "${env_file}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
line_number=0
|
||||
while IFS= read -r line || [[ -n "${line}" ]]; do
|
||||
line_number=$((line_number + 1))
|
||||
if [[ "${line_number}" -eq 1 ]]; then
|
||||
line="${line#"${utf8_bom}"}"
|
||||
fi
|
||||
line="${line%$'\r'}"
|
||||
|
||||
if [[ "${line}" =~ ^[[:space:]]*$ || "${line}" =~ ^[[:space:]]*# ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! "${line}" =~ ^[[:space:]]*(export[[:space:]]+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$ ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
parsed_key="${BASH_REMATCH[2]}"
|
||||
parsed_value="${BASH_REMATCH[3]}"
|
||||
if [[ "${parsed_key}" != "${key}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
value="${parsed_value}"
|
||||
if [[ "${#value}" -ge 2 && "${value:0:1}" == '"' && "${value: -1}" == '"' ]]; then
|
||||
value="${value:1:${#value}-2}"
|
||||
value="${value//\\\"/\"}"
|
||||
elif [[ "${#value}" -ge 2 && "${value:0:1}" == "'" && "${value: -1}" == "'" ]]; then
|
||||
value="${value:1:${#value}-2}"
|
||||
fi
|
||||
done <"${env_file}"
|
||||
done
|
||||
|
||||
printf "%s" "${value}"
|
||||
}
|
||||
|
||||
normalize_release_env_files() {
|
||||
local release_dir="$1"
|
||||
|
||||
@@ -113,6 +175,10 @@ CLEAR_DATABASE="0"
|
||||
MIGRATE_ON_CONFLICT="true"
|
||||
MIGRATION_DIR=""
|
||||
HOOK_WITH_SUDO="0"
|
||||
MIGRATION_EXPORT_TOKEN=""
|
||||
MIGRATION_IMPORT_TOKEN=""
|
||||
PRESERVED_MIGRATION_EXPORT_TOKEN=""
|
||||
PRESERVED_MIGRATION_IMPORT_TOKEN=""
|
||||
DEPLOY_ITEMS=(
|
||||
".env"
|
||||
".env.local"
|
||||
@@ -162,6 +228,14 @@ while [[ $# -gt 0 ]]; do
|
||||
MIGRATION_DIR="${2:?缺少 --migration-dir 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--migration-export-token)
|
||||
MIGRATION_EXPORT_TOKEN="${2:?缺少 --migration-export-token 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--migration-import-token)
|
||||
MIGRATION_IMPORT_TOKEN="${2:?缺少 --migration-import-token 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--hook-with-sudo)
|
||||
HOOK_WITH_SUDO="1"
|
||||
shift
|
||||
@@ -288,6 +362,8 @@ if [[ ! -f "${SOURCE_DIR}/start.sh" ]]; then
|
||||
fi
|
||||
|
||||
normalize_release_env_files "${SOURCE_DIR}"
|
||||
PRESERVED_MIGRATION_EXPORT_TOKEN="$(read_env_value "GENARRATIVE_SPACETIME_MIGRATION_EXPORT_TOKEN" "${DEPLOY_DIR}/.env" "${DEPLOY_DIR}/.env.local")"
|
||||
PRESERVED_MIGRATION_IMPORT_TOKEN="$(read_env_value "GENARRATIVE_SPACETIME_MIGRATION_IMPORT_TOKEN" "${DEPLOY_DIR}/.env" "${DEPLOY_DIR}/.env.local")"
|
||||
|
||||
if [[ -x "${DEPLOY_DIR}/stop.sh" ]]; then
|
||||
echo "[jenkins-deploy] 先停止旧版本: ${DEPLOY_DIR}"
|
||||
@@ -336,6 +412,26 @@ normalize_release_env_files "${DEPLOY_DIR}"
|
||||
write_env_override "${DEPLOY_DIR}/.env.local" "GENARRATIVE_WEB_PORT" "${WEB_PORT}"
|
||||
write_env_override "${DEPLOY_DIR}/.env.local" "GENARRATIVE_SPACETIME_MIGRATE_ON_CONFLICT" "${MIGRATE_ON_CONFLICT}"
|
||||
write_env_override "${DEPLOY_DIR}/.env.local" "GENARRATIVE_SPACETIME_MIGRATION_DIR" "${MIGRATION_DIR}"
|
||||
if [[ -n "${MIGRATION_EXPORT_TOKEN}" ]]; then
|
||||
write_env_override "${DEPLOY_DIR}/.env.local" "GENARRATIVE_SPACETIME_MIGRATION_EXPORT_TOKEN" "${MIGRATION_EXPORT_TOKEN}"
|
||||
elif [[ -n "${PRESERVED_MIGRATION_EXPORT_TOKEN}" ]] \
|
||||
&& [[ -z "$(read_env_value "GENARRATIVE_SPACETIME_MIGRATION_EXPORT_TOKEN" "${DEPLOY_DIR}/.env" "${DEPLOY_DIR}/.env.local")" ]]; then
|
||||
write_env_override "${DEPLOY_DIR}/.env.local" "GENARRATIVE_SPACETIME_MIGRATION_EXPORT_TOKEN" "${PRESERVED_MIGRATION_EXPORT_TOKEN}"
|
||||
fi
|
||||
if [[ -n "${MIGRATION_IMPORT_TOKEN}" ]]; then
|
||||
write_env_override "${DEPLOY_DIR}/.env.local" "GENARRATIVE_SPACETIME_MIGRATION_IMPORT_TOKEN" "${MIGRATION_IMPORT_TOKEN}"
|
||||
elif [[ -n "${PRESERVED_MIGRATION_IMPORT_TOKEN}" ]] \
|
||||
&& [[ -z "$(read_env_value "GENARRATIVE_SPACETIME_MIGRATION_IMPORT_TOKEN" "${DEPLOY_DIR}/.env" "${DEPLOY_DIR}/.env.local")" ]]; then
|
||||
write_env_override "${DEPLOY_DIR}/.env.local" "GENARRATIVE_SPACETIME_MIGRATION_IMPORT_TOKEN" "${PRESERVED_MIGRATION_IMPORT_TOKEN}"
|
||||
fi
|
||||
|
||||
DEPLOY_DATABASE="$(read_env_value "GENARRATIVE_SPACETIME_DATABASE" "${DEPLOY_DIR}/.env" "${DEPLOY_DIR}/.env.local")"
|
||||
if [[ -z "${DEPLOY_DATABASE}" ]]; then
|
||||
echo "[jenkins-deploy] 部署包未显式写入 GENARRATIVE_SPACETIME_DATABASE;将由 start.sh 使用构建时默认值。" >&2
|
||||
else
|
||||
validate_spacetime_database_name "${DEPLOY_DATABASE}"
|
||||
echo "[jenkins-deploy] SpacetimeDB 发布数据库: ${DEPLOY_DATABASE}"
|
||||
fi
|
||||
|
||||
echo "[jenkins-deploy] 启动新版本: ${DEPLOY_DIR}"
|
||||
if [[ "${CLEAR_DATABASE}" == "1" ]]; then
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
callSpacetimeProcedure,
|
||||
callSpacetimeProcedureViaCli,
|
||||
ensureProcedureOk,
|
||||
parseArgs,
|
||||
@@ -17,11 +18,16 @@ try {
|
||||
operator_identity_hex: options.operatorIdentity,
|
||||
note: options.note || '',
|
||||
};
|
||||
const result = await callSpacetimeProcedureViaCli(
|
||||
options,
|
||||
'authorize_database_migration_operator',
|
||||
input,
|
||||
);
|
||||
if (options.useHttp && !options.token) {
|
||||
throw new Error('--use-http 需要同时传入 --token。');
|
||||
}
|
||||
const result = options.useHttp
|
||||
? await callSpacetimeProcedure(options, 'authorize_database_migration_operator', input)
|
||||
: await callSpacetimeProcedureViaCli(
|
||||
options,
|
||||
'authorize_database_migration_operator',
|
||||
input,
|
||||
);
|
||||
ensureProcedureOk(result);
|
||||
|
||||
console.log(
|
||||
|
||||
@@ -57,16 +57,24 @@ async function prepareWebExportOptions(options) {
|
||||
`[spacetime:migration:export] 已通过 Web API 创建临时 identity: ${identity.identity}`,
|
||||
);
|
||||
|
||||
const authorizeResult = await callSpacetimeProcedureViaCli(
|
||||
options,
|
||||
'authorize_database_migration_operator',
|
||||
{
|
||||
bootstrap_secret: options.bootstrapSecret || '',
|
||||
operator_identity_hex: identity.identity,
|
||||
note: options.note || 'temporary web api migration export',
|
||||
},
|
||||
);
|
||||
ensureProcedureOk(authorizeResult);
|
||||
try {
|
||||
const authorizeResult = await callSpacetimeProcedureViaCli(
|
||||
options,
|
||||
'authorize_database_migration_operator',
|
||||
{
|
||||
bootstrap_secret: options.bootstrapSecret || '',
|
||||
operator_identity_hex: identity.identity,
|
||||
note: options.note || 'temporary web api migration export',
|
||||
},
|
||||
);
|
||||
ensureProcedureOk(authorizeResult);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`授权临时 Web API identity 失败。当前 spacetime CLI identity 必须已经是迁移操作员;如果旧库迁移操作员表不为空,bootstrap secret 不会越权授权新的操作员。可先用已有迁移操作员授权当前部署机 identity,或为导出脚本提供已有迁移操作员的 --token。原始错误: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
console.log(`[spacetime:migration:export] 已授权临时 Web API identity`);
|
||||
|
||||
return {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
@@ -11,6 +12,8 @@ import {
|
||||
parseArgs,
|
||||
} from './spacetime-migration-common.mjs';
|
||||
|
||||
const DEFAULT_MIGRATION_IMPORT_CHUNK_SIZE = 512 * 1024;
|
||||
|
||||
try {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (!options.in) {
|
||||
@@ -30,7 +33,7 @@ try {
|
||||
const webOptions = await prepareWebImportOptions(options);
|
||||
let result;
|
||||
try {
|
||||
result = await importMigrationJsonDirect(webOptions, migrationJson);
|
||||
result = await importMigrationJsonWithFallback(webOptions, migrationJson);
|
||||
} finally {
|
||||
await revokeTemporaryWebIdentity(webOptions);
|
||||
}
|
||||
@@ -58,16 +61,24 @@ async function prepareWebImportOptions(options) {
|
||||
`[spacetime:migration:import] 已通过 Web API 创建临时 identity: ${identity.identity}`,
|
||||
);
|
||||
|
||||
const authorizeResult = await callSpacetimeProcedureViaCli(
|
||||
options,
|
||||
'authorize_database_migration_operator',
|
||||
{
|
||||
bootstrap_secret: options.bootstrapSecret || '',
|
||||
operator_identity_hex: identity.identity,
|
||||
note: options.note || 'temporary web api migration import',
|
||||
},
|
||||
);
|
||||
ensureProcedureOk(authorizeResult);
|
||||
try {
|
||||
const authorizeResult = await callSpacetimeProcedureViaCli(
|
||||
options,
|
||||
'authorize_database_migration_operator',
|
||||
{
|
||||
bootstrap_secret: options.bootstrapSecret || '',
|
||||
operator_identity_hex: identity.identity,
|
||||
note: options.note || 'temporary web api migration import',
|
||||
},
|
||||
);
|
||||
ensureProcedureOk(authorizeResult);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`授权临时 Web API identity 失败。当前 spacetime CLI identity 必须已经是迁移操作员;如果目标库迁移操作员表不为空,bootstrap secret 不会越权授权新的操作员。可先用已有迁移操作员授权当前部署机 identity,或为导入脚本提供已有迁移操作员的 --token。原始错误: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
console.log(`[spacetime:migration:import] 已授权临时 Web API identity`);
|
||||
|
||||
return {
|
||||
@@ -78,6 +89,25 @@ async function prepareWebImportOptions(options) {
|
||||
};
|
||||
}
|
||||
|
||||
async function importMigrationJsonWithFallback(options, migrationJson) {
|
||||
const chunkSize = resolveChunkSize(options);
|
||||
if (Buffer.byteLength(migrationJson, 'utf8') > chunkSize) {
|
||||
return importMigrationJsonChunked(options, migrationJson, chunkSize);
|
||||
}
|
||||
|
||||
try {
|
||||
return await importMigrationJsonDirect(options, migrationJson);
|
||||
} catch (error) {
|
||||
if (!isRequestBodyTooLargeError(error)) {
|
||||
throw error;
|
||||
}
|
||||
console.warn(
|
||||
`[spacetime:migration:import] 直接导入触发 HTTP 413,改用 ${chunkSize} bytes 分片上传。`,
|
||||
);
|
||||
return importMigrationJsonChunked(options, migrationJson, chunkSize);
|
||||
}
|
||||
}
|
||||
|
||||
async function importMigrationJsonDirect(options, migrationJson) {
|
||||
const includeTables = resolveImportIncludeTables(options, migrationJson);
|
||||
const procedureName =
|
||||
@@ -100,6 +130,60 @@ async function importMigrationJsonDirect(options, migrationJson) {
|
||||
return callSpacetimeProcedure(options, procedureName, input);
|
||||
}
|
||||
|
||||
async function importMigrationJsonChunked(options, migrationJson, chunkSize) {
|
||||
const includeTables = resolveImportIncludeTables(options, migrationJson);
|
||||
const procedureName =
|
||||
options.incremental === true
|
||||
? 'import_database_migration_incremental_from_chunks'
|
||||
: 'import_database_migration_from_chunks';
|
||||
const uploadId = `migration-${Date.now()}-${randomUUID()}`;
|
||||
const chunks = splitStringByUtf8Bytes(migrationJson, chunkSize);
|
||||
console.log(
|
||||
`[spacetime:migration:import] 使用分片导入: upload_id=${uploadId}, chunks=${chunks.length}, chunk_size=${chunkSize}`,
|
||||
);
|
||||
if (options.replaceExisting === true) {
|
||||
console.log(
|
||||
`[spacetime:migration:import] replace-existing 仅覆盖本次文件内的表: ${includeTables.join(', ') || '无'}`,
|
||||
);
|
||||
} else if (options.incremental === true) {
|
||||
console.log(`[spacetime:migration:import] 使用增量模式,已存在或冲突的行会跳过`);
|
||||
}
|
||||
|
||||
let committed = false;
|
||||
try {
|
||||
for (let index = 0; index < chunks.length; index += 1) {
|
||||
const chunkResult = await callSpacetimeProcedure(
|
||||
options,
|
||||
'put_database_migration_import_chunk',
|
||||
{
|
||||
upload_id: uploadId,
|
||||
chunk_index: index,
|
||||
chunk_count: chunks.length,
|
||||
chunk: chunks[index],
|
||||
},
|
||||
);
|
||||
ensureProcedureOk(chunkResult);
|
||||
console.log(
|
||||
`[spacetime:migration:import] 已上传迁移分片 ${index + 1}/${chunks.length}`,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await callSpacetimeProcedure(options, procedureName, {
|
||||
upload_id: uploadId,
|
||||
include_tables: includeTables,
|
||||
replace_existing: options.replaceExisting === true,
|
||||
dry_run: options.dryRun === true,
|
||||
});
|
||||
ensureProcedureOk(result);
|
||||
committed = true;
|
||||
return result;
|
||||
} finally {
|
||||
if (!committed) {
|
||||
await clearMigrationChunksBestEffort(options, uploadId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resolveImportIncludeTables(options, migrationJson) {
|
||||
if (options.replaceExisting !== true) {
|
||||
return options.includeTables;
|
||||
@@ -144,6 +228,63 @@ function readMigrationTableNames(migrationJson) {
|
||||
return tableNames;
|
||||
}
|
||||
|
||||
function resolveChunkSize(options) {
|
||||
const chunkSize = options.chunkSize || DEFAULT_MIGRATION_IMPORT_CHUNK_SIZE;
|
||||
if (chunkSize > 1024 * 1024) {
|
||||
throw new Error('--chunk-size 不能超过 1048576,避免触发迁移分片 procedure 单片限制。');
|
||||
}
|
||||
return chunkSize;
|
||||
}
|
||||
|
||||
function splitStringByUtf8Bytes(value, maxBytes) {
|
||||
const chunks = [];
|
||||
let current = '';
|
||||
let currentBytes = 0;
|
||||
for (const character of value) {
|
||||
const characterBytes = Buffer.byteLength(character, 'utf8');
|
||||
if (characterBytes > maxBytes) {
|
||||
throw new Error(`单个字符超过 chunk-size,当前 chunk-size: ${maxBytes}`);
|
||||
}
|
||||
if (currentBytes + characterBytes > maxBytes && current) {
|
||||
chunks.push(current);
|
||||
current = '';
|
||||
currentBytes = 0;
|
||||
}
|
||||
current += character;
|
||||
currentBytes += characterBytes;
|
||||
}
|
||||
if (current) {
|
||||
chunks.push(current);
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function isRequestBodyTooLargeError(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return (
|
||||
message.includes('HTTP 413') ||
|
||||
message.toLowerCase().includes('length limit exceeded')
|
||||
);
|
||||
}
|
||||
|
||||
async function clearMigrationChunksBestEffort(options, uploadId) {
|
||||
try {
|
||||
const result = await callSpacetimeProcedure(
|
||||
options,
|
||||
'clear_database_migration_import_chunks',
|
||||
{ upload_id: uploadId },
|
||||
);
|
||||
ensureProcedureOk(result);
|
||||
console.warn(`[spacetime:migration:import] 已清理失败导入的临时分片: ${uploadId}`);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[spacetime:migration:import] 清理临时迁移分片失败: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function revokeTemporaryWebIdentity(options) {
|
||||
if (!options.temporaryWebIdentity) {
|
||||
return;
|
||||
|
||||
@@ -4,6 +4,10 @@ import path from 'node:path';
|
||||
|
||||
export function parseArgs(argv) {
|
||||
const options = {
|
||||
chunkSize: parseOptionalPositiveInteger(
|
||||
process.env.GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE,
|
||||
'GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE',
|
||||
),
|
||||
database:
|
||||
process.env.GENARRATIVE_SPACETIME_MAINCLOUD_DATABASE ||
|
||||
process.env.GENARRATIVE_SPACETIME_DATABASE ||
|
||||
@@ -48,6 +52,8 @@ export function parseArgs(argv) {
|
||||
options.token = readValue(arg);
|
||||
} else if (arg === '--bootstrap-secret') {
|
||||
options.bootstrapSecret = readValue(arg);
|
||||
} else if (arg === '--chunk-size') {
|
||||
options.chunkSize = parsePositiveInteger(readValue(arg), arg);
|
||||
} else if (arg === '--operator-identity') {
|
||||
options.operatorIdentity = readValue(arg);
|
||||
} else if (arg === '--note') {
|
||||
@@ -81,10 +87,30 @@ export function parseArgs(argv) {
|
||||
return options;
|
||||
}
|
||||
|
||||
export function parsePositiveInteger(value, name) {
|
||||
if (!/^[1-9][0-9]*$/u.test(String(value).trim())) {
|
||||
throw new Error(`${name} 必须是正整数。`);
|
||||
}
|
||||
|
||||
const parsed = Number.parseInt(String(value).trim(), 10);
|
||||
if (!Number.isSafeInteger(parsed)) {
|
||||
throw new Error(`${name} 超出安全整数范围。`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseOptionalPositiveInteger(value, name) {
|
||||
if (!value) {
|
||||
return 0;
|
||||
}
|
||||
return parsePositiveInteger(value, name);
|
||||
}
|
||||
|
||||
export function buildSpacetimeCallArgs(options, procedureName, input) {
|
||||
if (!options.database) {
|
||||
throw new Error('必须传入 --database。');
|
||||
}
|
||||
validateSpacetimeDatabaseName(options.database);
|
||||
|
||||
const args = [];
|
||||
if (options.rootDir) {
|
||||
@@ -108,6 +134,7 @@ export async function callSpacetimeProcedure(options, procedureName, input) {
|
||||
if (!options.database) {
|
||||
throw new Error('必须传入 --database,或设置 GENARRATIVE_SPACETIME_DATABASE。');
|
||||
}
|
||||
validateSpacetimeDatabaseName(options.database);
|
||||
|
||||
const serverUrl = resolveServerUrl(options).replace(/\/+$/u, '');
|
||||
const url = `${serverUrl}/v1/database/${encodeURIComponent(options.database)}/call/${encodeURIComponent(procedureName)}`;
|
||||
@@ -195,6 +222,14 @@ export async function callSpacetimeProcedureViaCli(options, procedureName, input
|
||||
return parseProcedureResult(output);
|
||||
}
|
||||
|
||||
export function validateSpacetimeDatabaseName(database) {
|
||||
if (!/^[a-z0-9]+(-[a-z0-9]+)*$/u.test(database)) {
|
||||
throw new Error(
|
||||
`SpacetimeDB 数据库名必须匹配 ^[a-z0-9]+(-[a-z0-9]+)*$,只能使用小写字母、数字,并用单个短横线分隔: ${database}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function parseProcedureResult(output) {
|
||||
const candidates = [];
|
||||
const trimmed = output.trim();
|
||||
|
||||
@@ -88,6 +88,15 @@ timestamp_slug() {
|
||||
node -e 'process.stdout.write(new Date().toISOString().replace(/[:.]/g, "-"));'
|
||||
}
|
||||
|
||||
validate_spacetime_database_name() {
|
||||
local database="$1"
|
||||
|
||||
if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
|
||||
echo "[spacetime:maincloud] --database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$,只能使用小写字母、数字,并用单个短横线分隔: ${database}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
is_publish_conflict_output() {
|
||||
local output="$1"
|
||||
[[ "${output}" == *"conflict"* ]] \
|
||||
@@ -209,6 +218,11 @@ if [[ -z "${SPACETIME_DATABASE}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_spacetime_database_name "${SPACETIME_DATABASE}"
|
||||
|
||||
echo "[spacetime:maincloud] SpacetimeDB 发布数据库: ${SPACETIME_DATABASE}"
|
||||
echo "[spacetime:maincloud] SpacetimeDB server: ${SPACETIME_SERVER_ALIAS} (${SPACETIME_SERVER_URL})"
|
||||
|
||||
if ! command -v cargo >/dev/null 2>&1; then
|
||||
echo "[spacetime:maincloud] 缺少 cargo 命令。" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user