Merge remote-tracking branch 'web/master' into feat/pixel_art2

# Conflicts:
#	docs/project-memory/shared-memory/decision-log.md
#	docs/project-memory/shared-memory/pitfalls.md
#	docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md
#	server-rs/crates/api-server/src/editor_project.rs
This commit is contained in:
2026-07-31 09:11:12 +00:00
570 changed files with 398466 additions and 2274 deletions
+83
View File
@@ -14,6 +14,7 @@ import {
cleanupHistoryCandidates,
collectDirectFileEntries,
createUploadBandwidthLimiter,
discoverDeferredArchiveUploads,
discoverHistoryPlan,
restoreDirectFilesBackup,
restoreDirectFilesLatest,
@@ -46,6 +47,7 @@ if (failures.length > 0) {
console.log('[check:database-backup] OK');
async function main() {
assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
assertInsufficientSpaceStopsBeforeServiceChanges();
assertArchiveFailureStillRestoresDependentServices();
@@ -73,6 +75,87 @@ async function main() {
await assertDirectFilesRestoreDownloadsCatalogAndObjects();
}
function assertDeferredArchiveDiscoveryIsBoundedAndDeterministic() {
const root = path.join(tmpRoot, 'deferred-archive-discovery');
mkdirSync(root, {recursive: true});
const createCandidate = ({name, status, database = 'test-db', withArchive = true}) => {
const archivePath = path.join(root, `${name}.tar.gz`);
const manifestPath = `${archivePath}.manifest.json`;
if (withArchive) {
writeFileSync(archivePath, name);
}
writeFileSync(manifestPath, `${JSON.stringify({
backupKind: 'spacetimedb-data-dir',
database,
archivePath,
uploadStatus: status,
})}\n`);
return {archivePath, manifestPath};
};
const later = createCandidate({name: 'test-db-20260731T020000Z', status: 'pending'});
const earlier = createCandidate({name: 'test-db-20260731T010000Z', status: 'deferred'});
const uploaded = createCandidate({name: 'test-db-20260731T000000Z', status: 'uploaded'});
createCandidate({name: 'other-db-20260731T000000Z', status: 'deferred', database: 'other-db'});
const missing = createCandidate({name: 'test-db-20260730T230000Z', status: 'deferred', withArchive: false});
const result = discoverDeferredArchiveUploads({workDir: root, database: 'test-db'});
assertEqual(
result.archives.map(({archivePath}) => archivePath).join(','),
[earlier.archivePath, later.archivePath].join(','),
'deferred/pending 扫描必须只返回同库现存归档,并按文件名稳定排序。',
);
assertEqual(result.missingArchives.length, 1, '缺失归档的 deferred 清单必须单独报告。');
assertEqual(result.missingArchives[0].manifestPath, missing.manifestPath, '缺失归档报告必须保留精确 manifest。');
const cleanupResult = discoverDeferredArchiveUploads({workDir: root, database: 'test-db', includeUploaded: true});
assertEqual(
cleanupResult.archives.map(({archivePath}) => archivePath).join(','),
[uploaded.archivePath, earlier.archivePath, later.archivePath].join(','),
'未要求保留本地归档时,补偿扫描必须同时收敛上传后未清理的本地归档。',
);
const cliDryRun = spawnSync(process.execPath, [
BACKUP_SCRIPT,
'--upload-deferred-dir', root,
'--database', 'test-db',
'--bucket', 'test-bucket',
'--endpoint', 'oss-cn-shanghai.aliyuncs.com',
'--access-key-id', 'test-id',
'--access-key-secret', 'test-secret',
'--keep-local',
'--dry-run',
], {encoding: 'utf8'});
assertStatus(cliDryRun, 0, 'deferred 补偿扫描 dry-run 必须可通过统一 CLI 入口执行。');
assertIncludes(cliDryRun.stdout, 'count=2', 'deferred 补偿扫描 CLI 必须报告待处理归档数量。');
assertTrue(existsSync(earlier.archivePath) && existsSync(later.archivePath), 'dry-run 不得删除 deferred 本地归档。');
const unsafeRoot = path.join(tmpRoot, 'deferred-archive-unsafe');
mkdirSync(unsafeRoot, {recursive: true});
const escapedArchive = path.join(tmpRoot, 'outside.tar.gz');
writeFileSync(escapedArchive, 'outside');
writeFileSync(
path.join(unsafeRoot, 'test-db-unsafe.tar.gz.manifest.json'),
`${JSON.stringify({database: 'test-db', archivePath: escapedArchive, uploadStatus: 'deferred'})}\n`,
);
assertThrows(
() => discoverDeferredArchiveUploads({workDir: unsafeRoot, database: 'test-db'}),
'路径与清单不匹配',
'deferred 扫描必须拒绝目录外归档或 manifest 名不匹配。',
);
const symlinkRoot = path.join(tmpRoot, 'deferred-archive-symlink');
mkdirSync(symlinkRoot, {recursive: true});
const symlinkArchive = path.join(symlinkRoot, 'test-db-symlink.tar.gz');
symlinkSync(escapedArchive, symlinkArchive);
writeFileSync(
`${symlinkArchive}.manifest.json`,
`${JSON.stringify({database: 'test-db', archivePath: symlinkArchive, uploadStatus: 'deferred'})}\n`,
);
assertThrows(
() => discoverDeferredArchiveUploads({workDir: symlinkRoot, database: 'test-db'}),
'非符号链接的普通文件',
'deferred 扫描必须拒绝符号链接归档。',
);
}
function readGzipJson(filePath) {
return JSON.parse(gunzipSync(readFileSync(filePath)).toString('utf8'));
}
File diff suppressed because it is too large Load Diff
@@ -206,6 +206,40 @@ const checks = [
includes: '按参数保持维护模式和旧运行时服务停止状态',
reason: '受控维护发布成功后不得自动重启旧运行时或退出维护模式。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: 'systemd-run',
reason:
'生产 Stdb publish 的异步 OSS 上传必须交给 systemd transient service,避免 Jenkins 结束时清理上传进程。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: '--unit="${unit_name}"',
reason: '生产 Stdb publish 必须为异步 OSS 上传创建独立、可追踪的 transient unit。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: '--collect',
reason: '异步 OSS 上传的 transient unit 结束后必须允许 systemd 回收。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: '--service-type=exec',
reason:
'异步 OSS 上传必须等待 systemd 确认上传进程 exec 成功,不能把启动失败误判为已接管。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: '--upload-deferred-dir "${log_dir}"',
reason:
'独立 OSS 上传服务必须补偿扫描历史 deferred/pending 归档,不能只处理当次归档。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
excludes: 'nohup ',
reason:
'生产 Stdb publish 不得恢复会继承 Jenkins 进程生命周期的 nohup 异步上传。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'ensure_runtime_bootstrap_secret_file_env',
@@ -7148,6 +7182,38 @@ for (const check of checks) {
}
}
const stdbPublishContent = readFileSync(
'scripts/deploy/production-stdb-publish.sh',
'utf8',
);
const asyncBackupUploadStart = stdbPublishContent.indexOf(
'start_async_backup_upload() {',
);
const asyncBackupUploadEnd = stdbPublishContent.indexOf(
'\nwait_for_spacetime_ready() {',
asyncBackupUploadStart,
);
const asyncBackupUploadFunction =
asyncBackupUploadStart >= 0 && asyncBackupUploadEnd > asyncBackupUploadStart
? stdbPublishContent.slice(asyncBackupUploadStart, asyncBackupUploadEnd)
: '';
const systemdRunFailureGuard = asyncBackupUploadFunction.match(
/if\s+!\s+(?:run_privileged\s+)?systemd-run\b[\s\S]*?\bthen\b(?:(?!\n\s*fi\b)[\s\S])*?\breturn\s+[1-9][0-9]*\b(?:(?!\n\s*fi\b)[\s\S])*?\n\s*fi\b/u,
);
const asyncBackupStatusClearOffset = asyncBackupUploadFunction.indexOf(
'rm -f "${ASYNC_BACKUP_STATUS_FILE}"',
);
if (
!systemdRunFailureGuard ||
asyncBackupStatusClearOffset <
(systemdRunFailureGuard.index ?? 0) + systemdRunFailureGuard[0].length
) {
failed = true;
console.error(
'[check:production-ops] production-stdb-publish 的 systemd-run 启动失败分支必须先返回非零,并且只能在 transient unit 启动成功后清理异步备份 status 文件。',
);
}
for (const file of jenkinsSourceCheckoutFiles) {
const content = readFileSync(file, 'utf8');
if (!content.includes(jenkinsLoopbackGitRemote)) {
+100
View File
@@ -64,6 +64,7 @@ function usage() {
npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload]
node -- scripts/database-backup-to-oss.mjs --upload-archive <path>
node -- scripts/database-backup-to-oss.mjs --upload-deferred-dir <path>
node -- scripts/database-backup-to-oss.mjs --publish-manifest <path>
node -- scripts/database-backup-to-oss.mjs --restore-files-state <path> --restore-dir <path>
node -- scripts/database-backup-to-oss.mjs --restore-files-latest --restore-dir <path> [--dry-run]
@@ -74,6 +75,7 @@ function usage() {
--storage-format files 不打包:按原相对路径建立 catalog,文件内容以 SHA-256 不可变对象上传;重复运行只上传新增或变化内容。
archive history 必须有已验真的 full baseline state;files history 必须复用同一 work-dir 中已发布的 full catalog state。
--defer-upload 只生成本地冷备份和 manifest,不上传;后续用 --upload-archive 异步上传。
--upload-deferred-dir 串行收敛目录内 deferred/pending 及已上传未清理归档;只有 OSS 上传与验真完成后才按 keep-local 规则删除。
默认读取 .env / .env.local / .env.secrets.local;生产服务可传 --env-file /etc/genarrative/api-server.env。
shell 环境变量优先级最高,不会被 env 文件覆盖。
@@ -157,6 +159,7 @@ function parseArgs(argv) {
dryRun: false,
deferUpload: false,
uploadArchive: '',
uploadDeferredDir: '',
manifestFile: '',
objectKey: '',
resultFile: '',
@@ -237,6 +240,9 @@ function parseArgs(argv) {
case '--upload-archive':
options.uploadArchive = readValue();
break;
case '--upload-deferred-dir':
options.uploadDeferredDir = readValue();
break;
case '--manifest-file':
options.manifestFile = readValue();
break;
@@ -2811,6 +2817,50 @@ export async function uploadHistoryArchiveWithCleanup({
return {result, uploadedManifest, cleanup, state};
}
export function discoverDeferredArchiveUploads({workDir, database, includeUploaded = false}) {
const resolvedWorkDir = resolvePath(workDir);
if (!existsSync(resolvedWorkDir)) {
return {archives: [], missingArchives: []};
}
const archives = [];
const missingArchives = [];
const manifestSuffix = '.tar.gz.manifest.json';
const expectedDatabase = String(database || '').trim();
const entries = readdirSync(resolvedWorkDir, {withFileTypes: true})
.filter((candidate) => candidate.isFile() && candidate.name.endsWith(manifestSuffix))
.sort((left, right) => left.name.localeCompare(right.name, 'en'));
for (const entry of entries) {
const manifestPath = join(resolvedWorkDir, entry.name);
const manifest = readManifest(manifestPath);
const uploadStatus = String(manifest.uploadStatus || '').trim();
if (!['deferred', 'pending'].includes(uploadStatus) && !(includeUploaded && uploadStatus === 'uploaded')) {
continue;
}
if (expectedDatabase && String(manifest.database || '').trim() !== expectedDatabase) {
continue;
}
if (!manifest.archivePath) {
throw new Error(`deferred 备份清单缺少 archivePath: ${manifestPath}`);
}
const archivePath = resolvePath(manifest.archivePath);
if (dirname(archivePath) !== resolvedWorkDir || manifestPath !== `${archivePath}.manifest.json`) {
throw new Error(`deferred 备份路径与清单不匹配: ${manifestPath}`);
}
const candidate = {archivePath, manifestPath, manifest};
if (existsSync(archivePath)) {
const archiveStat = lstatSync(archivePath);
if (!archiveStat.isFile() || archiveStat.isSymbolicLink()) {
throw new Error(`deferred 备份归档必须是非符号链接的普通文件: ${archivePath}`);
}
archives.push(candidate);
} else {
missingArchives.push(candidate);
}
}
return {archives, missingArchives};
}
async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix, bandwidthLimiter}) {
const archivePath = resolvePath(args.uploadArchive);
if (!existsSync(archivePath)) {
@@ -2897,6 +2947,37 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId,
}
}
async function uploadDeferredArchives({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix, database, bandwidthLimiter}) {
const workDir = resolvePath(args.uploadDeferredDir);
const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
const {archives, missingArchives} = discoverDeferredArchiveUploads({
workDir,
database,
includeUploaded: !keepLocal,
});
for (const {manifestPath} of missingArchives) {
console.warn(`[database-backup] deferred 清单对应的本地归档不存在,跳过: ${manifestPath}`);
}
if (archives.length === 0) {
console.log(`[database-backup] 没有可补偿的本地归档: ${workDir}`);
return;
}
console.log(`[database-backup] 开始串行上传待补偿本地归档: count=${archives.length}`);
for (const {archivePath, manifestPath} of archives) {
await uploadExistingArchive({
args: {...args, uploadArchive: archivePath, manifestFile: manifestPath},
env,
bucket,
endpoint,
accessKeyId,
accessKeySecret,
objectPrefix,
bandwidthLimiter,
});
}
console.log(`[database-backup] 待补偿本地归档上传完成: count=${archives.length}`);
}
async function publishExistingManifest({args, bucket, endpoint, accessKeyId, accessKeySecret, bandwidthLimiter}) {
const manifestPath = resolvePath(args.publishManifest);
const manifest = readManifest(manifestPath);
@@ -3108,6 +3189,7 @@ async function main() {
));
const workDir = resolvePath(firstNonEmpty(
args.workDir,
args.uploadDeferredDir,
env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR,
isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR,
));
@@ -3171,6 +3253,9 @@ async function main() {
if (args.restoreDir) {
throw new Error('--restore-dir 只能与 --restore-files-state 或 --restore-files-latest 一起使用。');
}
if (args.uploadArchive && args.uploadDeferredDir) {
throw new Error('--upload-archive 与 --upload-deferred-dir 不能同时使用。');
}
if (!args.dryRun) {
const lockPath = acquireBackupLock({workDir, database});
@@ -3196,6 +3281,21 @@ async function main() {
return;
}
if (args.uploadDeferredDir) {
await uploadDeferredArchives({
args,
env,
bucket,
endpoint,
accessKeyId,
accessKeySecret,
objectPrefix,
database,
bandwidthLimiter: uploadBandwidthLimiter,
});
return;
}
if (storageFormat === 'files') {
if (args.deferUpload) {
throw new Error('files 模式无需本地归档且不支持 --defer-upload;失败后使用同一 work-dir 重跑即可续传。');
+56 -9
View File
@@ -532,6 +532,11 @@ prepare_async_backup() {
}
start_async_backup_upload() {
local log_dir=""
local node_binary=""
local unit_name=""
local unit_suffix=""
if [[ -z "${ASYNC_BACKUP_STATUS_FILE}" || ! -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
echo "[production-stdb-publish] 警告:未找到可上传的本地备份状态文件,跳过异步上传" >&2
return 0
@@ -543,16 +548,58 @@ start_async_backup_upload() {
echo "[production-stdb-publish] 警告:备份状态文件缺少 archivePath 或 manifestPath,跳过异步上传" >&2
return 0
fi
if [[ "${ASYNC_BACKUP_ARCHIVE}" != /* || ! -f "${ASYNC_BACKUP_ARCHIVE}" || -L "${ASYNC_BACKUP_ARCHIVE}" ]]; then
echo "[production-stdb-publish] 警告:异步上传归档必须是现存、非符号链接的普通绝对路径文件,保留状态文件等待处理: ${ASYNC_BACKUP_ARCHIVE}" >&2
return 1
fi
if [[ "${ASYNC_BACKUP_MANIFEST}" != /* || ! -f "${ASYNC_BACKUP_MANIFEST}" || -L "${ASYNC_BACKUP_MANIFEST}" ]]; then
echo "[production-stdb-publish] 警告:异步上传 manifest 必须是现存、非符号链接的普通绝对路径文件,保留状态文件等待处理: ${ASYNC_BACKUP_MANIFEST}" >&2
return 1
fi
if ! command -v systemd-run >/dev/null 2>&1; then
echo "[production-stdb-publish] 警告:systemd-run 不可用,无法启动独立上传服务;保留状态文件等待处理" >&2
return 1
fi
mkdir -p "$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
ASYNC_BACKUP_LOG="$(dirname "${ASYNC_BACKUP_ARCHIVE}")/${DATABASE}-upload.log"
echo "[production-stdb-publish] 后台上传本地备份到 OSS: ${ASYNC_BACKUP_ARCHIVE}"
nohup node -- "${ASYNC_BACKUP_SCRIPT}" \
--env-file /etc/genarrative/api-server.env \
--upload-archive "${ASYNC_BACKUP_ARCHIVE}" \
--manifest-file "${ASYNC_BACKUP_MANIFEST}" \
>"${ASYNC_BACKUP_LOG}" 2>&1 &
echo "[production-stdb-publish] OSS 后台上传日志: ${ASYNC_BACKUP_LOG}"
node_binary="$(command -v node || true)"
if [[ "${node_binary}" != /* || ! -x "${node_binary}" ]]; then
echo "[production-stdb-publish] 警告:未找到可供 systemd 服务执行的绝对 node 路径;保留状态文件等待处理" >&2
return 1
fi
log_dir="$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
unit_suffix="$(date -u +%Y%m%dT%H%M%S%N)-$$-${RANDOM}"
unit_name="genarrative-stdb-backup-upload-${unit_suffix}.service"
if ! ASYNC_BACKUP_LOG="$(mktemp "${log_dir}/${DATABASE}-upload-${unit_suffix}.XXXXXX.log")"; then
echo "[production-stdb-publish] 警告:无法创建独立 OSS 上传日志,保留状态文件和本地归档等待处理" >&2
return 1
fi
if ! chmod 0600 "${ASYNC_BACKUP_LOG}"; then
echo "[production-stdb-publish] 警告:无法收紧独立 OSS 上传日志权限,保留状态文件和本地归档等待处理: ${ASYNC_BACKUP_LOG}" >&2
return 1
fi
echo "[production-stdb-publish] 通过独立 systemd 服务串行上传 deferred/pending 本地备份到 OSS: ${log_dir}"
if ! run_privileged systemd-run \
--no-ask-password \
--unit="${unit_name}" \
--description="Genarrative SpacetimeDB backup upload ${DATABASE}" \
--collect \
--service-type=exec \
--property="Restart=no" \
--property="UMask=0077" \
--property="StandardOutput=append:${ASYNC_BACKUP_LOG}" \
--property="StandardError=append:${ASYNC_BACKUP_LOG}" \
-- "${node_binary}" -- "${ASYNC_BACKUP_SCRIPT}" \
--env-file /etc/genarrative/api-server.env \
--database "${DATABASE}" \
--upload-deferred-dir "${log_dir}"; then
echo "[production-stdb-publish] 警告:独立 OSS 上传服务启动失败,保留状态文件和本地归档等待处理;启动日志: ${ASYNC_BACKUP_LOG}" >&2
return 1
fi
echo "[production-stdb-publish] OSS 上传服务已启动: ${unit_name}"
echo "[production-stdb-publish] OSS 上传日志: ${ASYNC_BACKUP_LOG}"
rm -f "${ASYNC_BACKUP_STATUS_FILE}"
ASYNC_BACKUP_STATUS_FILE=""
}
+199 -70
View File
@@ -152,12 +152,14 @@ const SERVICE_ALIASES = new Map([
['bgfilterWorker', 'bgfilter-worker'],
['admin', 'admin-web'],
['adminWeb', 'admin-web'],
['backend', 'backend'],
['all', 'all'],
]);
function usage() {
console.log(`用法:
npm run dev [-- --watch] [-- --api-port 8090]
npm run dev backend [-- --watch]
npm run dev:spacetime [-- --skip-publish]
npm run dev:api-server [-- --database genarrative-dev]
npm run dev:bgfilter-worker [-- --database genarrative-dev]
@@ -388,7 +390,7 @@ function parseArgs(argv, baseEnv) {
function normalizeServiceName(rawName) {
const alias = SERVICE_ALIASES.get(rawName);
const name = alias ?? rawName;
if (name === 'all' || SERVICE_NAMES.includes(name)) {
if (name === 'all' || name === 'backend' || SERVICE_NAMES.includes(name)) {
return name;
}
@@ -410,10 +412,11 @@ function buildDevStackSnapshot(runner, updatedAt = new Date().toISOString()) {
}
return {
schemaVersion: 1,
schemaVersion: 2,
command: runner.command ?? 'all',
repoRoot,
database: runner.options.database,
spacetimeDataDir: resolve(runner.options.spacetimeDataDir),
watch: Boolean(runner.options.watch),
updatedAt,
services,
@@ -716,12 +719,13 @@ function ensureRequiredFiles(command) {
command === 'api-server' ||
command === 'bgfilter-worker' ||
command === 'spacetime' ||
command === 'all'
command === 'all' ||
command === 'backend'
) {
requiredFiles.push([manifestPath, 'server-rs/Cargo.toml']);
}
if (command === 'spacetime' || command === 'all') {
if (command === 'spacetime' || command === 'all' || command === 'backend') {
requiredFiles.push([
resolve(modulePath, 'Cargo.toml'),
'spacetime-module Cargo.toml',
@@ -1197,13 +1201,14 @@ class DevRunner {
if (
command === 'api-server' ||
command === 'bgfilter-worker' ||
command === 'all'
command === 'all' ||
command === 'backend'
) {
requireCommand('cargo');
}
if (
command === 'spacetime' ||
(command === 'all' &&
((command === 'all' || command === 'backend') &&
(!this.options.skipSpacetime || !this.options.skipPublish))
) {
requireCommand('spacetime');
@@ -1275,6 +1280,9 @@ class DevRunner {
if (command === 'all') {
return !this.options.skipSpacetime || !this.options.skipPublish;
}
if (command === 'backend') {
return !this.options.skipSpacetime || !this.options.skipPublish;
}
if (command === 'api-server' || command === 'bgfilter-worker') {
return isLoopbackSpacetimeServer(this.state.spacetimeServer);
}
@@ -1289,6 +1297,7 @@ class DevRunner {
if (
this.options.spacetimeServerUrl &&
command !== 'all' &&
command !== 'backend' &&
command !== 'spacetime'
) {
return;
@@ -1383,7 +1392,7 @@ class DevRunner {
const portRangeFor = (optionName) =>
this.explicitOptions.has(optionName) ? null : this.state.portRange;
if (command === 'all' || command === 'spacetime') {
if (command === 'all' || command === 'backend' || command === 'spacetime') {
if (!options.skipSpacetime && !this.state.spacetimeReused) {
portConfig.spacetime = {
host: options.spacetimeHost,
@@ -1393,7 +1402,11 @@ class DevRunner {
}
}
if (command === 'all' || command === 'api-server') {
if (
command === 'all' ||
command === 'backend' ||
command === 'api-server'
) {
portConfig.api = {
host: options.apiHost,
preferredPort: options.apiPort,
@@ -1469,7 +1482,7 @@ class DevRunner {
this.state.bgfilterWorkerTargetHost = resolveClientHost(
options.bgfilterWorkerHost,
);
if (command === 'all' || command === 'spacetime') {
if (command === 'all' || command === 'backend' || command === 'spacetime') {
this.state.spacetimeServer = `http://${options.spacetimeHost}:${options.spacetimePort}`;
}
this.state.apiTarget = `http://${this.state.apiTargetHost}:${options.apiPort}`;
@@ -1594,6 +1607,14 @@ class DevRunner {
return;
}
if (command === 'backend') {
await this.startSpacetimeForFullStack();
await this.services.get('api-server').start();
await this.waitForApiServer();
this.startWatchers(['spacetime', 'api-server']);
return;
}
if (command === 'spacetime') {
await this.startSpacetimeForFullStack();
} else {
@@ -1636,12 +1657,11 @@ class DevRunner {
await this.publishSpacetimeModule();
} catch (error) {
if (isSpacetimePublishPermissionError(error)) {
console.warn(
`[dev:spacetime] 本地发布被当前 identity 拒绝,保留已启动的 standalone: ${error.message}`,
throw new Error(
`本地数据库不属于当前隔离 identity,已停止启动以避免 API 使用旧 schema 后持续重试订阅。请改用独立本地数据目录,或在确认无需保留旧开发数据后重建该目录。详情: ${error.message}`,
);
} else {
throw error;
}
throw error;
}
}
}
@@ -1772,8 +1792,10 @@ class DevRunner {
async publishSpacetimeModule() {
const env = buildLocalRustProcessEnv(this.baseEnv);
this.prepareMigrationBootstrapSecret(env);
const cliConfigPath = await this.prepareLocalSpacetimeCliIdentity(env);
const args = buildSpacetimePublishArgs({
cliConfigPath,
database: this.options.database,
preserveDatabase: this.options.preserveDatabase,
server: this.state.spacetimeServer,
@@ -1787,6 +1809,48 @@ class DevRunner {
});
}
async prepareLocalSpacetimeCliIdentity(env) {
if (!isLoopbackSpacetimeServer(this.state.spacetimeServer)) {
return '';
}
await this.ensureApiServerSpacetimeToken();
const cliConfigPath = resolve(
this.options.spacetimeDataDir,
'dev-cli',
'cli.toml',
);
ensureParentDir(cliConfigPath);
if (
existsSync(cliConfigPath) &&
resolveCurrentSpacetimeCliToken(cliConfigPath) === this.spacetimeApiToken
) {
chmodSync(cliConfigPath, 0o600);
console.log('[dev:spacetime] 已复用隔离的本地发布 identity');
return cliConfigPath;
}
await runForeground(
'spacetime',
[
'--config-path',
cliConfigPath,
'login',
'--token',
this.spacetimeApiToken,
],
{
cwd: serverRsDir,
env,
label: 'spacetime-login',
},
);
if (existsSync(cliConfigPath)) {
chmodSync(cliConfigPath, 0o600);
}
console.log('[dev:spacetime] 已配置隔离的本地发布 identity');
return cliConfigPath;
}
prepareMigrationBootstrapSecret(env) {
let runtimeServiceBootstrapSecret = '';
switch (this.options.migrationBootstrapSecretMode) {
@@ -2098,8 +2162,8 @@ class DevRunner {
if (await isHttpReady(readinessUrl, 500)) {
return;
}
const runtimeStatus = this.services.get('bgfilter-worker')?.runtime
?.status;
const runtimeStatus =
this.services.get('bgfilter-worker')?.runtime?.status;
if (runtimeStatus === 'failed' || runtimeStatus === 'stopped') {
throw new Error(
`bgfilter-worker 在 readiness 前退出,请检查 logs/bgfilter-worker/: ${readinessUrl}`,
@@ -2718,10 +2782,85 @@ function normalizeSpacetimeServerForIdentity(serverUrl) {
return url.href.replace(/\/$/u, '');
}
function resolveLocalSpacetimeApiIdentityPath(dataDir, serverUrl) {
const normalizedServer = normalizeSpacetimeServerForIdentity(serverUrl);
const serverKey = createHash('sha256').update(normalizedServer).digest('hex');
return resolve(dataDir, 'dev-api-identities', `${serverKey}.json`);
function resolveLocalSpacetimeApiIdentityPath(dataDir) {
return resolve(dataDir, 'dev-api-identities', 'local-node.json');
}
function readLocalSpacetimeApiIdentityRecord(identityPath, expected = {}) {
const stat = lstatSync(identityPath);
if (!stat.isFile() || stat.isSymbolicLink()) {
throw new Error('记录不是普通文件');
}
chmodSync(identityPath, 0o600);
const payload = JSON.parse(readFileSync(identityPath, 'utf8'));
const identity =
typeof payload.identity === 'string' ? payload.identity.trim() : '';
const token = typeof payload.token === 'string' ? payload.token.trim() : '';
if (!identity || !token) {
throw new Error('记录缺少 identity 或 token');
}
if (payload.schemaVersion === 2 && payload.scope === 'local-data-dir') {
return { identity, token };
}
if (
expected.allowLegacy &&
payload.schemaVersion === 1 &&
typeof payload.server === 'string' &&
isLoopbackSpacetimeServer(payload.server)
) {
return { identity, token };
}
throw new Error('记录格式或 data dir 作用域不匹配');
}
function migrateLegacyLocalSpacetimeApiIdentity(dataDir) {
const identityDir = resolve(dataDir, 'dev-api-identities');
if (!existsSync(identityDir)) {
return null;
}
const candidates = [];
for (const entry of readdirSync(identityDir, { withFileTypes: true })) {
if (!entry.isFile() || !entry.name.endsWith('.json')) {
continue;
}
const candidatePath = resolve(identityDir, entry.name);
if (candidatePath === resolveLocalSpacetimeApiIdentityPath(dataDir)) {
continue;
}
try {
candidates.push(
readLocalSpacetimeApiIdentityRecord(candidatePath, {
allowLegacy: true,
}),
);
} catch {
// 无效或非本地旧记录不参与迁移。
}
}
const uniqueCandidates = new Map(
candidates.map((candidate) => [
`${candidate.identity}\n${candidate.token}`,
candidate,
]),
);
if (uniqueCandidates.size === 0) {
return null;
}
if (uniqueCandidates.size > 1) {
throw new Error(
'同一 SpacetimeDB data dir 下发现多个旧 API identity,无法安全判断数据库 owner;请保留正确 owner 记录后重试',
);
}
const [identity] = uniqueCandidates.values();
writeLocalSpacetimeApiIdentity({ dataDir, ...identity });
console.log(
'[dev:spacetime] 已将旧端口作用域 API identity 迁移到 data dir 作用域',
);
return identity;
}
function resolveLocalSpacetimeRuntimeServiceBootstrapSecretPath(
@@ -2871,37 +3010,13 @@ function readLocalSpacetimeApiIdentity({ dataDir, serverUrl }) {
return null;
}
const normalizedServer = normalizeSpacetimeServerForIdentity(serverUrl);
const identityPath = resolveLocalSpacetimeApiIdentityPath(
dataDir,
normalizedServer,
);
const identityPath = resolveLocalSpacetimeApiIdentityPath(dataDir);
if (!existsSync(identityPath)) {
return null;
return migrateLegacyLocalSpacetimeApiIdentity(dataDir);
}
try {
const stat = lstatSync(identityPath);
if (!stat.isFile() || stat.isSymbolicLink()) {
throw new Error('记录不是普通文件');
}
chmodSync(identityPath, 0o600);
const payload = JSON.parse(readFileSync(identityPath, 'utf8'));
if (
payload.schemaVersion !== 1 ||
payload.server !== normalizedServer ||
typeof payload.identity !== 'string' ||
!payload.identity.trim() ||
typeof payload.token !== 'string' ||
!payload.token.trim()
) {
throw new Error('记录格式或 server 绑定不匹配');
}
return {
identity: payload.identity.trim(),
token: payload.token.trim(),
};
return readLocalSpacetimeApiIdentityRecord(identityPath);
} catch (error) {
console.warn(
`[dev:spacetime] 本地 API identity 记录不可用,将重新创建: ${error.message}`,
@@ -2910,17 +3025,8 @@ function readLocalSpacetimeApiIdentity({ dataDir, serverUrl }) {
}
}
function writeLocalSpacetimeApiIdentity({
dataDir,
serverUrl,
identity,
token,
}) {
const normalizedServer = normalizeSpacetimeServerForIdentity(serverUrl);
const identityPath = resolveLocalSpacetimeApiIdentityPath(
dataDir,
normalizedServer,
);
function writeLocalSpacetimeApiIdentity({ dataDir, identity, token }) {
const identityPath = resolveLocalSpacetimeApiIdentityPath(dataDir);
const tempPath = `${identityPath}.${process.pid}.${randomHex(8)}.tmp`;
ensureParentDir(identityPath);
@@ -2928,8 +3034,8 @@ function writeLocalSpacetimeApiIdentity({
writeFileSync(
tempPath,
`${JSON.stringify({
schemaVersion: 1,
server: normalizedServer,
schemaVersion: 2,
scope: 'local-data-dir',
identity,
token,
})}\n`,
@@ -3089,8 +3195,14 @@ function isLoopbackSpacetimeServer(serverUrl) {
}
}
function resolveCurrentSpacetimeCliToken() {
const result = spawnSync('spacetime', ['login', 'show', '--token'], {
function resolveCurrentSpacetimeCliToken(cliConfigPath = '') {
const args = [
...(cliConfigPath ? ['--config-path', cliConfigPath] : []),
'login',
'show',
'--token',
];
const result = spawnSync('spacetime', args, {
cwd: repoRoot,
encoding: 'utf8',
shell: process.platform === 'win32',
@@ -3114,13 +3226,21 @@ function trimPreview(text, maxLength = 300) {
function runForeground(command, args, { cwd, env, label }) {
return new Promise((resolveRun, rejectRun) => {
let capturedOutput = '';
const capture = (chunk, target) => {
target.write(chunk);
capturedOutput = `${capturedOutput}${String(chunk)}`.slice(-32_768);
};
const child = spawn(command, args, {
cwd,
env,
stdio: 'inherit',
stdio: ['inherit', 'pipe', 'pipe'],
shell: process.platform === 'win32',
});
child.stdout?.on('data', (chunk) => capture(chunk, process.stdout));
child.stderr?.on('data', (chunk) => capture(chunk, process.stderr));
child.on('error', rejectRun);
child.on('exit', (code, signal) => {
if (signal) {
@@ -3129,7 +3249,12 @@ function runForeground(command, args, { cwd, env, label }) {
}
if (code !== 0) {
rejectRun(new Error(`[dev:${label}] 退出码: ${code}`));
const detail = trimPreview(capturedOutput, 2_000);
rejectRun(
new Error(
`[dev:${label}] 退出码: ${code}${detail ? `: ${detail}` : ''}`,
),
);
return;
}
@@ -3189,8 +3314,14 @@ function isDirectModuleExecution(argv1, moduleUrl, resolvePath = safeRealpath) {
}
}
function buildSpacetimePublishArgs({ database, server, preserveDatabase }) {
function buildSpacetimePublishArgs({
cliConfigPath = '',
database,
server,
preserveDatabase,
}) {
const args = [
...(cliConfigPath ? ['--config-path', cliConfigPath] : []),
'publish',
database,
'--server',
@@ -3260,17 +3391,15 @@ function buildBgfilterWorkerProcessEnv({
GENARRATIVE_BGFILTER_WORKER_BASE_URL: state.bgfilterWorkerTarget,
GENARRATIVE_BGFILTER_INTERNAL_TOKEN: bgfilterInternalToken,
GENARRATIVE_BGFILTER_WORKER_CONCURRENCY:
String(
baseEnv.GENARRATIVE_BGFILTER_WORKER_CONCURRENCY ?? '',
).trim() || '16',
String(baseEnv.GENARRATIVE_BGFILTER_WORKER_CONCURRENCY ?? '').trim() ||
'16',
GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS:
String(
baseEnv.GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS ?? '',
).trim() || '5000',
GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS:
String(
baseEnv.GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS ?? '',
).trim() || '2048',
String(baseEnv.GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS ?? '').trim() ||
'2048',
GENARRATIVE_API_LOG: options.apiLog,
GENARRATIVE_SPACETIME_SERVER_URL: state.spacetimeServer,
GENARRATIVE_SPACETIME_DATABASE: options.database,
+117 -25
View File
@@ -11,7 +11,7 @@ import {
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { dirname, join, resolve } from 'node:path';
import { afterEach, describe, expect, test, vi } from 'vitest';
@@ -673,9 +673,12 @@ describe('dev scheduler stack state file', () => {
const snapshot = buildDevStackSnapshot(runner, updatedAt);
expect(snapshot.schemaVersion).toBe(1);
expect(snapshot.schemaVersion).toBe(2);
expect(snapshot.command).toBe('web');
expect(snapshot.database).toBe('genarrative-test');
expect(snapshot.spacetimeDataDir).toBe(
resolve('server-rs/.spacetimedb/local/data'),
);
expect(snapshot.services.web).toMatchObject({
status: 'running',
pid: 4321,
@@ -974,16 +977,20 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
).toBe(false);
});
test('发布 spacetime-module 时忽略 spacetime.json 以免覆盖显式数据库', () => {
test('发布 spacetime-module 时使用隔离身份配置并忽略 spacetime.json', () => {
const args = buildSpacetimePublishArgs({
cliConfigPath: '/tmp/genarrative-cli.toml',
database: 'xushi-p4wfr',
preserveDatabase: false,
server: 'http://127.0.0.1:3101',
});
expect(args).toContain('--no-config');
expect(args).not.toContain('--anonymous');
expect(args).toEqual(
expect.arrayContaining([
'--config-path',
'/tmp/genarrative-cli.toml',
'publish',
'xushi-p4wfr',
'--server',
@@ -993,6 +1000,17 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
);
});
test('远程 SpacetimeDB 发布继续使用默认登录身份', () => {
const args = buildSpacetimePublishArgs({
database: 'xushi-p4wfr',
preserveDatabase: true,
server: 'https://spacetime.example.com',
});
expect(args).not.toContain('--anonymous');
expect(args).not.toContain('--config-path');
});
test('手动刷新 spacetime 只重新发布模块,不重启 standalone 进程', async () => {
const { explicitOptions, options } = parseArgs([], {});
const runner = new DevRunner(options, {}, explicitOptions);
@@ -1025,26 +1043,18 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
expect(runner.publishSpacetimeModule).not.toHaveBeenCalled();
});
test('本地 API identity 路径同时绑定 data dir 和规范化 server', () => {
test('本地 API identity 路径只绑定 data dir', () => {
const first = resolveLocalSpacetimeApiIdentityPath(
'/tmp/genarrative-data-a',
'http://127.0.0.1:3101',
);
const normalizedEquivalent = resolveLocalSpacetimeApiIdentityPath(
const sameDataDir = resolveLocalSpacetimeApiIdentityPath(
'/tmp/genarrative-data-a',
'http://127.0.0.1:3101/',
);
const otherServer = resolveLocalSpacetimeApiIdentityPath(
'/tmp/genarrative-data-a',
'http://127.0.0.1:3102',
);
const otherDataDir = resolveLocalSpacetimeApiIdentityPath(
'/tmp/genarrative-data-b',
'http://127.0.0.1:3101',
);
expect(normalizedEquivalent).toBe(first);
expect(otherServer).not.toBe(first);
expect(sameDataDir).toBe(first);
expect(otherDataDir).not.toBe(first);
expect(first).toContain(join('genarrative-data-a', 'dev-api-identities'));
});
@@ -1071,10 +1081,7 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
await firstRunner.ensureApiServerSpacetimeToken();
const identityPath = resolveLocalSpacetimeApiIdentityPath(
tempDir,
firstRunner.state.spacetimeServer,
);
const identityPath = resolveLocalSpacetimeApiIdentityPath(tempDir);
expect(firstRunner.spacetimeApiToken).toBe('local-api-token');
expect(firstRunner.baseEnv.GENARRATIVE_SPACETIME_TOKEN).toBeUndefined();
expect(globalThis.fetch).toHaveBeenCalledWith(
@@ -1082,8 +1089,8 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
expect.objectContaining({ method: 'POST' }),
);
expect(JSON.parse(readFileSync(identityPath, 'utf8'))).toMatchObject({
schemaVersion: 1,
server: 'http://127.0.0.1:3101',
schemaVersion: 2,
scope: 'local-data-dir',
identity: 'c200localidentity',
token: 'local-api-token',
});
@@ -1093,7 +1100,7 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
}
const secondRunner = new DevRunner(options, {}, explicitOptions);
secondRunner.state.spacetimeServer = 'http://127.0.0.1:3101';
secondRunner.state.spacetimeServer = 'http://127.0.0.1:3199';
globalThis.fetch = vi.fn();
await secondRunner.ensureApiServerSpacetimeToken();
@@ -1112,6 +1119,94 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
}
});
test('旧端口作用域 API identity 会迁移为 data dir 作用域', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-api-identity-'));
try {
const legacyServer = 'http://127.0.0.1:3101';
const legacyKey = createHash('sha256').update(legacyServer).digest('hex');
const legacyPath = join(
tempDir,
'dev-api-identities',
`${legacyKey}.json`,
);
mkdirSync(dirname(legacyPath), { recursive: true });
writeFileSync(
legacyPath,
`${JSON.stringify({
schemaVersion: 1,
server: legacyServer,
identity: 'legacy-owner-identity',
token: 'legacy-owner-token',
})}\n`,
{ mode: 0o600 },
);
const { explicitOptions, options } = parseArgs(
['--spacetime-data-dir', tempDir],
{},
);
const runner = new DevRunner(options, {}, explicitOptions);
runner.state.spacetimeServer = 'http://127.0.0.1:3199';
globalThis.fetch = vi.fn();
await runner.ensureApiServerSpacetimeToken();
expect(runner.spacetimeApiToken).toBe('legacy-owner-token');
expect(globalThis.fetch).not.toHaveBeenCalled();
expect(
JSON.parse(
readFileSync(resolveLocalSpacetimeApiIdentityPath(tempDir), 'utf8'),
),
).toMatchObject({
schemaVersion: 2,
scope: 'local-data-dir',
identity: 'legacy-owner-identity',
});
} finally {
rmSync(tempDir, { recursive: true, force: true });
}
});
test('同一 data dir 存在多个旧 identity 时失败关闭', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-api-identity-'));
try {
for (const [port, identity] of [
[3101, 'legacy-owner-a'],
[3199, 'legacy-owner-b'],
] as const) {
const server = `http://127.0.0.1:${port}`;
const legacyPath = join(
tempDir,
'dev-api-identities',
`${createHash('sha256').update(server).digest('hex')}.json`,
);
mkdirSync(dirname(legacyPath), { recursive: true });
writeFileSync(
legacyPath,
`${JSON.stringify({
schemaVersion: 1,
server,
identity,
token: `${identity}-token`,
})}\n`,
{ mode: 0o600 },
);
}
const { explicitOptions, options } = parseArgs(
['--spacetime-data-dir', tempDir],
{},
);
const runner = new DevRunner(options, {}, explicitOptions);
globalThis.fetch = vi.fn();
await expect(runner.ensureApiServerSpacetimeToken()).rejects.toThrow(
'无法安全判断数据库 owner',
);
expect(globalThis.fetch).not.toHaveBeenCalled();
} finally {
rmSync(tempDir, { recursive: true, force: true });
}
});
test('外部显式 token 优先于已持久化的本地 API identity', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-api-identity-'));
const originalToken = process.env.GENARRATIVE_SPACETIME_TOKEN;
@@ -1167,10 +1262,7 @@ spacetimedb tool version 2.7.0; spacetimedb-lib version 2.7.0;
);
const runner = new DevRunner(options, {}, explicitOptions);
runner.state.spacetimeServer = 'http://127.0.0.1:3101';
const identityPath = resolveLocalSpacetimeApiIdentityPath(
tempDir,
runner.state.spacetimeServer,
);
const identityPath = resolveLocalSpacetimeApiIdentityPath(tempDir);
mkdirSync(dirname(identityPath), { recursive: true });
const linkedRecordPath = join(tempDir, 'linked-api-identity.json');
writeFileSync(