Merge remote-tracking branch 'origin/master' into editor-agent-abortable
Project CI / Repository checks (pull_request) Failing after 26s
Project CI / Frontend tests (pull_request) Failing after 48s
Project CI / Native shell tests (pull_request) Successful in 2m46s
Project CI / Backend tests (pull_request) Successful in 3m21s

# Conflicts:
#	docs/project-memory/shared-memory/decision-log.md
#	docs/【编辑器】画布Agent对话面板-2026-07-03.md
#	server-rs/crates/api-server/src/editor_agent/api.rs
#	src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx
#	src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx
#	src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx
This commit is contained in:
2026-07-28 15:55:11 +08:00
63 changed files with 5575 additions and 1493 deletions
+131 -4
View File
@@ -6,6 +6,7 @@ import {chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync,
import {tmpdir} from 'node:os';
import path from 'node:path';
import {Readable} from 'node:stream';
import {gunzipSync, gzipSync} from 'node:zlib';
import {
buildAuthorization,
@@ -65,12 +66,17 @@ async function main() {
await assertHistorySuccessfulUploadCleansAndIsIdempotent();
await assertHistoryResumeReverifiesArchiveAndManifest();
await assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload();
await assertDirectFilesMigratesLegacyStateAndPrunesEmbeddedCatalogs();
await assertDirectFilesConcurrencyIsBounded();
await assertDirectHistoryPublishesCatalogBeforeCleanup();
await assertDirectHistoryWithoutCandidatesPublishesLatest();
await assertDirectFilesRestoreDownloadsCatalogAndObjects();
}
function readGzipJson(filePath) {
return JSON.parse(gunzipSync(readFileSync(filePath)).toString('utf8'));
}
function createDirectOssHarness() {
const objects = new Map();
const uploadedKeys = [];
@@ -217,6 +223,13 @@ async function assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload(
const first = await runDirectFilesBackup(options);
assertEqual(first.uploadedCount, 2, '首次 files full 应上传全部普通文件。');
assertTrue(!Object.hasOwn(first.catalog, 'dataDir'), '远端 files catalog 不得绑定 staging 主机的绝对 data-dir。');
assertTrue(first.statePath.endsWith('.json.gz'), 'files state 必须使用 gzip 压缩文件。');
const compactState = readGzipJson(first.statePath);
assertEqual(compactState.schemaVersion, 2, 'files state 必须使用去重后的 v2 契约。');
assertTrue(!Object.hasOwn(compactState.baselineCatalog, 'files'), 'baseline ref 不得重复嵌入 files。');
assertTrue(!Object.hasOwn(compactState.latestCatalog, 'files'), 'latest ref 不得重复嵌入 files。');
assertTrue(!existsSync(first.catalogPath), '本地 full catalog 原始 JSON 应在成功后压缩。');
assertTrue(existsSync(`${first.catalogPath}.gz`), '本地应保留压缩后的 latest full catalog 供增量复用。');
const latestObjectKey = 'database-backups/test-db/latest.json';
const latest = JSON.parse(harness.objects.get(latestObjectKey).body.toString('utf8'));
assertEqual(latest.latestFullCatalog.catalogId, first.catalogId, 'latest pointer 必须指向已验真的最新 full catalog。');
@@ -231,11 +244,77 @@ async function assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload(
'相同 catalog 重跑不得重复 PUT 文件或 catalog,但应覆盖验真 latest pointer。',
);
rmSync(`${first.catalogPath}.gz`, {force: false});
writeFileSync(path.join(dataDir, 'control-db'), 'control changed');
writeFileSync(path.join(dataDir, 'new-program.bin'), 'new program');
const incremental = await runDirectFilesBackup(options);
assertEqual(incremental.uploadedCount, 2, '增量 files full 只应上传新增和变化文件。');
assertEqual(incremental.reusedCount, 1, '增量 files full 应复用未变化 snapshot 文件。');
assertEqual(incremental.reusedCount, 1, '本地 full catalog 缓存缺失时仍应通过 OSS HEAD 复用未变化文件。');
}
async function assertDirectFilesMigratesLegacyStateAndPrunesEmbeddedCatalogs() {
const root = path.join(tmpRoot, 'direct-files-state-migration');
const dataDir = path.join(root, 'stdb');
const workDir = path.join(root, 'work');
mkdirSync(dataDir, {recursive: true});
writeFileSync(path.join(dataDir, 'control-db'), 'control');
const harness = createDirectOssHarness();
const options = {
mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups',
uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn,
};
const first = await runDirectFilesBackup(options);
const compactState = readGzipJson(first.statePath);
const catalog = JSON.parse(gunzipSync(readFileSync(`${first.catalogPath}.gz`)).toString('utf8'));
const legacyCatalogRef = {
...compactState.latestCatalog,
files: catalog.files,
symlinks: catalog.symlinks,
};
const legacyStatePath = first.statePath.slice(0, -3);
writeFileSync(legacyStatePath, `${JSON.stringify({
...compactState,
schemaVersion: 1,
baselineCatalog: legacyCatalogRef,
latestCatalog: legacyCatalogRef,
}, null, 2)}\n`);
rmSync(first.statePath, {force: false});
const migrated = await runDirectFilesBackup(options);
assertTrue(migrated.unchanged, '旧 state 迁移不得改变相同 full catalog 的零上传语义。');
assertTrue(existsSync(migrated.statePath), '旧 state 成功运行后必须生成压缩 state。');
assertTrue(!existsSync(legacyStatePath), '压缩 state 原子落盘后应删除旧未压缩 state。');
const migratedState = readGzipJson(migrated.statePath);
assertEqual(migratedState.schemaVersion, 2, '旧 state 必须迁移到 v2。');
assertTrue(!Object.hasOwn(migratedState.latestCatalog, 'files'), '迁移后 state 不得保留重复 files 清单。');
const latestCatalogPath = `${migrated.catalogPath}.gz`;
const validCatalogBody = readFileSync(latestCatalogPath);
writeFileSync(latestCatalogPath, gzipSync(Buffer.from('{}\n')));
writeFileSync(path.join(dataDir, 'control-db'), 'control changed');
let corruptCatalogFailure = null;
try {
await runDirectFilesBackup(options);
} catch (error) {
corruptCatalogFailure = error;
}
assertIncludes(
corruptCatalogFailure?.message ?? '',
'长度或 SHA 与 state 引用不匹配',
'本地 latest full catalog 损坏时不得作为增量复用缓存。',
);
writeFileSync(latestCatalogPath, validCatalogBody);
writeFileSync(legacyStatePath, `${JSON.stringify({...migratedState, schemaVersion: 1})}\n`);
writeFileSync(migrated.statePath, 'not-a-gzip-state');
let corruptStateFailure = null;
try {
await runDirectFilesBackup(options);
} catch (error) {
corruptStateFailure = error;
}
assertTrue(corruptStateFailure instanceof Error, '压缩 state 损坏时必须失败。');
assertTrue(existsSync(legacyStatePath), '压缩 state 损坏时不得静默回退并删除旧 state。');
}
async function assertDirectFilesConcurrencyIsBounded() {
@@ -290,7 +369,14 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() {
uploadFn: harness.uploadFn,
verifyFn: harness.verifyFn,
};
await runDirectFilesBackup({...common, mode: 'full', uploadManifestFn: harness.uploadManifestFn});
const baseline = await runDirectFilesBackup({...common, mode: 'full', uploadManifestFn: harness.uploadManifestFn});
const legacyResultFile = path.join(fixture.workDir, 'legacy-full-result.json');
const legacyCatalogWithoutSymlinks = {...baseline.catalog};
delete legacyCatalogWithoutSymlinks.symlinks;
writeFileSync(legacyResultFile, `${JSON.stringify({
...baseline,
catalog: legacyCatalogWithoutSymlinks,
}, null, 2)}\n`);
const plan = discoverHistoryPlan({dataDir: fixture.dataDir});
let failure = null;
try {
@@ -330,7 +416,13 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() {
assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `latest pointer 发布失败不得删除: ${candidate.path}`);
}
const success = await runDirectFilesBackup({...common, mode: 'history', uploadManifestFn: harness.uploadManifestFn});
const resultFile = path.join(fixture.workDir, 'history-result.json');
const success = await runDirectFilesBackup({
...common,
mode: 'history',
resultFile,
uploadManifestFn: harness.uploadManifestFn,
});
assertEqual(success.uploadedCount, 0, 'history 文件已在 full CAS baseline 时不应重复上传内容。');
for (const file of success.catalog.files) {
assertTrue(
@@ -340,7 +432,19 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() {
}
assertEqual(success.cleanup?.deletedCount, plan.candidates.length, 'catalog 和 baseline 验真后才应清理全部安全候选。');
const state = JSON.parse(readFileSync(success.statePath, 'utf8'));
const state = readGzipJson(success.statePath);
assertEqual(state.schemaVersion, 2, 'files state 必须迁移为去重后的 v2 契约。');
assertTrue(!Object.hasOwn(state.latestCatalog, 'files'), 'files state latest ref 不得重复嵌入 files。');
assertTrue(state.historyCatalogs.every((catalog) => !Object.hasOwn(catalog, 'files')), 'files state history ref 不得重复嵌入 files。');
assertTrue(!existsSync(success.catalogPath), '已上传并验真的 history catalog 本地 JSON 应被清理。');
assertTrue(!existsSync(`${success.catalogPath}.gz`), 'history catalog 本地压缩副本也不应保留。');
const diskResult = JSON.parse(readFileSync(resultFile, 'utf8'));
assertTrue(!Object.hasOwn(diskResult.catalog, 'files'), 'files result 文件不得重复写入完整 files 清单。');
assertEqual(diskResult.catalog.fileCount, success.fileCount, '紧凑 result 仍应保留文件计数。');
const compactedLegacyResult = JSON.parse(readFileSync(legacyResultFile, 'utf8'));
assertTrue(!Object.hasOwn(compactedLegacyResult.catalog, 'files'), '旧 result 中重复的 files 清单应在成功运行后压缩。');
assertEqual(compactedLegacyResult.catalog.symlinkCount, 0, '缺少 symlinks 的旧 result 应按零个符号链接兼容迁移。');
assertTrue((success.metadataCleanup?.compactedResultCount ?? 0) >= 1, 'metadata 清理应报告已压缩旧 result。');
const historyCatalogObjectKey = state.historyCatalogs[0].objectKey;
harness.objects.delete(historyCatalogObjectKey);
let brokenHistoryFailure = null;
@@ -425,6 +529,29 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() {
assertTrue(lstatSync(path.join(restoreDir, 'bin', 'current')).isSymbolicLink(), 'files restore 必须重建符号链接。');
assertEqual(readlinkSync(path.join(restoreDir, 'bin', 'current'), 'utf8'), '2.6.0', 'files restore 必须保留符号链接目标。');
rmSync(restoreDir, {recursive: true, force: true});
const legacyRestoreStatePath = path.join(workDir, 'legacy-restore-state.json');
writeFileSync(legacyRestoreStatePath, `${JSON.stringify({
...readGzipJson(latestFull.statePath),
schemaVersion: 1,
})}\n`);
const legacyRestored = await restoreDirectFilesBackup({
statePath: legacyRestoreStatePath,
restoreDir,
database: 'test-db',
bucket: 'backup-bucket',
uploadOptions: {},
downloadBufferFn: async ({objectKey}) => Buffer.from(harness.objects.get(objectKey)?.body ?? ''),
downloadFileFn: async ({objectKey, destinationPath}) => {
const object = harness.objects.get(objectKey);
if (!object) {
throw new Error(`missing ${objectKey}`);
}
writeFileSync(destinationPath, object.body);
},
});
assertEqual(legacyRestored.downloadedCount, 1, 'files restore 必须继续兼容 v1 JSON state。');
rmSync(restoreDir, {recursive: true, force: true});
const downloadBufferFn = async ({objectKey}) => {
const object = harness.objects.get(objectKey);
+5
View File
@@ -148,6 +148,10 @@ function validateGatewayConfiguration() {
'try_files /page.html @genarrative_default_maintenance;',
'location @genarrative_default_maintenance',
'root /srv/genarrative/web;',
'location = /branding/taonier-maintenance-page.png',
'try_files /branding/taonier-maintenance-page.png =404;',
'location = /branding/taonier-product-ip.png',
'try_files /branding/taonier-product-ip.png =404;',
]) {
if (!nginxSnippet.includes(expected)) {
fail(`Nginx 维护页配置缺少运行态覆盖约束: ${expected}`);
@@ -161,6 +165,7 @@ function validateGatewayConfiguration() {
for (const expected of [
'GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_PAGE_FILE',
'maintenance_page_file',
'is_maintenance_page_asset(path)',
]) {
if (!pingoraSource.includes(expected)) {
fail(`Pingora 维护页配置缺少运行态覆盖约束: ${expected}`);
+4
View File
@@ -164,6 +164,10 @@ function validateMaintenanceInternalBypass() {
fail(`${MAINTENANCE_SNIPPET_PATH} 不应保留仅后台使用的维护变量。`);
}
for (const fragment of [
'location = /branding/taonier-maintenance-page.png {',
'try_files /branding/taonier-maintenance-page.png =404;',
'location = /branding/taonier-product-ip.png {',
'try_files /branding/taonier-product-ip.png =404;',
'location = /404.html {',
'if ($http_accept !~* "text/html") {',
'try_files /404.html =404;',
+23 -1
View File
@@ -347,6 +347,19 @@ async function prepareStaticRoots(webRoot, acmeRoot) {
path.join(webRoot, 'maintenance.html'),
'<main>default-maintenance</main>',
);
await mkdir(path.join(webRoot, 'branding'), { recursive: true });
for (const fileName of [
'taonier-maintenance-page.png',
'taonier-product-ip.png',
]) {
await writeFile(
path.join(webRoot, 'branding', fileName),
Buffer.concat([
PNG_MAGIC_BYTES,
Buffer.from([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52]),
]),
);
}
await writeFile(path.join(webRoot, '404.html'), '<main>not-found-page</main>');
await writeFile(
path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'),
@@ -1216,6 +1229,14 @@ async function runSmokeCases(
},
);
}
for (const path of [
'/branding/taonier-maintenance-page.png',
'/branding/taonier-product-ip.png',
]) {
await expectStaticPng(baseUrl, path, `维护模式放行品牌图片 ${path}`, {
headers: publicClientHeaders,
});
}
await rm(maintenancePageFile, { force: true });
await expectHttp(
baseUrl,
@@ -1445,9 +1466,10 @@ async function expectNotCompressedResponse(baseUrl, route, bodyNeedle, label) {
}
}
async function expectStaticPng(baseUrl, route, label) {
async function expectStaticPng(baseUrl, route, label, options = {}) {
console.log(`[pingora-gateway-smoke] ${label}`);
const response = await requestHttp(`${baseUrl}${route}`, {
headers: options.headers,
rawBody: true,
});
+256 -19
View File
@@ -26,6 +26,7 @@ import {Readable} from 'node:stream';
import {pipeline} from 'node:stream/promises';
import {setTimeout as sleep} from 'node:timers/promises';
import {fileURLToPath} from 'node:url';
import {gunzipSync, gzipSync} from 'node:zlib';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -53,7 +54,8 @@ const DIRECT_FILES_SINGLE_PUT_MAX_BYTES = 16 * 1024 * 1024;
const RETRYABLE_OSS_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
const HISTORY_STATE_SCHEMA_VERSION = 1;
const HISTORY_MANIFEST_SCHEMA_VERSION = 1;
const DIRECT_FILES_STATE_SCHEMA_VERSION = 1;
const DIRECT_FILES_STATE_SCHEMA_VERSION = 2;
const LEGACY_DIRECT_FILES_STATE_SCHEMA_VERSION = 1;
const DIRECT_FILES_CATALOG_SCHEMA_VERSION = 1;
const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1;
@@ -370,14 +372,23 @@ function buildBackupNames({database, dataDir, objectPrefix}) {
return {fileName, objectKey};
}
function atomicWriteJson(filePath, payload) {
function atomicWriteBuffer(filePath, body) {
mkdirSync(dirname(filePath), {recursive: true});
const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
writeFileSync(tempPath, `${JSON.stringify(payload, null, 2)}\n`, {encoding: 'utf8', mode: 0o600});
writeFileSync(tempPath, body, {mode: 0o600});
chmodSync(tempPath, 0o600);
renameSync(tempPath, filePath);
}
function atomicWriteJson(filePath, payload) {
atomicWriteBuffer(filePath, Buffer.from(`${JSON.stringify(payload, null, 2)}\n`, 'utf8'));
}
function atomicWriteGzipJson(filePath, payload) {
const body = Buffer.from(`${JSON.stringify(payload)}\n`, 'utf8');
atomicWriteBuffer(filePath, gzipSync(body, {level: 9}));
}
function processIsAlive(pid) {
try {
process.kill(pid, 0);
@@ -1070,9 +1081,127 @@ async function sha256FileHex(filePath) {
}
function directFilesStatePath({workDir, database}) {
return join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json.gz`);
}
function legacyDirectFilesStatePath({workDir, database}) {
return join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json`);
}
function directCatalogLocalPaths({workDir, database, catalog}) {
const baseName = `${sanitizeObjectPart(database, 'spacetimedb')}-${catalog.mode}-${catalog.catalogId}.catalog.json`;
return {
jsonPath: join(workDir, baseName),
gzipPath: join(workDir, `${baseName}.gz`),
};
}
function readLocalDirectCatalog({workDir, database, catalog}) {
const {jsonPath, gzipPath} = directCatalogLocalPaths({workDir, database, catalog});
let body = null;
if (existsSync(jsonPath)) {
body = readFileSync(jsonPath);
} else if (existsSync(gzipPath)) {
body = gunzipSync(readFileSync(gzipPath));
}
if (!body) {
return null;
}
if (body.length !== catalog.contentLength || sha256Hex(body) !== catalog.sha256) {
throw new Error(`本地 files catalog 长度或 SHA 与 state 引用不匹配: ${jsonPath}`);
}
const payload = JSON.parse(body.toString('utf8'));
if (
payload.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION
|| payload.database !== database
|| payload.mode !== catalog.mode
|| payload.catalogId !== catalog.catalogId
|| !Array.isArray(payload.files)
) {
throw new Error(`本地 files catalog 与 state 引用不匹配: ${jsonPath}`);
}
return payload;
}
function readJsonOrGzip(filePath) {
const body = readFileSync(filePath);
const decoded = filePath.endsWith('.gz') || (body[0] === 0x1f && body[1] === 0x8b)
? gunzipSync(body)
: body;
return JSON.parse(decoded.toString('utf8'));
}
function compactDirectCatalogFile({workDir, database, catalog}) {
const {jsonPath, gzipPath} = directCatalogLocalPaths({workDir, database, catalog});
if (!existsSync(jsonPath)) {
return existsSync(gzipPath) ? {compressed: false, gzipPath} : null;
}
const body = readFileSync(jsonPath);
atomicWriteBuffer(gzipPath, gzipSync(body, {level: 9}));
rmSync(jsonPath, {force: false});
return {compressed: true, gzipPath};
}
function compactDirectFilesLocalMetadata({workDir, database, nextState, transientCatalogPaths = []}) {
const keepCatalog = nextState.latestCatalog;
const keepCatalogIds = new Set([keepCatalog?.catalogId].filter(Boolean));
const databasePart = sanitizeObjectPart(database, 'spacetimedb');
const catalogPattern = new RegExp(`^${databasePart.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&')}-(full|history)-([a-f0-9]{64})\\.catalog\\.json(?:\\.gz)?$`, 'u');
let compressedCatalogCount = 0;
let deletedCatalogCount = 0;
let compactedResultCount = 0;
for (const entry of readdirSync(workDir, {withFileTypes: true})) {
if (!entry.isFile()) {
continue;
}
const match = catalogPattern.exec(entry.name);
if (!match) {
continue;
}
const catalog = {mode: match[1], catalogId: match[2]};
if (keepCatalogIds.has(catalog.catalogId) && catalog.mode === 'full') {
readLocalDirectCatalog({workDir, database, catalog: keepCatalog});
if (!entry.name.endsWith('.gz') && compactDirectCatalogFile({workDir, database, catalog})?.compressed) {
compressedCatalogCount += 1;
}
continue;
}
rmSync(join(workDir, entry.name), {force: false});
deletedCatalogCount += 1;
}
for (const filePath of transientCatalogPaths) {
if (existsSync(filePath)) {
rmSync(filePath, {force: false});
deletedCatalogCount += 1;
}
}
for (const entry of readdirSync(workDir, {withFileTypes: true})) {
if (!entry.isFile() || !entry.name.endsWith('.json') || entry.name.endsWith('.catalog.json')) {
continue;
}
const filePath = join(workDir, entry.name);
let payload = null;
try {
payload = JSON.parse(readFileSync(filePath, 'utf8'));
} catch {
continue;
}
if (
payload?.catalog?.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION
|| payload.catalog.database !== database
|| payload.catalog.bucket !== nextState.bucket
|| !Array.isArray(payload.catalog.files)
) {
continue;
}
atomicWriteJson(filePath, compactDirectFilesResult(payload));
compactedResultCount += 1;
}
const result = {compressedCatalogCount, deletedCatalogCount, compactedResultCount};
console.log(`[database-backup] files 本地元数据清理: ${JSON.stringify(result)}`);
return result;
}
function normalizeObjectPrefix(objectPrefix, database) {
const prefix = String(objectPrefix || 'database-backups')
.trim()
@@ -1185,12 +1314,16 @@ function directCatalogIdentity({mode, baselineCatalogId, rootName, directories,
}
function readDirectFilesState(statePath, {database, bucket}) {
if (!existsSync(statePath)) {
const candidates = statePath.endsWith('.gz')
? [statePath, statePath.slice(0, -3)]
: [statePath, `${statePath}.gz`];
const existingPath = candidates.find((candidate) => existsSync(candidate));
if (!existingPath) {
return null;
}
const state = readManifest(statePath);
const state = readJsonOrGzip(existingPath);
if (
state.schemaVersion !== DIRECT_FILES_STATE_SCHEMA_VERSION
![LEGACY_DIRECT_FILES_STATE_SCHEMA_VERSION, DIRECT_FILES_STATE_SCHEMA_VERSION].includes(state.schemaVersion)
|| state.backupKind !== 'spacetimedb-direct-files-state'
|| state.database !== database
|| state.bucket !== bucket
@@ -1200,6 +1333,16 @@ function readDirectFilesState(statePath, {database, bucket}) {
return state;
}
function directPreviousFiles({state, workDir, database}) {
if (!state?.latestCatalog) {
return [];
}
if (Array.isArray(state?.latestCatalog?.files)) {
return state.latestCatalog.files;
}
return readLocalDirectCatalog({workDir, database, catalog: state?.latestCatalog})?.files ?? [];
}
async function ensureDirectObject({
file,
dataDir,
@@ -1290,6 +1433,52 @@ function directCatalogRef(catalog) {
};
}
function normalizeDirectFilesState({state, dataDir, database, bucket}) {
return {
schemaVersion: DIRECT_FILES_STATE_SCHEMA_VERSION,
backupKind: 'spacetimedb-direct-files-state',
database,
dataDir,
bucket,
updatedAt: new Date().toISOString(),
baselineCatalog: assertDirectCatalogRef(state?.baselineCatalog, 'full', 'baseline full'),
latestCatalog: assertDirectCatalogRef(state?.latestCatalog, 'full', 'latest full'),
historyCatalogs: (state?.historyCatalogs ?? []).map((catalog) => (
assertDirectCatalogRef(catalog, 'history', 'history')
)),
};
}
function persistDirectFilesState({statePath, legacyStatePath, state}) {
atomicWriteGzipJson(statePath, state);
if (legacyStatePath !== statePath && existsSync(legacyStatePath)) {
rmSync(legacyStatePath, {force: false});
}
}
function compactDirectFilesResult(result) {
if (!result.catalog) {
return result;
}
const {catalog, ...rest} = result;
return {
...rest,
catalog: {
schemaVersion: catalog.schemaVersion,
backupKind: catalog.backupKind,
database: catalog.database,
bucket: catalog.bucket,
mode: catalog.mode,
catalogId: catalog.catalogId,
catalogObjectKey: catalog.catalogObjectKey,
baselineCatalogId: catalog.baselineCatalogId,
rootName: catalog.rootName,
fileCount: Array.isArray(catalog.files) ? catalog.files.length : 0,
symlinkCount: Array.isArray(catalog.symlinks) ? catalog.symlinks.length : 0,
},
};
}
function assertDirectCatalogRef(catalog, expectedMode, label) {
if (
!catalog
@@ -1397,6 +1586,7 @@ export async function runDirectFilesBackup({
}) {
mkdirSync(workDir, {recursive: true});
const statePath = directFilesStatePath({workDir, database});
const legacyStatePath = legacyDirectFilesStatePath({workDir, database});
const state = readDirectFilesState(statePath, {database, bucket});
if (mode === 'history' && (!state?.baselineCatalog || state?.latestCatalog?.mode !== 'full')) {
throw new Error(`files history 模式缺少已发布 full baseline catalog: ${statePath}`);
@@ -1464,10 +1654,26 @@ export async function runDirectFilesBackup({
uploadManifestFn,
verifyFn,
});
const compactedState = normalizeDirectFilesState({state, dataDir, database, bucket});
persistDirectFilesState({statePath, legacyStatePath, state: compactedState});
const metadataCleanup = compactDirectFilesLocalMetadata({
workDir,
database,
nextState: compactedState,
transientCatalogPaths: [catalogPath],
});
console.log('[database-backup] files history 没有可归档候选。');
const emptyResult = {...summary, catalog, latestPointer, uploadedCount: 0, reusedCount: 0, cleanup: null};
const emptyResult = {
...summary,
catalog,
latestPointer,
uploadedCount: 0,
reusedCount: 0,
cleanup: null,
metadataCleanup,
};
if (resultFile) {
atomicWriteJson(resolvePath(resultFile), emptyResult);
atomicWriteJson(resolvePath(resultFile), compactDirectFilesResult(emptyResult));
}
return emptyResult;
}
@@ -1485,8 +1691,23 @@ export async function runDirectFilesBackup({
uploadManifestFn,
verifyFn,
});
const compactedState = normalizeDirectFilesState({state, dataDir, database, bucket});
persistDirectFilesState({statePath, legacyStatePath, state: compactedState});
const metadataCleanup = compactDirectFilesLocalMetadata({
workDir,
database,
nextState: compactedState,
});
console.log('[database-backup] files catalog 未变化,无文件需要上传。');
return {...summary, catalog, latestPointer, uploadedCount: 0, reusedCount: collected.files.length, unchanged: true};
return {
...summary,
catalog,
latestPointer,
uploadedCount: 0,
reusedCount: collected.files.length,
unchanged: true,
metadataCleanup,
};
}
}
@@ -1499,7 +1720,7 @@ export async function runDirectFilesBackup({
});
}
const previousFiles = new Map((state?.latestCatalog?.files ?? []).map((file) => [file.path, file]));
const previousFiles = new Map(directPreviousFiles({state, workDir, database}).map((file) => [file.path, file]));
let uploadedCount = 0;
let reusedCount = 0;
let nextIndex = 0;
@@ -1555,8 +1776,6 @@ export async function runDirectFilesBackup({
contentLength: catalogUpload.contentLength,
sha256: catalogUpload.archiveSha256,
verifiedAt: catalogUpload.verifiedAt,
files: catalog.files,
symlinks: catalog.symlinks,
};
const nextState = {
schemaVersion: DIRECT_FILES_STATE_SCHEMA_VERSION,
@@ -1565,11 +1784,16 @@ export async function runDirectFilesBackup({
dataDir,
bucket,
updatedAt: new Date().toISOString(),
baselineCatalog: state?.baselineCatalog ?? catalogRef,
latestCatalog: mode === 'full' ? catalogRef : state.latestCatalog,
baselineCatalog: directCatalogRef(state?.baselineCatalog ?? catalogRef),
latestCatalog: directCatalogRef(mode === 'full' ? catalogRef : state.latestCatalog),
historyCatalogs: mode === 'history'
? [...(state.historyCatalogs ?? []).filter((item) => item.catalogId !== catalogId), catalogRef]
: (state?.historyCatalogs ?? []),
? [
...(state.historyCatalogs ?? [])
.filter((item) => item.catalogId !== catalogId)
.map((item) => directCatalogRef(item)),
directCatalogRef(catalogRef),
]
: (state?.historyCatalogs ?? []).map((item) => directCatalogRef(item)),
};
const latestPointer = await publishDirectFilesLatest({
workDir,
@@ -1581,14 +1805,27 @@ export async function runDirectFilesBackup({
uploadManifestFn,
verifyFn,
});
atomicWriteJson(statePath, nextState);
persistDirectFilesState({statePath, legacyStatePath, state: nextState});
const metadataCleanup = compactDirectFilesLocalMetadata({
workDir,
database,
nextState,
});
let cleanup = null;
if (mode === 'history') {
cleanup = cleanupHistoryCandidates({dataDir, candidates: plan.candidates});
}
const finalResult = {...summary, catalog, latestPointer, uploadedCount, reusedCount, cleanup};
const finalResult = {
...summary,
catalog,
latestPointer,
uploadedCount,
reusedCount,
cleanup,
metadataCleanup,
};
if (resultFile) {
atomicWriteJson(resolvePath(resultFile), finalResult);
atomicWriteJson(resolvePath(resultFile), compactDirectFilesResult(finalResult));
}
return finalResult;
}