Merge remote-tracking branch 'origin/master' into editor-agent-abortable
# Conflicts: # docs/project-memory/shared-memory/decision-log.md # docs/【编辑器】画布Agent对话面板-2026-07-03.md # server-rs/crates/api-server/src/editor_agent/api.rs # src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx # src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx # src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx
This commit is contained in:
@@ -6,6 +6,7 @@ import {chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync,
|
||||
import {tmpdir} from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {Readable} from 'node:stream';
|
||||
import {gunzipSync, gzipSync} from 'node:zlib';
|
||||
|
||||
import {
|
||||
buildAuthorization,
|
||||
@@ -65,12 +66,17 @@ async function main() {
|
||||
await assertHistorySuccessfulUploadCleansAndIsIdempotent();
|
||||
await assertHistoryResumeReverifiesArchiveAndManifest();
|
||||
await assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload();
|
||||
await assertDirectFilesMigratesLegacyStateAndPrunesEmbeddedCatalogs();
|
||||
await assertDirectFilesConcurrencyIsBounded();
|
||||
await assertDirectHistoryPublishesCatalogBeforeCleanup();
|
||||
await assertDirectHistoryWithoutCandidatesPublishesLatest();
|
||||
await assertDirectFilesRestoreDownloadsCatalogAndObjects();
|
||||
}
|
||||
|
||||
function readGzipJson(filePath) {
|
||||
return JSON.parse(gunzipSync(readFileSync(filePath)).toString('utf8'));
|
||||
}
|
||||
|
||||
function createDirectOssHarness() {
|
||||
const objects = new Map();
|
||||
const uploadedKeys = [];
|
||||
@@ -217,6 +223,13 @@ async function assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload(
|
||||
const first = await runDirectFilesBackup(options);
|
||||
assertEqual(first.uploadedCount, 2, '首次 files full 应上传全部普通文件。');
|
||||
assertTrue(!Object.hasOwn(first.catalog, 'dataDir'), '远端 files catalog 不得绑定 staging 主机的绝对 data-dir。');
|
||||
assertTrue(first.statePath.endsWith('.json.gz'), 'files state 必须使用 gzip 压缩文件。');
|
||||
const compactState = readGzipJson(first.statePath);
|
||||
assertEqual(compactState.schemaVersion, 2, 'files state 必须使用去重后的 v2 契约。');
|
||||
assertTrue(!Object.hasOwn(compactState.baselineCatalog, 'files'), 'baseline ref 不得重复嵌入 files。');
|
||||
assertTrue(!Object.hasOwn(compactState.latestCatalog, 'files'), 'latest ref 不得重复嵌入 files。');
|
||||
assertTrue(!existsSync(first.catalogPath), '本地 full catalog 原始 JSON 应在成功后压缩。');
|
||||
assertTrue(existsSync(`${first.catalogPath}.gz`), '本地应保留压缩后的 latest full catalog 供增量复用。');
|
||||
const latestObjectKey = 'database-backups/test-db/latest.json';
|
||||
const latest = JSON.parse(harness.objects.get(latestObjectKey).body.toString('utf8'));
|
||||
assertEqual(latest.latestFullCatalog.catalogId, first.catalogId, 'latest pointer 必须指向已验真的最新 full catalog。');
|
||||
@@ -231,11 +244,77 @@ async function assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload(
|
||||
'相同 catalog 重跑不得重复 PUT 文件或 catalog,但应覆盖验真 latest pointer。',
|
||||
);
|
||||
|
||||
rmSync(`${first.catalogPath}.gz`, {force: false});
|
||||
writeFileSync(path.join(dataDir, 'control-db'), 'control changed');
|
||||
writeFileSync(path.join(dataDir, 'new-program.bin'), 'new program');
|
||||
const incremental = await runDirectFilesBackup(options);
|
||||
assertEqual(incremental.uploadedCount, 2, '增量 files full 只应上传新增和变化文件。');
|
||||
assertEqual(incremental.reusedCount, 1, '增量 files full 应复用未变化 snapshot 文件。');
|
||||
assertEqual(incremental.reusedCount, 1, '本地 full catalog 缓存缺失时仍应通过 OSS HEAD 复用未变化文件。');
|
||||
}
|
||||
|
||||
async function assertDirectFilesMigratesLegacyStateAndPrunesEmbeddedCatalogs() {
|
||||
const root = path.join(tmpRoot, 'direct-files-state-migration');
|
||||
const dataDir = path.join(root, 'stdb');
|
||||
const workDir = path.join(root, 'work');
|
||||
mkdirSync(dataDir, {recursive: true});
|
||||
writeFileSync(path.join(dataDir, 'control-db'), 'control');
|
||||
const harness = createDirectOssHarness();
|
||||
const options = {
|
||||
mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups',
|
||||
uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn,
|
||||
};
|
||||
const first = await runDirectFilesBackup(options);
|
||||
const compactState = readGzipJson(first.statePath);
|
||||
const catalog = JSON.parse(gunzipSync(readFileSync(`${first.catalogPath}.gz`)).toString('utf8'));
|
||||
const legacyCatalogRef = {
|
||||
...compactState.latestCatalog,
|
||||
files: catalog.files,
|
||||
symlinks: catalog.symlinks,
|
||||
};
|
||||
const legacyStatePath = first.statePath.slice(0, -3);
|
||||
writeFileSync(legacyStatePath, `${JSON.stringify({
|
||||
...compactState,
|
||||
schemaVersion: 1,
|
||||
baselineCatalog: legacyCatalogRef,
|
||||
latestCatalog: legacyCatalogRef,
|
||||
}, null, 2)}\n`);
|
||||
rmSync(first.statePath, {force: false});
|
||||
|
||||
const migrated = await runDirectFilesBackup(options);
|
||||
assertTrue(migrated.unchanged, '旧 state 迁移不得改变相同 full catalog 的零上传语义。');
|
||||
assertTrue(existsSync(migrated.statePath), '旧 state 成功运行后必须生成压缩 state。');
|
||||
assertTrue(!existsSync(legacyStatePath), '压缩 state 原子落盘后应删除旧未压缩 state。');
|
||||
const migratedState = readGzipJson(migrated.statePath);
|
||||
assertEqual(migratedState.schemaVersion, 2, '旧 state 必须迁移到 v2。');
|
||||
assertTrue(!Object.hasOwn(migratedState.latestCatalog, 'files'), '迁移后 state 不得保留重复 files 清单。');
|
||||
|
||||
const latestCatalogPath = `${migrated.catalogPath}.gz`;
|
||||
const validCatalogBody = readFileSync(latestCatalogPath);
|
||||
writeFileSync(latestCatalogPath, gzipSync(Buffer.from('{}\n')));
|
||||
writeFileSync(path.join(dataDir, 'control-db'), 'control changed');
|
||||
let corruptCatalogFailure = null;
|
||||
try {
|
||||
await runDirectFilesBackup(options);
|
||||
} catch (error) {
|
||||
corruptCatalogFailure = error;
|
||||
}
|
||||
assertIncludes(
|
||||
corruptCatalogFailure?.message ?? '',
|
||||
'长度或 SHA 与 state 引用不匹配',
|
||||
'本地 latest full catalog 损坏时不得作为增量复用缓存。',
|
||||
);
|
||||
writeFileSync(latestCatalogPath, validCatalogBody);
|
||||
|
||||
writeFileSync(legacyStatePath, `${JSON.stringify({...migratedState, schemaVersion: 1})}\n`);
|
||||
writeFileSync(migrated.statePath, 'not-a-gzip-state');
|
||||
let corruptStateFailure = null;
|
||||
try {
|
||||
await runDirectFilesBackup(options);
|
||||
} catch (error) {
|
||||
corruptStateFailure = error;
|
||||
}
|
||||
assertTrue(corruptStateFailure instanceof Error, '压缩 state 损坏时必须失败。');
|
||||
assertTrue(existsSync(legacyStatePath), '压缩 state 损坏时不得静默回退并删除旧 state。');
|
||||
}
|
||||
|
||||
async function assertDirectFilesConcurrencyIsBounded() {
|
||||
@@ -290,7 +369,14 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() {
|
||||
uploadFn: harness.uploadFn,
|
||||
verifyFn: harness.verifyFn,
|
||||
};
|
||||
await runDirectFilesBackup({...common, mode: 'full', uploadManifestFn: harness.uploadManifestFn});
|
||||
const baseline = await runDirectFilesBackup({...common, mode: 'full', uploadManifestFn: harness.uploadManifestFn});
|
||||
const legacyResultFile = path.join(fixture.workDir, 'legacy-full-result.json');
|
||||
const legacyCatalogWithoutSymlinks = {...baseline.catalog};
|
||||
delete legacyCatalogWithoutSymlinks.symlinks;
|
||||
writeFileSync(legacyResultFile, `${JSON.stringify({
|
||||
...baseline,
|
||||
catalog: legacyCatalogWithoutSymlinks,
|
||||
}, null, 2)}\n`);
|
||||
const plan = discoverHistoryPlan({dataDir: fixture.dataDir});
|
||||
let failure = null;
|
||||
try {
|
||||
@@ -330,7 +416,13 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() {
|
||||
assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `latest pointer 发布失败不得删除: ${candidate.path}`);
|
||||
}
|
||||
|
||||
const success = await runDirectFilesBackup({...common, mode: 'history', uploadManifestFn: harness.uploadManifestFn});
|
||||
const resultFile = path.join(fixture.workDir, 'history-result.json');
|
||||
const success = await runDirectFilesBackup({
|
||||
...common,
|
||||
mode: 'history',
|
||||
resultFile,
|
||||
uploadManifestFn: harness.uploadManifestFn,
|
||||
});
|
||||
assertEqual(success.uploadedCount, 0, 'history 文件已在 full CAS baseline 时不应重复上传内容。');
|
||||
for (const file of success.catalog.files) {
|
||||
assertTrue(
|
||||
@@ -340,7 +432,19 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() {
|
||||
}
|
||||
assertEqual(success.cleanup?.deletedCount, plan.candidates.length, 'catalog 和 baseline 验真后才应清理全部安全候选。');
|
||||
|
||||
const state = JSON.parse(readFileSync(success.statePath, 'utf8'));
|
||||
const state = readGzipJson(success.statePath);
|
||||
assertEqual(state.schemaVersion, 2, 'files state 必须迁移为去重后的 v2 契约。');
|
||||
assertTrue(!Object.hasOwn(state.latestCatalog, 'files'), 'files state latest ref 不得重复嵌入 files。');
|
||||
assertTrue(state.historyCatalogs.every((catalog) => !Object.hasOwn(catalog, 'files')), 'files state history ref 不得重复嵌入 files。');
|
||||
assertTrue(!existsSync(success.catalogPath), '已上传并验真的 history catalog 本地 JSON 应被清理。');
|
||||
assertTrue(!existsSync(`${success.catalogPath}.gz`), 'history catalog 本地压缩副本也不应保留。');
|
||||
const diskResult = JSON.parse(readFileSync(resultFile, 'utf8'));
|
||||
assertTrue(!Object.hasOwn(diskResult.catalog, 'files'), 'files result 文件不得重复写入完整 files 清单。');
|
||||
assertEqual(diskResult.catalog.fileCount, success.fileCount, '紧凑 result 仍应保留文件计数。');
|
||||
const compactedLegacyResult = JSON.parse(readFileSync(legacyResultFile, 'utf8'));
|
||||
assertTrue(!Object.hasOwn(compactedLegacyResult.catalog, 'files'), '旧 result 中重复的 files 清单应在成功运行后压缩。');
|
||||
assertEqual(compactedLegacyResult.catalog.symlinkCount, 0, '缺少 symlinks 的旧 result 应按零个符号链接兼容迁移。');
|
||||
assertTrue((success.metadataCleanup?.compactedResultCount ?? 0) >= 1, 'metadata 清理应报告已压缩旧 result。');
|
||||
const historyCatalogObjectKey = state.historyCatalogs[0].objectKey;
|
||||
harness.objects.delete(historyCatalogObjectKey);
|
||||
let brokenHistoryFailure = null;
|
||||
@@ -425,6 +529,29 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() {
|
||||
assertTrue(lstatSync(path.join(restoreDir, 'bin', 'current')).isSymbolicLink(), 'files restore 必须重建符号链接。');
|
||||
assertEqual(readlinkSync(path.join(restoreDir, 'bin', 'current'), 'utf8'), '2.6.0', 'files restore 必须保留符号链接目标。');
|
||||
|
||||
rmSync(restoreDir, {recursive: true, force: true});
|
||||
const legacyRestoreStatePath = path.join(workDir, 'legacy-restore-state.json');
|
||||
writeFileSync(legacyRestoreStatePath, `${JSON.stringify({
|
||||
...readGzipJson(latestFull.statePath),
|
||||
schemaVersion: 1,
|
||||
})}\n`);
|
||||
const legacyRestored = await restoreDirectFilesBackup({
|
||||
statePath: legacyRestoreStatePath,
|
||||
restoreDir,
|
||||
database: 'test-db',
|
||||
bucket: 'backup-bucket',
|
||||
uploadOptions: {},
|
||||
downloadBufferFn: async ({objectKey}) => Buffer.from(harness.objects.get(objectKey)?.body ?? ''),
|
||||
downloadFileFn: async ({objectKey, destinationPath}) => {
|
||||
const object = harness.objects.get(objectKey);
|
||||
if (!object) {
|
||||
throw new Error(`missing ${objectKey}`);
|
||||
}
|
||||
writeFileSync(destinationPath, object.body);
|
||||
},
|
||||
});
|
||||
assertEqual(legacyRestored.downloadedCount, 1, 'files restore 必须继续兼容 v1 JSON state。');
|
||||
|
||||
rmSync(restoreDir, {recursive: true, force: true});
|
||||
const downloadBufferFn = async ({objectKey}) => {
|
||||
const object = harness.objects.get(objectKey);
|
||||
|
||||
@@ -148,6 +148,10 @@ function validateGatewayConfiguration() {
|
||||
'try_files /page.html @genarrative_default_maintenance;',
|
||||
'location @genarrative_default_maintenance',
|
||||
'root /srv/genarrative/web;',
|
||||
'location = /branding/taonier-maintenance-page.png',
|
||||
'try_files /branding/taonier-maintenance-page.png =404;',
|
||||
'location = /branding/taonier-product-ip.png',
|
||||
'try_files /branding/taonier-product-ip.png =404;',
|
||||
]) {
|
||||
if (!nginxSnippet.includes(expected)) {
|
||||
fail(`Nginx 维护页配置缺少运行态覆盖约束: ${expected}`);
|
||||
@@ -161,6 +165,7 @@ function validateGatewayConfiguration() {
|
||||
for (const expected of [
|
||||
'GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_PAGE_FILE',
|
||||
'maintenance_page_file',
|
||||
'is_maintenance_page_asset(path)',
|
||||
]) {
|
||||
if (!pingoraSource.includes(expected)) {
|
||||
fail(`Pingora 维护页配置缺少运行态覆盖约束: ${expected}`);
|
||||
|
||||
@@ -164,6 +164,10 @@ function validateMaintenanceInternalBypass() {
|
||||
fail(`${MAINTENANCE_SNIPPET_PATH} 不应保留仅后台使用的维护变量。`);
|
||||
}
|
||||
for (const fragment of [
|
||||
'location = /branding/taonier-maintenance-page.png {',
|
||||
'try_files /branding/taonier-maintenance-page.png =404;',
|
||||
'location = /branding/taonier-product-ip.png {',
|
||||
'try_files /branding/taonier-product-ip.png =404;',
|
||||
'location = /404.html {',
|
||||
'if ($http_accept !~* "text/html") {',
|
||||
'try_files /404.html =404;',
|
||||
|
||||
@@ -347,6 +347,19 @@ async function prepareStaticRoots(webRoot, acmeRoot) {
|
||||
path.join(webRoot, 'maintenance.html'),
|
||||
'<main>default-maintenance</main>',
|
||||
);
|
||||
await mkdir(path.join(webRoot, 'branding'), { recursive: true });
|
||||
for (const fileName of [
|
||||
'taonier-maintenance-page.png',
|
||||
'taonier-product-ip.png',
|
||||
]) {
|
||||
await writeFile(
|
||||
path.join(webRoot, 'branding', fileName),
|
||||
Buffer.concat([
|
||||
PNG_MAGIC_BYTES,
|
||||
Buffer.from([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52]),
|
||||
]),
|
||||
);
|
||||
}
|
||||
await writeFile(path.join(webRoot, '404.html'), '<main>not-found-page</main>');
|
||||
await writeFile(
|
||||
path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'),
|
||||
@@ -1216,6 +1229,14 @@ async function runSmokeCases(
|
||||
},
|
||||
);
|
||||
}
|
||||
for (const path of [
|
||||
'/branding/taonier-maintenance-page.png',
|
||||
'/branding/taonier-product-ip.png',
|
||||
]) {
|
||||
await expectStaticPng(baseUrl, path, `维护模式放行品牌图片 ${path}`, {
|
||||
headers: publicClientHeaders,
|
||||
});
|
||||
}
|
||||
await rm(maintenancePageFile, { force: true });
|
||||
await expectHttp(
|
||||
baseUrl,
|
||||
@@ -1445,9 +1466,10 @@ async function expectNotCompressedResponse(baseUrl, route, bodyNeedle, label) {
|
||||
}
|
||||
}
|
||||
|
||||
async function expectStaticPng(baseUrl, route, label) {
|
||||
async function expectStaticPng(baseUrl, route, label, options = {}) {
|
||||
console.log(`[pingora-gateway-smoke] ${label}`);
|
||||
const response = await requestHttp(`${baseUrl}${route}`, {
|
||||
headers: options.headers,
|
||||
rawBody: true,
|
||||
});
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import {Readable} from 'node:stream';
|
||||
import {pipeline} from 'node:stream/promises';
|
||||
import {setTimeout as sleep} from 'node:timers/promises';
|
||||
import {fileURLToPath} from 'node:url';
|
||||
import {gunzipSync, gzipSync} from 'node:zlib';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
@@ -53,7 +54,8 @@ const DIRECT_FILES_SINGLE_PUT_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const RETRYABLE_OSS_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
|
||||
const HISTORY_STATE_SCHEMA_VERSION = 1;
|
||||
const HISTORY_MANIFEST_SCHEMA_VERSION = 1;
|
||||
const DIRECT_FILES_STATE_SCHEMA_VERSION = 1;
|
||||
const DIRECT_FILES_STATE_SCHEMA_VERSION = 2;
|
||||
const LEGACY_DIRECT_FILES_STATE_SCHEMA_VERSION = 1;
|
||||
const DIRECT_FILES_CATALOG_SCHEMA_VERSION = 1;
|
||||
const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1;
|
||||
|
||||
@@ -370,14 +372,23 @@ function buildBackupNames({database, dataDir, objectPrefix}) {
|
||||
return {fileName, objectKey};
|
||||
}
|
||||
|
||||
function atomicWriteJson(filePath, payload) {
|
||||
function atomicWriteBuffer(filePath, body) {
|
||||
mkdirSync(dirname(filePath), {recursive: true});
|
||||
const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
|
||||
writeFileSync(tempPath, `${JSON.stringify(payload, null, 2)}\n`, {encoding: 'utf8', mode: 0o600});
|
||||
writeFileSync(tempPath, body, {mode: 0o600});
|
||||
chmodSync(tempPath, 0o600);
|
||||
renameSync(tempPath, filePath);
|
||||
}
|
||||
|
||||
function atomicWriteJson(filePath, payload) {
|
||||
atomicWriteBuffer(filePath, Buffer.from(`${JSON.stringify(payload, null, 2)}\n`, 'utf8'));
|
||||
}
|
||||
|
||||
function atomicWriteGzipJson(filePath, payload) {
|
||||
const body = Buffer.from(`${JSON.stringify(payload)}\n`, 'utf8');
|
||||
atomicWriteBuffer(filePath, gzipSync(body, {level: 9}));
|
||||
}
|
||||
|
||||
function processIsAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
@@ -1070,9 +1081,127 @@ async function sha256FileHex(filePath) {
|
||||
}
|
||||
|
||||
function directFilesStatePath({workDir, database}) {
|
||||
return join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json.gz`);
|
||||
}
|
||||
|
||||
function legacyDirectFilesStatePath({workDir, database}) {
|
||||
return join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json`);
|
||||
}
|
||||
|
||||
function directCatalogLocalPaths({workDir, database, catalog}) {
|
||||
const baseName = `${sanitizeObjectPart(database, 'spacetimedb')}-${catalog.mode}-${catalog.catalogId}.catalog.json`;
|
||||
return {
|
||||
jsonPath: join(workDir, baseName),
|
||||
gzipPath: join(workDir, `${baseName}.gz`),
|
||||
};
|
||||
}
|
||||
|
||||
function readLocalDirectCatalog({workDir, database, catalog}) {
|
||||
const {jsonPath, gzipPath} = directCatalogLocalPaths({workDir, database, catalog});
|
||||
let body = null;
|
||||
if (existsSync(jsonPath)) {
|
||||
body = readFileSync(jsonPath);
|
||||
} else if (existsSync(gzipPath)) {
|
||||
body = gunzipSync(readFileSync(gzipPath));
|
||||
}
|
||||
if (!body) {
|
||||
return null;
|
||||
}
|
||||
if (body.length !== catalog.contentLength || sha256Hex(body) !== catalog.sha256) {
|
||||
throw new Error(`本地 files catalog 长度或 SHA 与 state 引用不匹配: ${jsonPath}`);
|
||||
}
|
||||
const payload = JSON.parse(body.toString('utf8'));
|
||||
if (
|
||||
payload.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION
|
||||
|| payload.database !== database
|
||||
|| payload.mode !== catalog.mode
|
||||
|| payload.catalogId !== catalog.catalogId
|
||||
|| !Array.isArray(payload.files)
|
||||
) {
|
||||
throw new Error(`本地 files catalog 与 state 引用不匹配: ${jsonPath}`);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
function readJsonOrGzip(filePath) {
|
||||
const body = readFileSync(filePath);
|
||||
const decoded = filePath.endsWith('.gz') || (body[0] === 0x1f && body[1] === 0x8b)
|
||||
? gunzipSync(body)
|
||||
: body;
|
||||
return JSON.parse(decoded.toString('utf8'));
|
||||
}
|
||||
|
||||
function compactDirectCatalogFile({workDir, database, catalog}) {
|
||||
const {jsonPath, gzipPath} = directCatalogLocalPaths({workDir, database, catalog});
|
||||
if (!existsSync(jsonPath)) {
|
||||
return existsSync(gzipPath) ? {compressed: false, gzipPath} : null;
|
||||
}
|
||||
const body = readFileSync(jsonPath);
|
||||
atomicWriteBuffer(gzipPath, gzipSync(body, {level: 9}));
|
||||
rmSync(jsonPath, {force: false});
|
||||
return {compressed: true, gzipPath};
|
||||
}
|
||||
|
||||
function compactDirectFilesLocalMetadata({workDir, database, nextState, transientCatalogPaths = []}) {
|
||||
const keepCatalog = nextState.latestCatalog;
|
||||
const keepCatalogIds = new Set([keepCatalog?.catalogId].filter(Boolean));
|
||||
const databasePart = sanitizeObjectPart(database, 'spacetimedb');
|
||||
const catalogPattern = new RegExp(`^${databasePart.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&')}-(full|history)-([a-f0-9]{64})\\.catalog\\.json(?:\\.gz)?$`, 'u');
|
||||
let compressedCatalogCount = 0;
|
||||
let deletedCatalogCount = 0;
|
||||
let compactedResultCount = 0;
|
||||
for (const entry of readdirSync(workDir, {withFileTypes: true})) {
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
const match = catalogPattern.exec(entry.name);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const catalog = {mode: match[1], catalogId: match[2]};
|
||||
if (keepCatalogIds.has(catalog.catalogId) && catalog.mode === 'full') {
|
||||
readLocalDirectCatalog({workDir, database, catalog: keepCatalog});
|
||||
if (!entry.name.endsWith('.gz') && compactDirectCatalogFile({workDir, database, catalog})?.compressed) {
|
||||
compressedCatalogCount += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
rmSync(join(workDir, entry.name), {force: false});
|
||||
deletedCatalogCount += 1;
|
||||
}
|
||||
for (const filePath of transientCatalogPaths) {
|
||||
if (existsSync(filePath)) {
|
||||
rmSync(filePath, {force: false});
|
||||
deletedCatalogCount += 1;
|
||||
}
|
||||
}
|
||||
for (const entry of readdirSync(workDir, {withFileTypes: true})) {
|
||||
if (!entry.isFile() || !entry.name.endsWith('.json') || entry.name.endsWith('.catalog.json')) {
|
||||
continue;
|
||||
}
|
||||
const filePath = join(workDir, entry.name);
|
||||
let payload = null;
|
||||
try {
|
||||
payload = JSON.parse(readFileSync(filePath, 'utf8'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
payload?.catalog?.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION
|
||||
|| payload.catalog.database !== database
|
||||
|| payload.catalog.bucket !== nextState.bucket
|
||||
|| !Array.isArray(payload.catalog.files)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
atomicWriteJson(filePath, compactDirectFilesResult(payload));
|
||||
compactedResultCount += 1;
|
||||
}
|
||||
const result = {compressedCatalogCount, deletedCatalogCount, compactedResultCount};
|
||||
console.log(`[database-backup] files 本地元数据清理: ${JSON.stringify(result)}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeObjectPrefix(objectPrefix, database) {
|
||||
const prefix = String(objectPrefix || 'database-backups')
|
||||
.trim()
|
||||
@@ -1185,12 +1314,16 @@ function directCatalogIdentity({mode, baselineCatalogId, rootName, directories,
|
||||
}
|
||||
|
||||
function readDirectFilesState(statePath, {database, bucket}) {
|
||||
if (!existsSync(statePath)) {
|
||||
const candidates = statePath.endsWith('.gz')
|
||||
? [statePath, statePath.slice(0, -3)]
|
||||
: [statePath, `${statePath}.gz`];
|
||||
const existingPath = candidates.find((candidate) => existsSync(candidate));
|
||||
if (!existingPath) {
|
||||
return null;
|
||||
}
|
||||
const state = readManifest(statePath);
|
||||
const state = readJsonOrGzip(existingPath);
|
||||
if (
|
||||
state.schemaVersion !== DIRECT_FILES_STATE_SCHEMA_VERSION
|
||||
![LEGACY_DIRECT_FILES_STATE_SCHEMA_VERSION, DIRECT_FILES_STATE_SCHEMA_VERSION].includes(state.schemaVersion)
|
||||
|| state.backupKind !== 'spacetimedb-direct-files-state'
|
||||
|| state.database !== database
|
||||
|| state.bucket !== bucket
|
||||
@@ -1200,6 +1333,16 @@ function readDirectFilesState(statePath, {database, bucket}) {
|
||||
return state;
|
||||
}
|
||||
|
||||
function directPreviousFiles({state, workDir, database}) {
|
||||
if (!state?.latestCatalog) {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(state?.latestCatalog?.files)) {
|
||||
return state.latestCatalog.files;
|
||||
}
|
||||
return readLocalDirectCatalog({workDir, database, catalog: state?.latestCatalog})?.files ?? [];
|
||||
}
|
||||
|
||||
async function ensureDirectObject({
|
||||
file,
|
||||
dataDir,
|
||||
@@ -1290,6 +1433,52 @@ function directCatalogRef(catalog) {
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeDirectFilesState({state, dataDir, database, bucket}) {
|
||||
return {
|
||||
schemaVersion: DIRECT_FILES_STATE_SCHEMA_VERSION,
|
||||
backupKind: 'spacetimedb-direct-files-state',
|
||||
database,
|
||||
dataDir,
|
||||
bucket,
|
||||
updatedAt: new Date().toISOString(),
|
||||
baselineCatalog: assertDirectCatalogRef(state?.baselineCatalog, 'full', 'baseline full'),
|
||||
latestCatalog: assertDirectCatalogRef(state?.latestCatalog, 'full', 'latest full'),
|
||||
historyCatalogs: (state?.historyCatalogs ?? []).map((catalog) => (
|
||||
assertDirectCatalogRef(catalog, 'history', 'history')
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
function persistDirectFilesState({statePath, legacyStatePath, state}) {
|
||||
atomicWriteGzipJson(statePath, state);
|
||||
if (legacyStatePath !== statePath && existsSync(legacyStatePath)) {
|
||||
rmSync(legacyStatePath, {force: false});
|
||||
}
|
||||
}
|
||||
|
||||
function compactDirectFilesResult(result) {
|
||||
if (!result.catalog) {
|
||||
return result;
|
||||
}
|
||||
const {catalog, ...rest} = result;
|
||||
return {
|
||||
...rest,
|
||||
catalog: {
|
||||
schemaVersion: catalog.schemaVersion,
|
||||
backupKind: catalog.backupKind,
|
||||
database: catalog.database,
|
||||
bucket: catalog.bucket,
|
||||
mode: catalog.mode,
|
||||
catalogId: catalog.catalogId,
|
||||
catalogObjectKey: catalog.catalogObjectKey,
|
||||
baselineCatalogId: catalog.baselineCatalogId,
|
||||
rootName: catalog.rootName,
|
||||
fileCount: Array.isArray(catalog.files) ? catalog.files.length : 0,
|
||||
symlinkCount: Array.isArray(catalog.symlinks) ? catalog.symlinks.length : 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assertDirectCatalogRef(catalog, expectedMode, label) {
|
||||
if (
|
||||
!catalog
|
||||
@@ -1397,6 +1586,7 @@ export async function runDirectFilesBackup({
|
||||
}) {
|
||||
mkdirSync(workDir, {recursive: true});
|
||||
const statePath = directFilesStatePath({workDir, database});
|
||||
const legacyStatePath = legacyDirectFilesStatePath({workDir, database});
|
||||
const state = readDirectFilesState(statePath, {database, bucket});
|
||||
if (mode === 'history' && (!state?.baselineCatalog || state?.latestCatalog?.mode !== 'full')) {
|
||||
throw new Error(`files history 模式缺少已发布 full baseline catalog: ${statePath}`);
|
||||
@@ -1464,10 +1654,26 @@ export async function runDirectFilesBackup({
|
||||
uploadManifestFn,
|
||||
verifyFn,
|
||||
});
|
||||
const compactedState = normalizeDirectFilesState({state, dataDir, database, bucket});
|
||||
persistDirectFilesState({statePath, legacyStatePath, state: compactedState});
|
||||
const metadataCleanup = compactDirectFilesLocalMetadata({
|
||||
workDir,
|
||||
database,
|
||||
nextState: compactedState,
|
||||
transientCatalogPaths: [catalogPath],
|
||||
});
|
||||
console.log('[database-backup] files history 没有可归档候选。');
|
||||
const emptyResult = {...summary, catalog, latestPointer, uploadedCount: 0, reusedCount: 0, cleanup: null};
|
||||
const emptyResult = {
|
||||
...summary,
|
||||
catalog,
|
||||
latestPointer,
|
||||
uploadedCount: 0,
|
||||
reusedCount: 0,
|
||||
cleanup: null,
|
||||
metadataCleanup,
|
||||
};
|
||||
if (resultFile) {
|
||||
atomicWriteJson(resolvePath(resultFile), emptyResult);
|
||||
atomicWriteJson(resolvePath(resultFile), compactDirectFilesResult(emptyResult));
|
||||
}
|
||||
return emptyResult;
|
||||
}
|
||||
@@ -1485,8 +1691,23 @@ export async function runDirectFilesBackup({
|
||||
uploadManifestFn,
|
||||
verifyFn,
|
||||
});
|
||||
const compactedState = normalizeDirectFilesState({state, dataDir, database, bucket});
|
||||
persistDirectFilesState({statePath, legacyStatePath, state: compactedState});
|
||||
const metadataCleanup = compactDirectFilesLocalMetadata({
|
||||
workDir,
|
||||
database,
|
||||
nextState: compactedState,
|
||||
});
|
||||
console.log('[database-backup] files catalog 未变化,无文件需要上传。');
|
||||
return {...summary, catalog, latestPointer, uploadedCount: 0, reusedCount: collected.files.length, unchanged: true};
|
||||
return {
|
||||
...summary,
|
||||
catalog,
|
||||
latestPointer,
|
||||
uploadedCount: 0,
|
||||
reusedCount: collected.files.length,
|
||||
unchanged: true,
|
||||
metadataCleanup,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1499,7 +1720,7 @@ export async function runDirectFilesBackup({
|
||||
});
|
||||
}
|
||||
|
||||
const previousFiles = new Map((state?.latestCatalog?.files ?? []).map((file) => [file.path, file]));
|
||||
const previousFiles = new Map(directPreviousFiles({state, workDir, database}).map((file) => [file.path, file]));
|
||||
let uploadedCount = 0;
|
||||
let reusedCount = 0;
|
||||
let nextIndex = 0;
|
||||
@@ -1555,8 +1776,6 @@ export async function runDirectFilesBackup({
|
||||
contentLength: catalogUpload.contentLength,
|
||||
sha256: catalogUpload.archiveSha256,
|
||||
verifiedAt: catalogUpload.verifiedAt,
|
||||
files: catalog.files,
|
||||
symlinks: catalog.symlinks,
|
||||
};
|
||||
const nextState = {
|
||||
schemaVersion: DIRECT_FILES_STATE_SCHEMA_VERSION,
|
||||
@@ -1565,11 +1784,16 @@ export async function runDirectFilesBackup({
|
||||
dataDir,
|
||||
bucket,
|
||||
updatedAt: new Date().toISOString(),
|
||||
baselineCatalog: state?.baselineCatalog ?? catalogRef,
|
||||
latestCatalog: mode === 'full' ? catalogRef : state.latestCatalog,
|
||||
baselineCatalog: directCatalogRef(state?.baselineCatalog ?? catalogRef),
|
||||
latestCatalog: directCatalogRef(mode === 'full' ? catalogRef : state.latestCatalog),
|
||||
historyCatalogs: mode === 'history'
|
||||
? [...(state.historyCatalogs ?? []).filter((item) => item.catalogId !== catalogId), catalogRef]
|
||||
: (state?.historyCatalogs ?? []),
|
||||
? [
|
||||
...(state.historyCatalogs ?? [])
|
||||
.filter((item) => item.catalogId !== catalogId)
|
||||
.map((item) => directCatalogRef(item)),
|
||||
directCatalogRef(catalogRef),
|
||||
]
|
||||
: (state?.historyCatalogs ?? []).map((item) => directCatalogRef(item)),
|
||||
};
|
||||
const latestPointer = await publishDirectFilesLatest({
|
||||
workDir,
|
||||
@@ -1581,14 +1805,27 @@ export async function runDirectFilesBackup({
|
||||
uploadManifestFn,
|
||||
verifyFn,
|
||||
});
|
||||
atomicWriteJson(statePath, nextState);
|
||||
persistDirectFilesState({statePath, legacyStatePath, state: nextState});
|
||||
const metadataCleanup = compactDirectFilesLocalMetadata({
|
||||
workDir,
|
||||
database,
|
||||
nextState,
|
||||
});
|
||||
let cleanup = null;
|
||||
if (mode === 'history') {
|
||||
cleanup = cleanupHistoryCandidates({dataDir, candidates: plan.candidates});
|
||||
}
|
||||
const finalResult = {...summary, catalog, latestPointer, uploadedCount, reusedCount, cleanup};
|
||||
const finalResult = {
|
||||
...summary,
|
||||
catalog,
|
||||
latestPointer,
|
||||
uploadedCount,
|
||||
reusedCount,
|
||||
cleanup,
|
||||
metadataCleanup,
|
||||
};
|
||||
if (resultFile) {
|
||||
atomicWriteJson(resolvePath(resultFile), finalResult);
|
||||
atomicWriteJson(resolvePath(resultFile), compactDirectFilesResult(finalResult));
|
||||
}
|
||||
return finalResult;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user