Files
Genarrative/scripts/check-database-backup-to-oss.mjs
T
kdletters aeca412cdb 修复数据库冷备分片上传
冷备归档改用 OSS Multipart 顺序分片上传并加入可重试请求。

Complete 后通过签名 HEAD 严格核对远端对象长度,再更新清单或清理本地归档。

补齐签名查询、分片重试、缺失 ETag、Complete 歧义和 Abort 回归测试。

记录数据库冷备上传与清理边界的长期运维决策。
2026-07-13 11:13:57 +08:00

516 lines
19 KiB
JavaScript

#!/usr/bin/env node
import {spawnSync} from 'node:child_process';
import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
import {tmpdir} from 'node:os';
import path from 'node:path';
import {buildAuthorization, buildCanonicalQuery, uploadArchive} from './database-backup-to-oss.mjs';
const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs');
const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-'));
const failures = [];
try {
await main();
} finally {
rmSync(tmpRoot, {recursive: true, force: true});
}
if (failures.length > 0) {
console.error('[check:database-backup] FAILED');
for (const failure of failures) {
console.error(`- ${failure}`);
}
process.exit(1);
}
console.log('[check:database-backup] OK');
async function main() {
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
assertInsufficientSpaceStopsBeforeServiceChanges();
assertArchiveFailureStillRestoresDependentServices();
await assertMultipartUploadRetriesAndVerifiesRemoteLength();
await assertMissingPartEtagAbortsMultipartUpload();
await assertCompleteResponseAmbiguityUsesHeadVerification();
await assertHeadLengthMismatchAbortsMultipartUpload();
}
function assertCanonicalQueryAndAuthorizationIncludeMultipartParameters() {
assertEqual(buildCanonicalQuery({uploads: null}), 'uploads', 'InitiateMultipartUpload 必须使用无等号的 uploads 参数。');
assertEqual(
buildCanonicalQuery({uploadId: 'abc+/= xyz', partNumber: 12}),
'partNumber=12&uploadId=abc%2B%2F%3D%20xyz',
'multipart query 必须按 key 排序并使用 RFC3986 编码。',
);
const date = new Date('2026-07-13T10:20:30.000Z');
const headers = {
host: 'genarrative-test.oss-cn-shanghai.aliyuncs.com',
'content-type': 'application/octet-stream',
'x-oss-content-sha256': 'UNSIGNED-PAYLOAD',
'x-oss-date': '20260713T102030Z',
};
const withoutQuery = buildAuthorization({
method: 'PUT',
bucket: 'genarrative-test',
endpoint: 'oss-cn-shanghai.aliyuncs.com',
objectKey: 'database-backups/test/archive.tar.gz',
accessKeyId: 'test-access-key',
accessKeySecret: 'test-access-secret',
headers,
date,
});
const withQuery = buildAuthorization({
method: 'PUT',
bucket: 'genarrative-test',
endpoint: 'oss-cn-shanghai.aliyuncs.com',
objectKey: 'database-backups/test/archive.tar.gz',
accessKeyId: 'test-access-key',
accessKeySecret: 'test-access-secret',
headers,
date,
queries: {partNumber: 12, uploadId: 'abc+/= xyz'},
});
assertNotEqual(withQuery, withoutQuery, 'multipart query 必须参与 V4 Authorization 计算。');
assertEqual(
withQuery,
'OSS4-HMAC-SHA256 Credential=test-access-key/20260713/cn-shanghai/oss/aliyun_v4_request,AdditionalHeaders=host,Signature=9323dd3b7272b52f416c4d32115fcc00460eaccdcdaf011575c2502a63a27b1f',
'multipart V4 Authorization 必须保持固定签名向量。',
);
}
function assertInsufficientSpaceStopsBeforeServiceChanges() {
const fixture = createFixture('insufficient-space');
const result = runBackup(fixture, [
'--stop-service',
'spacetimedb.service',
'--restart-service-after',
'genarrative-api.service',
'--min-free-bytes',
'999999999999999999',
]);
assertStatus(result, 1, '空间不足时必须失败。');
assertIncludes(result.stdout, '备份空间预检', '空间不足失败前应打印空间预检。');
assertIncludes(result.stderr, '剩余空间不足', '空间不足失败应说明剩余空间不足。');
assertFileMissing(fixture.systemctlLog, '空间不足时不能调用 systemctl。');
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
}
function assertArchiveFailureStillRestoresDependentServices() {
const fixture = createFixture('tar-failure');
const result = runBackup(fixture, [
'--stop-service',
'spacetimedb.service',
'--restart-service-after',
'genarrative-api.service',
'--restart-service-after',
'genarrative-external-generation-worker@1.service',
'--restart-service-after',
'genarrative-external-generation-controller.service',
'--min-free-bytes',
'1',
]);
assertStatus(result, 1, 'tar 失败时备份脚本必须失败。');
assertIncludes(result.stderr, 'fake tar failure', 'tar 失败原因应保留在错误输出中。');
const systemctlLog = readFile(fixture.systemctlLog);
const expectedCommands = [
'systemctl stop spacetimedb.service',
'systemctl start spacetimedb.service',
'systemctl restart genarrative-api.service',
'systemctl restart genarrative-external-generation-worker@1.service',
'systemctl restart genarrative-external-generation-controller.service',
];
for (const command of expectedCommands) {
assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`);
}
}
async function assertMultipartUploadRetriesAndVerifiesRemoteLength() {
const root = path.join(tmpRoot, 'multipart-success');
const archivePath = path.join(root, 'backup.tar.gz');
const partSizeBytes = 100 * 1024;
const payload = Buffer.concat([
Buffer.alloc(partSizeBytes, 'a'),
Buffer.alloc(partSizeBytes, 'b'),
Buffer.alloc(17, 'c'),
]);
mkdirSync(root, {recursive: true});
writeFileSync(archivePath, payload);
const requests = [];
const retryDelays = [];
let firstPartAttempts = 0;
const uploadId = 'upload+/= id';
const fetchImpl = async (url, options) => {
const body = await readRequestBody(options.body);
requests.push({url, method: options.method, headers: options.headers, body});
const parsedUrl = new URL(url);
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
return new Response(`<InitiateMultipartUploadResult><UploadId>${uploadId}</UploadId></InitiateMultipartUploadResult>`, {status: 200});
}
if (options.method === 'PUT') {
const partNumber = Number(parsedUrl.searchParams.get('partNumber'));
if (partNumber === 1) {
firstPartAttempts += 1;
if (firstPartAttempts === 1) {
return new Response('<Error><Code>ServiceUnavailable</Code></Error>', {status: 503});
}
}
return new Response('', {status: 200, headers: {etag: `"etag-${partNumber}"`}});
}
if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) {
return new Response('<CompleteMultipartUploadResult/>', {status: 200, headers: {etag: '"complete-etag"'}});
}
if (options.method === 'HEAD') {
return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}});
}
throw new Error(`unexpected request: ${options.method} ${url}`);
};
const result = await uploadArchive({
archivePath,
bucket: 'genarrative-test',
endpoint: 'oss-cn-shanghai.aliyuncs.com',
objectKey: 'database-backups/test/backup.tar.gz',
accessKeyId: 'test-access-key',
accessKeySecret: 'test-access-secret',
partSizeBytes,
maxAttempts: 3,
retryBaseDelayMs: 1,
retryMaxDelayMs: 1,
fetchImpl,
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
sleepImpl: async (delayMs) => retryDelays.push(delayMs),
randomFn: () => 0,
});
assertEqual(result.uploadMode, 'multipart', '上传结果必须记录 multipart 模式。');
assertEqual(result.partCount, 3, 'multipart 应按配置大小切成三段。');
assertEqual(result.contentLength, payload.length, '上传结果应保留完整归档长度。');
assertEqual(result.etag, 'complete-etag', '上传结果应保留 CompleteMultipartUpload ETag。');
assertEqual(firstPartAttempts, 2, '503 后应仅重试失败的第一段。');
assertEqual(retryDelays.length, 1, '一次可重试失败应触发一次退避。');
const initiateRequest = requests[0];
assertTrue(initiateRequest.url.endsWith('?uploads'), 'InitiateMultipartUpload URL 必须使用裸 uploads 参数。');
assertTrue(!initiateRequest.url.endsWith('?uploads='), 'InitiateMultipartUpload URL 不能把裸参数写成 uploads=。');
const firstPartRequests = requests.filter(({method, url}) => method === 'PUT' && new URL(url).searchParams.get('partNumber') === '1');
assertEqual(firstPartRequests.length, 2, '第一段应产生原请求和一次重试。');
assertBufferEqual(firstPartRequests[0].body, payload.subarray(0, partSizeBytes), '第一段原请求内容必须完整。');
assertBufferEqual(firstPartRequests[1].body, payload.subarray(0, partSizeBytes), '第一段重试必须重新创建并完整读取 stream。');
assertTrue(
firstPartRequests[0].url.includes('?partNumber=1&uploadId=upload%2B%2F%3D%20id'),
'UploadPart URL 必须使用排序并编码后的 canonical query。',
);
const completeRequest = requests.find(({method, url}) => method === 'POST' && new URL(url).searchParams.has('uploadId'));
assertIncludes(completeRequest?.body.toString('utf8') ?? '', '<PartNumber>1</PartNumber><ETag>&quot;etag-1&quot;</ETag>', 'Complete XML 应包含第一段 ETag。');
assertIncludes(completeRequest?.body.toString('utf8') ?? '', '<PartNumber>3</PartNumber><ETag>&quot;etag-3&quot;</ETag>', 'Complete XML 应包含最后一段 ETag。');
assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 后必须执行签名 HEAD 验证。');
for (const request of requests) {
assertTrue(String(request.headers.authorization ?? '').startsWith('OSS4-HMAC-SHA256 '), `${request.method} 请求必须携带 V4 Authorization。`);
}
}
async function assertHeadLengthMismatchAbortsMultipartUpload() {
const root = path.join(tmpRoot, 'multipart-head-mismatch');
const archivePath = path.join(root, 'backup.tar.gz');
const partSizeBytes = 100 * 1024;
const payload = Buffer.alloc(partSizeBytes + 1, 'x');
mkdirSync(root, {recursive: true});
writeFileSync(archivePath, payload);
const requests = [];
const fetchImpl = async (url, options) => {
await readRequestBody(options.body);
requests.push({url, method: options.method});
const parsedUrl = new URL(url);
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
return new Response('<InitiateMultipartUploadResult><UploadId>mismatch-upload</UploadId></InitiateMultipartUploadResult>', {status: 200});
}
if (options.method === 'PUT') {
return new Response('', {status: 200, headers: {etag: `"etag-${parsedUrl.searchParams.get('partNumber')}"`}});
}
if (options.method === 'POST') {
return new Response('<CompleteMultipartUploadResult/>', {status: 200, headers: {etag: '"complete-etag"'}});
}
if (options.method === 'HEAD') {
return new Response(null, {status: 200, headers: {'content-length': String(payload.length - 1)}});
}
if (options.method === 'DELETE') {
return new Response(null, {status: 204});
}
throw new Error(`unexpected request: ${options.method} ${url}`);
};
let uploadError = null;
try {
await uploadArchive({
archivePath,
bucket: 'genarrative-test',
endpoint: 'oss-cn-shanghai.aliyuncs.com',
objectKey: 'database-backups/test/mismatch.tar.gz',
accessKeyId: 'test-access-key',
accessKeySecret: 'test-access-secret',
partSizeBytes,
maxAttempts: 2,
retryBaseDelayMs: 1,
retryMaxDelayMs: 1,
fetchImpl,
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
sleepImpl: async () => {},
randomFn: () => 0,
});
} catch (error) {
uploadError = error;
}
assertTrue(uploadError instanceof Error, 'HEAD 长度不一致时上传必须失败。');
assertIncludes(uploadError?.message ?? '', 'HEAD 验证长度不一致', 'HEAD 长度不一致错误应保留本地和远端长度。');
const abortRequest = requests.find(({method}) => method === 'DELETE');
assertTrue(Boolean(abortRequest), 'HEAD 长度不一致后必须 best-effort AbortMultipartUpload。');
assertTrue(abortRequest?.url.endsWith('?uploadId=mismatch-upload'), 'AbortMultipartUpload 必须携带同一 uploadId。');
}
async function assertMissingPartEtagAbortsMultipartUpload() {
const root = path.join(tmpRoot, 'multipart-missing-etag');
const archivePath = path.join(root, 'backup.tar.gz');
mkdirSync(root, {recursive: true});
writeFileSync(archivePath, Buffer.alloc(100 * 1024, 'e'));
const requests = [];
const fetchImpl = async (url, options) => {
await readRequestBody(options.body);
requests.push({url, method: options.method});
const parsedUrl = new URL(url);
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
return new Response('<InitiateMultipartUploadResult><UploadId>missing-etag-upload</UploadId></InitiateMultipartUploadResult>', {status: 200});
}
if (options.method === 'PUT') {
return new Response('', {status: 200});
}
if (options.method === 'DELETE') {
return new Response(null, {status: 204});
}
throw new Error(`unexpected request: ${options.method} ${url}`);
};
let uploadError = null;
try {
await uploadArchive({
archivePath,
bucket: 'genarrative-test',
endpoint: 'oss-cn-shanghai.aliyuncs.com',
objectKey: 'database-backups/test/missing-etag.tar.gz',
accessKeyId: 'test-access-key',
accessKeySecret: 'test-access-secret',
partSizeBytes: 100 * 1024,
maxAttempts: 1,
fetchImpl,
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
sleepImpl: async () => {},
randomFn: () => 0,
});
} catch (error) {
uploadError = error;
}
assertIncludes(uploadError?.message ?? '', '响应缺少 ETag', 'UploadPart 缺少 ETag 时必须失败。');
assertTrue(
requests.some(({method, url}) => method === 'DELETE' && url.endsWith('?uploadId=missing-etag-upload')),
'UploadPart 缺少 ETag 后必须 AbortMultipartUpload。',
);
}
async function assertCompleteResponseAmbiguityUsesHeadVerification() {
const root = path.join(tmpRoot, 'multipart-complete-ambiguity');
const archivePath = path.join(root, 'backup.tar.gz');
const payload = Buffer.alloc(100 * 1024, 'c');
mkdirSync(root, {recursive: true});
writeFileSync(archivePath, payload);
const requests = [];
let completeAttempts = 0;
const fetchImpl = async (url, options) => {
await readRequestBody(options.body);
requests.push({url, method: options.method});
const parsedUrl = new URL(url);
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
return new Response('<InitiateMultipartUploadResult><UploadId>ambiguous-upload</UploadId></InitiateMultipartUploadResult>', {status: 200});
}
if (options.method === 'PUT') {
return new Response('', {status: 200, headers: {etag: '"part-etag"'}});
}
if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) {
completeAttempts += 1;
if (completeAttempts === 1) {
throw new TypeError('socket closed after remote complete');
}
return new Response('<Error><Code>NoSuchUpload</Code></Error>', {status: 404});
}
if (options.method === 'HEAD') {
return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}});
}
if (options.method === 'DELETE') {
return new Response(null, {status: 204});
}
throw new Error(`unexpected request: ${options.method} ${url}`);
};
const result = await uploadArchive({
archivePath,
bucket: 'genarrative-test',
endpoint: 'oss-cn-shanghai.aliyuncs.com',
objectKey: 'database-backups/test/complete-ambiguity.tar.gz',
accessKeyId: 'test-access-key',
accessKeySecret: 'test-access-secret',
partSizeBytes: 100 * 1024,
maxAttempts: 2,
retryBaseDelayMs: 1,
retryMaxDelayMs: 1,
fetchImpl,
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
sleepImpl: async () => {},
randomFn: () => 0,
});
assertEqual(completeAttempts, 2, 'Complete 网络错误后应按策略重试。');
assertEqual(result.contentLength, payload.length, 'Complete 结果不确定时应以 HEAD 长度验真收口。');
assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 结果不确定时必须执行 HEAD 验真。');
assertTrue(!requests.some(({method}) => method === 'DELETE'), 'HEAD 已证实对象完整时不得 Abort 已完成上传。');
}
async function readRequestBody(body) {
if (body === undefined || body === null) {
return Buffer.alloc(0);
}
if (typeof body === 'string') {
return Buffer.from(body);
}
if (Buffer.isBuffer(body)) {
return body;
}
const chunks = [];
for await (const chunk of body) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
function createFixture(name) {
const root = path.join(tmpRoot, name);
const binDir = path.join(root, 'bin');
const dataDir = path.join(root, 'data');
const workDir = path.join(root, 'work');
const systemctlLog = path.join(root, 'systemctl.log');
const tarLog = path.join(root, 'tar.log');
mkdirSync(binDir, {recursive: true});
mkdirSync(dataDir, {recursive: true});
writeFileSync(path.join(dataDir, 'sample.bin'), 'sample backup payload\n', 'utf8');
writeExecutable(
path.join(binDir, 'systemctl'),
`#!/usr/bin/env bash
printf 'systemctl %s\\n' "$*" >> "${systemctlLog}"
exit 0
`,
);
writeExecutable(
path.join(binDir, 'tar'),
`#!/usr/bin/env bash
printf 'tar %s\\n' "$*" >> "${tarLog}"
echo 'fake tar failure' >&2
exit 2
`,
);
return {root, binDir, dataDir, workDir, systemctlLog, tarLog};
}
function runBackup(fixture, extraArgs = []) {
return spawnSync(
process.execPath,
[
BACKUP_SCRIPT,
'--data-dir',
fixture.dataDir,
'--work-dir',
fixture.workDir,
'--bucket',
'genarrative-test',
'--endpoint',
'oss-cn-shanghai.aliyuncs.com',
'--access-key-id',
'test',
'--access-key-secret',
'test',
...extraArgs,
],
{
cwd: process.cwd(),
encoding: 'utf8',
env: {
...process.env,
PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`,
},
},
);
}
function writeExecutable(filePath, content) {
writeFileSync(filePath, content, 'utf8');
spawnSync('chmod', ['0755', filePath], {encoding: 'utf8'});
}
function readFile(filePath) {
return existsSync(filePath) ? readFileSync(filePath, 'utf8') : '';
}
function assertStatus(result, expected, reason) {
const actual = result.status ?? 0;
if (actual !== expected) {
failures.push(
`${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
);
}
}
function assertIncludes(content, expected, reason) {
if (!String(content).includes(expected)) {
failures.push(`${reason} 缺少: ${expected}`);
}
}
function assertEqual(actual, expected, reason) {
if (actual !== expected) {
failures.push(`${reason} 预期: ${String(expected)},实际: ${String(actual)}`);
}
}
function assertNotEqual(actual, expected, reason) {
if (actual === expected) {
failures.push(`${reason} 两者均为: ${String(actual)}`);
}
}
function assertTrue(condition, reason) {
if (!condition) {
failures.push(reason);
}
}
function assertBufferEqual(actual, expected, reason) {
if (!Buffer.isBuffer(actual) || !actual.equals(expected)) {
failures.push(`${reason} 预期 ${expected.length} bytes,实际 ${actual?.length ?? '<missing>'} bytes。`);
}
}
function assertFileMissing(filePath, reason) {
if (existsSync(filePath)) {
failures.push(`${reason} 实际存在: ${filePath}\n${readFile(filePath)}`);
}
}