#!/usr/bin/env node import {spawnSync} from 'node:child_process'; import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import path from 'node:path'; import {buildAuthorization, buildCanonicalQuery, uploadArchive} from './database-backup-to-oss.mjs'; const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs'); const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-')); const failures = []; try { await main(); } finally { rmSync(tmpRoot, {recursive: true, force: true}); } if (failures.length > 0) { console.error('[check:database-backup] FAILED'); for (const failure of failures) { console.error(`- ${failure}`); } process.exit(1); } console.log('[check:database-backup] OK'); async function main() { assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); await assertMissingPartEtagAbortsMultipartUpload(); await assertCompleteResponseAmbiguityUsesHeadVerification(); await assertHeadLengthMismatchAbortsMultipartUpload(); } function assertCanonicalQueryAndAuthorizationIncludeMultipartParameters() { assertEqual(buildCanonicalQuery({uploads: null}), 'uploads', 'InitiateMultipartUpload 必须使用无等号的 uploads 参数。'); assertEqual( buildCanonicalQuery({uploadId: 'abc+/= xyz', partNumber: 12}), 'partNumber=12&uploadId=abc%2B%2F%3D%20xyz', 'multipart query 必须按 key 排序并使用 RFC3986 编码。', ); const date = new Date('2026-07-13T10:20:30.000Z'); const headers = { host: 'genarrative-test.oss-cn-shanghai.aliyuncs.com', 'content-type': 'application/octet-stream', 'x-oss-content-sha256': 'UNSIGNED-PAYLOAD', 'x-oss-date': '20260713T102030Z', }; const withoutQuery = buildAuthorization({ method: 'PUT', bucket: 'genarrative-test', endpoint: 'oss-cn-shanghai.aliyuncs.com', objectKey: 'database-backups/test/archive.tar.gz', accessKeyId: 'test-access-key', accessKeySecret: 'test-access-secret', headers, date, }); const withQuery = buildAuthorization({ method: 'PUT', bucket: 'genarrative-test', endpoint: 'oss-cn-shanghai.aliyuncs.com', objectKey: 'database-backups/test/archive.tar.gz', accessKeyId: 'test-access-key', accessKeySecret: 'test-access-secret', headers, date, queries: {partNumber: 12, uploadId: 'abc+/= xyz'}, }); assertNotEqual(withQuery, withoutQuery, 'multipart query 必须参与 V4 Authorization 计算。'); assertEqual( withQuery, 'OSS4-HMAC-SHA256 Credential=test-access-key/20260713/cn-shanghai/oss/aliyun_v4_request,AdditionalHeaders=host,Signature=9323dd3b7272b52f416c4d32115fcc00460eaccdcdaf011575c2502a63a27b1f', 'multipart V4 Authorization 必须保持固定签名向量。', ); } function assertInsufficientSpaceStopsBeforeServiceChanges() { const fixture = createFixture('insufficient-space'); const result = runBackup(fixture, [ '--stop-service', 'spacetimedb.service', '--restart-service-after', 'genarrative-api.service', '--min-free-bytes', '999999999999999999', ]); assertStatus(result, 1, '空间不足时必须失败。'); assertIncludes(result.stdout, '备份空间预检', '空间不足失败前应打印空间预检。'); assertIncludes(result.stderr, '剩余空间不足', '空间不足失败应说明剩余空间不足。'); assertFileMissing(fixture.systemctlLog, '空间不足时不能调用 systemctl。'); assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。'); } function assertArchiveFailureStillRestoresDependentServices() { const fixture = createFixture('tar-failure'); const result = runBackup(fixture, [ '--stop-service', 'spacetimedb.service', '--restart-service-after', 'genarrative-api.service', '--restart-service-after', 'genarrative-external-generation-worker@1.service', '--restart-service-after', 'genarrative-external-generation-controller.service', '--min-free-bytes', '1', ]); assertStatus(result, 1, 'tar 失败时备份脚本必须失败。'); assertIncludes(result.stderr, 'fake tar failure', 'tar 失败原因应保留在错误输出中。'); const systemctlLog = readFile(fixture.systemctlLog); const expectedCommands = [ 'systemctl stop spacetimedb.service', 'systemctl start spacetimedb.service', 'systemctl restart genarrative-api.service', 'systemctl restart genarrative-external-generation-worker@1.service', 'systemctl restart genarrative-external-generation-controller.service', ]; for (const command of expectedCommands) { assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`); } } async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { const root = path.join(tmpRoot, 'multipart-success'); const archivePath = path.join(root, 'backup.tar.gz'); const partSizeBytes = 100 * 1024; const payload = Buffer.concat([ Buffer.alloc(partSizeBytes, 'a'), Buffer.alloc(partSizeBytes, 'b'), Buffer.alloc(17, 'c'), ]); mkdirSync(root, {recursive: true}); writeFileSync(archivePath, payload); const requests = []; const retryDelays = []; let firstPartAttempts = 0; const uploadId = 'upload+/= id'; const fetchImpl = async (url, options) => { const body = await readRequestBody(options.body); requests.push({url, method: options.method, headers: options.headers, body}); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { return new Response(`${uploadId}`, {status: 200}); } if (options.method === 'PUT') { const partNumber = Number(parsedUrl.searchParams.get('partNumber')); if (partNumber === 1) { firstPartAttempts += 1; if (firstPartAttempts === 1) { return new Response('ServiceUnavailable', {status: 503}); } } return new Response('', {status: 200, headers: {etag: `"etag-${partNumber}"`}}); } if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) { return new Response('', {status: 200, headers: {etag: '"complete-etag"'}}); } if (options.method === 'HEAD') { return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}}); } throw new Error(`unexpected request: ${options.method} ${url}`); }; const result = await uploadArchive({ archivePath, bucket: 'genarrative-test', endpoint: 'oss-cn-shanghai.aliyuncs.com', objectKey: 'database-backups/test/backup.tar.gz', accessKeyId: 'test-access-key', accessKeySecret: 'test-access-secret', partSizeBytes, maxAttempts: 3, retryBaseDelayMs: 1, retryMaxDelayMs: 1, fetchImpl, nowFn: () => new Date('2026-07-13T10:20:30.000Z'), sleepImpl: async (delayMs) => retryDelays.push(delayMs), randomFn: () => 0, }); assertEqual(result.uploadMode, 'multipart', '上传结果必须记录 multipart 模式。'); assertEqual(result.partCount, 3, 'multipart 应按配置大小切成三段。'); assertEqual(result.contentLength, payload.length, '上传结果应保留完整归档长度。'); assertEqual(result.etag, 'complete-etag', '上传结果应保留 CompleteMultipartUpload ETag。'); assertEqual(firstPartAttempts, 2, '503 后应仅重试失败的第一段。'); assertEqual(retryDelays.length, 1, '一次可重试失败应触发一次退避。'); const initiateRequest = requests[0]; assertTrue(initiateRequest.url.endsWith('?uploads'), 'InitiateMultipartUpload URL 必须使用裸 uploads 参数。'); assertTrue(!initiateRequest.url.endsWith('?uploads='), 'InitiateMultipartUpload URL 不能把裸参数写成 uploads=。'); const firstPartRequests = requests.filter(({method, url}) => method === 'PUT' && new URL(url).searchParams.get('partNumber') === '1'); assertEqual(firstPartRequests.length, 2, '第一段应产生原请求和一次重试。'); assertBufferEqual(firstPartRequests[0].body, payload.subarray(0, partSizeBytes), '第一段原请求内容必须完整。'); assertBufferEqual(firstPartRequests[1].body, payload.subarray(0, partSizeBytes), '第一段重试必须重新创建并完整读取 stream。'); assertTrue( firstPartRequests[0].url.includes('?partNumber=1&uploadId=upload%2B%2F%3D%20id'), 'UploadPart URL 必须使用排序并编码后的 canonical query。', ); const completeRequest = requests.find(({method, url}) => method === 'POST' && new URL(url).searchParams.has('uploadId')); assertIncludes(completeRequest?.body.toString('utf8') ?? '', '1"etag-1"', 'Complete XML 应包含第一段 ETag。'); assertIncludes(completeRequest?.body.toString('utf8') ?? '', '3"etag-3"', 'Complete XML 应包含最后一段 ETag。'); assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 后必须执行签名 HEAD 验证。'); for (const request of requests) { assertTrue(String(request.headers.authorization ?? '').startsWith('OSS4-HMAC-SHA256 '), `${request.method} 请求必须携带 V4 Authorization。`); } } async function assertHeadLengthMismatchAbortsMultipartUpload() { const root = path.join(tmpRoot, 'multipart-head-mismatch'); const archivePath = path.join(root, 'backup.tar.gz'); const partSizeBytes = 100 * 1024; const payload = Buffer.alloc(partSizeBytes + 1, 'x'); mkdirSync(root, {recursive: true}); writeFileSync(archivePath, payload); const requests = []; const fetchImpl = async (url, options) => { await readRequestBody(options.body); requests.push({url, method: options.method}); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { return new Response('mismatch-upload', {status: 200}); } if (options.method === 'PUT') { return new Response('', {status: 200, headers: {etag: `"etag-${parsedUrl.searchParams.get('partNumber')}"`}}); } if (options.method === 'POST') { return new Response('', {status: 200, headers: {etag: '"complete-etag"'}}); } if (options.method === 'HEAD') { return new Response(null, {status: 200, headers: {'content-length': String(payload.length - 1)}}); } if (options.method === 'DELETE') { return new Response(null, {status: 204}); } throw new Error(`unexpected request: ${options.method} ${url}`); }; let uploadError = null; try { await uploadArchive({ archivePath, bucket: 'genarrative-test', endpoint: 'oss-cn-shanghai.aliyuncs.com', objectKey: 'database-backups/test/mismatch.tar.gz', accessKeyId: 'test-access-key', accessKeySecret: 'test-access-secret', partSizeBytes, maxAttempts: 2, retryBaseDelayMs: 1, retryMaxDelayMs: 1, fetchImpl, nowFn: () => new Date('2026-07-13T10:20:30.000Z'), sleepImpl: async () => {}, randomFn: () => 0, }); } catch (error) { uploadError = error; } assertTrue(uploadError instanceof Error, 'HEAD 长度不一致时上传必须失败。'); assertIncludes(uploadError?.message ?? '', 'HEAD 验证长度不一致', 'HEAD 长度不一致错误应保留本地和远端长度。'); const abortRequest = requests.find(({method}) => method === 'DELETE'); assertTrue(Boolean(abortRequest), 'HEAD 长度不一致后必须 best-effort AbortMultipartUpload。'); assertTrue(abortRequest?.url.endsWith('?uploadId=mismatch-upload'), 'AbortMultipartUpload 必须携带同一 uploadId。'); } async function assertMissingPartEtagAbortsMultipartUpload() { const root = path.join(tmpRoot, 'multipart-missing-etag'); const archivePath = path.join(root, 'backup.tar.gz'); mkdirSync(root, {recursive: true}); writeFileSync(archivePath, Buffer.alloc(100 * 1024, 'e')); const requests = []; const fetchImpl = async (url, options) => { await readRequestBody(options.body); requests.push({url, method: options.method}); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { return new Response('missing-etag-upload', {status: 200}); } if (options.method === 'PUT') { return new Response('', {status: 200}); } if (options.method === 'DELETE') { return new Response(null, {status: 204}); } throw new Error(`unexpected request: ${options.method} ${url}`); }; let uploadError = null; try { await uploadArchive({ archivePath, bucket: 'genarrative-test', endpoint: 'oss-cn-shanghai.aliyuncs.com', objectKey: 'database-backups/test/missing-etag.tar.gz', accessKeyId: 'test-access-key', accessKeySecret: 'test-access-secret', partSizeBytes: 100 * 1024, maxAttempts: 1, fetchImpl, nowFn: () => new Date('2026-07-13T10:20:30.000Z'), sleepImpl: async () => {}, randomFn: () => 0, }); } catch (error) { uploadError = error; } assertIncludes(uploadError?.message ?? '', '响应缺少 ETag', 'UploadPart 缺少 ETag 时必须失败。'); assertTrue( requests.some(({method, url}) => method === 'DELETE' && url.endsWith('?uploadId=missing-etag-upload')), 'UploadPart 缺少 ETag 后必须 AbortMultipartUpload。', ); } async function assertCompleteResponseAmbiguityUsesHeadVerification() { const root = path.join(tmpRoot, 'multipart-complete-ambiguity'); const archivePath = path.join(root, 'backup.tar.gz'); const payload = Buffer.alloc(100 * 1024, 'c'); mkdirSync(root, {recursive: true}); writeFileSync(archivePath, payload); const requests = []; let completeAttempts = 0; const fetchImpl = async (url, options) => { await readRequestBody(options.body); requests.push({url, method: options.method}); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { return new Response('ambiguous-upload', {status: 200}); } if (options.method === 'PUT') { return new Response('', {status: 200, headers: {etag: '"part-etag"'}}); } if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) { completeAttempts += 1; if (completeAttempts === 1) { throw new TypeError('socket closed after remote complete'); } return new Response('NoSuchUpload', {status: 404}); } if (options.method === 'HEAD') { return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}}); } if (options.method === 'DELETE') { return new Response(null, {status: 204}); } throw new Error(`unexpected request: ${options.method} ${url}`); }; const result = await uploadArchive({ archivePath, bucket: 'genarrative-test', endpoint: 'oss-cn-shanghai.aliyuncs.com', objectKey: 'database-backups/test/complete-ambiguity.tar.gz', accessKeyId: 'test-access-key', accessKeySecret: 'test-access-secret', partSizeBytes: 100 * 1024, maxAttempts: 2, retryBaseDelayMs: 1, retryMaxDelayMs: 1, fetchImpl, nowFn: () => new Date('2026-07-13T10:20:30.000Z'), sleepImpl: async () => {}, randomFn: () => 0, }); assertEqual(completeAttempts, 2, 'Complete 网络错误后应按策略重试。'); assertEqual(result.contentLength, payload.length, 'Complete 结果不确定时应以 HEAD 长度验真收口。'); assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 结果不确定时必须执行 HEAD 验真。'); assertTrue(!requests.some(({method}) => method === 'DELETE'), 'HEAD 已证实对象完整时不得 Abort 已完成上传。'); } async function readRequestBody(body) { if (body === undefined || body === null) { return Buffer.alloc(0); } if (typeof body === 'string') { return Buffer.from(body); } if (Buffer.isBuffer(body)) { return body; } const chunks = []; for await (const chunk of body) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); } return Buffer.concat(chunks); } function createFixture(name) { const root = path.join(tmpRoot, name); const binDir = path.join(root, 'bin'); const dataDir = path.join(root, 'data'); const workDir = path.join(root, 'work'); const systemctlLog = path.join(root, 'systemctl.log'); const tarLog = path.join(root, 'tar.log'); mkdirSync(binDir, {recursive: true}); mkdirSync(dataDir, {recursive: true}); writeFileSync(path.join(dataDir, 'sample.bin'), 'sample backup payload\n', 'utf8'); writeExecutable( path.join(binDir, 'systemctl'), `#!/usr/bin/env bash printf 'systemctl %s\\n' "$*" >> "${systemctlLog}" exit 0 `, ); writeExecutable( path.join(binDir, 'tar'), `#!/usr/bin/env bash printf 'tar %s\\n' "$*" >> "${tarLog}" echo 'fake tar failure' >&2 exit 2 `, ); return {root, binDir, dataDir, workDir, systemctlLog, tarLog}; } function runBackup(fixture, extraArgs = []) { return spawnSync( process.execPath, [ BACKUP_SCRIPT, '--data-dir', fixture.dataDir, '--work-dir', fixture.workDir, '--bucket', 'genarrative-test', '--endpoint', 'oss-cn-shanghai.aliyuncs.com', '--access-key-id', 'test', '--access-key-secret', 'test', ...extraArgs, ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`, }, }, ); } function writeExecutable(filePath, content) { writeFileSync(filePath, content, 'utf8'); spawnSync('chmod', ['0755', filePath], {encoding: 'utf8'}); } function readFile(filePath) { return existsSync(filePath) ? readFileSync(filePath, 'utf8') : ''; } function assertStatus(result, expected, reason) { const actual = result.status ?? 0; if (actual !== expected) { failures.push( `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ); } } function assertIncludes(content, expected, reason) { if (!String(content).includes(expected)) { failures.push(`${reason} 缺少: ${expected}`); } } function assertEqual(actual, expected, reason) { if (actual !== expected) { failures.push(`${reason} 预期: ${String(expected)},实际: ${String(actual)}`); } } function assertNotEqual(actual, expected, reason) { if (actual === expected) { failures.push(`${reason} 两者均为: ${String(actual)}`); } } function assertTrue(condition, reason) { if (!condition) { failures.push(reason); } } function assertBufferEqual(actual, expected, reason) { if (!Buffer.isBuffer(actual) || !actual.equals(expected)) { failures.push(`${reason} 预期 ${expected.length} bytes,实际 ${actual?.length ?? ''} bytes。`); } } function assertFileMissing(filePath, reason) { if (existsSync(filePath)) { failures.push(`${reason} 实际存在: ${filePath}\n${readFile(filePath)}`); } }