修复数据库冷备分片上传
冷备归档改用 OSS Multipart 顺序分片上传并加入可重试请求。 Complete 后通过签名 HEAD 严格核对远端对象长度,再更新清单或清理本地归档。 补齐签名查询、分片重试、缺失 ETag、Complete 歧义和 Abort 回归测试。 记录数据库冷备上传与清理边界的长期运维决策。
This commit is contained in:
@@ -3997,3 +3997,11 @@
|
||||
- 参数刷新:Jenkinsfile 是参数事实源。推送后必须让 Full 与 API Deploy live Job 安全加载一次新 Jenkinsfile,再只读确认两个参数已进入 `config.xml`;只在 Jenkins UI 手工加参数不是持久修复。
|
||||
- 影响范围:Full / API Deploy Jenkinsfile、API 发布脚本、生产 API deploy fixture、生产运维门禁与 live Job 参数 schema。
|
||||
- 验证方式:`bash -n scripts/deploy/production-api-deploy.sh`、`npm run check:production-api-deploy`、`npm run check:production-ops`、`npm run check:encoding`、`git diff --check`。
|
||||
|
||||
## 2026-07-13 数据库冷备使用 OSS Multipart 上传并在清理前验真
|
||||
|
||||
- 背景:SpacetimeDB 冷备归档已经超过 OSS 单次 PutObject 的 5 GiB 上限,单请求上传会稳定失败并让本地归档持续积压;网络中断还可能让 CompleteMultipartUpload 的客户端结果不确定。
|
||||
- 决策:`scripts/database-backup-to-oss.mjs` 对备份归档统一使用 OSS Multipart Upload,默认按 128 MiB 顺序分片;每个分片请求重新创建文件流、时间和 V4 签名,仅对网络错误、HTTP 408 / 429 / 5xx 做有限重试。V4 canonical query 必须同时支持无等号的 `uploads` 子资源和带值的 `partNumber` / `uploadId` 参数。
|
||||
- 验真与清理边界:Complete 后必须发送签名 HEAD,并严格核对 OSS `Content-Length` 与本地归档大小;Complete 响应不确定时也先用 HEAD 判定对象是否已经完整落盘。只有验真成功后才能把 manifest 标记为 `uploaded`,并按 `keepLocal` 决定是否删除本地归档;失败时 best-effort AbortMultipartUpload,不得提前更新 manifest 或清理本地文件。
|
||||
- 影响范围:数据库备份 OSS 上传实现、备份回归门禁、release 本地归档保留与 timer 恢复流程。
|
||||
- 验证方式:`npm run check:database-backup`、`npm run check:production-ops`、`npm run check:encoding`、`git diff --check`;线上先对既有归档使用 `--upload-archive ... --keep-local`,确认 OSS 对象长度和可恢复性后再清理积压并恢复 timer。
|
||||
|
||||
@@ -5,12 +5,14 @@ import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync}
|
||||
import {tmpdir} from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import {buildAuthorization, buildCanonicalQuery, uploadArchive} from './database-backup-to-oss.mjs';
|
||||
|
||||
const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs');
|
||||
const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-'));
|
||||
const failures = [];
|
||||
|
||||
try {
|
||||
main();
|
||||
await main();
|
||||
} finally {
|
||||
rmSync(tmpRoot, {recursive: true, force: true});
|
||||
}
|
||||
@@ -25,9 +27,58 @@ if (failures.length > 0) {
|
||||
|
||||
console.log('[check:database-backup] OK');
|
||||
|
||||
function main() {
|
||||
async function main() {
|
||||
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
|
||||
assertInsufficientSpaceStopsBeforeServiceChanges();
|
||||
assertArchiveFailureStillRestoresDependentServices();
|
||||
await assertMultipartUploadRetriesAndVerifiesRemoteLength();
|
||||
await assertMissingPartEtagAbortsMultipartUpload();
|
||||
await assertCompleteResponseAmbiguityUsesHeadVerification();
|
||||
await assertHeadLengthMismatchAbortsMultipartUpload();
|
||||
}
|
||||
|
||||
function assertCanonicalQueryAndAuthorizationIncludeMultipartParameters() {
|
||||
assertEqual(buildCanonicalQuery({uploads: null}), 'uploads', 'InitiateMultipartUpload 必须使用无等号的 uploads 参数。');
|
||||
assertEqual(
|
||||
buildCanonicalQuery({uploadId: 'abc+/= xyz', partNumber: 12}),
|
||||
'partNumber=12&uploadId=abc%2B%2F%3D%20xyz',
|
||||
'multipart query 必须按 key 排序并使用 RFC3986 编码。',
|
||||
);
|
||||
|
||||
const date = new Date('2026-07-13T10:20:30.000Z');
|
||||
const headers = {
|
||||
host: 'genarrative-test.oss-cn-shanghai.aliyuncs.com',
|
||||
'content-type': 'application/octet-stream',
|
||||
'x-oss-content-sha256': 'UNSIGNED-PAYLOAD',
|
||||
'x-oss-date': '20260713T102030Z',
|
||||
};
|
||||
const withoutQuery = buildAuthorization({
|
||||
method: 'PUT',
|
||||
bucket: 'genarrative-test',
|
||||
endpoint: 'oss-cn-shanghai.aliyuncs.com',
|
||||
objectKey: 'database-backups/test/archive.tar.gz',
|
||||
accessKeyId: 'test-access-key',
|
||||
accessKeySecret: 'test-access-secret',
|
||||
headers,
|
||||
date,
|
||||
});
|
||||
const withQuery = buildAuthorization({
|
||||
method: 'PUT',
|
||||
bucket: 'genarrative-test',
|
||||
endpoint: 'oss-cn-shanghai.aliyuncs.com',
|
||||
objectKey: 'database-backups/test/archive.tar.gz',
|
||||
accessKeyId: 'test-access-key',
|
||||
accessKeySecret: 'test-access-secret',
|
||||
headers,
|
||||
date,
|
||||
queries: {partNumber: 12, uploadId: 'abc+/= xyz'},
|
||||
});
|
||||
assertNotEqual(withQuery, withoutQuery, 'multipart query 必须参与 V4 Authorization 计算。');
|
||||
assertEqual(
|
||||
withQuery,
|
||||
'OSS4-HMAC-SHA256 Credential=test-access-key/20260713/cn-shanghai/oss/aliyun_v4_request,AdditionalHeaders=host,Signature=9323dd3b7272b52f416c4d32115fcc00460eaccdcdaf011575c2502a63a27b1f',
|
||||
'multipart V4 Authorization 必须保持固定签名向量。',
|
||||
);
|
||||
}
|
||||
|
||||
function assertInsufficientSpaceStopsBeforeServiceChanges() {
|
||||
@@ -78,6 +129,279 @@ function assertArchiveFailureStillRestoresDependentServices() {
|
||||
}
|
||||
}
|
||||
|
||||
async function assertMultipartUploadRetriesAndVerifiesRemoteLength() {
|
||||
const root = path.join(tmpRoot, 'multipart-success');
|
||||
const archivePath = path.join(root, 'backup.tar.gz');
|
||||
const partSizeBytes = 100 * 1024;
|
||||
const payload = Buffer.concat([
|
||||
Buffer.alloc(partSizeBytes, 'a'),
|
||||
Buffer.alloc(partSizeBytes, 'b'),
|
||||
Buffer.alloc(17, 'c'),
|
||||
]);
|
||||
mkdirSync(root, {recursive: true});
|
||||
writeFileSync(archivePath, payload);
|
||||
|
||||
const requests = [];
|
||||
const retryDelays = [];
|
||||
let firstPartAttempts = 0;
|
||||
const uploadId = 'upload+/= id';
|
||||
const fetchImpl = async (url, options) => {
|
||||
const body = await readRequestBody(options.body);
|
||||
requests.push({url, method: options.method, headers: options.headers, body});
|
||||
const parsedUrl = new URL(url);
|
||||
|
||||
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
|
||||
return new Response(`<InitiateMultipartUploadResult><UploadId>${uploadId}</UploadId></InitiateMultipartUploadResult>`, {status: 200});
|
||||
}
|
||||
if (options.method === 'PUT') {
|
||||
const partNumber = Number(parsedUrl.searchParams.get('partNumber'));
|
||||
if (partNumber === 1) {
|
||||
firstPartAttempts += 1;
|
||||
if (firstPartAttempts === 1) {
|
||||
return new Response('<Error><Code>ServiceUnavailable</Code></Error>', {status: 503});
|
||||
}
|
||||
}
|
||||
return new Response('', {status: 200, headers: {etag: `"etag-${partNumber}"`}});
|
||||
}
|
||||
if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) {
|
||||
return new Response('<CompleteMultipartUploadResult/>', {status: 200, headers: {etag: '"complete-etag"'}});
|
||||
}
|
||||
if (options.method === 'HEAD') {
|
||||
return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}});
|
||||
}
|
||||
throw new Error(`unexpected request: ${options.method} ${url}`);
|
||||
};
|
||||
|
||||
const result = await uploadArchive({
|
||||
archivePath,
|
||||
bucket: 'genarrative-test',
|
||||
endpoint: 'oss-cn-shanghai.aliyuncs.com',
|
||||
objectKey: 'database-backups/test/backup.tar.gz',
|
||||
accessKeyId: 'test-access-key',
|
||||
accessKeySecret: 'test-access-secret',
|
||||
partSizeBytes,
|
||||
maxAttempts: 3,
|
||||
retryBaseDelayMs: 1,
|
||||
retryMaxDelayMs: 1,
|
||||
fetchImpl,
|
||||
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
|
||||
sleepImpl: async (delayMs) => retryDelays.push(delayMs),
|
||||
randomFn: () => 0,
|
||||
});
|
||||
|
||||
assertEqual(result.uploadMode, 'multipart', '上传结果必须记录 multipart 模式。');
|
||||
assertEqual(result.partCount, 3, 'multipart 应按配置大小切成三段。');
|
||||
assertEqual(result.contentLength, payload.length, '上传结果应保留完整归档长度。');
|
||||
assertEqual(result.etag, 'complete-etag', '上传结果应保留 CompleteMultipartUpload ETag。');
|
||||
assertEqual(firstPartAttempts, 2, '503 后应仅重试失败的第一段。');
|
||||
assertEqual(retryDelays.length, 1, '一次可重试失败应触发一次退避。');
|
||||
|
||||
const initiateRequest = requests[0];
|
||||
assertTrue(initiateRequest.url.endsWith('?uploads'), 'InitiateMultipartUpload URL 必须使用裸 uploads 参数。');
|
||||
assertTrue(!initiateRequest.url.endsWith('?uploads='), 'InitiateMultipartUpload URL 不能把裸参数写成 uploads=。');
|
||||
const firstPartRequests = requests.filter(({method, url}) => method === 'PUT' && new URL(url).searchParams.get('partNumber') === '1');
|
||||
assertEqual(firstPartRequests.length, 2, '第一段应产生原请求和一次重试。');
|
||||
assertBufferEqual(firstPartRequests[0].body, payload.subarray(0, partSizeBytes), '第一段原请求内容必须完整。');
|
||||
assertBufferEqual(firstPartRequests[1].body, payload.subarray(0, partSizeBytes), '第一段重试必须重新创建并完整读取 stream。');
|
||||
assertTrue(
|
||||
firstPartRequests[0].url.includes('?partNumber=1&uploadId=upload%2B%2F%3D%20id'),
|
||||
'UploadPart URL 必须使用排序并编码后的 canonical query。',
|
||||
);
|
||||
|
||||
const completeRequest = requests.find(({method, url}) => method === 'POST' && new URL(url).searchParams.has('uploadId'));
|
||||
assertIncludes(completeRequest?.body.toString('utf8') ?? '', '<PartNumber>1</PartNumber><ETag>"etag-1"</ETag>', 'Complete XML 应包含第一段 ETag。');
|
||||
assertIncludes(completeRequest?.body.toString('utf8') ?? '', '<PartNumber>3</PartNumber><ETag>"etag-3"</ETag>', 'Complete XML 应包含最后一段 ETag。');
|
||||
assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 后必须执行签名 HEAD 验证。');
|
||||
for (const request of requests) {
|
||||
assertTrue(String(request.headers.authorization ?? '').startsWith('OSS4-HMAC-SHA256 '), `${request.method} 请求必须携带 V4 Authorization。`);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertHeadLengthMismatchAbortsMultipartUpload() {
|
||||
const root = path.join(tmpRoot, 'multipart-head-mismatch');
|
||||
const archivePath = path.join(root, 'backup.tar.gz');
|
||||
const partSizeBytes = 100 * 1024;
|
||||
const payload = Buffer.alloc(partSizeBytes + 1, 'x');
|
||||
mkdirSync(root, {recursive: true});
|
||||
writeFileSync(archivePath, payload);
|
||||
|
||||
const requests = [];
|
||||
const fetchImpl = async (url, options) => {
|
||||
await readRequestBody(options.body);
|
||||
requests.push({url, method: options.method});
|
||||
const parsedUrl = new URL(url);
|
||||
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
|
||||
return new Response('<InitiateMultipartUploadResult><UploadId>mismatch-upload</UploadId></InitiateMultipartUploadResult>', {status: 200});
|
||||
}
|
||||
if (options.method === 'PUT') {
|
||||
return new Response('', {status: 200, headers: {etag: `"etag-${parsedUrl.searchParams.get('partNumber')}"`}});
|
||||
}
|
||||
if (options.method === 'POST') {
|
||||
return new Response('<CompleteMultipartUploadResult/>', {status: 200, headers: {etag: '"complete-etag"'}});
|
||||
}
|
||||
if (options.method === 'HEAD') {
|
||||
return new Response(null, {status: 200, headers: {'content-length': String(payload.length - 1)}});
|
||||
}
|
||||
if (options.method === 'DELETE') {
|
||||
return new Response(null, {status: 204});
|
||||
}
|
||||
throw new Error(`unexpected request: ${options.method} ${url}`);
|
||||
};
|
||||
|
||||
let uploadError = null;
|
||||
try {
|
||||
await uploadArchive({
|
||||
archivePath,
|
||||
bucket: 'genarrative-test',
|
||||
endpoint: 'oss-cn-shanghai.aliyuncs.com',
|
||||
objectKey: 'database-backups/test/mismatch.tar.gz',
|
||||
accessKeyId: 'test-access-key',
|
||||
accessKeySecret: 'test-access-secret',
|
||||
partSizeBytes,
|
||||
maxAttempts: 2,
|
||||
retryBaseDelayMs: 1,
|
||||
retryMaxDelayMs: 1,
|
||||
fetchImpl,
|
||||
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
|
||||
sleepImpl: async () => {},
|
||||
randomFn: () => 0,
|
||||
});
|
||||
} catch (error) {
|
||||
uploadError = error;
|
||||
}
|
||||
|
||||
assertTrue(uploadError instanceof Error, 'HEAD 长度不一致时上传必须失败。');
|
||||
assertIncludes(uploadError?.message ?? '', 'HEAD 验证长度不一致', 'HEAD 长度不一致错误应保留本地和远端长度。');
|
||||
const abortRequest = requests.find(({method}) => method === 'DELETE');
|
||||
assertTrue(Boolean(abortRequest), 'HEAD 长度不一致后必须 best-effort AbortMultipartUpload。');
|
||||
assertTrue(abortRequest?.url.endsWith('?uploadId=mismatch-upload'), 'AbortMultipartUpload 必须携带同一 uploadId。');
|
||||
}
|
||||
|
||||
async function assertMissingPartEtagAbortsMultipartUpload() {
|
||||
const root = path.join(tmpRoot, 'multipart-missing-etag');
|
||||
const archivePath = path.join(root, 'backup.tar.gz');
|
||||
mkdirSync(root, {recursive: true});
|
||||
writeFileSync(archivePath, Buffer.alloc(100 * 1024, 'e'));
|
||||
|
||||
const requests = [];
|
||||
const fetchImpl = async (url, options) => {
|
||||
await readRequestBody(options.body);
|
||||
requests.push({url, method: options.method});
|
||||
const parsedUrl = new URL(url);
|
||||
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
|
||||
return new Response('<InitiateMultipartUploadResult><UploadId>missing-etag-upload</UploadId></InitiateMultipartUploadResult>', {status: 200});
|
||||
}
|
||||
if (options.method === 'PUT') {
|
||||
return new Response('', {status: 200});
|
||||
}
|
||||
if (options.method === 'DELETE') {
|
||||
return new Response(null, {status: 204});
|
||||
}
|
||||
throw new Error(`unexpected request: ${options.method} ${url}`);
|
||||
};
|
||||
|
||||
let uploadError = null;
|
||||
try {
|
||||
await uploadArchive({
|
||||
archivePath,
|
||||
bucket: 'genarrative-test',
|
||||
endpoint: 'oss-cn-shanghai.aliyuncs.com',
|
||||
objectKey: 'database-backups/test/missing-etag.tar.gz',
|
||||
accessKeyId: 'test-access-key',
|
||||
accessKeySecret: 'test-access-secret',
|
||||
partSizeBytes: 100 * 1024,
|
||||
maxAttempts: 1,
|
||||
fetchImpl,
|
||||
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
|
||||
sleepImpl: async () => {},
|
||||
randomFn: () => 0,
|
||||
});
|
||||
} catch (error) {
|
||||
uploadError = error;
|
||||
}
|
||||
|
||||
assertIncludes(uploadError?.message ?? '', '响应缺少 ETag', 'UploadPart 缺少 ETag 时必须失败。');
|
||||
assertTrue(
|
||||
requests.some(({method, url}) => method === 'DELETE' && url.endsWith('?uploadId=missing-etag-upload')),
|
||||
'UploadPart 缺少 ETag 后必须 AbortMultipartUpload。',
|
||||
);
|
||||
}
|
||||
|
||||
async function assertCompleteResponseAmbiguityUsesHeadVerification() {
|
||||
const root = path.join(tmpRoot, 'multipart-complete-ambiguity');
|
||||
const archivePath = path.join(root, 'backup.tar.gz');
|
||||
const payload = Buffer.alloc(100 * 1024, 'c');
|
||||
mkdirSync(root, {recursive: true});
|
||||
writeFileSync(archivePath, payload);
|
||||
|
||||
const requests = [];
|
||||
let completeAttempts = 0;
|
||||
const fetchImpl = async (url, options) => {
|
||||
await readRequestBody(options.body);
|
||||
requests.push({url, method: options.method});
|
||||
const parsedUrl = new URL(url);
|
||||
if (options.method === 'POST' && parsedUrl.search === '?uploads') {
|
||||
return new Response('<InitiateMultipartUploadResult><UploadId>ambiguous-upload</UploadId></InitiateMultipartUploadResult>', {status: 200});
|
||||
}
|
||||
if (options.method === 'PUT') {
|
||||
return new Response('', {status: 200, headers: {etag: '"part-etag"'}});
|
||||
}
|
||||
if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) {
|
||||
completeAttempts += 1;
|
||||
if (completeAttempts === 1) {
|
||||
throw new TypeError('socket closed after remote complete');
|
||||
}
|
||||
return new Response('<Error><Code>NoSuchUpload</Code></Error>', {status: 404});
|
||||
}
|
||||
if (options.method === 'HEAD') {
|
||||
return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}});
|
||||
}
|
||||
if (options.method === 'DELETE') {
|
||||
return new Response(null, {status: 204});
|
||||
}
|
||||
throw new Error(`unexpected request: ${options.method} ${url}`);
|
||||
};
|
||||
|
||||
const result = await uploadArchive({
|
||||
archivePath,
|
||||
bucket: 'genarrative-test',
|
||||
endpoint: 'oss-cn-shanghai.aliyuncs.com',
|
||||
objectKey: 'database-backups/test/complete-ambiguity.tar.gz',
|
||||
accessKeyId: 'test-access-key',
|
||||
accessKeySecret: 'test-access-secret',
|
||||
partSizeBytes: 100 * 1024,
|
||||
maxAttempts: 2,
|
||||
retryBaseDelayMs: 1,
|
||||
retryMaxDelayMs: 1,
|
||||
fetchImpl,
|
||||
nowFn: () => new Date('2026-07-13T10:20:30.000Z'),
|
||||
sleepImpl: async () => {},
|
||||
randomFn: () => 0,
|
||||
});
|
||||
|
||||
assertEqual(completeAttempts, 2, 'Complete 网络错误后应按策略重试。');
|
||||
assertEqual(result.contentLength, payload.length, 'Complete 结果不确定时应以 HEAD 长度验真收口。');
|
||||
assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 结果不确定时必须执行 HEAD 验真。');
|
||||
assertTrue(!requests.some(({method}) => method === 'DELETE'), 'HEAD 已证实对象完整时不得 Abort 已完成上传。');
|
||||
}
|
||||
|
||||
async function readRequestBody(body) {
|
||||
if (body === undefined || body === null) {
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
if (typeof body === 'string') {
|
||||
return Buffer.from(body);
|
||||
}
|
||||
if (Buffer.isBuffer(body)) {
|
||||
return body;
|
||||
}
|
||||
const chunks = [];
|
||||
for await (const chunk of body) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
function createFixture(name) {
|
||||
const root = path.join(tmpRoot, name);
|
||||
const binDir = path.join(root, 'bin');
|
||||
@@ -160,6 +484,30 @@ function assertIncludes(content, expected, reason) {
|
||||
}
|
||||
}
|
||||
|
||||
function assertEqual(actual, expected, reason) {
|
||||
if (actual !== expected) {
|
||||
failures.push(`${reason} 预期: ${String(expected)},实际: ${String(actual)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertNotEqual(actual, expected, reason) {
|
||||
if (actual === expected) {
|
||||
failures.push(`${reason} 两者均为: ${String(actual)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertTrue(condition, reason) {
|
||||
if (!condition) {
|
||||
failures.push(reason);
|
||||
}
|
||||
}
|
||||
|
||||
function assertBufferEqual(actual, expected, reason) {
|
||||
if (!Buffer.isBuffer(actual) || !actual.equals(expected)) {
|
||||
failures.push(`${reason} 预期 ${expected.length} bytes,实际 ${actual?.length ?? '<missing>'} bytes。`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertFileMissing(filePath, reason) {
|
||||
if (existsSync(filePath)) {
|
||||
failures.push(`${reason} 实际存在: ${filePath}\n${readFile(filePath)}`);
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import {spawnSync} from 'node:child_process';
|
||||
import {createHash, createHmac} from 'node:crypto';
|
||||
import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, statfsSync, writeFileSync} from 'node:fs';
|
||||
import {createReadStream, existsSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync, statfsSync, writeFileSync} from 'node:fs';
|
||||
import {basename, dirname, isAbsolute, resolve} from 'node:path';
|
||||
import {setTimeout as sleep} from 'node:timers/promises';
|
||||
import {fileURLToPath} from 'node:url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
@@ -18,6 +19,14 @@ const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
|
||||
const OSS_SERVICE = 'oss';
|
||||
const OSS_REQUEST = 'aliyun_v4_request';
|
||||
const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD';
|
||||
const DEFAULT_OSS_MULTIPART_PART_SIZE_BYTES = 128 * 1024 * 1024;
|
||||
const OSS_MIN_MULTIPART_PART_SIZE_BYTES = 100 * 1024;
|
||||
const OSS_MAX_MULTIPART_PART_SIZE_BYTES = 5 * 1024 * 1024 * 1024;
|
||||
const OSS_MAX_MULTIPART_PARTS = 10_000;
|
||||
const DEFAULT_OSS_REQUEST_MAX_ATTEMPTS = 5;
|
||||
const DEFAULT_OSS_RETRY_BASE_DELAY_MS = 1_000;
|
||||
const DEFAULT_OSS_RETRY_MAX_DELAY_MS = 30_000;
|
||||
const RETRYABLE_OSS_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
|
||||
|
||||
function usage() {
|
||||
console.log(`用法:
|
||||
@@ -484,11 +493,33 @@ function encodePath(path) {
|
||||
.join('/');
|
||||
}
|
||||
|
||||
function encodeQueryComponent(value) {
|
||||
return encodeURIComponent(String(value)).replace(
|
||||
/[!'()*]/gu,
|
||||
(char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildCanonicalQuery(queries = {}) {
|
||||
return Object.entries(queries)
|
||||
.map(([key, value]) => [encodeQueryComponent(key), value === null ? null : encodeQueryComponent(value)])
|
||||
.sort(([leftKey, leftValue], [rightKey, rightValue]) => {
|
||||
if (leftKey !== rightKey) {
|
||||
return leftKey < rightKey ? -1 : 1;
|
||||
}
|
||||
const left = leftValue ?? '';
|
||||
const right = rightValue ?? '';
|
||||
return left === right ? 0 : left < right ? -1 : 1;
|
||||
})
|
||||
.map(([key, value]) => value === null ? key : `${key}=${value}`)
|
||||
.join('&');
|
||||
}
|
||||
|
||||
function canonicalHeaderValue(value) {
|
||||
return String(value).trim().replace(/\s+/gu, ' ');
|
||||
}
|
||||
|
||||
function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date}) {
|
||||
export function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date, queries = {}}) {
|
||||
const region = regionFromEndpoint(endpoint);
|
||||
const scopeDate = formatScopeDate(date);
|
||||
const scope = `${scopeDate}/${region}/${OSS_SERVICE}/${OSS_REQUEST}`;
|
||||
@@ -504,7 +535,7 @@ function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, a
|
||||
const canonicalRequest = [
|
||||
method,
|
||||
canonicalUri,
|
||||
'',
|
||||
buildCanonicalQuery(queries),
|
||||
canonicalHeaders,
|
||||
additionalHeaders,
|
||||
UNSIGNED_PAYLOAD,
|
||||
@@ -518,56 +549,304 @@ function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, a
|
||||
return `${OSS_ALGORITHM} Credential=${accessKeyId}/${scope},AdditionalHeaders=${additionalHeaders},Signature=${finalSignature}`;
|
||||
}
|
||||
|
||||
async function uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}) {
|
||||
function buildOssUrl({bucket, endpoint, objectKey, queries = {}}) {
|
||||
const canonicalQuery = buildCanonicalQuery(queries);
|
||||
return `https://${bucket}.${endpoint}/${encodePath(objectKey)}${canonicalQuery ? `?${canonicalQuery}` : ''}`;
|
||||
}
|
||||
|
||||
function isRetryableOssStatus(status) {
|
||||
return RETRYABLE_OSS_HTTP_STATUSES.has(status);
|
||||
}
|
||||
|
||||
function retryDelayMs({attempt, baseDelayMs, maxDelayMs, randomFn}) {
|
||||
const ceiling = Math.min(maxDelayMs, baseDelayMs * (2 ** Math.max(0, attempt - 1)));
|
||||
return Math.floor(randomFn() * ceiling);
|
||||
}
|
||||
|
||||
async function signedOssRequest({
|
||||
method,
|
||||
bucket,
|
||||
endpoint,
|
||||
objectKey,
|
||||
accessKeyId,
|
||||
accessKeySecret,
|
||||
queries = {},
|
||||
headers = {},
|
||||
bodyFactory,
|
||||
contentLength,
|
||||
operation,
|
||||
fetchImpl,
|
||||
nowFn,
|
||||
sleepImpl,
|
||||
randomFn,
|
||||
maxAttempts,
|
||||
retryBaseDelayMs,
|
||||
retryMaxDelayMs,
|
||||
}) {
|
||||
const targetUrl = buildOssUrl({bucket, endpoint, objectKey, queries});
|
||||
let lastError = null;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
const now = nowFn();
|
||||
const signedHeaders = {
|
||||
host: `${bucket}.${endpoint}`,
|
||||
...headers,
|
||||
'x-oss-content-sha256': UNSIGNED_PAYLOAD,
|
||||
'x-oss-date': formatOssDate(now),
|
||||
};
|
||||
const authorization = buildAuthorization({
|
||||
method,
|
||||
bucket,
|
||||
endpoint,
|
||||
objectKey,
|
||||
accessKeyId,
|
||||
accessKeySecret,
|
||||
headers: signedHeaders,
|
||||
date: now,
|
||||
queries,
|
||||
});
|
||||
const requestHeaders = {...signedHeaders, authorization};
|
||||
if (contentLength !== undefined) {
|
||||
requestHeaders['content-length'] = String(contentLength);
|
||||
}
|
||||
const body = bodyFactory ? bodyFactory() : undefined;
|
||||
const requestOptions = {method, headers: requestHeaders};
|
||||
if (body !== undefined) {
|
||||
requestOptions.body = body;
|
||||
requestOptions.duplex = 'half';
|
||||
}
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(targetUrl, requestOptions);
|
||||
} catch (error) {
|
||||
lastError = new Error(`OSS ${operation}请求失败: oss://${bucket}/${objectKey}`, {cause: error});
|
||||
}
|
||||
|
||||
if (response?.ok) {
|
||||
return response;
|
||||
}
|
||||
if (response) {
|
||||
const responseText = await response.text();
|
||||
const requestId = response.headers.get('x-oss-request-id');
|
||||
lastError = new Error(
|
||||
`OSS ${operation}失败 HTTP ${response.status}${requestId ? ` requestId=${requestId}` : ''}: ${responseText.slice(0, 500)}`,
|
||||
);
|
||||
lastError.status = response.status;
|
||||
}
|
||||
|
||||
const retryable = !response || isRetryableOssStatus(response.status);
|
||||
if (!retryable || attempt >= maxAttempts) {
|
||||
throw lastError;
|
||||
}
|
||||
const delayMs = retryDelayMs({attempt, baseDelayMs: retryBaseDelayMs, maxDelayMs: retryMaxDelayMs, randomFn});
|
||||
console.warn(`[database-backup] OSS ${operation}失败,${delayMs}ms 后重试 (${attempt}/${maxAttempts})`);
|
||||
await sleepImpl(delayMs);
|
||||
}
|
||||
|
||||
throw lastError ?? new Error(`OSS ${operation}失败: oss://${bucket}/${objectKey}`);
|
||||
}
|
||||
|
||||
function readXmlTag(xml, tagName) {
|
||||
const match = new RegExp(`<${tagName}>([\\s\\S]*?)</${tagName}>`, 'u').exec(xml);
|
||||
if (!match) {
|
||||
return '';
|
||||
}
|
||||
return match[1]
|
||||
.replace(/</gu, '<')
|
||||
.replace(/>/gu, '>')
|
||||
.replace(/"/gu, '"')
|
||||
.replace(/'/gu, "'")
|
||||
.replace(/&/gu, '&')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function escapeXml(value) {
|
||||
return String(value)
|
||||
.replace(/&/gu, '&')
|
||||
.replace(/</gu, '<')
|
||||
.replace(/>/gu, '>')
|
||||
.replace(/"/gu, '"')
|
||||
.replace(/'/gu, ''');
|
||||
}
|
||||
|
||||
function buildCompleteMultipartBody(parts) {
|
||||
const partXml = parts
|
||||
.map(({partNumber, etag}) => [
|
||||
'<Part>',
|
||||
`<PartNumber>${partNumber}</PartNumber>`,
|
||||
`<ETag>${escapeXml(etag)}</ETag>`,
|
||||
'</Part>',
|
||||
].join(''))
|
||||
.join('');
|
||||
return `<?xml version="1.0" encoding="UTF-8"?><CompleteMultipartUpload>${partXml}</CompleteMultipartUpload>`;
|
||||
}
|
||||
|
||||
function resolveMultipartPartSize(fileSize, configuredPartSize) {
|
||||
if (!Number.isSafeInteger(configuredPartSize) || configuredPartSize < OSS_MIN_MULTIPART_PART_SIZE_BYTES) {
|
||||
throw new Error(`OSS multipart part size 必须是 >= ${OSS_MIN_MULTIPART_PART_SIZE_BYTES} 的安全整数`);
|
||||
}
|
||||
const minimumForPartLimit = Math.ceil(fileSize / OSS_MAX_MULTIPART_PARTS);
|
||||
const partSize = Math.max(configuredPartSize, minimumForPartLimit);
|
||||
if (partSize > OSS_MAX_MULTIPART_PART_SIZE_BYTES) {
|
||||
throw new Error(`OSS multipart part size 超过 5GiB: ${partSize}`);
|
||||
}
|
||||
return partSize;
|
||||
}
|
||||
|
||||
async function verifyUploadedObject({requestOptions, expectedContentLength}) {
|
||||
const response = await signedOssRequest({
|
||||
...requestOptions,
|
||||
method: 'HEAD',
|
||||
operation: 'HEAD 验证',
|
||||
});
|
||||
const contentLengthHeader = response.headers.get('content-length');
|
||||
if (!contentLengthHeader || !/^\d+$/u.test(contentLengthHeader)) {
|
||||
throw new Error(`OSS HEAD 验证缺少有效 content-length: ${contentLengthHeader ?? '<missing>'}`);
|
||||
}
|
||||
const remoteContentLength = Number(contentLengthHeader);
|
||||
if (remoteContentLength !== expectedContentLength) {
|
||||
throw new Error(`OSS HEAD 验证长度不一致: local=${expectedContentLength}, remote=${remoteContentLength}`);
|
||||
}
|
||||
return {verifiedAt: new Date().toISOString(), remoteContentLength};
|
||||
}
|
||||
|
||||
async function abortMultipartUpload({requestOptions, uploadId}) {
|
||||
try {
|
||||
await signedOssRequest({
|
||||
...requestOptions,
|
||||
method: 'DELETE',
|
||||
queries: {uploadId},
|
||||
operation: 'AbortMultipartUpload',
|
||||
maxAttempts: Math.min(2, requestOptions.maxAttempts),
|
||||
});
|
||||
console.warn(`[database-backup] 已清理失败的 multipart upload: ${uploadId}`);
|
||||
} catch (error) {
|
||||
console.warn(`[database-backup] 清理 multipart upload 失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadArchive({
|
||||
archivePath,
|
||||
bucket,
|
||||
endpoint,
|
||||
objectKey,
|
||||
accessKeyId,
|
||||
accessKeySecret,
|
||||
partSizeBytes = DEFAULT_OSS_MULTIPART_PART_SIZE_BYTES,
|
||||
maxAttempts = DEFAULT_OSS_REQUEST_MAX_ATTEMPTS,
|
||||
retryBaseDelayMs = DEFAULT_OSS_RETRY_BASE_DELAY_MS,
|
||||
retryMaxDelayMs = DEFAULT_OSS_RETRY_MAX_DELAY_MS,
|
||||
fetchImpl = globalThis.fetch,
|
||||
nowFn = () => new Date(),
|
||||
sleepImpl = sleep,
|
||||
randomFn = Math.random,
|
||||
}) {
|
||||
const fileStat = statSync(archivePath);
|
||||
const now = new Date();
|
||||
const targetUrl = `https://${bucket}.${endpoint}/${encodePath(objectKey)}`;
|
||||
const headers = {
|
||||
host: `${bucket}.${endpoint}`,
|
||||
'content-type': 'application/gzip',
|
||||
'x-oss-content-sha256': UNSIGNED_PAYLOAD,
|
||||
'x-oss-date': formatOssDate(now),
|
||||
'x-oss-meta-backup-kind': 'spacetimedb-data-dir',
|
||||
};
|
||||
const authorization = buildAuthorization({
|
||||
method: 'PUT',
|
||||
if (!fileStat.isFile() || fileStat.size <= 0) {
|
||||
throw new Error(`待上传备份必须是非空文件: ${archivePath}`);
|
||||
}
|
||||
const partSize = resolveMultipartPartSize(fileStat.size, partSizeBytes);
|
||||
const partCount = Math.ceil(fileStat.size / partSize);
|
||||
const requestOptions = {
|
||||
bucket,
|
||||
endpoint,
|
||||
objectKey,
|
||||
accessKeyId,
|
||||
accessKeySecret,
|
||||
headers,
|
||||
date: now,
|
||||
});
|
||||
|
||||
console.log(`[database-backup] 上传 OSS: oss://${bucket}/${objectKey}`);
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(targetUrl, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
...headers,
|
||||
authorization,
|
||||
'content-length': String(fileStat.size),
|
||||
},
|
||||
body: createReadStream(archivePath),
|
||||
duplex: 'half',
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(`OSS 上传请求失败: oss://${bucket}/${objectKey}`, {cause: error});
|
||||
}
|
||||
|
||||
const responseText = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`OSS 上传失败 HTTP ${response.status}: ${responseText.slice(0, 500)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
bucket,
|
||||
objectKey,
|
||||
contentLength: fileStat.size,
|
||||
etag: response.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '',
|
||||
fetchImpl,
|
||||
nowFn,
|
||||
sleepImpl,
|
||||
randomFn,
|
||||
maxAttempts,
|
||||
retryBaseDelayMs,
|
||||
retryMaxDelayMs,
|
||||
};
|
||||
let uploadId = '';
|
||||
let uploadCompleted = false;
|
||||
|
||||
console.log(`[database-backup] multipart 上传 OSS: oss://${bucket}/${objectKey} (${partCount} parts)`);
|
||||
try {
|
||||
const initiateResponse = await signedOssRequest({
|
||||
...requestOptions,
|
||||
method: 'POST',
|
||||
queries: {uploads: null},
|
||||
headers: {
|
||||
'content-type': 'application/gzip',
|
||||
'x-oss-meta-backup-kind': 'spacetimedb-data-dir',
|
||||
},
|
||||
operation: 'InitiateMultipartUpload',
|
||||
});
|
||||
uploadId = readXmlTag(await initiateResponse.text(), 'UploadId');
|
||||
if (!uploadId) {
|
||||
throw new Error('OSS InitiateMultipartUpload 响应缺少 UploadId');
|
||||
}
|
||||
|
||||
const parts = [];
|
||||
for (let partNumber = 1; partNumber <= partCount; partNumber += 1) {
|
||||
const start = (partNumber - 1) * partSize;
|
||||
const end = Math.min(fileStat.size, start + partSize) - 1;
|
||||
const contentLength = end - start + 1;
|
||||
const response = await signedOssRequest({
|
||||
...requestOptions,
|
||||
method: 'PUT',
|
||||
queries: {partNumber, uploadId},
|
||||
headers: {'content-type': 'application/octet-stream'},
|
||||
contentLength,
|
||||
bodyFactory: () => createReadStream(archivePath, {start, end}),
|
||||
operation: `UploadPart ${partNumber}/${partCount}`,
|
||||
});
|
||||
const etag = response.headers.get('etag');
|
||||
if (!etag) {
|
||||
throw new Error(`OSS UploadPart ${partNumber}/${partCount} 响应缺少 ETag`);
|
||||
}
|
||||
parts.push({partNumber, etag});
|
||||
console.log(`[database-backup] multipart 进度: ${partNumber}/${partCount}`);
|
||||
}
|
||||
|
||||
const completeBody = buildCompleteMultipartBody(parts);
|
||||
let completeResponse;
|
||||
try {
|
||||
completeResponse = await signedOssRequest({
|
||||
...requestOptions,
|
||||
method: 'POST',
|
||||
queries: {uploadId},
|
||||
headers: {'content-type': 'application/xml'},
|
||||
contentLength: Buffer.byteLength(completeBody),
|
||||
bodyFactory: () => completeBody,
|
||||
operation: 'CompleteMultipartUpload',
|
||||
});
|
||||
const completeResponseText = await completeResponse.text();
|
||||
if (/<Error(?:\s|>)/u.test(completeResponseText)) {
|
||||
throw new Error(`OSS CompleteMultipartUpload 返回错误: ${completeResponseText.slice(0, 500)}`);
|
||||
}
|
||||
} catch (completeError) {
|
||||
try {
|
||||
await verifyUploadedObject({requestOptions, expectedContentLength: fileStat.size});
|
||||
completeResponse = null;
|
||||
} catch {
|
||||
throw completeError;
|
||||
}
|
||||
}
|
||||
|
||||
const verification = await verifyUploadedObject({requestOptions, expectedContentLength: fileStat.size});
|
||||
uploadCompleted = true;
|
||||
return {
|
||||
bucket,
|
||||
objectKey,
|
||||
contentLength: fileStat.size,
|
||||
etag: completeResponse?.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '',
|
||||
uploadMode: 'multipart',
|
||||
partCount,
|
||||
partSizeBytes: partSize,
|
||||
verifiedAt: verification.verifiedAt,
|
||||
};
|
||||
} catch (error) {
|
||||
if (uploadId && !uploadCompleted) {
|
||||
await abortMultipartUpload({requestOptions, uploadId});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix}) {
|
||||
@@ -603,6 +882,10 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId,
|
||||
objectKey: result.objectKey,
|
||||
contentLength: result.contentLength,
|
||||
etag: result.etag,
|
||||
uploadMode: result.uploadMode,
|
||||
partCount: result.partCount,
|
||||
partSizeBytes: result.partSizeBytes,
|
||||
verifiedAt: result.verifiedAt,
|
||||
uploadedAt,
|
||||
uploadStatus: 'uploaded',
|
||||
},
|
||||
@@ -736,6 +1019,10 @@ async function main() {
|
||||
archivePath,
|
||||
contentLength: result.contentLength,
|
||||
etag: result.etag,
|
||||
uploadMode: result.uploadMode,
|
||||
partCount: result.partCount,
|
||||
partSizeBytes: result.partSizeBytes,
|
||||
verifiedAt: result.verifiedAt,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
uploadStatus: 'uploaded',
|
||||
},
|
||||
@@ -791,9 +1078,11 @@ function describeError(error) {
|
||||
return lines;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
for (const line of describeError(error)) {
|
||||
console.error(`[database-backup] ${line}`);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
if (process.argv[1] && realpathSync(resolve(process.argv[1])) === realpathSync(__filename)) {
|
||||
main().catch((error) => {
|
||||
for (const line of describeError(error)) {
|
||||
console.error(`[database-backup] ${line}`);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user