From 94014392ed5dc155be4b155d76350c5591417300 Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 16 Jul 2026 18:28:04 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=80=90=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=B0=8F=E5=AF=B9=E8=B1=A1=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 小于等于16MiB的CAS对象改用单次PUT并执行HEAD验真 大对象继续保留multipart上传与失败清理 补齐单次PUT请求路径测试和运维说明 --- ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- scripts/check-database-backup-to-oss.mjs | 35 +++++++ scripts/database-backup-to-oss.mjs | 97 ++++++++++++++++++- 3 files changed, 132 insertions(+), 2 deletions(-) diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 30164b5fa..cb4f4912e 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -340,7 +340,7 @@ GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= `files-history` 使用仓库模板 `deploy/systemd/genarrative-database-backup-files-history.conf` 覆盖主 service 的 `ExecStart`，从 `/etc/genarrative/api-server.env` 读取 data-dir、database、bucket、prefix 与 OSS 凭据，不在 unit 写死环境目标，也不传 `--stop-service`。Server-Provision 在改动 drop-in 前，先用 current release 的同一脚本、同一 env 和 `DATABASE_BACKUP_FILES_HISTORY_WORK_DIR` 执行一次 history `--dry-run`；缺少已发布 full catalog 的 files state、current 脚本过旧或配置不匹配都会在安装 drop-in 和 `daemon-reload` 前失败。选择 `archive-full` 会主动删除仓库托管的 `10-files-history.conf` 与 dev 试点遗留的 `10-dev-files.conf`，防止 systemd 继续合并旧覆盖。dev 可继续指定已有 `/var/lib/genarrative/database-backups/dev-files`，release 建议先在 `/var/lib/genarrative/database-backups/release-files` 建立自己的 full baseline；两台机器不得复用或互传本地 state 目录冒充本机基线。启用时通过 Server-Provision Job 选择目标、`DATABASE_BACKUP_PROFILE=files-history` 和对应 work-dir，先保持 `DRY_RUN=true` 核对，再以同参数正式 provision。不要直接在 `/etc/systemd/system` 手写第二份 drop-in。 -files full 会递归扫描 data-dir，保留空目录、每个普通文件的相对路径，以及目标仍位于 data-dir 内部的相对符号链接；绝对链接或解析后越界的链接直接拒绝。文件按 SHA-256 上传到不可变对象 key，catalog 记录目录、路径、长度、SHA、对象 key 和相对链接目标，不写 staging 主机的绝对路径。相同 catalog 重跑不重复 PUT；新增或变化文件先 HEAD CAS 对象，存在且长度/SHA 元数据一致就复用，否则上传。对象 PUT/HEAD 默认以 16 路并行执行，可用 `GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=1..64` 调整；并发只缩短传输与验真时间，不改变“全部对象、catalog 与 latest pointer 成功后才推进 state/清理”的顺序。full 基线必须来自停库后的 data-dir 或已通过恢复验证的冻结副本；源文件上传前后 stat 虽会复核，但在线扫描不能保证大量文件属于同一跨文件一致时点。catalog 验真后，脚本把最新 full/history 引用发布到固定 `//latest.json`，全新机器不需要本地 state 即可自动发现恢复入口。 +files full 会递归扫描 data-dir，保留空目录、每个普通文件的相对路径，以及目标仍位于 data-dir 内部的相对符号链接；绝对链接或解析后越界的链接直接拒绝。文件按 SHA-256 上传到不可变对象 key，catalog 记录目录、路径、长度、SHA、对象 key 和相对链接目标，不写 staging 主机的绝对路径。相同 catalog 重跑不重复 PUT；新增或变化文件先 HEAD CAS 对象，存在且长度/SHA 元数据一致就复用，否则上传。16 MiB 及以下对象使用单次 PUT 后 HEAD 验真，大对象继续使用 multipart；对象操作默认以 16 路并行执行，可用 `GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=1..64` 调整。并发和单次 PUT 只缩短传输与验真时间，不改变“全部对象、catalog 与 latest pointer 成功后才推进 state/清理”的顺序。full 基线必须来自停库后的 data-dir 或已通过恢复验证的冻结副本；源文件上传前后 stat 虽会复核，但在线扫描不能保证大量文件属于同一跨文件一致时点。catalog 验真后，脚本把最新 full/history 引用发布到固定 `//latest.json`，全新机器不需要本地 state 即可自动发现恢复入口。 history 的安全边界按每个 replica 独立计算。设最新完整且未锁定的 snapshot offset 为 `S`；数字更大但缺少同 offset `.snapshot_bsatn`、仍存在同名 `.lock` 的目录不能参与边界计算。脚本必须保留起始 offset 小于等于 `S` 的最后一个 commitlog segment，以及它之后的全部 segment；只处理更早的 `.stdb.log` / `.stdb.ofs`，snapshot 只处理最新目录之前的旧目录。files history 会递归展开候选目录，逐对象复用或上传，随后依次验真候选对象、history catalog 与 full baseline catalog，再重新扫描边界和 stat fingerprint，最后覆盖发布并验真 `latest.json`；任何一步失败都不推进 state 或删除源文件。脚本在 work-dir 使用 PID lock 拒绝同库并发上传，SSH 超时后必须先检查原进程，不能直接重跑。 diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index 7688c470e..31f685693 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -17,6 +17,7 @@ import { resumeUploadedHistoryBatch, runDirectFilesBackup, uploadArchive, + uploadDirectFile, uploadHistoryArchiveWithCleanup, uploadManifestFile, } from './database-backup-to-oss.mjs'; @@ -46,6 +47,7 @@ async function main() { assertInsufficientSpaceStopsBeforeServiceChanges(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); + await assertDirectSmallFileUsesSinglePut(); await assertMissingPartEtagAbortsMultipartUpload(); await assertCompleteResponseAmbiguityUsesHeadVerification(); await assertHeadLengthMismatchAbortsMultipartUpload(); @@ -101,6 +103,39 @@ function createDirectOssHarness() { return {objects, uploadedKeys, verifiedKeys, uploadFn, uploadManifestFn, verifyFn}; } +async function assertDirectSmallFileUsesSinglePut() { + const filePath = path.join(tmpRoot, 'direct-small-file.bin'); + const body = Buffer.from('small direct object'); + const sha256 = createHash('sha256').update(body).digest('hex'); + writeFileSync(filePath, body); + const methods = []; + const fetchImpl = async (_url, options) => { + methods.push(options.method); + if (options.method === 'PUT') { + return new Response('', {status: 200, headers: {etag: '"single-etag"'}}); + } + if (options.method === 'HEAD') { + return new Response(null, {status: 200, headers: { + 'content-length': String(body.length), + 'x-oss-meta-file-sha256': sha256, + }}); + } + throw new Error(`unexpected method ${options.method}`); + }; + const result = await uploadDirectFile({ + archivePath: filePath, + bucket: 'backup-bucket', + endpoint: 'oss-cn-shanghai.aliyuncs.com', + objectKey: 'database-backups/test-db/files/small', + accessKeyId: 'test-id', + accessKeySecret: 'test-secret', + archiveSha256: sha256, + fetchImpl, + }); + assertEqual(result.uploadMode, 'single', '小型逐文件对象必须使用单次 PUT。'); + assertEqual(methods.join(','), 'PUT,HEAD', '小型逐文件对象只能执行 PUT 后 HEAD 验真，不得进入 multipart。'); +} + async function assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload() { const root = path.join(tmpRoot, 'direct-files-incremental'); const dataDir = path.join(root, 'stdb'); diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index dd273cb3d..4c7a92aa8 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -49,6 +49,7 @@ const DEFAULT_OSS_RETRY_BASE_DELAY_MS = 1_000; const DEFAULT_OSS_RETRY_MAX_DELAY_MS = 30_000; const DEFAULT_DIRECT_FILES_CONCURRENCY = 16; const MAX_DIRECT_FILES_CONCURRENCY = 64; +const DIRECT_FILES_SINGLE_PUT_MAX_BYTES = 16 * 1024 * 1024; const RETRYABLE_OSS_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504]); const HISTORY_STATE_SCHEMA_VERSION = 1; const HISTORY_MANIFEST_SCHEMA_VERSION = 1; @@ -1348,7 +1349,7 @@ export async function runDirectFilesBackup({ dryRun = false, resultFile = '', uploadOptions, - uploadFn = uploadArchive, + uploadFn = uploadDirectFile, uploadManifestFn = uploadManifestFile, verifyFn = verifyOssObject, concurrency = 1, @@ -2279,6 +2280,100 @@ export async function uploadArchive({ } } +export async function uploadDirectFile({ + archivePath, + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + fetchImpl = globalThis.fetch, + nowFn = () => new Date(), + sleepImpl = sleep, + randomFn = Math.random, + maxAttempts = DEFAULT_OSS_REQUEST_MAX_ATTEMPTS, + retryBaseDelayMs = DEFAULT_OSS_RETRY_BASE_DELAY_MS, + retryMaxDelayMs = DEFAULT_OSS_RETRY_MAX_DELAY_MS, + backupKind = 'spacetimedb-direct-file', + archiveSha256 = '', + contentType = 'application/octet-stream', + allowEmpty = true, +}) { + const fileStat = statSync(archivePath); + if (!fileStat.isFile() || (!allowEmpty && fileStat.size <= 0)) { + throw new Error(`待上传备份必须是${allowEmpty ? '' : '非空'}普通文件: ${archivePath}`); + } + if (fileStat.size > DIRECT_FILES_SINGLE_PUT_MAX_BYTES) { + return uploadArchive({ + archivePath, + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + fetchImpl, + nowFn, + sleepImpl, + randomFn, + maxAttempts, + retryBaseDelayMs, + retryMaxDelayMs, + backupKind, + archiveSha256, + contentType, + allowEmpty, + }); + } + const verifiedArchiveSha256 = archiveSha256 || await sha256FileHex(archivePath); + if (!/^[a-f0-9]{64}$/u.test(verifiedArchiveSha256)) { + throw new Error(`归档 SHA-256 无效: ${verifiedArchiveSha256}`); + } + const requestOptions = { + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + fetchImpl, + nowFn, + sleepImpl, + randomFn, + maxAttempts, + retryBaseDelayMs, + retryMaxDelayMs, + }; + await signedOssRequest({ + ...requestOptions, + method: 'PUT', + headers: { + 'content-type': contentType, + 'x-oss-meta-archive-sha256': verifiedArchiveSha256, + 'x-oss-meta-file-sha256': verifiedArchiveSha256, + 'x-oss-meta-file-size': String(fileStat.size), + 'x-oss-meta-backup-kind': backupKind, + }, + contentLength: fileStat.size, + bodyFactory: () => fileStat.size === 0 ? Buffer.alloc(0) : createReadStream(archivePath), + operation: '上传逐文件对象', + }); + const verification = await verifyUploadedObject({ + requestOptions, + expectedContentLength: fileStat.size, + expectedArchiveSha256: verifiedArchiveSha256, + }); + return { + bucket, + objectKey, + contentLength: fileStat.size, + archiveSha256: verifiedArchiveSha256, + etag: '', + uploadMode: 'single', + partCount: 1, + partSizeBytes: fileStat.size, + verifiedAt: verification.verifiedAt, + }; +} + export async function uploadManifestFile({ manifestPath, bucket,