From b1ef9eef810dab5ac20521530dbf2808306a215f Mon Sep 17 00:00:00 2001 From: Linghong Date: Tue, 22 Sep 2026 08:49:58 +0000 Subject: [PATCH] =?UTF-8?q?=E5=A4=8D=E7=94=A8=20master=20CI=20=E4=BA=A7?= =?UTF-8?q?=E7=89=A9=E8=87=AA=E5=8A=A8=E6=9B=B4=E6=96=B0=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅由 master CI 导出新增缓存对象，宿主合并去重并组装限额快照，避免重复预热编译。 跟踪任务领取与最终上报，仅在安全空闲时切换镜像，移除管理员 API 依赖。 排除取消及不完整产物，定向清理过期镜像、归档和中断上传残块。 同步部署文档、共享记忆及缓存发布、合并和清理的定向测试。 --- .gitea/workflows/project-ci.yml | 36 + deploy/container/README.md | 29 +- .../gitea-ci-cache.config.example.json | 2 +- .../shared-memory/development-workflow.md | 4 +- docs/project-memory/shared-memory/pitfalls.md | 3 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 6 +- scripts/ci-rust-cache.sh | 16 +- scripts/ci-rust-cache.test.mjs | 25 +- scripts/export-gitea-rust-cache.py | 274 ++++++++ scripts/gitea-runner-fetch-gate.py | 181 ++++- scripts/gitea_cache_snapshot.py | 649 ++++++++++++++++++ scripts/gitea_cache_upload_cleanup.py | 115 ++++ scripts/maintain-gitea-rust-cache.py | 309 +++++++-- scripts/project-ci-workflow.test.ts | 32 + scripts/test_gitea_cache_export.py | 205 ++++++ scripts/test_gitea_cache_gate.py | 154 ++++- scripts/test_gitea_cache_maintenance.py | 190 ++++- scripts/test_gitea_cache_snapshot.py | 240 +++++++ scripts/test_gitea_cache_upload_cleanup.py | 90 +++ 19 files changed, 2473 insertions(+), 87 deletions(-) create mode 100644 scripts/export-gitea-rust-cache.py create mode 100644 scripts/gitea_cache_snapshot.py create mode 100644 scripts/gitea_cache_upload_cleanup.py create mode 100644 scripts/test_gitea_cache_export.py create mode 100644 scripts/test_gitea_cache_snapshot.py create mode 100644 scripts/test_gitea_cache_upload_cleanup.py diff --git a/.gitea/workflows/project-ci.yml b/.gitea/workflows/project-ci.yml index 1d9723bec..b84b9f901 100644 --- a/.gitea/workflows/project-ci.yml +++ b/.gitea/workflows/project-ci.yml @@ -98,6 +98,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py ai-game-creator-shell-rust-lane-2: name: AI game creator shell Rust lane 2/2 @@ -141,6 +147,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py # agent-run smoke 会 spawn `cargo run`（走壳自己的 manifest），同样不装 npm 依赖， # 单独一个 job，免得把已经压到 4 分钟级的片 job 拖长。 @@ -183,6 +195,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py # AGC 壳依赖的共享 / 平台和编辑器插件 crate 各自预热独立 manifest，再运行对应测试。 ai-game-creator-shell-rust-crates: @@ -291,6 +309,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py backend-tests: name: Backend tests @@ -380,6 +404,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py # 客户端的壳级与契约门禁：静态契约断言、H5 / 微信 / 移动 / 桌面壳运行时门禁， # 以及依赖发布产物的构建 smoke。 @@ -442,6 +472,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py frontend-tests: name: Frontend tests diff --git a/deploy/container/README.md b/deploy/container/README.md index 34a064572..8e9dece50 100644 --- a/deploy/container/README.md +++ b/deploy/container/README.md @@ -97,13 +97,13 @@ runner 配置保留原 `ubuntu-latest` 映射，`genarrative-ci` 继续映射到 ### Rust 测试组编译对象快照 -自动维护由宿主 systemd timer 调用 `scripts/maintain-gitea-rust-cache.py`，只管理 Gitea CI 测试镜像，不修改 Jenkins、生产发布、本地开发或客户端发行构建。维护器固定最新 master SHA，**不要求该提交事先全绿**；编译/镜像校验失败则继续使用旧版。任务串行执行，编译输入未变化时跳过，CI 繁忙时延后预热；每次只取当时最新 master，不为中间提交建立队列。构建过程中固定 SHA，不混入后续提交。 +自动维护由宿主 systemd timer 调用 `scripts/maintain-gitea-rust-cache.py`，只管理 Gitea CI 测试镜像，不修改 Jenkins、生产发布、本地开发或客户端发行构建。六个 Rust job 仅在 master push 中导出本次 CI 新增的 sccache 对象；已命中的继承对象只上传新近使用时间，通过 Gitea 原生 V4 artifact 接口上传；PR 不发布。维护器选择已结束且六组产物完整的最新 master run，校验提交、任务尝试、工具链与来源镜像，与六组实际使用的同一镜像快照合并去重，并按新近使用时间限制快照总容量为 4 GiB，然后从无对象缓存基础镜像组装新镜像，**不重复执行 Cargo 预热编译，也不要求源 run 事先全绿**。缺组、取消或校验失败时保留现役版，不混合不同 run 的对象来假装完整快照。 -切换先通过专属入口阻断新的 FetchTask，确认已转发的领取请求全部收到完整上游响应，再确认全局 runner 不 busy、Gitea 没有 in_progress run、内层 Docker 没有容器。有任务即恢复领取并延后，不停止任务。切换后等待使用该 Image ID 的完整真实 master push CI 通过，才允许下一次升级及定向清理；不会自动重跑失败用例或为了验收额外触发整轮 CI。首次接管的历史镜像默认不归自动清理管理。 +切换先通过专属入口阻断新的 FetchTask，确认已转发的领取请求全部收到完整上游响应，并检查入口持久化跟踪的已领取任务全部结束、内层 Docker 没有活动容器。任务终态必须依据 Runner 的执行结束及最终上报协议，不能由容器暂时为空、API 已取消或请求超时推断。有任务即恢复领取并延后，不停止任务；状态未知拒绝切换。维护器只需普通账号的 `write:repository` Token（包括查询、下载及定向删除 artifact），不访问全局 Runner 管理 API。切换后等待使用该 Image ID 的完整真实 master push CI 通过，才允许下一次升级及旧镜像清理；不会自动重跑失败用例或为了验收额外触发整轮 CI。首次接管的历史镜像默认不归自动清理管理。 维护状态、凭据、归档和配置备份保存在仓库外。当前版、回滚版、待验证候选、它们的基础镜像及容器引用的镜像均受保护。清理只针对维护器登记的专属 tag、完整 Image ID 和专用目录中的归档；禁止全局 prune。API、构建、验证或空闲检查失败时保留现役镜像与回滚资料，不以失败重跑制造全绿结果。 -`scripts/build-gitea-rust-cache.sh` 在已验证的 job 镜像上生成候选镜像，覆盖 AGC Rust 两条 lane、crates、agent-run smoke、Backend 和 Native shell 的桌面壳测试。Native shell 的 release build smoke 显式清空两个 wrapper，不消费测试快照。固定 sccache `0.18.0` 的 Linux x64 musl 归档并校验 SHA-256，维护者从 origin/master 的确定提交预热编译对象，PR job 没有生成/发布公共快照的权限。 +`scripts/build-gitea-rust-cache.sh` 仅用于首次缺少可消费快照时的人工 bootstrap，在已验证的 job 镜像上生成候选镜像，覆盖 AGC Rust 两条 lane、crates、agent-run smoke、Backend 和 Native shell 的桌面壳测试。Native shell 的 release build smoke 显式清空两个 wrapper，不消费测试快照。固定 sccache `0.18.0` 的 Linux x64 musl 归档并校验 SHA-256，维护者从 origin/master 的确定提交预热编译对象，PR job 没有生成/发布公共快照的权限。 基础镜像必须不含 `/opt/genarrative-ci/rust-cache`；脚本在拉取源码、下载工具和预热前执行只读、断网检查，发现已有对象快照就拒绝构建。不能在旧缓存镜像上删除目录再叠加新快照，删除操作不会释放旧镜像层。切换且真实 CI 验证通过后，维护器定向清理自己登记的更旧缓存镜像与导出归档，保留当前版、一个回滚版、所需基础镜像及容器引用的版本；接管前的试验镜像/归档仍由维护者确认后人工清理。对象缓存上限不覆盖这些宿主文件，不使用全局 prune，也不清理其它构建的 Docker build cache。 @@ -114,11 +114,15 @@ bash scripts/gitea-ci-job-image.sh export /仓库外受控路径/ci-rust-cache.t bash scripts/gitea-ci-job-image.sh load-runner genarrative/gitea-project-ci:rust-cache-candidate ``` -自动维护调用构建脚本时增加第三个参数 `<完整 master SHA>`。脚本确认该 SHA 是抓取到的 master 祖先，源码和 `ci-rust-cache.sh` 均来自该提交；两参数人工调用仍默认使用最新 master。基础镜像的 `revision` 由同一份 build context 清单计算；输入变化时维护器先生成新的无对象缓存基础镜像。Rust 输入指纹保守包含代码、配置、资源、脚本和内嵌 skill，仅排除一般 `docs/` 及几个根说明文档，`docs/openapi/` 始终参与。 +自动维护不调用预热脚本；以下人工 bootstrap 脚本接受第三个参数 `<完整 master SHA>`。脚本确认该 SHA 是抓取到的 master 祖先，源码和 `ci-rust-cache.sh` 均来自该提交；两参数人工调用仍默认使用最新 master。基础镜像的 `revision` 由同一份 build context 清单计算；输入变化时维护器先生成新的无对象缓存基础镜像。Rust 输入指纹保守包含代码、配置、资源、脚本和内嵌 skill，仅排除一般 `docs/` 及几个根说明文档，`docs/openapi/` 始终参与。 + +缓存产物通过六个 `Publish master Rust cache artifact` 步骤发布，daemon 成功停止才导出。源 run 必须结束，且六组为 success/failure 并有完整上传证据；整轮或任一 Rust 组取消、旧 attempt、缺组、混用来源镜像均不采用。上传失败仅告警，不改变测试结论。每组仅导出相对其启动快照的新 key，命中只上报触达时间；宿主按真实 job 日志确认同一来源 Image ID，再复用其对象，不信任产物中的可执行文件。合并校验 tar/zip 路径、对象大小与 SHA256，同 key 内容冲突拒绝发布。sccache 可执行文件取自可信来源镜像，不从 artifact 执行代码。 + +候选已校验、导出并装载内层 Docker 后，可定向删除该 run 已收集的六份上传产物；旧镜像清理仍须等真实 CI 验证。专属缓存 artifact 保留 7 天，宿主也清理超过 7 天且源 run 已结束的遗留项，不删普通构建产物、run 或日志。Gitea 1.26.4 不清理未 finalized 的上传块：上传器使用专属双层编码块标识，宿主从 `artifact_storage_dir/tmp-upload/run--v4/` 定向清理本上传器的普通文件，要求所属仓库 master run 已结束超过 7 天且文件本身也超过 7 天。未知、年轻、符号链接或不属于本上传器的文件受到保护，不改数据库、不删除目录。已完成产物通过 API 删除；未完成上传留下的数据库元数据仍归 Gitea 管理。宿主的未完成下载/合并临时文件仅在登记的私有目录内清理。 ### 自动维护首次部署与恢复 -首次部署在本变更合入 master 后进行。维护脚本安装到 `/opt/genarrative-ci-cache/scripts/`；运行状态、源码专用 clone、日志和归档放在 `/var/lib/genarrative-ci-cache/`，token 放在 `/etc/genarrative-ci-cache/api-token`（root 所有、0600），均不进入 Git。复制 `gitea-ci-cache.config.example.json` 为该目录的 `config.json` 并核实 runner ID；当前全局 runner 的只读 API 需要管理员账号的 `read:admin`，仓库 Actions/日志读取需要对应的只读权限，不能复用只有普通仓库权限的 token。维护器没有写 Gitea API 的调用。 +首次部署在本变更合入 master 后进行。维护脚本安装到 `/opt/genarrative-ci-cache/scripts/`；运行状态、源码专用 clone、日志和归档放在 `/var/lib/genarrative-ci-cache/`，token 放在 `/etc/genarrative-ci-cache/api-token`（root 所有、0600），均不进入 Git。复制 `gitea-ci-cache.config.example.json` 为该目录的 `config.json` 并核实仓库、Runner 容器与网关地址。Token 使用具有目标仓库 Actions 读写权限的普通账号，授予 `write:repository` 即可，不需要 `admin`；写操作仅定向删除本维护器的缓存 artifact，不重跑或取消任务。宿主运行账号还须能通过 `clone_url` 拉取专属 clone，API Token 不自动传给 Git。`artifact_storage_dir` 必须指向当前 Gitea 本地 ActionsArtifacts 存储的真实宿主路径（不是容器路径）；按当前 `/data` bind mount 和 `APP_DATA_PATH`，模板为 `/opt/gitea-stack/data/gitea/gitea/actions_artifacts`。部署时由 root 核实路径与 Gitea `[actions.artifacts]`/storage 配置一致，不能指向其它目录；此残块清理实现只支持本地存储，外部对象存储需要另行适配。 领取入口由 `gitea-runner-fetch-gate.compose.yml` 启动，使用已验证且含 Python 3 的无对象缓存 CI Image ID（`GITEA_FETCH_GATE_IMAGE`）。它只连接现有 `gitea-actions` 内部网络，不发布宿主端口、不挂 Docker socket；仅转发 `/api/actions/` RPC，控制 socket 位于独立私有目录 `/var/lib/genarrative-ci-cache-gate/`。该目录必须由 root 持有且权限为 0700。普通 job 无法访问控制 socket。部署前先运行下述测试，不直接启用 timer。 @@ -127,15 +131,16 @@ python3 -m unittest discover -s scripts -p 'test_gitea_cache_*.py' install -d -m 700 /etc/genarrative-ci-cache /var/lib/genarrative-ci-cache /var/lib/genarrative-ci-cache-gate install -d /opt/genarrative-ci-cache/scripts install -m 755 scripts/maintain-gitea-rust-cache.py scripts/gitea-runner-fetch-gate.py /opt/genarrative-ci-cache/scripts/ +install -m 644 scripts/gitea_cache_snapshot.py scripts/gitea_cache_upload_cleanup.py /opt/genarrative-ci-cache/scripts/ install -m 600 deploy/container/gitea-ci-cache.config.example.json /etc/genarrative-ci-cache/config.json -# 由维护者写入只读 API token，并核实 config.json；不要在终端回显 token。 +# 由维护者写入 write:repository API token，并核实 config.json；不要在终端回显 token。 # 设置 GITEA_FETCH_GATE_IMAGE 为已验证基础镜像的完整 Image ID 后： docker compose -f deploy/container/gitea-runner-fetch-gate.compose.yml up -d ``` -**首次接入需要空闲维护窗口**：确认无活跃 CI 且暂停新 CI 触发，再备份现有 runner 配置和注册文件，在 runner 部署的 `GITEA_INSTANCE_URL` 及 `/data/.runner` 的 `address` 中改用 `http://gitea-runner-fetch-gate:8080`，保留其余注册字段。同时在 `/data/config.yaml` 的 `runner.envs` 中设置 `GENARRATIVE_GITEA_REPOSITORY_URL: "http://gitea:3000/GenarrativeAI/Genarrative.git"`，与维护器的 `repository_url` 一致，沿用现有 job 已可达的内部 Git 通道。此专用变量也覆盖旧 PR 的 checkout；不要用同名 GITHUB_SERVER_URL 环境变量代替，Runner 会再次覆盖它。按原流程重启并验证 runner 注册。 +**首次接入或从不跟踪任务的旧网关升级，需要空闲维护窗口**：确认无活跃 CI 且暂停新 CI 触发，再备份现有 runner 配置和注册文件，在 runner 部署的 `GITEA_INSTANCE_URL` 及 `/data/.runner` 的 `address` 中改用 `http://gitea-runner-fetch-gate:8080`，保留其余注册字段。同时在 `/data/config.yaml` 的 `runner.envs` 中设置 `GENARRATIVE_GITEA_REPOSITORY_URL: "http://gitea:3000/GenarrativeAI/Genarrative.git"`，与维护器的 `repository_url` 一致，沿用现有 job 已可达的内部 Git 通道。此专用变量也覆盖旧 PR 的 checkout；不要用同名 GITHUB_SERVER_URL 环境变量代替，Runner 会再次覆盖它。按原流程重启并验证 runner 注册。 -不能仅改磁盘文件却不让进程加载；维护器还会检查独立 checkout URL，并确认入口实际见到了当前容器本次启动后的 FetchTask 来源 IP。此一次接入不由维护器冒险猜测空闲，也不对运行中 CI 动手。实例 API 根地址仍使用配置中的 HTTPS Gitea 地址，不能指向只支持 runner RPC 的入口。当前 Project CI 没有 `uses:` 或直接依赖 `github.server_url` 的 API 请求；后续新增这类调用时需沿用独立真实 Git/API 地址，不能假定只支持 RPC 的入口也是通用 Gitea 地址。 +不能仅改磁盘文件却不让进程加载；维护器还会检查独立 checkout URL，并确认入口实际见到了当前容器本次启动后的 FetchTask 来源 IP。此一次接入不由维护器冒险猜测空闲，也不对运行中 CI 动手。实例 API 根地址仍使用配置中的 HTTPS Gitea 地址，不能指向只支持 runner RPC 的入口。上传脚本从独立 `GENARRATIVE_GITEA_REPOSITORY_URL` 推导真实 Gitea 地址，使用本任务临时凭据调用原生 V4 API；不依赖被网关覆盖的 `GITHUB_SERVER_URL` / `ACTIONS_RUNTIME_URL`。Gitea 1.26.4 的仓库 REST 下载接口只支持 V4，不能换回 V3 上传。宿主下载只跟随同一 HTTPS Gitea origin 的签名重定向，不转发长期 Token；如配置外部对象存储直出，需另行适配下载来源。 ```bash # --apply 缺省时只检查 API、runner、master 和入口路径，不修改配置或镜像。 @@ -146,15 +151,15 @@ systemctl enable --now genarrative-ci-cache.timer journalctl -u genarrative-ci-cache.service -n 50 ``` -timer 在上次执行结束后约 5 分钟再次检查，文件锁防止人工与定时执行重叠。构建日志位于状态目录 `artifacts//build.log`。同 SHA 构建失败后不每 5 分钟重复消耗资源；新 master 到来后自动重试，也可修复环境后显式运行 `--apply --retry`。切换事务及暂停归属先落状态文件；`ExecStopPost --resume` 恢复本维护器暂停的领取，下次执行再收敛中断的切换。其它人暂停的入口或 disabled runner 不由维护器擅自恢复。 +timer 在上次执行结束后约 5 分钟再次检查，文件锁防止人工与定时执行重叠。构建日志位于状态目录 `artifacts//build.log`。同一 run 组装失败后不每 5 分钟重复消耗资源；新的完整 master run 到来后自动尝试，也可修复环境后显式运行 `--apply --retry`。切换事务及暂停归属先落状态文件；`ExecStopPost --resume` 恢复本维护器暂停的领取，下次执行再收敛中断的切换。其它人暂停的入口不由维护器擅自恢复；不修改 Gitea 的 Runner disabled 设置。 候选切换后的验收读取真实 **master push** 的完整九个 job，要求全部 success、每个 job 均使用目标 Image ID，六个 Rust job 有启用缓存、正命中数和零缓存错误。测试失败、旧 PR 缺 prepare、混用镜像或缺日志均不清理旧版，也不伪造“缓存已验收”。仍保留源代码失败需要修复的原始结果。 -入口遇到“已发送 FetchTask，但上游响应未完整结束”会持久化 `uncertain` 并拒绝继续领取/自动切换；不会因为客户端的 5 秒超时就认定服务端事务已回滚。其它 RPC 和任务上报仍继续转发。维护者需先核实 Gitea 在途领取事务与该 runner 的任务全部收敛，在维护窗口停止入口，清理它的 `uncertain` / `inflight` 标记后再启动并恢复领取。禁止自动删这些标记绕过屏障。停用自动维护先 `systemctl disable --now genarrative-ci-cache.timer`；不要为了停 timer 停止运行中的 CI 容器。已接入的领取入口继续运行，不影响普通 CI。 +入口遇到“已发送 FetchTask，但上游响应未完整结束”会持久化 `uncertain` 并拒绝继续领取/自动切换；不会因为客户端的 5 秒超时就认定服务端事务已回滚。其它 RPC 和任务上报仍继续转发。维护者需先核实 Gitea 在途领取事务与该 runner 的任务全部收敛，在维护窗口停止入口，核实并修复它的 `tasks.json` 任务账本及 `uncertain` / `inflight` 标记后再启动并恢复领取。网关按 Gitea 1.26.4 / Runner 2.0.0 的 Connect Protobuf 协议，在最终日志确认及 Runner 执行结束后的最终任务上报确认后才清账；取消响应不等于进程停止，任务 ID 不按超时自动删除。禁止自动删这些标记绕过屏障。停用自动维护先 `systemctl disable --now genarrative-ci-cache.timer`；不要为了停 timer 停止运行中的 CI 容器。已接入的领取入口继续运行，不影响普通 CI。 -预热容器上限为 4 核、12 GiB，移除 capabilities，不挂宿主目录/socket，也不注入 Git/OSS/Jenkins 凭据。源码通过 `git archive` 复制，当前工作区、ignored 文件和 `.git` 不进入容器。最终从原镜像重新组装，仅复制 `/opt/genarrative-ci/rust-cache` 的 sccache、对象和来源元数据，不提交含源码/target 的预热容器；镜像本身的下载缓存与工具链校验保持原样。 +人工 bootstrap 预热容器上限为 4 核、12 GiB，移除 capabilities，不挂宿主目录/socket，也不注入 Git/OSS/Jenkins 凭据。源码通过 `git archive` 复制，当前工作区、ignored 文件和 `.git` 不进入容器。最终从原镜像重新组装，仅复制 `/opt/genarrative-ci/rust-cache` 的 sccache、对象和来源元数据，不提交含源码/target 的预热容器；镜像本身的下载缓存与工具链校验保持原样。 -快照由固定 Image ID 分发，每个 job 仅修改容器自己的写时复制层，缓存上限 4 GiB，构建末尾输出实际对象体积，结束后不回传。`ci-rust-cache.sh prepare` 清空继承的 `SCCACHE_*` 远程配置，使用独立配置和 Unix socket；旧镜像、工具链不匹配或限时 wrapper 探测失败时使用直接 rustc，正式编译启用 sccache 的 server IO 错误回退。真实编译/测试失败保留非零退出码。`report` 输出命中统计并停止本 job daemon，分片日志输出独立编译耗时。sccache 0.18.0 的只读模式在 miss 后仍打包再拒绝写入，不能用它宣称零 miss 开销；普通 CI 继续只向容器层写入。 +快照由固定 Image ID 分发，每个 job 仅修改容器自己的写时复制层，缓存上限 4 GiB，只有 master push 在结束前回传新增对象；PR 不扫描基线、不打包、不回传。`ci-rust-cache.sh prepare` 清空继承的 `SCCACHE_*` 远程配置，使用独立配置和 Unix socket；旧镜像、工具链不匹配或限时 wrapper 探测失败时使用直接 rustc，正式编译启用 sccache 的 server IO 错误回退。真实编译/测试失败保留非零退出码。`report` 输出命中统计并停止本 job daemon，分片日志输出独立编译耗时。sccache 0.18.0 的只读模式在 miss 后仍打包再拒绝写入，不能用它宣称零 miss 开销；普通 CI 继续只向容器层写入。 生成候选不会改变 runner 配置。线上有活跃 CI 时禁止停止 job、重启 runner 或切换标签；只在确认空闲后按上节流程切换固定 Image ID。全组启用须重建完整快照；旧 AGC 单目标快照不能作为其它组预热成功的证据。按组用相同源码、独立干净 target 比较编译命中与耗时；分片测试内容和前置检查不同，不能直接用两条 lane 总耗时推断缓存收益。回滚时可移除各 job 的 prepare/report，或恢复无对象缓存的原基础镜像 ID，均不需要改变 incremental 或测试分片。 diff --git a/deploy/container/gitea-ci-cache.config.example.json b/deploy/container/gitea-ci-cache.config.example.json index f8972af94..c10600468 100644 --- a/deploy/container/gitea-ci-cache.config.example.json +++ b/deploy/container/gitea-ci-cache.config.example.json @@ -4,9 +4,9 @@ "repository_url": "http://gitea:3000/GenarrativeAI/Genarrative.git", "clone_url": "https://git.genarrative.world/git/GenarrativeAI/Genarrative.git", "runner_container": "gitea-runner", - "runner_api_path": "admin/actions/runners/2", "token_file": "/etc/genarrative-ci-cache/api-token", "state_dir": "/var/lib/genarrative-ci-cache", + "artifact_storage_dir": "/opt/gitea-stack/data/gitea/gitea/actions_artifacts", "gate_socket": "/var/lib/genarrative-ci-cache-gate/gate.sock", "gate_url": "http://gitea-runner-fetch-gate:8080" } diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index 1d7c6c3af..46c640c03 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -96,8 +96,8 @@ SpacetimeDB 任务统一先读取 `.codex/skills/genarrative-spacetimedb/SKILL.m ## Gitea CI 依赖闭合 -Gitea Rust 缓存自动维护由宿主 `genarrative-ci-cache.timer` 调用 `scripts/maintain-gitea-rust-cache.py`：最新 master 固定 SHA 构建，不要求源提交先全绿；输入未变跳过，失败保留现役版。只有专属 FetchTask 入口已暂停且在途请求完整结束、全局 runner/API/内层 Docker 均空闲时才切换；随后真实 master push 的完整九个 job 与六组缓存统计通过，才清理自己登记的过期镜像/归档。保留当前、一个回滚版、所需基础镜像和容器引用。首次部署需要空闲窗口和只读管理员 API 凭据，操作入口见 `deploy/container/README.md`；合并代码不会自动启用宿主服务。范围仅 Gitea CI 测试镜像，不改 Jenkins、生产、本地或发行构建。 +Gitea Rust 缓存自动维护由宿主 `genarrative-ci-cache.timer` 收集同一 master push run 六个 Rust job 的原生 V4 缓存产物，不重复执行 Cargo 预热。只传本轮新 key，命中对象只传使用时间；宿主与真实来源镜像对象合并、去重、按新近使用时间裁剪到 4 GiB，从无对象缓存基础镜像重新组装。源 run 不要求全绿，但取消、缺组、旧 attempt、未完成上传或混用来源镜像不得采用。网关暂停新 FetchTask、在途领取结束、持久化任务账本清空且内层活动容器为空才切换，不打断运行中的 CI。首次接入/升级网关需空闲窗口；Token 只需普通仓库 `write:repository`，不查管理员 API。候选装载后清理已收集 artifact，遗留项保留 7 天；真实 master CI 验证后才清理旧镜像，保留当前、一个回滚版、基础镜像及容器引用。部署入口见 `deploy/container/README.md`，合并代码不等于服务启用。 -AGC Rust 两条 lane、crates、smoke、Backend 和桌面壳测试使用镜像内可信 sccache 对象快照；Native shell release step 显式清空双 wrapper，前端/repository checks 不启用。维护者通过 `scripts/build-gitea-rust-cache.sh` 从远端 master 在限额、无宿主挂载的临时容器中按实际 cwd/profile/目标预热全部测试组，仅编译、不执行测试/应用；后端 workspace 与 spacetime-module 保持独立，AGC 的三个 cwd 入口之间清理预热 target，防止 fresh 判断漏产缓存键。最终镜像只追加 sccache、对象和来源元数据，不包含源码或 target。容量上限 4 GiB，不替代宿主旧镜像/归档清理。PR 只写当前容器层、不回传，不开放 Docker API/发布权限；继续禁用 incremental。`ci-rust-cache.sh` 在快照缺失、工具链不符或 wrapper 探测失败时直接编译，并隔离远程缓存配置和 daemon。分片日志记录编译耗时，收尾输出命中统计；两个 lane 的测试和前置检查不同，耗时差不是严格 A/B。线上存在活跃 CI 时不得重启 runner 或切换标签；全组启用前须刷新完整快照并逐组验证，详见开发运维文档。 +AGC Rust 两条 lane、crates、smoke、Backend 和桌面壳测试使用镜像内可信 sccache 对象快照；Native shell release step 显式清空双 wrapper，前端/repository checks 不启用。仅首次人工 bootstrap 时，维护者通过 `scripts/build-gitea-rust-cache.sh` 从远端 master 在限额、无宿主挂载的临时容器中按实际 cwd/profile/目标预热全部测试组，仅编译、不执行测试/应用；后端 workspace 与 spacetime-module 保持独立，AGC 的三个 cwd 入口之间清理预热 target，防止 fresh 判断漏产缓存键。最终镜像只追加 sccache、对象和来源元数据，不包含源码或 target。容量上限 4 GiB，不替代宿主旧镜像/归档清理。PR 只写当前容器层、不回传，不开放 Docker API/发布权限；继续禁用 incremental。`ci-rust-cache.sh` 在快照缺失、工具链不符或 wrapper 探测失败时直接编译，并隔离远程缓存配置和 daemon。分片日志记录编译耗时，收尾输出命中统计；两个 lane 的测试和前置检查不同，耗时差不是严格 A/B。线上存在活跃 CI 时不得重启 runner 或切换标签；全组启用前须刷新完整快照并逐组验证，详见开发运维文档。 `.gitea/workflows/project-ci.yml` 的客户端门禁拆成 lane 与功能 job，每个 job 只预热自己会构建的那几份依赖：`AI game creator shell Rust lane 1/2`、`lane 2/2` 各自预取一次 AGC 壳 manifest，并顺序运行两片 Rust bin 单测；`AI game creator shell Rust smoke` 同样只预取 AGC 壳 manifest（`agent-run` smoke 会用 `src-tauri/Cargo.toml` spawn `cargo run`），`AI game creator shell Rust crates` 预取 `server-rs/Cargo.toml` 与独立 crate，`Native shell tests` 预取桌面壳与 AGC 壳 manifest，`AI game creator shell web tests` 不触碰 Cargo，不预热。两条 Rust lane、smoke job 与 crates job 只用 cargo 与 node 内建模块，因此不执行 `npm ci`。两个被 `server-rs/Cargo.toml` 排除、且没有提交 `Cargo.lock` 的独立 crate（`agent-runtime-core`、`agent-runtime-orchestration`）只能在 `AI game creator shell Rust crates` 里用不带锁标志的 fetch。AGC 壳的 bin target 单测（约 2466 条）由 `apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs` 编译后按 `--list` 名单分 4 片：每次分片调用用 `--shard-index=` 只跑自己那片，片内保持 `--test-threads=1` 并使用独立 `TMPDIR`；两条 lane 之间并发，lane 内顺序运行两片，避免重复依赖预热和同一容器内多进程争抢。不要改回「一个 job 内多进程并行这几片」——同一容器里它们会争抢共享 `HOME`、target 目录与固定临时路径，实测比整套串行还慢。每个分片调用都会自校验「片并集等于全集且互斥」，因此改分片规则不会静默漏跑。Backend host workspace tests 使用 `cargo test --locked --workspace --exclude spacetime-module --no-fail-fast`，避免 `spacetime-module` 的 `spacetime-types` feature 统一污染普通领域 crate 的 host 测试；随后单独执行 `cargo test --locked -p spacetime-module --no-fail-fast`，由 `spacetime-module/src/active.rs` 在 host 测试构建期间提供仅测试期的 SpacetimeDB ABI 链接支持，使该 crate 的纯单元测试也纳入 Backend 门禁。`spacetime-module` 的 reducer / procedure 运行时行为仍必须通过真实 SpacetimeDB runtime/integration harness 验证，host 链接支持不得被当作运行时替身。Backend 另外执行 `cargo check --locked -p spacetime-module` 验证模块源码。AGC 壳检查还会运行 `platform-llm` 与 `shared-contracts` 的 server-rs workspace 测试，这些命令以及 AGC 壳测试必须带 `--locked`，避免在测试阶段重新解析 registry index；锁文件发生变化时应先更新受信任 CI 镜像缓存，再重跑门禁。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 3708b11cd..c7cd7884d 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -5955,5 +5955,6 @@ Cocos Creator 根目录由 `package.json.creator.version` 与普通 `assets/` - 验证：相同源码、资源上限和独立干净 target 下分别记录无缓存、冷缓存、热缓存的编译耗时和 hit/miss；只有真实热命中有净收益才切换候选镜像。PR 的写入始终留在 job 容器层，公共快照仍由可信维护流程生成。 - 统计：job 私有 daemon 设置 `SCCACHE_IDLE_TIMEOUT=0`，由 `report` 显式停止；最终测试 bin 的不可缓存编译或测试可能超过一分钟，短 idle timeout 会让 daemon 提前退出，结尾查询启动新 daemon 后误报零次请求。容器销毁仍会回收该 job 的全部进程。 - 磁盘：快照构建拒绝含 `/opt/genarrative-ci/rust-cache` 的基础镜像，始终从无对象缓存的镜像重建；容器内删除旧对象不能释放 Docker 底层。对象缓存容量上限不涵盖宿主旧镜像及导出归档。自动维护只回收自己预先登记的 Image ID/tag 和专属归档，保留当前、一个回滚版、基础镜像及所有容器引用；内层按 ID 导入的镜像可能没有 tag，不能只按 tag 判断已清理。接管前历史试验版本仍需人工确认。 -- 自动切换：Gitea 1.26.4 的 disabled 检查与 FetchTask 创建任务事务之间没有共同锁，Runner 2.0.0 的 fetch_timeout 也不能保证服务端事务回滚。不能以“禁用后睡几秒”或反复 docker ps 冒充领取屏障。专属入口阻断新 FetchTask，并完整读取在途响应；未知完成状态持久化 uncertain 并停止自动切换。Runner 的 .runner 文件还会因 label 更新回写，mtime 不等于地址变更时间；要结合已配置地址、当前容器启动时间与入口实际见到的 FetchTask 来源确认路由已生效。 +- CI 产物清理：Gitea 1.26.4 的仓库 REST 仅列出 finalized/expired V4 artifact，内置到期清理不回收上传中断的 tmp-upload 分块。缓存上传块须带专属标识，宿主只清理目标仓库已结束且超过 7 天的 master run 中同样过期的自有普通文件，未知文件/符号链接保护，不改数据库或全局 prune。Artifact.workflow_run 仅含 ID/SHA，判断过期产物所属事件和状态须再读 run API，不能当作完整 run 使用。 +- 自动切换：Gitea 1.26.4 的 disabled 检查与 FetchTask 事务不原子，Runner 客户端超时不能证明服务端回滚，容器暂时为空也不能证明没有已领取任务。网关必须解析实际 Connect Protobuf/gzip，转发 FetchTask 结果前持久化任务 ID，仅在最终日志及执行清理后的最终 UpdateTask 确认后清账；取消响应不能提前释放。暂停新领取、在途为零、账本为零且内层活动容器为空才可切换，无需全局 Runner admin API。未知协议/响应或崩溃遗留标记停止切换；旧网关缺 active_tasks 不能默认零。首次接入与账本升级须空闲窗口。.runner 的 mtime 不证明地址已加载，应核验真实 FetchTask 来源及本次容器启动时间。 - 扩展：预热所有 Rust 测试组时保留各自 cwd、profile、features 和锁策略；同一临时 target 的 Cargo fresh 不代表不同 cwd 都已生成缓存键，AGC 提示词契约、分片和 smoke 切换入口前清理预热 target。不要把 workspace 与 spacetime-module 合并成一次编译；Native shell release step 清空双 wrapper，避免将测试缓存扩展成发布缓存。当前 sccache 0.18.0 的 READ_ONLY 在 miss 后仍打包产物并产生 cache write error，不适合用来承诺“未命中无开销”。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index e8030833e..06e81f3cc 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -334,17 +334,17 @@ master 日常交付必须禁止直接 push，只允许经 PR 在最近一次 Pro #### Rust 测试组的隔离编译缓存 -每次生成快照必须使用不含对象缓存的原始 CI 基础镜像；构建脚本在拉取源码和预热前拒绝已存在 `/opt/genarrative-ci/rust-cache` 的基础镜像，避免重复叠加不可释放的旧对象层。宿主自动维护器 `scripts/maintain-gitea-rust-cache.py` 固定最新 master SHA 构建，不要求该提交事先全绿；源码或镜像校验失败保留旧版。切换后通过真实 master CI 才按 `deploy/container/README.md` 自动定向清理自己登记的旧镜像及归档，保护当前版、一个回滚版、所需基础镜像及全部容器引用；首次接管前的手工试验版本不自动认领。 +每次生成快照必须使用不含对象缓存的原始 CI 基础镜像，避免叠加不可释放的旧层。宿主自动维护器 `scripts/maintain-gitea-rust-cache.py` 收集完整 master push CI 的六组 V4 增量产物，复用其真实来源镜像对象并合并去重，总容量 4 GiB，不重复执行 Cargo 预热编译。测试失败可收集，取消、缺组、未完成上传、旧 attempt 或混用来源镜像不可发布。候选校验、导出和装载后删除已收集 artifact，遗留项保留 7 天；切换后通过真实 master CI 才定向清理受管旧镜像/归档，保护当前、一个回滚版、基础镜像及容器引用。 AGC Rust 两条 lane、crates、agent-run smoke、Backend 和 Native shell 的桌面壳测试均启用 sccache。Native shell 的 release build smoke 显式清空两个 wrapper，保持发布构建原有 profile/features 与资源 staging；前端和 repository checks 不启用对象缓存。继续设置 `CARGO_INCREMENTAL=0`，不共享 target、不恢复 Actions 可写缓存。可信快照通过已有固定 Image ID 分发：镜像只增加固定版本的 sccache、编译对象和来源元数据，不包含源码、target 或凭据；容器写时复制层承接本 job 的新增对象，job 删除后丢弃，PR 没有 Docker API 或快照发布权限。该权限边界由 runner 基础设施保证，不能仅用 workflow 的分支条件替代。 -维护者在受信任 checkout 中运行 `bash scripts/build-gitea-rust-cache.sh <已验证基础镜像> <候选镜像tag> [完整master-SHA]`。自动维护显式传入固定 SHA，脚本验证它为抓取到的 master 祖先，源码及缓存辅助脚本来自同一归档；人工省略 SHA 时默认最新 master。在无宿主目录挂载、无凭据且有 CPU/内存上限的临时容器中预热各组实际 Cargo 目标，只编译、不执行测试或应用。后端 workspace 仍排除 spacetime-module，再单独预热该模块，禁止为缓存合并 features。AGC 独立 crate 与插件沿用各自 manifest/锁文件策略；未提交锁文件的 crate 仍可能因新版本解析产生 miss。最终镜像从原基础镜像重新组装，只导出 sccache 对象，不提交预热容器。公共快照不接受 PR 上传，也不复用 Jenkins 发布缓存。工具链或系统依赖变化时重新生成快照，缓存命中仍由 sccache 的编译输入校验决定，不能省略 Cargo 构建。 +人工 bootstrap 在没有可消费快照时使用 `bash scripts/build-gitea-rust-cache.sh <已验证基础镜像> <候选镜像tag> [完整master-SHA]`；自动维护不调用它。bootstrap 固定 master 归档，在无宿主挂载、无凭据、有资源限额的容器中只编译预热，不执行测试/应用，Cargo features 和工作目录保持实际 CI 口径。日常更新由 master CI 导出新 key，命中继承对象只传使用时间；PR 不导出、不扫描。宿主验证来源、大小和哈希，不从上传产物执行程序，只从可信来源镜像复制固定 sccache。工具链或基础镜像输入变化时重建无对象缓存基础镜像，缓存工具链必须匹配。公共快照不接受 PR，不复用 Jenkins 发布缓存。 各 Rust job 在编译前执行 `scripts/ci-rust-cache.sh prepare`：检查快照与 rustc 身份，隔离 sccache 配置和 daemon，限时探测真实 wrapper。旧镜像没有快照或探测失败时保留空 wrapper，输出 fallback 原因；缓存故障不得把真实编译/测试失败改成成功，也不允许重跑整个测试组掩盖失败。结束时 `report` 输出命中统计；分片日志单独记录 Cargo 编译耗时。对象缓存上限为 4 GiB，快照构建完成后输出实际体积；最终测试 bin 的链接仍须执行。全组开启时必须生成覆盖全部目标的新快照，不能把旧 AGC 单目标快照当作后端/桌面壳的预热验收。 候选镜像沿用 `gitea-ci-job-image.sh verify/export/load-runner` 验证和装载。现役 CI 正在运行时只允许准备候选镜像，不停止 job、不重启 runner、不切换标签；待确认无运行中的 job 后再按上述镜像更新顺序切换。验收分别记录独立 target 的无缓存、冷缓存和热缓存构建耗时及命中率，并检查失效/故障回退、两容器写入互不影响。未完成真实 CI 验证前不能宣称提速或推广到其它 job。 -自动维护首次部署及故障恢复统一见 `deploy/container/README.md` 的“自动维护首次部署与恢复”。systemd timer 约每 5 分钟检查一次，持有文件锁，使用独立 clone/状态/归档目录；只管理 Gitea CI 测试镜像，不改变 Jenkins、生产、本地或客户端发行构建，Native shell release smoke 继续清空双 wrapper。自动切换使用专属 FetchTask 入口的暂停与在途计数屏障，不能将 Gitea disabled 或客户端超时当成事务完成证据；屏障收敛后还检查全局 runner busy、Gitea in_progress 和内层容器。首次接入入口必须安排空闲维护窗口，不在运行中的 CI 上试切；脚本合并不等于服务已在线启用。 +自动维护首次部署及故障恢复统一见 `deploy/container/README.md`。timer 约每 5 分钟检查，文件锁串行执行，使用专属 clone/状态/归档目录；只管理 Gitea CI 测试镜像，Native shell release smoke 继续清空双 wrapper。凭据只需普通账号 `write:repository`，读取 run/job/日志/产物并定向删除缓存 artifact，不需要管理员权限。切换依赖 FetchTask 暂停与在途屏障、任务持久账本和内层 Docker 空闲；Gitea 显示 cancelled、disabled 或客户端超时都不能证明执行已经结束。网关依据最终日志与执行收尾后的最终任务上报确认释放任务。首次接入/旧网关升级须等待 CI 自然结束的空闲窗口，不在运行中的 CI 上试切；脚本合并不等于服务已部署。 Rust 对象 key 包含编译工作目录。预热必须使用已核实的 Gitea checkout 路径 `/workspace/GenarrativeAI/Genarrative`：后端、独立 crate、插件、桌面壳及提示词契约从仓库根目录启动 Cargo，AGC 分片从 `apps/ai-game-creator-shell/src-tauri` 启动，agent-run smoke 从 `apps/ai-game-creator-shell` 启动。切换 AGC 编译入口前清理该临时容器的 AGC target，防止 Cargo 的 fresh 判断跳过新 cwd 所需对象；CI 各 job 的 target 本来就独立。快照保存 `workspace.txt`，job 根路径不符时直接编译。不要仅设置 `SCCACHE_BASEDIRS` 就假定 Rust 可以跨 cwd 命中，也不为缓存改写 `RUSTFLAGS` 或源码路径语义。固定 sccache `0.18.0` 的基础设施错误码 `2` 会由 wrapper 回退执行本次 rustc；其它退出码原样返回，升级 sccache/Rust 时须复核此约定。 diff --git a/scripts/ci-rust-cache.sh b/scripts/ci-rust-cache.sh index f93aacfc0..c19459da9 100644 --- a/scripts/ci-rust-cache.sh +++ b/scripts/ci-rust-cache.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# 仅消费镜像内的可信快照；所有写入留在当前容器的可写层。 +# 消费镜像内可信快照；写入留在任务容器，master push 可在任务结束后导出。 set -euo pipefail cache_root="${GENARRATIVE_CI_RUST_CACHE_ROOT:-/opt/genarrative-ci/rust-cache}" @@ -22,7 +22,7 @@ configure_local_cache() { case "${1:-}" in prepare) : "${GITHUB_ENV:?GITHUB_ENV is required}" - printf 'RUSTC_WRAPPER=\nCARGO_BUILD_RUSTC_WRAPPER=\nGENARRATIVE_CI_RUST_CACHE_STATE=\n' >> "${GITHUB_ENV}" + printf 'RUSTC_WRAPPER=\nCARGO_BUILD_RUSTC_WRAPPER=\nGENARRATIVE_CI_RUST_CACHE_STATE=\nGENARRATIVE_CI_RUST_CACHE_EXPORT_READY=\n' >> "${GITHUB_ENV}" fallback() { printf '[rust-cache] mode=direct reason=%s\n' "$1"; exit 0; } [[ -x "${cache_binary}" && -d "${cache_root}/objects" && -f "${cache_root}/rustc.txt" && -f "${cache_root}/source-commit.txt" && -f "${cache_root}/workspace.txt" ]] \ || fallback snapshot-unavailable @@ -44,6 +44,10 @@ case "${1:-}" in fallback wrapper-probe-failed fi script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/$(basename "${BASH_SOURCE[0]}")" + if [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_REF:-}" == refs/heads/master ]]; then + # 只记 key/大小/时间，不读取对象正文；PR 没有扫描与打包开销。 + python3 "$(dirname "${script_path}")/export-gitea-rust-cache.py" baseline "${state}/baseline.json" + fi # wrapper 路径也参与 Rust cache key；固定容器内路径，隔离由 job 容器保证。 wrapper_path="${cache_root}/rustc-wrapper" printf '#!/usr/bin/env bash\nexec bash %q "$@"\n' "${script_path}" > "${wrapper_path}" @@ -59,7 +63,13 @@ case "${1:-}" in if [[ -n "${state}" && -d "${state}" ]]; then configure_local_cache timeout --kill-after=2 5 "${cache_binary}" --show-stats || true - timeout --kill-after=2 5 "${cache_binary}" --stop-server >/dev/null 2>&1 || true + # 只有成功停服后才允许读取对象；失败不能把仍在写入的目录发布成完整快照。 + if timeout --kill-after=2 15 "${cache_binary}" --stop-server >/dev/null 2>&1; then + if [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_REF:-}" == refs/heads/master && ! -f "${state}/disabled" && -f "${state}/baseline.json" ]]; then + mv -- "${state}/baseline.json" "${cache_root}/export-baseline.json" + printf 'GENARRATIVE_CI_RUST_CACHE_EXPORT_READY=1\n' >> "${GITHUB_ENV}" + fi + fi rm -rf -- "${state}" else printf '[rust-cache] mode=direct\n' diff --git a/scripts/ci-rust-cache.test.mjs b/scripts/ci-rust-cache.test.mjs index 2b4f4f0ad..84ab6d9f9 100644 --- a/scripts/ci-rust-cache.test.mjs +++ b/scripts/ci-rust-cache.test.mjs @@ -35,7 +35,8 @@ function fixture(t) { `#!/bin/bash set -eu case "$1" in - --stop-server|--show-stats) exit 0 ;; + --stop-server) exit "\${STOP_FAILURE:-0}" ;; + --show-stats) exit 0 ;; esac if [[ "$*" == *-vV ]]; then [[ "\${PROBE_FAILURE:-}" != 1 ]] || exit 1 @@ -215,3 +216,25 @@ linuxTest('lost local cache state still invokes the real compiler', (t) => { const f = fixture(t); assert.equal(f.run(['/bin/bash', '-c', 'exit 43']).status, 43); }); + +linuxTest('only a cleanly stopped master cache can export objects', (t) => { + const master = { + GITHUB_EVENT_NAME: 'push', + GITHUB_REF: 'refs/heads/master', + }; + for (const stopFailure of ['0', '1']) { + const f = fixture(t); + const prepared = f.run(['prepare'], master); + assert.equal(prepared.status, 0, prepared.stderr); + const result = f.run(['report'], { + ...f.preparedEnv(), + ...master, + STOP_FAILURE: stopFailure, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal( + f.preparedEnv().GENARRATIVE_CI_RUST_CACHE_EXPORT_READY, + stopFailure === '0' ? '1' : '', + ); + } +}); diff --git a/scripts/export-gitea-rust-cache.py b/scripts/export-gitea-rust-cache.py new file mode 100644 index 000000000..3ad38e606 --- /dev/null +++ b/scripts/export-gitea-rust-cache.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""只把 master push 的任务内 sccache 对象导出为 Gitea Actions 产物。""" + +import base64 +from datetime import datetime, timedelta, timezone +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile + + +MAX_OBJECT_BYTES = 4 * 1024**3 +CHUNK_BYTES = 8 * 1024**2 +JOBS = { + "ai-game-creator-shell-rust-lane-1", + "ai-game-creator-shell-rust-lane-2", + "ai-game-creator-shell-rust-smoke", + "ai-game-creator-shell-rust-crates", + "backend-tests", + "native-shell-tests", +} + + +def object_path(name): + parts = name.split("/") + return ( + len(parts) == 3 + and re.fullmatch(r"[a-f0-9]{64}", parts[2]) is not None + and parts[0] == parts[2][0] + and parts[1] == parts[2][1] + ) + + +class HashingReader: + def __init__(self, stream): + self.stream = stream + self.digest = hashlib.sha256() + + def read(self, size=-1): + data = self.stream.read(size) + self.digest.update(data) + return data + + +def scan_objects(root): + objects = root / "objects" + if objects.is_symlink() or not objects.is_dir(): + raise ValueError("cache object directory is unavailable") + candidates = [] + for directory, directories, files in os.walk(objects, followlinks=False): + directories[:] = [ + name for name in directories if not (Path(directory) / name).is_symlink() + ] + for filename in files: + path = Path(directory) / filename + relative = path.relative_to(objects).as_posix() + info = path.lstat() + if object_path(relative) and stat.S_ISREG(info.st_mode): + candidates.append((path, relative, info)) + return candidates + + +def save_baseline(root, destination): + # sccache 命中会更新 mtime，不能把时间变化当成内容变化，否则又变成全量上传。 + baseline = { + relative: {"size": info.st_size, "mtime_ns": info.st_mtime_ns} + for _, relative, info in scan_objects(root) + } + destination.write_text(json.dumps(baseline), encoding="utf-8") + + +def pack_snapshot(root, destination, metadata, baseline, limit=MAX_OBJECT_BYTES): + """仅归档新 key；命中的已有对象只传递新近使用时间，不传输对象内容。""" + candidates = scan_objects(root) + candidates.sort(key=lambda item: (-item[2].st_mtime_ns, item[1])) + entries = [] + touched = [] + total = 0 + with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_STORED) as bundle, \ + bundle.open("snapshot.tar", "w", force_zip64=True) as tar_stream, \ + tarfile.open(fileobj=tar_stream, mode="w|", format=tarfile.USTAR_FORMAT) as archive: + for path, relative, info in candidates: + previous = baseline.get(relative) + if previous is not None and previous["size"] == info.st_size: + if previous["mtime_ns"] != info.st_mtime_ns: + touched.append({"path": "objects/" + relative, "mtime_ns": info.st_mtime_ns}) + continue + if info.st_size <= 0 or total + info.st_size > limit: + continue + # report 已成功停止 daemon；打开后再核实对象，避免导出变化中的文件。 + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(descriptor, "rb") as stream: + actual = os.fstat(stream.fileno()) + if not stat.S_ISREG(actual.st_mode) or ( + actual.st_size, actual.st_mtime_ns, actual.st_ino + ) != (info.st_size, info.st_mtime_ns, info.st_ino): + raise ValueError("cache object changed during export") + member = tarfile.TarInfo("objects/" + relative) + member.size = info.st_size + member.mode = 0o644 + member.mtime = int(info.st_mtime) + reader = HashingReader(stream) + archive.addfile(member, reader) + entries.append({ + "path": member.name, + "size": info.st_size, + "sha256": reader.digest.hexdigest(), + "mtime_ns": info.st_mtime_ns, + }) + total += info.st_size + manifest = dict(metadata, schema=1, mode="delta", objects=entries, touched=touched) + data = json.dumps(manifest, ensure_ascii=False, sort_keys=True).encode("utf-8") + member = tarfile.TarInfo("manifest.json") + member.size = len(data) + member.mode = 0o644 + archive.addfile(member, io.BytesIO(data)) + return len(entries), total + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # runtime token 只允许发往明确配置的 Gitea 地址。 + return None + + +def request(method, url, token, data=None, headers=None, attempts=3): + opener = urllib.request.build_opener(NoRedirect()) + for attempt in range(attempts): + req = urllib.request.Request(url, data=data, method=method, headers={ + **({"Authorization": "Bearer " + token} if token else {}), + "Content-Type": "application/json", + **(headers or {}), + }) + try: + with opener.open(req, timeout=120) as response: + body = response.read(1024 * 1024) + return json.loads(body) if body else {} + except urllib.error.HTTPError as error: + if error.code not in (408, 429, 500, 502, 503, 504) or attempt == attempts - 1: + raise RuntimeError(f"artifact {method} failed: HTTP {error.code}") from None + except (urllib.error.URLError, TimeoutError, ConnectionError): + if attempt == attempts - 1: + raise RuntimeError(f"artifact {method} connection failed") from None + time.sleep(2 ** attempt) + + +def artifact_base_url(env): + # .runner.address 可指向仅支持 RPC 的领取网关，不能用 GITHUB_SERVER_URL。 + repository = env["GITHUB_REPOSITORY"] + clone_url = env["GENARRATIVE_GITEA_REPOSITORY_URL"] + suffix = "/" + repository + ".git" + parsed = urllib.parse.urlsplit(clone_url) + if ( + parsed.scheme not in ("http", "https") + or not parsed.netloc + or parsed.username or parsed.password or parsed.query or parsed.fragment + or not parsed.path.endswith(suffix) + ): + raise ValueError("a credential-free HTTP Gitea repository URL is required") + return urllib.parse.urlunsplit(( + parsed.scheme, parsed.netloc, parsed.path[:-len(suffix)], "", "" + )) + + +def upload_snapshot(path, name, run_id, base_url, token): + """Gitea 1.26.4 原生 v4：只有完成全部块并校验 SHA256 后才发布。""" + producer_match = re.fullmatch(r"rust-cache-v1-([a-z0-9-]+)-attempt-(0|[1-9][0-9]*)", name) + if not producer_match or producer_match[1] not in JOBS: + raise ValueError("unexpected cache artifact name") + producer = f"{producer_match[1]}:{producer_match[2]}" + endpoint = f"{base_url}/twirp/github.actions.results.api.v1.ArtifactService" + # Gitea 从任务凭据确定 job，只要求请求中的 run ID 与任务所属 run 一致。 + identity = {"workflowRunBackendId": str(run_id), "name": name} + created = request("POST", endpoint + "/CreateArtifact", token, json.dumps({ + **identity, "version": 4, + # Gitea 将剩余小时向下取整成天；多给一天，服务端才实际保留至少七天。 + "expiresAt": (datetime.now(timezone.utc) + timedelta(days=8)).isoformat(), + }).encode()) + received = urllib.parse.urlsplit(created.get("signedUploadUrl", "")) + expected_path = urllib.parse.urlsplit(endpoint).path + "/UploadArtifact" + if not created.get("ok") or received.path != expected_path or not received.query: + raise ValueError("unexpected artifact upload path") + # 服务可能返回外部 AppURL；保留签名，但只连接配置的内部 Gitea 地址。 + upload_url = endpoint + "/UploadArtifact?" + received.query + size = path.stat().st_size + digest = hashlib.sha256() + blocks = [] + with path.open("rb") as source: + while chunk := source.read(CHUNK_BYTES): + # 双层编码中的内层保留所属任务；宿主只回收带此前缀的过期未完成块。 + block = base64.b64encode( + f"genarrative-rust-cache-v1:{producer}:{len(blocks):08d}".encode() + ).decode() + blocks.append(block) + digest.update(chunk) + request("PUT", upload_url + "&" + urllib.parse.urlencode({ + "comp": "block", "blockid": block, + }), None, chunk, { + "Content-Type": "application/octet-stream", + }) + blocklist = "" + "".join(f"{block}" for block in blocks) + "" + request("PUT", upload_url + "&comp=blocklist", None, blocklist.encode(), { + "Content-Type": "application/xml", + }) + # Finalize 会消耗服务端块列表；响应不确定时不盲目重试、也不宣告成功。 + finalized = request("POST", endpoint + "/FinalizeArtifact", token, json.dumps({ + **identity, "size": str(size), "hash": "sha256:" + digest.hexdigest(), + }).encode(), attempts=1) + if not finalized.get("ok"): + raise ValueError("artifact was not finalized") + + +def export(env): + # 双重限定：手工调用、PR 和其它分支不会扫描、打包或上传对象。 + if env.get("GITHUB_EVENT_NAME") != "push" or env.get("GITHUB_REF") != "refs/heads/master": + return + if env.get("GENARRATIVE_CI_RUST_CACHE_EXPORT_READY") != "1": + print("[rust-cache] export skipped: cache daemon did not stop cleanly") + return + job = env.get("GITHUB_JOB") + if job not in JOBS: + raise ValueError("unexpected Rust cache job") + sha = env["GITHUB_SHA"] + if not re.fullmatch(r"[a-f0-9]{40}", sha): + raise ValueError("a complete source SHA is required") + run_id, attempt = int(env["GITHUB_RUN_ID"]), int(env["GITHUB_RUN_ATTEMPT"]) + if run_id <= 0 or attempt < 0: + raise ValueError("invalid run identity") + token = env.get("ACTIONS_RUNTIME_TOKEN") or env["GENARRATIVE_GITEA_TOKEN"] + base_url = artifact_base_url(env) + root = Path(env.get("GENARRATIVE_CI_RUST_CACHE_ROOT", "/opt/genarrative-ci/rust-cache")) + baseline_path = root / "export-baseline.json" + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + rustc = subprocess.check_output(["rustc", "-vV"], text=True) + workspace = str(Path.cwd().resolve()) + if rustc != (root / "rustc.txt").read_text() or workspace != (root / "workspace.txt").read_text().strip(): + raise ValueError("cache provenance changed after prepare") + metadata = { + "repository": env["GITHUB_REPOSITORY"], + "run_id": run_id, "run_attempt": attempt, "job": job, "source_sha": sha, + "rustc": rustc, "workspace": workspace, + "inherited_source_sha": (root / "source-commit.txt").read_text().strip(), + "sccache_version": subprocess.check_output([str(root / "sccache"), "--version"], text=True).strip(), + } + if (root / "base-image.txt").is_file(): + metadata["base_image"] = (root / "base-image.txt").read_text().strip() + name = f"rust-cache-v1-{job}-attempt-{attempt}" + try: + with tempfile.TemporaryDirectory(prefix="ci-rust-cache-export-") as directory: + path = Path(directory) / "snapshot.zip" + count, size = pack_snapshot(root, path, metadata, baseline) + upload_snapshot(path, name, run_id, base_url, token) + finally: + baseline_path.unlink(missing_ok=True) + print(f"[rust-cache] artifact={name} objects={count} bytes={size} complete=true") + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "baseline": + save_baseline(Path(os.environ.get("GENARRATIVE_CI_RUST_CACHE_ROOT", "/opt/genarrative-ci/rust-cache")), Path(sys.argv[2])) + else: + export(os.environ) diff --git a/scripts/gitea-runner-fetch-gate.py b/scripts/gitea-runner-fetch-gate.py index 8e68cb6db..7f8346e65 100644 --- a/scripts/gitea-runner-fetch-gate.py +++ b/scripts/gitea-runner-fetch-gate.py @@ -3,6 +3,8 @@ import http.client import http.server +import gzip +import io import json import os from pathlib import Path @@ -10,12 +12,90 @@ import socketserver import threading import time import urllib.parse +import zlib MAX_BODY = 32 * 1024 * 1024 HOP_HEADERS = { "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade", "host", "content-length", } +RPC_PREFIX = "/api/actions/runner.v1.RunnerService/" + + +def protobuf_fields(data): + """只解码已核对的 actions-proto-go v0.4.1 字段，不引入 protobuf 运行时。""" + fields = {} + offset = 0 + + def varint(): + nonlocal offset + value = 0 + for shift in range(0, 70, 7): + if offset >= len(data): + raise ValueError("truncated protobuf") + byte = data[offset] + offset += 1 + value |= (byte & 127) << shift + if byte < 128: + return value + raise ValueError("invalid protobuf varint") + + while offset < len(data): + tag = varint() + number, wire = tag >> 3, tag & 7 + if not number: + raise ValueError("invalid protobuf tag") + if wire == 0: + value = varint() + elif wire in (1, 2, 5): + length = varint() if wire == 2 else (8 if wire == 1 else 4) + if offset + length > len(data): + raise ValueError("truncated protobuf field") + value = data[offset:offset + length] + offset += length + else: + raise ValueError("unsupported protobuf wire type") + fields.setdefault(number, []).append(value) + return fields + + +def single(fields, number, default=None): + values = fields.get(number, []) + if len(values) > 1: + raise ValueError("ambiguous tracking field") + return values[0] if values else default + + +def rpc_fields(body, headers): + encoding = headers.get("Content-Encoding", "identity").lower() + if encoding == "gzip": + with gzip.GzipFile(fileobj=io.BytesIO(body)) as stream: + body = stream.read(MAX_BODY + 1) + elif encoding != "identity": + raise ValueError("unsupported RPC encoding") + if len(body) > MAX_BODY: + raise ValueError("decoded RPC too large") + if headers.get("Content-Type", "").split(";", 1)[0] != "application/proto": + raise ValueError("task tracking requires Connect protobuf") + return protobuf_fields(body) + + +def task_state(fields): + nested = single(fields, 1) + if not isinstance(nested, bytes): + raise ValueError("missing task state") + state = protobuf_fields(nested) + task_id = positive_id(single(state, 1)) + result = single(state, 2, 0) + if type(result) is not int or result not in range(5): + raise ValueError("unknown task result") + return task_id, result + + +def positive_id(value): + if type(value) is not int or not 0 < value < 2 ** 63: + raise ValueError("invalid task ID") + return str(value) class Gate: @@ -26,7 +106,19 @@ class Gate: self.inflight = 0 self.last_fetch_peer = None self.last_fetch_at = None + self.tasks = {} self.uncertain = (self.directory / "uncertain").exists() + try: + if (self.directory / "tasks.json").exists(): + tasks = json.loads((self.directory / "tasks.json").read_text()) + if (not isinstance(tasks, dict) or any( + positive_id(int(key)) != key or type(value) is not bool + for key, value in tasks.items())): + raise ValueError("invalid task ledger") + self.tasks = tasks + except (ValueError, TypeError, OSError): + self.uncertain = True + self.mark("uncertain") if (self.directory / "inflight").exists(): self.uncertain = True self.mark("uncertain") @@ -48,9 +140,71 @@ class Gate: def snapshot(self): return {"paused": self.paused, "inflight": self.inflight, + "active_tasks": len(self.tasks), "task_ids": sorted(self.tasks), "uncertain": self.uncertain, "last_fetch_peer": self.last_fetch_peer, "last_fetch_at": self.last_fetch_at} + def save_tasks(self): + temporary = self.directory / "tasks.json.tmp" + with temporary.open("w", encoding="ascii") as stream: + json.dump(self.tasks, stream, sort_keys=True) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self.directory / "tasks.json") + self.sync_directory() + + def fail_closed(self): + with self.lock: + self.uncertain = self.paused = True + self.mark("uncertain") + + def assigned(self, fields): + task = single(fields, 1) + if task is None: + return + if not isinstance(task, bytes): + raise ValueError("invalid fetched task") + task_id = positive_id(single(protobuf_fields(task), 1)) + with self.lock: + self.tasks[task_id] = False + # 必须先持久化，再把领取结果交给 runner；仅保存 ID，不保存 secrets。 + self.save_tasks() + + def reported(self, method, request, response): + if method == "UpdateLog": + task_id = positive_id(single(request, 1)) + index = single(request, 2, 0) + ack = single(response, 1, 0) + no_more = single(request, 4, 0) + if (type(index) is not int or type(ack) is not int + or index < 0 or ack < 0 or no_more not in (0, 1)): + raise ValueError("invalid log acknowledgement") + finalized = no_more == 1 and ack == index + len(request.get(3, [])) + else: + task_id, result = task_state(request) + response_id, response_result = task_state(response) + if response_id != task_id: + raise ValueError("mismatched task acknowledgement") + # 取消响应可能出现在任务执行途中，必须等 runner 自己报告终态。 + finalized = result != 0 and response_result != 0 + output_keys = {single(protobuf_fields(entry), 1, b"") + for entry in request.get(2, [])} + finalized = finalized and output_keys.issubset(set(response.get(2, []))) + with self.lock: + if task_id not in self.tasks: + # 客户端可能未读到已成功发出的终态响应而重试；v2.0.0 仅在 + # executor 清理后的 Close 中发送终态，不为幂等重报增加墓碑账本。 + if method == "UpdateTask" and finalized: + return + raise ValueError("report for untracked task; idle bootstrap required") + if method == "UpdateLog" and finalized: + self.tasks[task_id] = True + self.save_tasks() + elif method == "UpdateTask" and finalized and self.tasks[task_id]: + # act_runner Reporter.Close 在 executor 清理之后先封存日志再报终态。 + del self.tasks[task_id] + self.save_tasks() + def record_fetch(self, peer): with self.lock: self.last_fetch_peer = peer @@ -142,6 +296,7 @@ class Proxy(http.server.BaseHTTPRequestHandler): pass # 不输出 RPC 认证头、请求内容或带认证信息的 URL。 def reply(self, status, body, headers=()): + delivered = False try: self.send_response(status) excluded = HOP_HEADERS | { @@ -155,9 +310,12 @@ class Proxy(http.server.BaseHTTPRequestHandler): self.send_header("Connection", "close") self.end_headers() self.wfile.write(body) + self.wfile.flush() + delivered = True except (OSError, ValueError): pass self.close_connection = True + return delivered def do_POST(self): path = urllib.parse.urlsplit(self.path) @@ -170,7 +328,8 @@ class Proxy(http.server.BaseHTTPRequestHandler): except (ValueError, OSError): self.reply(400, b"invalid request body\n") return - is_fetch = path.path.endswith("/FetchTask") + method = path.path.removeprefix(RPC_PREFIX) if path.path.startswith(RPC_PREFIX) else "" + is_fetch = method == "FetchTask" if is_fetch: # 包括暂停时被拒绝的请求；只记录网络来源与时间以证明 daemon 路由。 self.server.gate.record_fetch(self.client_address[0]) @@ -209,12 +368,23 @@ class Proxy(http.server.BaseHTTPRequestHandler): raise http.client.IncompleteRead(bytes(result), response.length) completed = True if is_fetch: + try: + if oversized or response.status != 200: + raise ValueError("unknown FetchTask result") + self.server.gate.assigned(rpc_fields(bytes(result), response.headers)) + except (ValueError, TypeError, OSError, EOFError, zlib.error): + self.server.gate.fail_closed() self.server.gate.leave(True) is_fetch = False if oversized: self.reply(502, b"upstream response too large\n") else: - self.reply(response.status, bytes(result), response.getheaders()) + # 日志封存先记账再返回，避免 runner 立即发终态时抢先读到旧账本。 + if method == "UpdateLog" and response.status == 200: + self.observe_report(method, body, bytes(result), response.headers) + delivered = self.reply(response.status, bytes(result), response.getheaders()) + if method == "UpdateTask" and response.status == 200 and delivered: + self.observe_report(method, body, bytes(result), response.headers) except (OSError, http.client.HTTPException, ValueError): self.reply(502, b"runner upstream unavailable\n") finally: @@ -223,6 +393,13 @@ class Proxy(http.server.BaseHTTPRequestHandler): if is_fetch: self.server.gate.leave(completed) + def observe_report(self, method, request, response, headers): + try: + self.server.gate.reported(method, rpc_fields(request, self.headers), + rpc_fields(response, headers)) + except (ValueError, TypeError, OSError, EOFError, zlib.error): + self.server.gate.fail_closed() + class Control(socketserver.StreamRequestHandler): def handle(self): diff --git a/scripts/gitea_cache_snapshot.py b/scripts/gitea_cache_snapshot.py new file mode 100644 index 000000000..46e117e15 --- /dev/null +++ b/scripts/gitea_cache_snapshot.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +"""Validate and merge Gitea CI sccache artifacts into one object snapshot.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import tarfile +import tempfile +from typing import BinaryIO, Mapping, Sequence +import zipfile + + +GIB = 1024 ** 3 +DEFAULT_MAX_COMBINED_BYTES = 4 * GIB +MAX_INPUT_OBJECT_BYTES = 4 * GIB +MAX_MANIFEST_BYTES = 16 * 1024 ** 2 +MAX_OBJECTS = 100_000 +MAX_BASE_OBJECTS = 1_000_000 +MAX_TAR_OVERHEAD_BYTES = 128 * 1024 ** 2 +OBJECT_PATH = re.compile(r"objects/([0-9a-f])/([0-9a-f])/([0-9a-f]{64})\Z") +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +TEMP_PREFIX = ".gitea-cache-snapshot-" + + +class SnapshotError(ValueError): + """The downloaded snapshot does not satisfy the trusted artifact contract.""" + + +@dataclass(frozen=True) +class ArtifactIdentity: + repository: str + run_id: int + run_attempt: int + job: str + source_sha: str + + +@dataclass(frozen=True) +class ArtifactInput: + archive: Path + expected: ArtifactIdentity + + +@dataclass(frozen=True) +class MergeResult: + object_count: int + total_bytes: int + source_sha: str + rustc: str + workspace: str + base_image: str | None + sccache_version: str + + +@dataclass(frozen=True) +class _Object: + path: str + size: int + sha256: str | None + mtime_ns: int + source_index: int | None + + +@dataclass(frozen=True) +class _Touch: + path: str + mtime_ns: int + + +@dataclass(frozen=True) +class _Archive: + input: ArtifactInput + manifest: Mapping[str, object] + objects: tuple[_Object, ...] + touched: tuple[_Touch, ...] + signature: tuple[int, int, int] + + +def _is_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _require_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value or "\0" in value: + raise SnapshotError(f"manifest {field} must be a non-empty string") + return value + + +def _strict_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise SnapshotError(f"manifest contains duplicate JSON key: {key}") + result[key] = value + return result + + +def _archive_signature(path: Path) -> tuple[int, int, int]: + try: + status = path.stat() + except OSError as error: + raise SnapshotError(f"cannot stat artifact archive: {path}") from error + if path.is_symlink() or not path.is_file(): + raise SnapshotError(f"artifact archive must be a regular file: {path}") + return status.st_size, status.st_mtime_ns, status.st_ino + + +def _safe_zip_path(name: str) -> tuple[str, ...]: + if not name or "\0" in name or "\\" in name or name.startswith("/"): + raise SnapshotError(f"unsafe ZIP member path: {name!r}") + parts = PurePosixPath(name.rstrip("/")).parts + if not parts or any(part in {"", ".", ".."} for part in parts): + raise SnapshotError(f"unsafe ZIP member path: {name!r}") + return parts + + +def _snapshot_member(archive: Path) -> tuple[zipfile.ZipFile, zipfile.ZipInfo]: + try: + bundle = zipfile.ZipFile(archive) + except (OSError, zipfile.BadZipFile) as error: + raise SnapshotError(f"invalid artifact ZIP: {archive}") from error + + try: + infos = bundle.infolist() + if len({info.filename for info in infos}) != len(infos): + raise SnapshotError("ZIP contains duplicate members") + files: list[zipfile.ZipInfo] = [] + directories: list[tuple[str, ...]] = [] + for info in infos: + parts = _safe_zip_path(info.filename) + mode = info.external_attr >> 16 + if info.flag_bits & 1: + raise SnapshotError("encrypted ZIP members are not accepted") + if stat.S_ISLNK(mode): + raise SnapshotError("ZIP links are not accepted") + if info.is_dir(): + directories.append(parts) + else: + files.append(info) + if len(files) != 1: + raise SnapshotError("ZIP must contain exactly one snapshot.tar file") + member = files[0] + parts = _safe_zip_path(member.filename) + if not (parts == ("snapshot.tar",) or + (len(parts) == 2 and parts[1] == "snapshot.tar")): + raise SnapshotError("ZIP member must be snapshot.tar or /snapshot.tar") + expected_directories = set() if len(parts) == 1 else {(parts[0],)} + if set(directories) - expected_directories: + raise SnapshotError("ZIP contains unexpected directory members") + maximum_tar_size = ( + MAX_INPUT_OBJECT_BYTES + MAX_MANIFEST_BYTES + MAX_TAR_OVERHEAD_BYTES + ) + if member.file_size > maximum_tar_size: + raise SnapshotError("snapshot.tar exceeds the per-job input bound") + if member.compress_type != zipfile.ZIP_STORED: + raise SnapshotError("snapshot.tar ZIP member must use ZIP_STORED") + return bundle, member + except Exception: + bundle.close() + raise + + +def _tar_stream(archive: Path): + bundle, member = _snapshot_member(archive) + try: + raw = bundle.open(member, "r") + except Exception: + bundle.close() + raise + try: + tar = tarfile.open(fileobj=raw, mode="r|") + except (OSError, tarfile.TarError) as error: + raw.close() + bundle.close() + raise SnapshotError(f"invalid snapshot.tar in {archive}") from error + return bundle, raw, tar + + +def _hash_stream(stream: BinaryIO, expected_size: int) -> tuple[int, str]: + digest = hashlib.sha256() + total = 0 + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > expected_size: + raise SnapshotError("tar member contains more bytes than declared") + digest.update(chunk) + if total != expected_size: + raise SnapshotError("tar member is truncated") + return total, digest.hexdigest() + + +def _validate_object_path(path: str) -> None: + match = OBJECT_PATH.fullmatch(path) + if not match or match[1] != match[3][0] or match[2] != match[3][1]: + raise SnapshotError(f"invalid sccache object path: {path!r}") + + +def _parse_manifest( + contents: bytes, + expected: ArtifactIdentity, + expected_inherited_source_sha: str, +) -> dict[str, object]: + try: + manifest = json.loads(contents, object_pairs_hook=_strict_object) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise SnapshotError("manifest.json is not valid UTF-8 JSON") from error + if not isinstance(manifest, dict): + raise SnapshotError("manifest.json must contain an object") + required = { + "schema", "repository", "run_id", "run_attempt", "job", "source_sha", + "rustc", "workspace", "sccache_version", "mode", "inherited_source_sha", + "objects", "touched", + } + allowed = required | {"base_image"} + if set(manifest) != required and set(manifest) != allowed: + raise SnapshotError("manifest.json has missing or unexpected fields") + if not _is_int(manifest["schema"]) or manifest["schema"] != 1: + raise SnapshotError("manifest schema must be 1") + for field in ("repository", "job", "source_sha", "rustc", "workspace", + "sccache_version", "mode", "inherited_source_sha"): + _require_string(manifest[field], field) + if manifest["mode"] != "delta": + raise SnapshotError("manifest mode must be delta") + if manifest["inherited_source_sha"] != expected_inherited_source_sha: + raise SnapshotError("manifest inherited_source_sha does not match expected source image") + for field in ("run_id", "run_attempt"): + if not _is_int(manifest[field]) or manifest[field] < 1: + raise SnapshotError(f"manifest {field} must be a positive integer") + if "base_image" in manifest: + _require_string(manifest["base_image"], "base_image") + + for field in ("repository", "run_id", "run_attempt", "job", "source_sha"): + if manifest[field] != getattr(expected, field): + raise SnapshotError(f"manifest {field} does not match expected identity") + if not isinstance(manifest["objects"], list): + raise SnapshotError("manifest objects must be a list") + if not isinstance(manifest["touched"], list): + raise SnapshotError("manifest touched must be a list") + return manifest + + +def _manifest_objects(manifest: Mapping[str, object], source_index: int) -> tuple[_Object, ...]: + rows = manifest["objects"] + assert isinstance(rows, list) + if len(rows) > MAX_OBJECTS: + raise SnapshotError("manifest contains too many objects") + objects: list[_Object] = [] + seen: set[str] = set() + for row in rows: + if not isinstance(row, dict) or set(row) != {"path", "size", "sha256", "mtime_ns"}: + raise SnapshotError("manifest object has missing or unexpected fields") + path = row["path"] + checksum = row["sha256"] + if not isinstance(path, str): + raise SnapshotError("manifest object path must be a string") + _validate_object_path(path) + if path in seen: + raise SnapshotError(f"manifest contains duplicate object: {path}") + seen.add(path) + if not _is_int(row["size"]) or row["size"] < 0: + raise SnapshotError(f"manifest object size is invalid: {path}") + if not isinstance(checksum, str) or not SHA256.fullmatch(checksum): + raise SnapshotError(f"manifest object sha256 is invalid: {path}") + if not _is_int(row["mtime_ns"]) or row["mtime_ns"] < 0: + raise SnapshotError(f"manifest object mtime_ns is invalid: {path}") + objects.append(_Object( + path=path, + size=row["size"], + sha256=checksum, + mtime_ns=row["mtime_ns"], + source_index=source_index, + )) + return tuple(objects) + + +def _manifest_touches(manifest: Mapping[str, object]) -> tuple[_Touch, ...]: + rows = manifest["touched"] + assert isinstance(rows, list) + if len(rows) > MAX_OBJECTS: + raise SnapshotError("manifest contains too many touched objects") + touched: list[_Touch] = [] + seen: set[str] = set() + for row in rows: + if not isinstance(row, dict) or set(row) != {"path", "mtime_ns"}: + raise SnapshotError("manifest touched object has missing or unexpected fields") + path = row["path"] + if not isinstance(path, str): + raise SnapshotError("manifest touched path must be a string") + _validate_object_path(path) + if path in seen: + raise SnapshotError(f"manifest contains duplicate touched object: {path}") + seen.add(path) + if not _is_int(row["mtime_ns"]) or row["mtime_ns"] < 0: + raise SnapshotError(f"manifest touched mtime_ns is invalid: {path}") + touched.append(_Touch(path=path, mtime_ns=row["mtime_ns"])) + return tuple(touched) + + +def _validate_archive( + item: ArtifactInput, + source_index: int, + expected_inherited_source_sha: str, +) -> _Archive: + path = Path(item.archive) + signature = _archive_signature(path) + bundle, raw, tar = _tar_stream(path) + actual: dict[str, tuple[int, str]] = {} + manifest_contents: bytes | None = None + total_bytes = 0 + member_count = 0 + try: + try: + for member in tar: + member_count += 1 + if member_count > MAX_OBJECTS + 1: + raise SnapshotError("snapshot.tar contains too many members") + if manifest_contents is not None: + raise SnapshotError("manifest.json must be the last tar member") + if not member.isreg(): + raise SnapshotError(f"tar member must be a regular file: {member.name!r}") + if member.name == "manifest.json": + if member.size < 0 or member.size > MAX_MANIFEST_BYTES: + raise SnapshotError("manifest.json exceeds its size bound") + stream = tar.extractfile(member) + if stream is None: + raise SnapshotError("cannot read manifest.json") + manifest_contents = stream.read(MAX_MANIFEST_BYTES + 1) + if len(manifest_contents) != member.size: + raise SnapshotError("manifest.json is truncated") + continue + _validate_object_path(member.name) + if member.size < 0: + raise SnapshotError(f"tar member has a negative size: {member.name}") + if member.name in actual: + raise SnapshotError(f"snapshot.tar contains duplicate member: {member.name}") + total_bytes += member.size + if total_bytes > MAX_INPUT_OBJECT_BYTES: + raise SnapshotError("snapshot exceeds the per-job object byte bound") + stream = tar.extractfile(member) + if stream is None: + raise SnapshotError(f"cannot read tar member: {member.name}") + actual[member.name] = _hash_stream(stream, member.size) + except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: + raise SnapshotError(f"cannot stream snapshot archive: {path}") from error + finally: + tar.close() + raw.close() + bundle.close() + + if manifest_contents is None: + raise SnapshotError("snapshot.tar is missing final manifest.json") + manifest = _parse_manifest( + manifest_contents, item.expected, expected_inherited_source_sha, + ) + objects = _manifest_objects(manifest, source_index) + touched = _manifest_touches(manifest) + if {obj.path for obj in objects} & {touch.path for touch in touched}: + raise SnapshotError("an object cannot be both uploaded and touched") + described = {obj.path: (obj.size, obj.sha256) for obj in objects} + if actual != described: + raise SnapshotError("manifest object list, sizes, or checksums do not match snapshot.tar") + return _Archive( + input=item, manifest=manifest, objects=objects, touched=touched, + signature=signature, + ) + + +def _consistent_metadata(archives: Sequence[_Archive]) -> tuple[str, str, str, str, str | None]: + first = archives[0].manifest + fields = ( + "source_sha", "rustc", "workspace", "sccache_version", "base_image", + "inherited_source_sha", + ) + expected = tuple(first.get(field) for field in fields) + for archive in archives[1:]: + if tuple(archive.manifest.get(field) for field in fields) != expected: + raise SnapshotError("artifact compiler, workspace, source, or base-image metadata differs") + source_sha, rustc, workspace, sccache_version, base_image, _ = expected + assert isinstance(source_sha, str) + assert isinstance(rustc, str) + assert isinstance(workspace, str) + assert isinstance(sccache_version, str) + assert base_image is None or isinstance(base_image, str) + return source_sha, rustc, workspace, sccache_version, base_image + + +def _validated_base_objects(base_objects: os.PathLike[str] | str) -> tuple[Path, dict[str, _Object]]: + requested = Path(base_objects) + if not requested.is_absolute(): + raise SnapshotError("base_objects must be absolute") + try: + root = requested.resolve(strict=True) + except OSError as error: + raise SnapshotError("base_objects must exist") from error + if requested != root or requested.is_symlink() or not root.is_dir(): + raise SnapshotError("base_objects must be a real canonical directory") + + objects: dict[str, _Object] = {} + for first in root.iterdir(): + if first.is_symlink() or not first.is_dir() or not re.fullmatch(r"[0-9a-f]", first.name): + raise SnapshotError(f"noncanonical entry in base objects: {first}") + for second in first.iterdir(): + if second.is_symlink() or not second.is_dir() or not re.fullmatch(r"[0-9a-f]", second.name): + raise SnapshotError(f"noncanonical entry in base objects: {second}") + for entry in second.iterdir(): + relative = f"objects/{first.name}/{second.name}/{entry.name}" + _validate_object_path(relative) + if entry.is_symlink() or not entry.is_file(): + raise SnapshotError(f"base object must be a regular file: {entry}") + status = entry.stat() + objects[relative] = _Object( + path=relative, + size=status.st_size, + sha256=None, + mtime_ns=status.st_mtime_ns, + source_index=None, + ) + if len(objects) > MAX_BASE_OBJECTS: + raise SnapshotError("base_objects contains too many files") + return root, objects + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + return digest.hexdigest() + digest.update(chunk) + + +def _select_objects( + archives: Sequence[_Archive], + base_root: Path, + base: Mapping[str, _Object], + maximum: int, +) -> tuple[_Object, ...]: + merged = dict(base) + delta_paths = {obj.path for archive in archives for obj in archive.objects} + touched_paths = {touch.path for archive in archives for touch in archive.touched} + overlap = delta_paths & touched_paths + if overlap: + raise SnapshotError(f"object appears as both delta and touched: {min(overlap)}") + for archive in archives: + for touch in archive.touched: + current = merged.get(touch.path) + if current is None or current.source_index is not None: + raise SnapshotError(f"touched object is absent from inherited base: {touch.path}") + if touch.mtime_ns > current.mtime_ns: + merged[touch.path] = _Object( + path=current.path, + size=current.size, + sha256=current.sha256, + mtime_ns=touch.mtime_ns, + source_index=None, + ) + + base_hashes: dict[str, str] = {} + for archive in archives: + for candidate in archive.objects: + current = merged.get(candidate.path) + if current is None: + merged[candidate.path] = candidate + elif current.source_index is None: + if current.size != candidate.size: + raise SnapshotError(f"conflicting content for duplicate object: {candidate.path}") + checksum = base_hashes.get(candidate.path) + if checksum is None: + checksum = _file_sha256( + base_root.joinpath(*PurePosixPath(candidate.path).parts[1:]) + ) + base_hashes[candidate.path] = checksum + if checksum != candidate.sha256: + raise SnapshotError(f"conflicting content for duplicate object: {candidate.path}") + merged[candidate.path] = _Object( + path=current.path, + size=current.size, + sha256=checksum, + mtime_ns=max(current.mtime_ns, candidate.mtime_ns), + source_index=None, + ) + elif (current.size, current.sha256) != (candidate.size, candidate.sha256): + raise SnapshotError(f"conflicting content for duplicate object: {candidate.path}") + elif candidate.mtime_ns > current.mtime_ns: + merged[candidate.path] = _Object( + path=current.path, + size=current.size, + sha256=current.sha256, + mtime_ns=candidate.mtime_ns, + source_index=current.source_index, + ) + + selected: list[_Object] = [] + total = 0 + for candidate in sorted(merged.values(), key=lambda obj: (-obj.mtime_ns, obj.path)): + if candidate.size <= maximum - total: + selected.append(candidate) + total += candidate.size + return tuple(selected) + + +def _copy_selected(archive: _Archive, selected: Mapping[str, _Object], root: Path) -> None: + if _archive_signature(Path(archive.input.archive)) != archive.signature: + raise SnapshotError(f"artifact archive changed while merging: {archive.input.archive}") + bundle, raw, tar = _tar_stream(Path(archive.input.archive)) + remaining = set(selected) + try: + try: + for member in tar: + target_object = selected.get(member.name) + if target_object is None: + continue + if not member.isreg() or member.size != target_object.size: + raise SnapshotError(f"selected object changed while merging: {member.name}") + source = tar.extractfile(member) + if source is None: + raise SnapshotError(f"cannot read selected object: {member.name}") + destination = root.joinpath(*PurePosixPath(member.name).parts) + destination.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + written = 0 + with destination.open("xb") as output: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > target_object.size: + raise SnapshotError(f"selected object grew while merging: {member.name}") + digest.update(chunk) + output.write(chunk) + if written != target_object.size or digest.hexdigest() != target_object.sha256: + raise SnapshotError(f"selected object checksum changed while merging: {member.name}") + os.utime(destination, ns=(target_object.mtime_ns, target_object.mtime_ns)) + remaining.remove(member.name) + except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: + raise SnapshotError(f"cannot copy selected objects from {archive.input.archive}") from error + finally: + tar.close() + raw.close() + bundle.close() + if remaining: + raise SnapshotError(f"selected objects disappeared from {archive.input.archive}") + + +def _copy_base(base_root: Path, selected: Mapping[str, _Object], root: Path) -> None: + for relative, obj in selected.items(): + source = base_root.joinpath(*PurePosixPath(relative).parts[1:]) + if source.is_symlink() or not source.is_file() or source.stat().st_size != obj.size: + raise SnapshotError(f"inherited base object changed while merging: {relative}") + destination = root.joinpath(*PurePosixPath(relative).parts) + destination.parent.mkdir(parents=True, exist_ok=True) + with source.open("rb") as input_stream, destination.open("xb") as output_stream: + shutil.copyfileobj(input_stream, output_stream, length=1024 * 1024) + if destination.stat().st_size != obj.size: + raise SnapshotError(f"inherited base object was truncated while merging: {relative}") + os.utime(destination, ns=(obj.mtime_ns, obj.mtime_ns)) + + +def _write_metadata(path: Path, value: str) -> None: + path.write_text(value if value.endswith("\n") else value + "\n", encoding="utf-8") + + +def _validated_output(output_dir: os.PathLike[str] | str) -> tuple[Path, Path]: + output = Path(output_dir) + if not output.is_absolute(): + raise SnapshotError("output_dir must be absolute") + if output.exists() or output.is_symlink(): + raise SnapshotError("output_dir must not already exist") + try: + parent = output.parent.resolve(strict=True) + except OSError as error: + raise SnapshotError("output_dir parent must already exist") from error + if output.parent.is_symlink() or not parent.is_dir(): + raise SnapshotError("output_dir parent must be a real directory") + if output.parent != parent: + raise SnapshotError("output_dir parent must use its canonical absolute path") + return output, parent + + +def merge_snapshots( + inputs: Sequence[ArtifactInput], + output_dir: os.PathLike[str] | str, + *, + base_objects: os.PathLike[str] | str, + expected_inherited_source_sha: str, + max_combined_bytes: int = DEFAULT_MAX_COMBINED_BYTES, +) -> MergeResult: + """Validate artifacts and atomically create a bounded sccache object snapshot.""" + if not inputs: + raise SnapshotError("at least one artifact is required") + if not _is_int(max_combined_bytes) or max_combined_bytes < 0: + raise SnapshotError("max_combined_bytes must be a non-negative integer") + _require_string(expected_inherited_source_sha, "expected_inherited_source_sha") + output, parent = _validated_output(output_dir) + + base_root, base = _validated_base_objects(base_objects) + archives = tuple( + _validate_archive(item, index, expected_inherited_source_sha) + for index, item in enumerate(inputs) + ) + source_sha, rustc, workspace, sccache_version, base_image = _consistent_metadata(archives) + selected = _select_objects(archives, base_root, base, max_combined_bytes) + by_source: dict[int, dict[str, _Object]] = {} + from_base: dict[str, _Object] = {} + for obj in selected: + if obj.source_index is None: + from_base[obj.path] = obj + else: + by_source.setdefault(obj.source_index, {})[obj.path] = obj + + temporary = Path(tempfile.mkdtemp(prefix=TEMP_PREFIX, dir=parent)) + try: + snapshot = temporary / "snapshot" + (snapshot / "objects").mkdir(parents=True) + _copy_base(base_root, from_base, snapshot) + for source_index, objects in by_source.items(): + _copy_selected(archives[source_index], objects, snapshot) + _write_metadata(snapshot / "rustc.txt", rustc) + _write_metadata(snapshot / "workspace.txt", workspace) + _write_metadata(snapshot / "source-commit.txt", source_sha) + if base_image is not None: + _write_metadata(snapshot / "base-image.txt", base_image) + snapshot.replace(output) + finally: + if temporary.parent != parent or not temporary.name.startswith(TEMP_PREFIX): + raise RuntimeError("refusing to clean an unexpected temporary directory") + shutil.rmtree(temporary, ignore_errors=False) + + return MergeResult( + object_count=len(selected), + total_bytes=sum(obj.size for obj in selected), + source_sha=source_sha, + rustc=rustc, + workspace=workspace, + base_image=base_image, + sccache_version=sccache_version, + ) diff --git a/scripts/gitea_cache_upload_cleanup.py b/scripts/gitea_cache_upload_cleanup.py new file mode 100644 index 000000000..6801d193a --- /dev/null +++ b/scripts/gitea_cache_upload_cleanup.py @@ -0,0 +1,115 @@ +"""定向回收 Gitea 1.26.4 不会自动过期的本仓库缓存上传块。""" + +import base64 +import binascii +import math +from pathlib import Path +import re +import stat + + +JOBS = { + "ai-game-creator-shell-rust-lane-1", + "ai-game-creator-shell-rust-lane-2", + "ai-game-creator-shell-rust-smoke", + "ai-game-creator-shell-rust-crates", + "backend-tests", + "native-shell-tests", +} +CHUNK = re.compile(r"block-([1-9][0-9]*)-([1-9][0-9]*)-(0|[1-9][0-9]*)-([A-Za-z0-9_-]+={0,2})") +BLOCKLIST = re.compile(r"([1-9][0-9]*)-([1-9][0-9]*)-blocklist") +MARKER = re.compile(r"genarrative-rust-cache-v1:([a-z0-9-]+):(0|[1-9][0-9]*):([0-9]{8})") + + +def decode_owner(encoded): + """Gitea 文件名编码为 URL-base64(生产者上传的标准 base64 blockid)。""" + try: + block_id = base64.b64decode(encoded, altchars=b"-_", validate=True) + if base64.urlsafe_b64encode(block_id).decode() != encoded: + return None + marker = base64.b64decode(block_id, validate=True) + if base64.b64encode(marker) != block_id: + return None + match = MARKER.fullmatch(marker.decode("ascii")) + if match and match[1] in JOBS: + return match[1], int(match[2]) + except (binascii.Error, UnicodeError, ValueError): + pass + return None + + +def _unchanged_old_regular(path, previous, cutoff): + try: + current = path.lstat() + except FileNotFoundError: + return False + return ( + stat.S_ISREG(current.st_mode) + and current.st_mtime < cutoff + and (current.st_ino, current.st_size, current.st_mtime_ns) + == (previous.st_ino, previous.st_size, previous.st_mtime_ns) + ) + + +def cleanup_upload_chunks(storage_root: Path, eligible_runs: set[int], cutoff_timestamp: float) -> int: + """调用方先用仓库 API 确认这些 master run 已结束超过保留期限。""" + storage_root = Path(storage_root) + if not storage_root.is_absolute() or storage_root.resolve(strict=True) != storage_root: + raise ValueError("artifact storage root must be an existing absolute real directory") + if not stat.S_ISDIR(storage_root.lstat().st_mode): + raise ValueError("artifact storage root must be a directory") + if not math.isfinite(cutoff_timestamp) or cutoff_timestamp <= 0: + raise ValueError("invalid artifact cleanup cutoff") + if any(type(run_id) is not int or run_id <= 0 for run_id in eligible_runs): + raise ValueError("invalid eligible run ID") + temporary = storage_root / "tmp-upload" + if not temporary.exists() and not temporary.is_symlink(): + return 0 + if temporary.is_symlink() or not temporary.is_dir(): + raise ValueError("artifact temporary path must be a real directory") + + deleted = 0 + for run_id in sorted(eligible_runs): + directory = temporary / f"run-{run_id}-v4" + if directory.is_symlink() or not directory.is_dir(): + continue + groups = {} + unknown = False + for path in directory.iterdir(): + chunk = CHUNK.fullmatch(path.name) + blocklist = BLOCKLIST.fullmatch(path.name) + match = chunk or blocklist + if match and int(match[1]) == run_id: + artifact_id = int(match[2]) + owner = decode_owner(chunk[4]) if chunk else None + owned = owner is not None if chunk else True + else: + # 若仍能识别 artifact ID，仅保护该组；无法识别时保护整个 run。 + prefix = re.match(rf"(?:block-)?{run_id}-([1-9][0-9]*)-", path.name) + if not prefix: + unknown = True + break + artifact_id, owner, owned = int(prefix[1]), None, False + group = groups.setdefault(artifact_id, {"files": [], "owners": set(), "safe": True}) + try: + info = path.lstat() + except FileNotFoundError: + group["safe"] = False + continue + group["safe"] &= owned and stat.S_ISREG(info.st_mode) and info.st_mtime < cutoff_timestamp + if owner: + group["owners"].add(owner) + group["files"].append((path, info)) + if unknown: + continue + for group in groups.values(): + if not group["safe"] or len(group["owners"]) != 1: + continue + files = group["files"] + if not all(_unchanged_old_regular(path, info, cutoff_timestamp) for path, info in files): + continue + # 保留目录，不使用递归删除；没有所属块证明的孤立 blocklist 也不会删除。 + for path, _ in files: + path.unlink() + deleted += 1 + return deleted diff --git a/scripts/maintain-gitea-rust-cache.py b/scripts/maintain-gitea-rust-cache.py index 3dc066423..8dd0691bd 100644 --- a/scripts/maintain-gitea-rust-cache.py +++ b/scripts/maintain-gitea-rust-cache.py @@ -9,13 +9,18 @@ import os from pathlib import Path import re import socket +import shutil import subprocess import sys +import tempfile import time import urllib.error import urllib.parse import urllib.request +from gitea_cache_snapshot import ArtifactIdentity, ArtifactInput, merge_snapshots +from gitea_cache_upload_cleanup import cleanup_upload_chunks + IMAGE = re.compile(r"sha256:[0-9a-f]{64}\Z") SHA = re.compile(r"[0-9a-f]{40}\Z") @@ -29,6 +34,17 @@ JOBS = { "Repository checks", "AI game creator shell web tests", } RUST_JOBS = {name for name in JOBS if "Rust" in name or name in {"Backend tests", "Native shell tests"}} +RUST_JOB_IDS = { + "AI game creator shell Rust lane 1/2": "ai-game-creator-shell-rust-lane-1", + "AI game creator shell Rust lane 2/2": "ai-game-creator-shell-rust-lane-2", + "AI game creator shell Rust smoke": "ai-game-creator-shell-rust-smoke", + "AI game creator shell Rust crates": "ai-game-creator-shell-rust-crates", + "Backend tests": "backend-tests", + "Native shell tests": "native-shell-tests", +} +ARTIFACT_PREFIX = "rust-cache-v1-" +MAX_DOWNLOAD = 4 * 1024 ** 3 + 129 * 1024 ** 2 +EXPORT_STEP = "Publish master Rust cache artifact" def now(): @@ -127,14 +143,14 @@ class Api: raise ValueError("api_url must use HTTPS") self.token_file = Path(token_file) - def request(self, path, body=None, raw=False): + def request(self, path, body=None, raw=False, method=None): token = self.token_file.read_text().strip() if not token or "\n" in token: raise ValueError("invalid token file") request = urllib.request.Request( self.url + "/" + path.lstrip("/"), data=None if body is None else json.dumps(body).encode(), - method="GET" if body is None else "PATCH", + method=method or ("GET" if body is None else "PATCH"), headers={"Authorization": "token " + token, "Content-Type": "application/json"}, ) try: @@ -143,8 +159,41 @@ class Api: with opener.open(request, timeout=30) as response: content = response.read().decode() except urllib.error.HTTPError as error: + error.close() + if method == "DELETE" and error.code == 404: + return None raise RuntimeError(f"Gitea API HTTP {error.code}") from None - return content if raw else json.loads(content) + return content if raw else (json.loads(content) if content else None) + + def download(self, path, destination): + """REST V4 archive redirects to a signed URL; never forward the API token.""" + token = self.token_file.read_text().strip() + url = self.url + "/" + path.lstrip("/") + opener = urllib.request.build_opener(NoRedirect()) + request = urllib.request.Request(url, headers={"Authorization": "token " + token}) + try: + response = opener.open(request, timeout=60) + except urllib.error.HTTPError as error: + error.close() + if error.code not in (301, 302, 303, 307, 308): + raise RuntimeError(f"artifact download HTTP {error.code}") from None + target = urllib.parse.urljoin(url, error.headers.get("Location", "")) + parsed, origin = urllib.parse.urlsplit(target), urllib.parse.urlsplit(self.url) + if (parsed.scheme != "https" or parsed.netloc != origin.netloc + or parsed.username or parsed.password or target == url): + raise RuntimeError("artifact redirect must stay on configured HTTPS Gitea origin") from None + response = opener.open(target, timeout=60) + try: + with response, destination.open("wb") as out: + total = 0 + while chunk := response.read(1024 * 1024): + total += len(chunk) + if total > MAX_DOWNLOAD: + raise RuntimeError("artifact exceeds per-job size limit") + out.write(chunk) + except Exception: + destination.unlink(missing_ok=True) + raise def pages(self, path, key): separator = "&" if "?" in path else "?" @@ -175,9 +224,6 @@ class Maintenance: self.runner = config.get("runner_container", "gitea-runner") self.api = Api(config["api_url"], config["token_file"]) self.repo_api = "repos/" + config["repository"] - self.runner_api = config["runner_api_path"].strip("/") - if not re.fullmatch(r"admin/actions/runners/\d+", self.runner_api): - raise ValueError("this service requires the global runner admin endpoint") self.state_path = self.root / "state.json" self.state = json.loads(self.state_path.read_text()) if self.state_path.exists() else { "versions": [], "current": None, "rollback": None, "candidate": None, @@ -250,40 +296,149 @@ class Maintenance: command("bash", str(self.repo / "scripts" / script), *args, cwd=self.repo, env=env, output=out, timeout=7200) - def build(self, sha, inputs): + def master_run(self, run): + return (run.get("path") == "project-ci.yml@refs/heads/master" + and run.get("event") == "push" and run.get("head_branch") == "master" + and SHA.fullmatch(run.get("head_sha", "")) is not None) + + def source_run(self): + """Only complete exports from the latest eligible master run; never mix runs.""" + current = self.version(self.state["current"]) + current_inputs = (cache_inputs(command("git", "ls-tree", "-rz", current["source"], cwd=self.repo)) + if current.get("source") else None) + for run in self.api.pages(self.repo_api + "/actions/runs?branch=master&event=push", "workflow_runs"): + if (not self.master_run(run) or run.get("status") != "completed" + or run.get("conclusion") not in {"success", "failure"} + or run["id"] <= current.get("run_id", 0)): + continue + sha = run["head_sha"] + if sha == current.get("source"): + continue + ancestry = subprocess.run(["git", "merge-base", "--is-ancestor", sha, "FETCH_HEAD"], + cwd=self.repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if ancestry.returncode: + continue + if current.get("source"): + ancestry = subprocess.run(["git", "merge-base", "--is-ancestor", current["source"], sha], + cwd=self.repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if ancestry.returncode: + continue + inputs = cache_inputs(command("git", "ls-tree", "-rz", sha, cwd=self.repo)) + if inputs == current_inputs: + continue + jobs = list(self.api.pages(self.repo_api + f'/actions/runs/{run["id"]}/jobs', "jobs")) + rust_jobs = [job for job in jobs if job["name"] in RUST_JOB_IDS] + if len(rust_jobs) != 6 or {job["name"] for job in rust_jobs} != RUST_JOBS: + continue + if any(job.get("status") != "completed" or job.get("conclusion") not in {"success", "failure"} + or job.get("head_sha") != sha + or not any(step.get("name") == EXPORT_STEP and step.get("conclusion") == "success" + for step in job.get("steps", [])) for job in rust_jobs): + continue + artifacts = list(self.api.pages(self.repo_api + f'/actions/runs/{run["id"]}/artifacts', "artifacts")) + selected = [] + images = set() + for job in rust_jobs: + name = f'{ARTIFACT_PREFIX}{RUST_JOB_IDS[job["name"]]}-attempt-{job["run_attempt"]}' + matches = [item for item in artifacts if item["name"] == name and not item.get("expired") + and item.get("workflow_run", {}).get("id") == run["id"]] + if len(matches) != 1: + break + content = self.api.request(self.repo_api + f'/actions/jobs/{job["id"]}/logs', raw=True) + if not re.search(re.escape(f"[rust-cache] artifact={name}") + + r" objects=\d+ bytes=\d+ complete=true", content): + break + used = set(re.findall(r"(?m)^\S+ image: (sha256:[0-9a-f]{64})\s*$", content)) + if len(used) != 1: + break + images.update(used) + selected.append({"id": matches[0]["id"], "name": name, + "job": RUST_JOB_IDS[job["name"]], "attempt": job["run_attempt"]}) + if len(selected) != 6 or len(images) != 1: + continue + source_image = images.pop() + if source_image not in {row["image"] for row in self.state["versions"]}: + continue + return {"run_id": run["id"], "source": sha, "inputs": inputs, "source_image": source_image, + "exports": selected} + return None + + def build(self, source): + """Assemble existing CI objects; no cargo warm-up or test execution.""" + sha = source["source"] + command("git", "checkout", "--detach", "--force", sha, cwd=self.repo) + command("git", "clean", "-ffdx", cwd=self.repo) artifact = self.root / "artifacts" / sha artifact.mkdir(parents=True, exist_ok=True) build_log = artifact / "build.log" tag = "genarrative/gitea-project-ci:rust-cache-auto-" + sha base_tag = "genarrative/gitea-project-ci:base-auto-" + sha - self.state.setdefault("attempts", []).append({ - "source": sha, "inputs": inputs, "tag": tag, - "base_tag": base_tag, "artifact": str(artifact), - }) + attempt = {**source, "tag": tag, "base_tag": base_tag, "artifact": str(artifact)} + self.state.setdefault("attempts", []).append(attempt) self.save() env = {**os.environ, "GENARRATIVE_GITEA_RUNNER_CONTAINER": self.runner} env.pop("CI", None) - current = self.version(self.state["current"]) - base = current["base"] + labels = self.image_info(source["source_image"])["Config"].get("Labels") or {} + inherited_source = labels.get("world.genarrative.ci.rust-cache-source") + base = labels.get("world.genarrative.ci.rust-cache-base") + if not IMAGE.fullmatch(base or "") or not SHA.fullmatch(inherited_source or ""): + raise RuntimeError("source image must contain a trusted cache snapshot") revision = command("bash", "scripts/gitea-ci-job-image.sh", "revision", cwd=self.repo).strip() - labels = self.image_info(base)["Config"].get("Labels") or {} - if labels.get("com.genarrative.ci.definition-sha256") != revision: + base_labels = self.image_info(base)["Config"].get("Labels") or {} + if base_labels.get("com.genarrative.ci.definition-sha256") != revision: env["GENARRATIVE_GITEA_CI_IMAGE_TAG"] = base_tag self.build_command(build_log, "gitea-ci-job-image.sh", "build", env=env) base = self.image_info(base_tag)["Id"] self.state.setdefault("bases", {})[base] = base_tag self.save() - self.build_command(build_log, "build-gitea-rust-cache.sh", base, tag, sha, env=env) + # 与旧缓存镜像分离；绝不把 Docker 可写层、源码或 target commit 成镜像。 + self.docker("run", "--rm", "--network", "none", "--read-only", "--cap-drop=ALL", + "--entrypoint", "bash", base, "-c", "test ! -e /opt/genarrative-ci/rust-cache") + with tempfile.TemporaryDirectory(prefix="assemble-", dir=artifact) as temporary: + work = Path(temporary) + inherited = work / "inherited" + inherited.mkdir() + container = self.docker("create", source["source_image"]).strip() + try: + self.docker("cp", container + ":/opt/genarrative-ci/rust-cache/.", str(inherited), timeout=600) + finally: + self.docker("rm", "--volumes", container) + inputs = [] + for export in source["exports"]: + archive = work / (str(export["id"]) + ".zip") + self.api.download(self.repo_api + f'/actions/artifacts/{export["id"]}/zip', archive) + inputs.append(ArtifactInput(archive, ArtifactIdentity( + self.config["repository"], source["run_id"], export["attempt"], export["job"], sha))) + snapshot = work / "snapshot" + merged = merge_snapshots(inputs, snapshot, base_objects=inherited / "objects", + expected_inherited_source_sha=inherited_source) + if merged.sccache_version != "sccache 0.18.0": + raise RuntimeError("unsupported sccache version") + if merged.base_image is not None and merged.base_image != labels["world.genarrative.ci.rust-cache-base"]: + raise RuntimeError("artifact base differs from its actual source image") + rustc = self.docker("run", "--rm", "--network", "none", "--read-only", base, "rustc", "-vV") + if rustc.strip() != merged.rustc.strip(): + raise RuntimeError("artifact toolchain differs from target base image") + if merged.workspace != "/workspace/" + self.config["repository"]: + raise RuntimeError("artifact workspace differs from CI checkout") + shutil.copyfile(inherited / "sccache", snapshot / "sccache") + (snapshot / "sccache").chmod(0o755) + (snapshot / "base-image.txt").write_text(base + "\n") + (work / "Dockerfile").write_text( + f"FROM {base}\nCOPY snapshot/ /opt/genarrative-ci/rust-cache/\n" + f'LABEL world.genarrative.ci.rust-cache-source="{sha}"\n' + f'LABEL world.genarrative.ci.rust-cache-base="{base}"\n') + (work / ".dockerignore").write_text("**\n!Dockerfile\n!snapshot/\n!snapshot/**\n") + with build_log.open("a") as out: + self.docker("build", "--pull=false", "--tag", tag, str(work), output=out, timeout=1800) + self.build_command(build_log, "gitea-ci-job-image.sh", "verify", tag, env=env) image = self.image_info(tag)["Id"] - record = {"image": image, "tag": tag, "source": sha, "inputs": inputs, - "base": base, "owned": True, "verified_run": None, - "artifact": str(artifact)} - # 先登记再导出；进程中断后下次可以复用候选，不留下无归属成功镜像。 - self.state["versions"].append(record) + self.state["versions"].append({**attempt, "image": image, "base": base, + "owned": True, "verified_run": None}) self.state["candidate"] = image self.state["attempts"] = [row for row in self.state["attempts"] if row["source"] != sha] self.save() - log(f"candidate built source={sha} image={image}") + log(f"candidate assembled run={source['run_id']} source={sha} image={image}") def stage_candidate(self): candidate = self.version(self.state["candidate"]) @@ -309,10 +464,14 @@ class Maintenance: self.save() def idle(self): - runner = self.api.request(self.runner_api) - active = self.api.request("admin/actions/runs?status=in_progress&limit=1") - return (runner.get("busy") is False and active["total_count"] == 0 - and not self.docker("ps", "-q", inner=True).strip()) + gate = self.gate("status") + if type(gate.get("active_tasks")) is not int or gate["active_tasks"] < 0: + raise RuntimeError("gate must support durable task tracking before automatic switching") + # 已领取但尚未建容器、正在收尾上报的任务都由入口跟踪，不查管理员 API。 + return (gate.get("uncertain") is False and gate.get("active_tasks") == 0 + and not self.docker("ps", "-q", "--filter", "status=running", + "--filter", "status=created", "--filter", "status=restarting", + "--filter", "status=paused", inner=True).strip()) def verify_current(self): current = self.version(self.state["current"]) @@ -402,12 +561,62 @@ class Maintenance: del self.state["bases"][image] self.save() + def cleanup_exports(self): + """Only our named artifacts; keep logs/runs and every unrelated artifact.""" + protected_runs = {row["run_id"] for row in self.state.get("attempts", []) if row.get("run_id")} + for row in self.state["versions"]: + if row.get("run_id") and not row.get("staged"): + protected_runs.add(row["run_id"]) + if not row.get("staged"): + continue + for item in row.get("exports", []): + if item.get("deleted"): + continue + self.api.request(self.repo_api + f'/actions/artifacts/{item["id"]}', method="DELETE") + item["deleted"] = True + self.save() + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=7) + # 先收集再删除，避免按页删除让下一页位置前移、漏掉旧产物。 + artifacts = list(self.api.pages(self.repo_api + "/actions/artifacts", "artifacts")) + pattern = re.compile(re.escape(ARTIFACT_PREFIX) + "(" + "|".join(RUST_JOB_IDS.values()) + + r")-attempt-\d+\Z") + runs = {} + for item in artifacts: + run_id = (item.get("workflow_run") or {}).get("id") + if (not pattern.fullmatch(item["name"]) or not run_id or run_id in protected_runs + or timestamp(item["created_at"]) >= cutoff): + continue + # Artifact.workflow_run 在 Gitea 1.26.4 中只有 id/repository_id/head_sha。 + if run_id not in runs: + runs[run_id] = self.api.request(self.repo_api + f"/actions/runs/{run_id}") + run = runs[run_id] + if not self.master_run(run) or run.get("status") != "completed": + continue + self.api.request(self.repo_api + f'/actions/artifacts/{item["id"]}', method="DELETE") + + def cleanup_pending_uploads(self): + # Gitea 1.26.4 的过期/DELETE API 不会清理未 finalized 的 V4 分块。 + # 只处理本上传器命名的块，且 run 和文件本身均已过保留期限。 + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=7) + eligible = set() + protected = {row["run_id"] for row in self.state.get("attempts", []) if row.get("run_id")} + for run in self.api.pages(self.repo_api + "/actions/runs?branch=master&event=push", "workflow_runs"): + if (self.master_run(run) and run.get("status") == "completed" + and run["id"] not in protected and run.get("completed_at") + and timestamp(run["completed_at"]) < cutoff): + eligible.add(run["id"]) + removed = cleanup_upload_chunks(Path(self.config["artifact_storage_dir"]), eligible, cutoff.timestamp()) + if removed: + log(f"removed {removed} expired owned upload fragments") + def tick(self, retry=False): if not self.recover_switch(): return self.adopt_current() - sha, inputs = self.fetch_source() + self.fetch_source() self.recover_builds() + self.cleanup_exports() + self.cleanup_pending_uploads() if not self.verify_current(): return self.cleanup() @@ -415,35 +624,42 @@ class Maintenance: self.stage_candidate() self.activate() return - current = self.version(self.state["current"]) - if "inputs" not in current and current.get("source"): - current["inputs"] = cache_inputs(command("git", "ls-tree", "-rz", current["source"], cwd=self.repo)) - self.save() - if current.get("inputs") == inputs: - log(f"unchanged compilation inputs at master={sha}") + source = self.source_run() + if source is None: + log("waiting for a complete set of master CI cache exports") return - if not retry and self.state.get("failed_source") == sha: - log(f"previous build failed at {sha}; waiting for new master or --retry") + if not retry and self.state.get("failed_run") == source["run_id"]: + log(f'previous assembly failed at run={source["run_id"]}; waiting for new run or --retry') return - # 新的 CI 波峰不额外争抢预热资源，等下一次维护周期。 + # 下载、合并和镜像装载也消耗宿主 IO；繁忙时留给 CI，下轮再收集。 if not self.idle(): log("CI active; defer refresh") return try: - self.build(sha, inputs) + self.build(source) except Exception: - self.state["failed_source"] = sha + self.state["failed_run"] = source["run_id"] self.save() raise - self.state.pop("failed_source", None) + self.state.pop("failed_run", None) self.save() self.stage_candidate() + self.cleanup_exports() self.activate() def recover_builds(self): # 宕机可能发生在 docker build 完成之后、登记 Image ID 之前。 # 只接管预先登记的确定性 tag；恢复的候选仍须经过 stage 的 verify/load。 for attempt in list(self.state.get("attempts", [])): + artifact = self.root / "artifacts" / attempt["source"] + if (not SHA.fullmatch(attempt["source"]) or Path(attempt["artifact"]).resolve() != artifact + or artifact.is_symlink() or artifact.parent.is_symlink()): + raise RuntimeError("interrupted assembly directory is outside managed artifacts") + # 持有维护锁，只有此前中断的组装可能遗留这些私有工作目录。 + for directory in artifact.glob("assemble-*"): + if directory.is_symlink() or not directory.is_dir(): + raise RuntimeError("unexpected interrupted assembly entry") + shutil.rmtree(directory) for kind in ("base_tag", "tag"): ids = set(self.docker("image", "ls", "--no-trunc", "--quiet", attempt[kind]).split()) if not ids: @@ -508,16 +724,14 @@ class Maintenance: gate = self.gate("status") # .runner 会因 label 更新而回写；mtime 不能证明内存中的地址。 # 要求入口实际见到本次容器启动后、来自它的 FetchTask。 - if gate.get("last_fetch_peer") not in addresses or gate.get("last_fetch_at", 0) < started: + if gate.get("last_fetch_peer") not in addresses or (gate.get("last_fetch_at") or 0) < started: raise RuntimeError("gate has not observed FetchTask from this runner startup") + if type(gate.get("active_tasks")) is not int or gate["active_tasks"] < 0: + raise RuntimeError("gate must support durable task tracking before automatic switching") def activate(self): self.check_gate_route() candidate = self.version(self.state["candidate"]) - runner = self.api.request(self.runner_api) - if runner.get("disabled") is not False: - log("runner disabled by operator; defer switch") - return False config = self.read_config() gate = self.gate("status") if gate.get("paused") is not False and not self.state.get("pause_owned"): @@ -608,7 +822,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True) parser.add_argument("--apply", action="store_true", help="执行维护；默认只检查连接和配置") - parser.add_argument("--retry", action="store_true", help="重试同一 master 上失败的构建") + parser.add_argument("--retry", action="store_true", help="重试同一 master run 上失败的缓存组装") parser.add_argument("--resume", action="store_true", help="仅恢复本维护器暂停的领取；用于 ExecStopPost") args = parser.parse_args() if os.environ.get("CI") == "true": @@ -617,14 +831,15 @@ def main(): config = json.loads(Path(args.config).read_text()) maintenance = Maintenance(config) if not args.apply and not args.resume: - runner = maintenance.api.request(maintenance.runner_api) + maintenance.api.request(maintenance.repo_api + "/actions/artifacts?limit=1") image = configured_image(maintenance.read_config()) head = command("git", "ls-remote", config["clone_url"], "refs/heads/master").split()[0] maintenance.check_gate_route() + cleanup_upload_chunks(Path(config["artifact_storage_dir"]), set(), time.time()) gate = maintenance.gate("status") if gate.get("uncertain"): raise RuntimeError("runner gate needs manual inspection") - log(f'check master={head} image={image} runner_disabled={runner["disabled"]} busy={runner["busy"]}') + log(f'check master={head} image={image} active_tasks={gate.get("active_tasks")}') return import fcntl # Linux 宿主；纯逻辑测试仍可在 Windows 上导入。 maintenance.root.mkdir(parents=True, exist_ok=True) diff --git a/scripts/project-ci-workflow.test.ts b/scripts/project-ci-workflow.test.ts index 4fe601c93..7304408dc 100644 --- a/scripts/project-ci-workflow.test.ts +++ b/scripts/project-ci-workflow.test.ts @@ -142,6 +142,38 @@ function backendStepIndex(stepName: string) { } describe('project CI workflow', () => { + it('publishes cache deltas only after Rust reporting on non-cancelled master pushes', () => { + const producers = [ + 'ai-game-creator-shell-rust-lane-1', + 'ai-game-creator-shell-rust-lane-2', + 'ai-game-creator-shell-rust-smoke', + 'ai-game-creator-shell-rust-crates', + 'backend-tests', + 'native-shell-tests', + ]; + for (const job of jobNames) { + if (!producers.includes(job)) { + expect(jobSection(job)).not.toContain('export-gitea-rust-cache.py'); + continue; + } + const publish = stepSection(job, 'Publish master Rust cache artifact'); + expect(publish).toContain( + "if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }}", + ); + expect(publish).toContain('GENARRATIVE_GITEA_TOKEN: ${{ github.token }}'); + expect(publish).toContain('continue-on-error: true'); + expect(publish).toContain( + 'run: python3 scripts/export-gitea-rust-cache.py', + ); + const section = jobSection(job); + expect( + section.indexOf('Publish master Rust cache artifact'), + ).toBeGreaterThan( + section.indexOf('run: bash scripts/ci-rust-cache.sh report'), + ); + } + }); + it('uses isolated compilation caching for every Rust test job', () => { const firstRustSteps = { 'ai-game-creator-shell-rust-lane-1': diff --git a/scripts/test_gitea_cache_export.py b/scripts/test_gitea_cache_export.py new file mode 100644 index 000000000..0ab36732c --- /dev/null +++ b/scripts/test_gitea_cache_export.py @@ -0,0 +1,205 @@ +"""master 产物的增量归档与 Gitea v4 上传协议回归。""" + +import hashlib +import base64 +from contextlib import redirect_stdout +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +import io +import json +import os +from pathlib import Path +import tarfile +import tempfile +import threading +import unittest +from unittest.mock import patch +import urllib.parse +import zipfile + +from gitea_cache_upload_cleanup import decode_owner + + +spec = importlib.util.spec_from_file_location("cache_export", Path(__file__).with_name("export-gitea-rust-cache.py")) +exporter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(exporter) + + +class ExportTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / "objects").mkdir() + + def object(self, key, value, mtime): + path = self.root / "objects" / key[0] / key[1] / key + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + os.utime(path, ns=(mtime * 10**9, mtime * 10**9)) + return path + + def unpack(self, baseline, limit=1024): + destination = self.root / "snapshot.zip" + result = exporter.pack_snapshot(self.root, destination, {"job": "fixture"}, baseline, limit) + with zipfile.ZipFile(destination) as bundle: + self.assertEqual(bundle.namelist(), ["snapshot.tar"]) + with tarfile.open(fileobj=io.BytesIO(bundle.read("snapshot.tar"))) as archive: + files = {member.name: archive.extractfile(member).read() for member in archive} + return result, json.loads(files.pop("manifest.json")), files + + def test_delta_omits_hit_bytes_preserves_recency_and_exports_new_keys(self): + inherited = self.object("a" * 64, b"old", 10) + baseline_path = self.root / "baseline.json" + exporter.save_baseline(self.root, baseline_path) + baseline = json.loads(baseline_path.read_text()) + # sccache 对命中对象更新 mtime，不能因此重复上传已有对象。 + os.utime(inherited, ns=(20 * 10**9, 20 * 10**9)) + self.object("b" * 64, b"new", 30) + result, manifest, files = self.unpack(baseline) + key = "objects/b/b/" + "b" * 64 + self.assertEqual(result, (1, 3)) + self.assertEqual(files, {key: b"new"}) + self.assertEqual(manifest["mode"], "delta") + self.assertEqual(manifest["touched"], [{"path": "objects/a/a/" + "a" * 64, "mtime_ns": 20 * 10**9}]) + self.assertEqual(manifest["objects"][0]["sha256"], hashlib.sha256(b"new").hexdigest()) + + def test_empty_delta_is_valid_and_changed_size_is_exported(self): + path = self.object("a" * 64, b"old", 10) + baseline = {"a/a/" + "a" * 64: {"size": 3, "mtime_ns": 10 * 10**9}} + self.assertEqual(self.unpack(baseline)[0], (0, 0)) + path.write_bytes(b"replacement") + self.assertEqual(self.unpack(baseline)[0], (1, 11)) + + def test_capacity_prefers_newest_objects_and_excludes_unrelated_files(self): + self.object("a" * 64, b"old", 10) + self.object("b" * 64, b"new", 20) + (self.root / "objects" / "credentials").write_bytes(b"secret") + (self.root / "sccache").write_bytes(b"binary") + result, _, files = self.unpack({}, limit=3) + self.assertEqual(result, (1, 3)) + self.assertEqual(list(files.values()), [b"new"]) + + @unittest.skipUnless(os.name == "posix", "symlink fixture requires POSIX") + def test_symlink_objects_and_subdirectories_are_not_exported(self): + outside = self.root / "secret" + outside.write_bytes(b"secret") + linked = self.object("a" * 64, b"old", 10) + linked.unlink() + linked.symlink_to(outside) + (self.root / "objects" / "b").symlink_to(self.root, target_is_directory=True) + self.assertEqual(self.unpack({})[0], (0, 0)) + + def test_non_master_and_unclean_shutdown_do_not_scan_or_upload(self): + with patch.object(exporter, "scan_objects", side_effect=AssertionError("must not scan")), \ + patch.object(exporter, "upload_snapshot", side_effect=AssertionError("must not upload")): + exporter.export({"GITHUB_EVENT_NAME": "pull_request", "GITHUB_REF": "refs/heads/master"}) + exporter.export({"GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/feature"}) + exporter.export({"GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/master"}) + + def test_internal_repository_route_bypasses_rpc_only_gateway(self): + env = { + "GITHUB_REPOSITORY": "owner/repo", + "GITHUB_SERVER_URL": "http://fetch-gate:8080", + "GENARRATIVE_GITEA_REPOSITORY_URL": "http://gitea:3000/sub/owner/repo.git", + } + self.assertEqual(exporter.artifact_base_url(env), "http://gitea:3000/sub") + env["GENARRATIVE_GITEA_REPOSITORY_URL"] = "http://token@gitea:3000/owner/repo.git" + with self.assertRaises(ValueError): + exporter.artifact_base_url(env) + + def test_native_v4_chunks_finalize_with_verified_size_and_hash(self): + calls = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + calls.append((self.command, self.path, dict(self.headers), body)) + payload = {"ok": True} + if self.path.endswith("CreateArtifact"): + payload["signedUploadUrl"] = "https://external.invalid/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=private&taskID=4&artifactID=8" + else: + payload["artifactId"] = "8" + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps(payload).encode()) + + def do_PUT(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + calls.append((self.command, self.path, dict(self.headers), body)) + self.send_response(201) + self.end_headers() + self.wfile.write(b'"created"') + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + path = self.root / "test.zip" + path.write_bytes(b"123456789") + with patch.object(exporter, "CHUNK_BYTES", 4): + exporter.upload_snapshot(path, "rust-cache-v1-backend-tests-attempt-1", 123, f"http://127.0.0.1:{server.server_port}", "ephemeral") + finally: + server.shutdown() + server.server_close() + thread.join() + self.assertEqual(len(calls), 6) + self.assertEqual(calls[0][3]["workflowRunBackendId"], "123") + remaining = datetime.fromisoformat(calls[0][3]["expiresAt"]) - datetime.now(timezone.utc) + self.assertGreater(remaining.total_seconds(), 7 * 24 * 3600) + self.assertLessEqual(remaining.total_seconds(), 8 * 24 * 3600) + blocks = calls[1:4] + self.assertEqual(b"".join(call[3] for call in blocks), b"123456789") + self.assertEqual(len({urllib.parse.parse_qs(urllib.parse.urlsplit(call[1]).query)["blockid"][0] for call in blocks}), 3) + for call in blocks: + block_id = urllib.parse.parse_qs(urllib.parse.urlsplit(call[1]).query)["blockid"][0] + gitea_filename_suffix = base64.urlsafe_b64encode(block_id.encode()).decode() + self.assertEqual(decode_owner(gitea_filename_suffix), ("backend-tests", 1)) + self.assertTrue(all("Authorization" not in call[2] for call in calls[1:5])) + self.assertEqual(calls[-1][2]["Authorization"], "Bearer ephemeral") + self.assertEqual(calls[-1][3]["size"], "9") + self.assertEqual(calls[-1][3]["hash"], "sha256:" + hashlib.sha256(b"123456789").hexdigest()) + + def test_upload_failure_never_finalizes_an_incomplete_artifact(self): + path = self.root / "test.zip" + path.write_bytes(b"data") + with patch.object(exporter, "request", side_effect=[ + {"ok": True, "signedUploadUrl": "http://gitea/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=x"}, + RuntimeError("upload failed"), + ]) as request: + with self.assertRaisesRegex(RuntimeError, "upload failed"): + exporter.upload_snapshot(path, "rust-cache-v1-backend-tests-attempt-1", 123, "http://gitea", "ephemeral") + self.assertEqual(request.call_count, 2) + + def test_failed_optional_export_never_emits_host_completion_marker(self): + (self.root / "export-baseline.json").write_text("{}") + (self.root / "rustc.txt").write_text("rustc fixture\n") + (self.root / "workspace.txt").write_text(str(Path.cwd().resolve())) + (self.root / "source-commit.txt").write_text("a" * 40) + self.object("b" * 64, b"new", 20) + env = { + "GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/master", + "GENARRATIVE_CI_RUST_CACHE_EXPORT_READY": "1", + "GITHUB_JOB": "backend-tests", "GITHUB_SHA": "b" * 40, + "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "1", + "GENARRATIVE_GITEA_TOKEN": "ephemeral", + "GITHUB_REPOSITORY": "owner/repo", + "GENARRATIVE_GITEA_REPOSITORY_URL": "http://gitea/owner/repo.git", + "GENARRATIVE_CI_RUST_CACHE_ROOT": str(self.root), + } + output = io.StringIO() + with patch.object(exporter.subprocess, "check_output", side_effect=["rustc fixture\n", "sccache 0.18.0\n"]), \ + patch.object(exporter, "upload_snapshot", side_effect=RuntimeError("upload failed")), \ + redirect_stdout(output): + with self.assertRaisesRegex(RuntimeError, "upload failed"): + exporter.export(env) + self.assertNotIn("complete=true", output.getvalue()) + self.assertFalse((self.root / "export-baseline.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_gitea_cache_gate.py b/scripts/test_gitea_cache_gate.py index 01a86b6c1..ddc3a4a2c 100644 --- a/scripts/test_gitea_cache_gate.py +++ b/scripts/test_gitea_cache_gate.py @@ -2,6 +2,7 @@ import http.client import http.server +import gzip import importlib.util import json import os @@ -21,6 +22,29 @@ MODULE = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(MODULE) FETCH = "/api/actions/runner.v1.RunnerService/FetchTask" UPDATE = "/api/actions/runner.v1.RunnerService/UpdateTask" +LOG = "/api/actions/runner.v1.RunnerService/UpdateLog" +PING = "/api/actions/ping.v1.PingService/Ping" + + +def varint(value): + data = bytearray() + while value > 127: + data.append((value & 127) | 128) + value >>= 7 + data.append(value) + return bytes(data) + + +def field(number, value): + # actions-proto-go v0.4.1: task/state ID=1; result=2; log task_id=1, + # index=2, rows=3, no_more=4; log response ack_index=1. + if isinstance(value, bytes): + return varint(number * 8 + 2) + varint(len(value)) + value + return varint(number * 8) + varint(value) + + +def state(task_id, result=0): + return field(1, field(1, task_id) + field(2, result)) def core_state(state): @@ -50,8 +74,17 @@ class Upstream(http.server.BaseHTTPRequestHandler): self.wfile.write(b"incomplete") self.close_connection = True return - body = b'{"task":null}' - self.send_response(200) + body = self.server.responses.get(self.path, b"") + if self.path == UPDATE and self.path not in self.server.responses: + body = state(*map(int, MODULE.task_state(MODULE.protobuf_fields(data)))) + if self.path == LOG and self.path not in self.server.responses: + fields = MODULE.protobuf_fields(data) + body = field(1, MODULE.single(fields, 2, 0) + len(fields.get(3, []))) + self.send_response(self.server.statuses.get(self.path, 200)) + self.send_header("Content-Type", self.server.content_type) + if self.server.compressed: + body = gzip.compress(body) + self.send_header("Content-Encoding", "gzip") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -66,6 +99,10 @@ class GateTests(unittest.TestCase): self.upstream.entered = threading.Event() self.upstream.release = threading.Event() self.upstream.truncated = False + self.upstream.responses = {} + self.upstream.statuses = {} + self.upstream.content_type = "application/proto" + self.upstream.compressed = False self.proxy = MODULE.create_proxy(("127.0.0.1", 0), f"http://127.0.0.1:{self.upstream.server_port}", self.gate) self.servers = [self.upstream, self.proxy] @@ -79,13 +116,15 @@ class GateTests(unittest.TestCase): server.server_close() self.temp.cleanup() - def request(self, path, body=b"{}", chunked=False): + def request(self, path, body=b"", chunked=False): connection = http.client.HTTPConnection("127.0.0.1", self.proxy.server_port, timeout=5) try: if chunked: - connection.request("POST", path, body=[body], encode_chunked=True) + connection.request("POST", path, body=[body], encode_chunked=True, + headers={"Content-Type": "application/proto"}) else: - connection.request("POST", path, body=body) + connection.request("POST", path, body=body, + headers={"Content-Type": "application/proto"}) response = connection.getresponse() result = response.status, response.read() return result @@ -112,7 +151,7 @@ class GateTests(unittest.TestCase): self.assertEqual(core_state(self.gate.control("pause")), {"paused": True, "inflight": 1, "uncertain": False}) self.assertEqual(self.request(FETCH)[0], 503) - self.assertEqual(self.request(UPDATE)[0], 200) + self.assertEqual(self.request(PING)[0], 200) self.assertEqual(sum(path == FETCH for path, _ in self.upstream.requests), 1) self.upstream.release.set() thread.join(3) @@ -159,8 +198,8 @@ class GateTests(unittest.TestCase): self.assertFalse(MODULE.Gate(self.temp.name).control("status")["paused"]) def test_chunked_body_is_decoded_and_non_rpc_path_rejected(self): - self.assertEqual(self.request(UPDATE, b'{"state":"running"}', chunked=True)[0], 200) - self.assertEqual(self.upstream.requests[-1], (UPDATE, b'{"state":"running"}')) + self.assertEqual(self.request(PING, b"ping", chunked=True)[0], 200) + self.assertEqual(self.upstream.requests[-1], (PING, b"ping")) self.assertEqual(self.request("/api/v1/repos")[0], 404) self.assertEqual(self.request("/api/actions/../v1/repos")[0], 404) @@ -200,12 +239,109 @@ class GateTests(unittest.TestCase): self.assertEqual(state["last_fetch_peer"], "127.0.0.1") self.assertGreaterEqual(state["last_fetch_at"], before) self.assertEqual(self.upstream.requests, []) - self.assertEqual(self.request(UPDATE)[0], 200) + self.assertEqual(self.request(PING)[0], 200) self.assertEqual(self.gate.control("status")["last_fetch_at"], state["last_fetch_at"]) restarted = MODULE.Gate(self.temp.name).control("status") self.assertIsNone(restarted["last_fetch_peer"]) self.assertIsNone(restarted["last_fetch_at"]) + def fetch_task(self, task_id=42): + self.upstream.responses[FETCH] = field(1, field(1, task_id)) + field(2, 10) + self.upstream.release.set() + self.assertEqual(self.request(FETCH)[0], 200) + self.assertEqual(self.gate.control("status")["task_ids"], [str(task_id)]) + + def finalize_log(self, task_id=42): + self.assertEqual(self.request(LOG, field(1, task_id) + field(2, 3) + + field(3, b"log row") + field(4, 1))[0], 200) + self.wait_for(lambda: self.gate.tasks.get(str(task_id)) is True) + + def test_assigned_task_survives_pause_until_runner_finished_reporting(self): + self.fetch_task() + self.assertEqual(self.gate.control("pause")["active_tasks"], 1) + self.assertEqual(self.request(FETCH)[0], 503) + self.assertEqual(self.request(UPDATE, state(42, 1))[0], 200) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.finalize_log() + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.assertEqual(self.request(UPDATE, state(42, 1))[0], 200) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + self.assertFalse(self.gate.control("status")["uncertain"]) + # 终态响应在客户端超时后重试，不应再次阻断后续任务领取。 + self.request(UPDATE, state(42, 1)) + self.assertFalse(self.gate.control("status")["uncertain"]) + + def test_compressed_fetch_and_task_ledger_survive_restart(self): + self.upstream.compressed = True + self.fetch_task() + restored = MODULE.Gate(self.temp.name) + self.assertEqual(restored.control("status")["task_ids"], ["42"]) + self.assertFalse(restored.control("status")["uncertain"]) + self.proxy.gate = self.gate = restored + self.finalize_log() + self.request(UPDATE, state(42, 2)) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + self.assertEqual(MODULE.Gate(self.temp.name).control("status")["active_tasks"], 0) + + def test_server_cancellation_does_not_retire_executing_task(self): + self.fetch_task() + self.upstream.responses[UPDATE] = state(42, 3) + self.request(UPDATE, state(42)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.finalize_log() + self.request(UPDATE, state(42)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.request(UPDATE, state(42, 3)) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + + def test_partial_final_log_acknowledgement_keeps_task_active(self): + self.fetch_task() + self.upstream.responses[LOG] = field(1, 3) + self.request(LOG, field(1, 42) + field(2, 3) + field(3, b"row") + field(4, 1)) + self.request(UPDATE, state(42, 1)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + del self.upstream.responses[LOG] + self.finalize_log() + self.request(UPDATE, state(42, 1)) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + + def test_final_state_waits_for_success_and_output_acknowledgement(self): + self.fetch_task() + self.finalize_log() + self.upstream.statuses[UPDATE] = 500 + self.request(UPDATE, state(42, 1)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.upstream.statuses[UPDATE] = 200 + output = field(2, field(1, b"key") + field(2, b"value")) + self.request(UPDATE, state(42, 1) + output) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.upstream.responses[UPDATE] = state(42, 1) + field(2, b"key") + self.request(UPDATE, state(42, 1) + output) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + + def test_unknown_or_invalid_fetch_result_prevents_idle(self): + self.upstream.release.set() + self.upstream.responses[FETCH] = b"not protobuf" + self.assertEqual(self.request(FETCH)[0], 200) + self.assertTrue(self.gate.control("status")["uncertain"]) + self.assertTrue(MODULE.Gate(self.temp.name).control("status")["uncertain"]) + + def test_task_report_without_observed_assignment_requires_idle_bootstrap(self): + self.assertEqual(self.request(UPDATE, state(42))[0], 200) + self.wait_for(lambda: self.gate.control("status")["uncertain"]) + self.assertIn("error", self.gate.control("resume")) + + def test_disconnected_fetch_client_still_leaves_assigned_task_busy(self): + self.upstream.responses[FETCH] = field(1, field(1, 42)) + client = socket.create_connection(self.proxy.server_address, timeout=3) + client.sendall(f"POST {FETCH} HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n".encode()) + self.assertTrue(self.upstream.entered.wait(3)) + client.close() + self.gate.control("pause") + self.upstream.release.set() + self.wait_for(lambda: self.gate.control("status")["inflight"] == 0) + self.assertEqual(self.gate.control("status")["task_ids"], ["42"]) + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_gitea_cache_maintenance.py b/scripts/test_gitea_cache_maintenance.py index 4f515b6ef..ac6650ba7 100644 --- a/scripts/test_gitea_cache_maintenance.py +++ b/scripts/test_gitea_cache_maintenance.py @@ -4,6 +4,7 @@ from __future__ import annotations import importlib.util +import io import json from pathlib import Path import tempfile @@ -28,14 +29,11 @@ def runner_config(image: str) -> str: class FakeApi: - def __init__(self, disabled: bool = False): - self.disabled = disabled + def __init__(self): self.requests: list[str] = [] def request(self, path, body=None, raw=False): self.requests.append(path) - if path.startswith("admin/actions/runners/"): - return {"disabled": self.disabled, "busy": False} raise AssertionError(f"unexpected API request: {path}") @@ -51,7 +49,6 @@ class GiteaCacheMaintenanceTest(unittest.TestCase): "token_file": str(self.token), "repository": "team/project", "repository_url": "http://gitea:3000/team/project.git", - "runner_api_path": "admin/actions/runners/1", "runner_container": "gitea-runner", "gate_socket": str(self.root / "gate.sock"), "gate_url": "https://gitea.example.test", @@ -168,7 +165,8 @@ class GiteaCacheMaintenanceTest(unittest.TestCase): return json.dumps([{"State": {"StartedAt": started}, "NetworkSettings": { "Networks": {"internal": {"IPAddress": "10.0.0.2"}}}}]) instance.docker = docker - proof = {"last_fetch_peer": "10.0.0.2", "last_fetch_at": maintenance_module.timestamp(started).timestamp() + 1} + proof = {"last_fetch_peer": "10.0.0.2", "last_fetch_at": maintenance_module.timestamp(started).timestamp() + 1, + "active_tasks": 0} instance.gate = lambda action: proof instance.check_gate_route() proof["last_fetch_at"] -= 2 @@ -225,6 +223,186 @@ class GiteaCacheMaintenanceTest(unittest.TestCase): self.assertEqual(instance.state["candidate"], IMAGE) self.assertNotIn("pause_owned", instance.state) + def test_idle_requires_task_ledger_and_all_live_container_states(self): + instance = self.maintenance({"versions": [], "current": None}) + state = {"uncertain": False, "active_tasks": 1} + instance.gate = lambda _: state + calls = [] + instance.docker = lambda *args, **kw: calls.append((args, kw)) or "" + self.assertFalse(instance.idle()) # assigned task before its first container + self.assertEqual(calls, []) + state["active_tasks"] = 0 + self.assertTrue(instance.idle()) + self.assertIn("status=created", calls[-1][0]) + self.assertTrue(calls[-1][1]["inner"]) + instance.docker = lambda *_a, **_k: "created-container\n" + self.assertFalse(instance.idle()) + del state["active_tasks"] + with self.assertRaisesRegex(RuntimeError, "durable task tracking"): + instance.idle() + + def source_fixture(self): + old = self.record(OLD_IMAGE, source="b" * 40, owned=False, verified_run=1) + instance = self.maintenance({"versions": [old], "current": OLD_IMAGE}) + run = {"id": 44, "status": "completed", "conclusion": "failure", "event": "push", + "path": "project-ci.yml@refs/heads/master", "head_branch": "master", "head_sha": SHA} + jobs = [{"id": i, "name": name, "status": "completed", "conclusion": "failure", + "head_sha": SHA, "run_attempt": 1, + "steps": [{"name": maintenance_module.EXPORT_STEP, "conclusion": "success"}]} + for i, name in enumerate(maintenance_module.RUST_JOB_IDS, start=1)] + artifacts = [{"id": job["id"], "name": "rust-cache-v1-" + maintenance_module.RUST_JOB_IDS[job["name"]] + + "-attempt-1", "expired": False, "workflow_run": run} for job in jobs] + class Api: + def pages(self, path, key): + return iter({"workflow_runs": [run], "jobs": jobs, "artifacts": artifacts}[key]) + def request(self, path, raw=False): + job_id = int(path.split("/jobs/")[1].split("/")[0]) + name = next(item["name"] for item in artifacts if item["id"] == job_id) + return f"worker image: {OLD_IMAGE}\n[rust-cache] artifact={name} objects=1 bytes=10 complete=true\n" + instance.api = Api() + return instance, run, jobs, artifacts + + def select_source(self, instance): + with patch.object(maintenance_module.subprocess, "run", return_value=type("Result", (), {"returncode": 0})()), \ + patch.object(maintenance_module, "command", lambda *args, **kw: "100644 blob " + args[-1] + "\tsrc.rs\0"): + return instance.source_run() + + def test_complete_failed_master_can_supply_cache_but_pr_and_missing_groups_cannot(self): + instance, run, jobs, artifacts = self.source_fixture() + result = self.select_source(instance) + self.assertEqual(result["run_id"], 44) + self.assertEqual(len(result["exports"]), 6) + run["event"] = "pull_request" + self.assertIsNone(self.select_source(instance)) + run["event"] = "push" + artifacts.pop() + self.assertIsNone(self.select_source(instance)) + + def test_stale_attempt_cancelled_job_or_failed_export_cannot_supply_cache(self): + instance, run, jobs, artifacts = self.source_fixture() + run["conclusion"] = "cancelled" + self.assertIsNone(self.select_source(instance)) # all six artifacts may already exist + run["conclusion"] = "failure" + jobs[0]["run_attempt"] = 2 + self.assertIsNone(self.select_source(instance)) + jobs[0]["run_attempt"] = 1 + jobs[0]["conclusion"] = "cancelled" + self.assertIsNone(self.select_source(instance)) + jobs[0]["conclusion"] = "success" + jobs[0]["steps"][0]["conclusion"] = "failure" + self.assertIsNone(self.select_source(instance)) + + def test_exports_using_different_images_are_not_combined(self): + instance, _, _, _ = self.source_fixture() + original = instance.api.request + instance.api.request = lambda path, raw=False: original(path, raw).replace(OLD_IMAGE, IMAGE) if '/jobs/1/' in path else original(path, raw) + self.assertIsNone(self.select_source(instance)) + + def test_completed_artifact_without_upload_completion_log_is_not_used(self): + instance, _, _, _ = self.source_fixture() + instance.api.request = lambda *_a, **_k: f"worker image: {OLD_IMAGE}\n" + self.assertIsNone(self.select_source(instance)) + + def test_assembly_uses_ci_objects_and_trusted_binary_without_warming_compiler(self): + instance, _, _, _ = self.source_fixture() + source = self.select_source(instance) + shell_calls, docker_calls = [], [] + instance.build_command = lambda log, script, *args, **kw: shell_calls.append((script, args)) + def info(image, inner=False): + return {"Id": IMAGE if image.startswith("genarrative/") else image, "Config": {"Labels": { + "world.genarrative.ci.rust-cache-source": "b" * 40, + "world.genarrative.ci.rust-cache-base": BASE_IMAGE, + "com.genarrative.ci.definition-sha256": "definition", + }}} + instance.image_info = info + def docker(*args, **kw): + docker_calls.append(args) + if args[0] == "create": + return "temporary-copy-container" + if args[0] == "cp": + root = Path(args[-1]) + (root / "objects").mkdir() + (root / "sccache").write_bytes(b"trusted binary") + if args[-2:] == ("rustc", "-vV"): + return "rustc test\n" + return "" + instance.docker = docker + instance.api.download = lambda path, destination: destination.write_bytes(b"download") + def merge(inputs, output, **kwargs): + self.assertEqual(len(inputs), 6) + self.assertEqual(kwargs["expected_inherited_source_sha"], "b" * 40) + output.mkdir() + (output / "objects").mkdir() + return type("Merged", (), {"sccache_version": "sccache 0.18.0", "rustc": "rustc test\n", + "workspace": "/workspace/team/project", "base_image": BASE_IMAGE})() + with patch.object(maintenance_module, "command", return_value="definition\n"), \ + patch.object(maintenance_module, "merge_snapshots", merge): + instance.build(source) + self.assertEqual(shell_calls, [("gitea-ci-job-image.sh", ("verify", "genarrative/gitea-project-ci:rust-cache-auto-" + SHA))]) + self.assertIn(("create", OLD_IMAGE), docker_calls) + self.assertIn(("rm", "--volumes", "temporary-copy-container"), docker_calls) + self.assertEqual(instance.state["candidate"], IMAGE) + self.assertEqual(instance.version(IMAGE)["run_id"], 44) + + def test_export_cleanup_only_deletes_collected_or_expired_owned_master_artifacts(self): + current = self.record(OLD_IMAGE, owned=False, verified_run=1) + current.update(staged=True, run_id=44, exports=[{"id": 1}]) + instance = self.maintenance({"versions": [current], "current": OLD_IMAGE}) + old_run = {"id": 9, "status": "completed", "path": "project-ci.yml@refs/heads/master", + "event": "push", "head_branch": "master", "head_sha": SHA} + artifact = {"id": 2, "name": "rust-cache-v1-backend-tests-attempt-1", + "workflow_run": {"id": 9}, "created_at": "2020-01-01T00:00:00Z"} + rows = [artifact, {**artifact, "id": 3, "name": "release-binary"}, + {**artifact, "id": 4, "workflow_run": {"id": 10}}, + {**artifact, "id": 5, "workflow_run": {"id": 11}}] + deleted = [] + instance.api.pages = lambda *_: iter(rows) + def request(path, method=None): + if method == "DELETE": + return deleted.append((path, method)) + return {9: old_run, 10: {**old_run, "event": "pull_request"}, + 11: {**old_run, "status": "in_progress"}}[int(path.rsplit("/", 1)[1])] + instance.api.request = request + instance.cleanup_exports() + self.assertEqual(deleted, [(instance.repo_api + "/actions/artifacts/1", "DELETE"), + (instance.repo_api + "/actions/artifacts/2", "DELETE")]) + self.assertTrue(instance.version(OLD_IMAGE)["exports"][0]["deleted"]) + + def test_signed_download_drops_token_and_rejects_other_origins(self): + api = maintenance_module.Api(self.config["api_url"], self.token) + destination = self.root / "artifact.zip" + seen = [] + target = ["https://gitea.example.test/api/v1/repos/team/project/actions/artifacts/1/zip/raw?sig=test"] + class Opener: + def open(self, request, timeout): + seen.append(request) + if not isinstance(request, str): + raise maintenance_module.urllib.error.HTTPError(request.full_url, 302, "Found", {"Location": target[0]}, None) + return io.BytesIO(b"archive") + with patch.object(maintenance_module.urllib.request, "build_opener", return_value=Opener()): + api.download("repos/team/project/actions/artifacts/1/zip", destination) + self.assertEqual(destination.read_bytes(), b"archive") + self.assertEqual(seen[0].get_header("Authorization"), "token test-token") + self.assertIsInstance(seen[1], str) # signed URL request has no Authorization header + target[0] = "https://other.example.test/download" + with self.assertRaisesRegex(RuntimeError, "configured HTTPS Gitea origin"): + api.download("repos/team/project/actions/artifacts/1/zip", self.root / "rejected.zip") + self.assertFalse((self.root / "rejected.zip").exists()) + + def test_pending_chunk_cleanup_requires_old_terminal_master_run(self): + instance = self.maintenance({"versions": [], "current": None, "attempts": [{"run_id": 4}]}) + instance.config["artifact_storage_dir"] = str(self.root / "storage") + run = {"id": 1, "status": "completed", "path": "project-ci.yml@refs/heads/master", + "event": "push", "head_branch": "master", "head_sha": SHA, + "completed_at": "2020-01-01T00:00:00Z"} + rows = [run, {**run, "id": 2, "event": "pull_request"}, + {**run, "id": 3, "status": "in_progress"}, {**run, "id": 4}, + {**run, "id": 5, "completed_at": "2999-01-01T00:00:00Z"}] + instance.api.pages = lambda *_: iter(rows) + with patch.object(maintenance_module, "cleanup_upload_chunks", return_value=0) as cleanup: + instance.cleanup_pending_uploads() + self.assertEqual(cleanup.call_args.args[1], {1}) + def test_uncertain_or_inflight_gate_never_restarts(self): instance, _, restarts = self.activation_maintenance(idle_values=[True]) statuses = 0 diff --git a/scripts/test_gitea_cache_snapshot.py b/scripts/test_gitea_cache_snapshot.py new file mode 100644 index 000000000..135cd903b --- /dev/null +++ b/scripts/test_gitea_cache_snapshot.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Tests for streamed Gitea sccache snapshot validation and merging.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import os +import sys +import tarfile +import tempfile +import unittest +import zipfile + + +SCRIPT = Path(__file__).with_name("gitea_cache_snapshot.py") +SPEC = importlib.util.spec_from_file_location("gitea_cache_snapshot", SCRIPT) +assert SPEC and SPEC.loader +snapshot = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = snapshot +SPEC.loader.exec_module(snapshot) + +REPOSITORY = "team/project" +SOURCE_SHA = "a" * 40 +INHERITED_SOURCE_SHA = "c" * 40 +BASE_IMAGE = "sha256:" + "b" * 64 +RUSTC = "rustc 1.90.0\nbinary: rustc\n" +WORKSPACE = "/workspace/GenarrativeAI/Genarrative" +SCCACHE_VERSION = "0.18.0" + + +def object_path(character: str) -> str: + key = character * 64 + return f"objects/{key[0]}/{key[1]}/{key}" + + +class SnapshotMergeTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + self.root = Path(self.temporary_directory.name).resolve() + self.base = self.root / "base-objects" + self.base.mkdir() + + def tearDown(self) -> None: + self.temporary_directory.cleanup() + + def identity(self, job: str, run_id: int) -> object: + return snapshot.ArtifactIdentity(REPOSITORY, run_id, 1, job, SOURCE_SHA) + + def archive( + self, + name: str, + job: str, + run_id: int, + objects: list[tuple[str, bytes, int]], + *, + prefix: bool = True, + manifest_update=None, + extra_member: tuple[str, bytes, str] | None = None, + touched: list[tuple[str, int]] | None = None, + ) -> object: + rows = [ + { + "path": path, + "size": len(contents), + "sha256": hashlib.sha256(contents).hexdigest(), + "mtime_ns": mtime_ns, + } + for path, contents, mtime_ns in objects + ] + manifest = { + "schema": 1, + "repository": REPOSITORY, + "run_id": run_id, + "run_attempt": 1, + "job": job, + "source_sha": SOURCE_SHA, + "base_image": BASE_IMAGE, + "rustc": RUSTC, + "workspace": WORKSPACE, + "sccache_version": SCCACHE_VERSION, + "mode": "delta", + "inherited_source_sha": INHERITED_SOURCE_SHA, + "objects": rows, + "touched": [ + {"path": path, "mtime_ns": mtime_ns} + for path, mtime_ns in (touched or []) + ], + } + if manifest_update is not None: + manifest_update(manifest) + + tar_bytes = io.BytesIO() + with tarfile.open(fileobj=tar_bytes, mode="w") as bundle: + for path, contents, _ in objects: + info = tarfile.TarInfo(path) + info.size = len(contents) + bundle.addfile(info, io.BytesIO(contents)) + if extra_member is not None: + path, contents, kind = extra_member + info = tarfile.TarInfo(path) + if kind == "symlink": + info.type = tarfile.SYMTYPE + info.linkname = "manifest.json" + bundle.addfile(info) + else: + info.size = len(contents) + bundle.addfile(info, io.BytesIO(contents)) + manifest_bytes = json.dumps(manifest, separators=(",", ":")).encode() + info = tarfile.TarInfo("manifest.json") + info.size = len(manifest_bytes) + bundle.addfile(info, io.BytesIO(manifest_bytes)) + + archive = self.root / f"{name}.zip" + member = f"{name}/snapshot.tar" if prefix else "snapshot.tar" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_STORED) as bundle: + bundle.writestr(member, tar_bytes.getvalue()) + return snapshot.ArtifactInput(archive, self.identity(job, run_id)) + + def base_object(self, path: str, contents: bytes, mtime_ns: int) -> None: + target = self.base.joinpath(*Path(path).parts[1:]) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(contents) + os.utime(target, ns=(mtime_ns, mtime_ns)) + + def merge(self, inputs, output=None, maximum=1024 ** 3): + return snapshot.merge_snapshots( + inputs, + output or self.root / "merged", + base_objects=self.base, + expected_inherited_source_sha=INHERITED_SOURCE_SHA, + max_combined_bytes=maximum, + ) + + def test_merges_groups_deduplicates_newest_mtime_and_prunes_to_bound(self) -> None: + first_path = object_path("a") + second_path = object_path("b") + third_path = object_path("c") + inherited_path = object_path("d") + self.base_object(inherited_path, b"dd", 50) + inputs = [ + self.archive("first", "Backend tests", 41, [ + (first_path, b"aaaa", 100), + (second_path, b"bbbb", 200), + ]), + self.archive("second", "Native shell tests", 41, [ + (first_path, b"aaaa", 300), + (third_path, b"ccc", 250), + ], prefix=False, touched=[(inherited_path, 400)]), + ] + + output = self.root / "merged" + result = self.merge(inputs, output, maximum=9) + + self.assertEqual((result.object_count, result.total_bytes), (3, 9)) + self.assertEqual((output / inherited_path).read_bytes(), b"dd") + self.assertEqual((output / first_path).read_bytes(), b"aaaa") + self.assertEqual((output / third_path).read_bytes(), b"ccc") + self.assertFalse((output / second_path).exists()) + self.assertEqual((output / "rustc.txt").read_text(), RUSTC) + self.assertEqual((output / "workspace.txt").read_text(), WORKSPACE + "\n") + self.assertEqual((output / "source-commit.txt").read_text(), SOURCE_SHA + "\n") + self.assertEqual((output / "base-image.txt").read_text(), BASE_IMAGE + "\n") + self.assertFalse((output / "sccache").exists()) + + def test_rejects_traversal_member_without_creating_output(self) -> None: + item = self.archive( + "traversal", "Backend tests", 42, [], + extra_member=("../outside", b"bad", "file"), + ) + output = self.root / "merged" + with self.assertRaisesRegex(snapshot.SnapshotError, "invalid sccache object path"): + self.merge([item], output) + self.assertFalse(output.exists()) + self.assertFalse((self.root / "outside").exists()) + + def test_rejects_checksum_mismatch(self) -> None: + path = object_path("d") + + def corrupt(manifest): + manifest["objects"][0]["sha256"] = "0" * 64 + + item = self.archive( + "checksum", "Backend tests", 43, [(path, b"content", 1)], + manifest_update=corrupt, + ) + with self.assertRaisesRegex(snapshot.SnapshotError, "checksums do not match"): + self.merge([item]) + + def test_rejects_manifest_identity_mismatch(self) -> None: + item = self.archive( + "identity", "Backend tests", 44, [], + manifest_update=lambda manifest: manifest.update(run_id=999), + ) + with self.assertRaisesRegex(snapshot.SnapshotError, "run_id does not match"): + self.merge([item]) + + def test_rejects_conflicting_duplicate_key(self) -> None: + path = object_path("e") + inputs = [ + self.archive("conflict-one", "Backend tests", 45, [(path, b"one", 1)]), + self.archive("conflict-two", "Native shell tests", 45, [(path, b"two", 2)]), + ] + with self.assertRaisesRegex(snapshot.SnapshotError, "conflicting content"): + self.merge(inputs) + + def test_rejects_delta_that_conflicts_with_inherited_key(self) -> None: + path = object_path("f") + self.base_object(path, b"base", 1) + item = self.archive("base-conflict", "Backend tests", 46, [(path, b"evil", 2)]) + with self.assertRaisesRegex(snapshot.SnapshotError, "conflicting content"): + self.merge([item]) + + def test_accepts_empty_delta(self) -> None: + item = self.archive("empty", "Backend tests", 47, []) + output = self.root / "merged" + result = self.merge([item], output) + self.assertEqual((result.object_count, result.total_bytes), (0, 0)) + self.assertEqual(list((output / "objects").iterdir()), []) + + def test_rejects_noncanonical_base_entry(self) -> None: + (self.base / "unexpected").write_bytes(b"not an object") + item = self.archive("base-path", "Backend tests", 48, []) + with self.assertRaisesRegex(snapshot.SnapshotError, "noncanonical entry"): + self.merge([item]) + + def test_rejects_tar_links(self) -> None: + item = self.archive( + "link", "Backend tests", 49, [], + extra_member=(object_path("9"), b"", "symlink"), + ) + with self.assertRaisesRegex(snapshot.SnapshotError, "regular file"): + self.merge([item]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_gitea_cache_upload_cleanup.py b/scripts/test_gitea_cache_upload_cleanup.py new file mode 100644 index 000000000..6ebe0d200 --- /dev/null +++ b/scripts/test_gitea_cache_upload_cleanup.py @@ -0,0 +1,90 @@ +"""缓存上传残块清理只触碰已确认结束的过期自有文件。""" + +import base64 +import os +from pathlib import Path +import tempfile +import unittest + +from gitea_cache_upload_cleanup import cleanup_upload_chunks, decode_owner + + +class UploadCleanupTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + + def file(self, name, run=12, age=10): + directory = self.root / "tmp-upload" / f"run-{run}-v4" + directory.mkdir(parents=True, exist_ok=True) + path = directory / name + path.write_bytes(b"block data") + os.utime(path, (age, age)) + return path + + def block(self, artifact=34, run=12, age=10, marker="genarrative-rust-cache-v1:backend-tests:1:00000000"): + encoded = base64.urlsafe_b64encode(base64.b64encode(marker.encode())).decode() + return self.file(f"block-{run}-{artifact}-10-{encoded}", run, age) + + def test_old_owned_blocks_and_associated_blocklist_are_deleted(self): + block = self.block() + blocklist = self.file("12-34-blocklist") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 2) + self.assertFalse(block.exists()) + self.assertFalse(blocklist.exists()) + self.assertTrue(block.parent.is_dir()) + + def test_other_runs_artifacts_and_unproven_blocklists_remain(self): + own = self.block() + other_run = self.block(run=13) + other_artifact = self.block(artifact=35, marker="unrelated-upload") + unproven = self.file("12-36-blocklist") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 1) + self.assertFalse(own.exists()) + self.assertTrue(all(path.exists() for path in (other_run, other_artifact, unproven))) + + def test_young_or_unknown_member_protects_entire_artifact_group(self): + own = self.block() + young = self.file("12-34-blocklist", age=20) + other = self.block(artifact=35) + unknown = self.file("block-12-35-invalid-size-unknown") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 0) + self.assertTrue(all(path.exists() for path in (own, young, other, unknown))) + + def test_completely_unknown_file_protects_entire_run(self): + block = self.block() + self.file("unknown") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 0) + self.assertTrue(block.exists()) + + @unittest.skipUnless(os.name == "posix", "symlink fixture requires POSIX") + def test_symlinks_never_traversed_or_deleted(self): + block = self.block() + outside = self.root / "secret" + outside.write_text("keep") + (block.parent / "12-34-blocklist").symlink_to(outside) + (self.root / "tmp-upload" / "run-13-v4").symlink_to(block.parent, target_is_directory=True) + self.assertEqual(cleanup_upload_chunks(self.root, {12, 13}, 20), 0) + self.assertTrue(block.exists()) + self.assertEqual(outside.read_text(), "keep") + linked = self.root / "root-link" + linked.symlink_to(self.root, target_is_directory=True) + with self.assertRaises(ValueError): + cleanup_upload_chunks(linked, {12}, 20) + + def test_root_missing_or_relative_is_an_error(self): + with self.assertRaises(FileNotFoundError): + cleanup_upload_chunks(self.root / "missing", {12}, 20) + with self.assertRaises(ValueError): + cleanup_upload_chunks(Path("relative"), {12}, 20) + + def test_noncanonical_encoding_and_unknown_job_are_not_owned(self): + for marker in ("genarrative-rust-cache-v1:other-job:1:00000000", "genarrative-rust-cache-v1:backend-tests:01:00000000"): + encoded = base64.urlsafe_b64encode(base64.b64encode(marker.encode())).decode() + self.assertIsNone(decode_owner(encoded)) + self.assertIsNone(decode_owner("not-base64")) + + +if __name__ == "__main__": + unittest.main()