diff --git a/.gitea/workflows/project-ci.yml b/.gitea/workflows/project-ci.yml index 707e5f7a6..b84b9f901 100644 --- a/.gitea/workflows/project-ci.yml +++ b/.gitea/workflows/project-ci.yml @@ -98,6 +98,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py ai-game-creator-shell-rust-lane-2: name: AI game creator shell Rust lane 2/2 @@ -141,6 +147,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py # agent-run smoke 会 spawn `cargo run`（走壳自己的 manifest），同样不装 npm 依赖， # 单独一个 job，免得把已经压到 4 分钟级的片 job 拖长。 @@ -183,6 +195,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py # AGC 壳依赖的共享 / 平台和编辑器插件 crate 各自预热独立 manifest，再运行对应测试。 ai-game-creator-shell-rust-crates: @@ -291,6 +309,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py backend-tests: name: Backend tests @@ -380,6 +404,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py # 客户端的壳级与契约门禁：静态契约断言、H5 / 微信 / 移动 / 桌面壳运行时门禁， # 以及依赖发布产物的构建 smoke。 @@ -442,6 +472,12 @@ jobs: - name: Report isolated Rust compilation cache if: always() run: bash scripts/ci-rust-cache.sh report + - name: Publish master Rust cache artifact + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: python3 scripts/export-gitea-rust-cache.py frontend-tests: name: Frontend tests @@ -524,6 +560,9 @@ jobs: - name: Install npm dependencies run: bash scripts/ci-npm-ci-with-retry.sh + - name: Validate CI cache maintenance behavior + run: python3 -m unittest discover -s scripts -p 'test_gitea_cache_*.py' + - name: Run repository checks run: npm run check:repository-ci diff --git a/deploy/container/README.md b/deploy/container/README.md index a129d2d1d..8e9dece50 100644 --- a/deploy/container/README.md +++ b/deploy/container/README.md @@ -89,7 +89,7 @@ runner 配置保留原 `ubuntu-latest` 映射，`genarrative-ci` 继续映射到 2. 执行 `load-runner`，确认该 ID 已进入 runner 内层 Docker。 3. 确认没有活跃 job，将当前 runner config 备份到仓库外的受控位置；备份不得进入 Git，也不得在文档或日志中回显注册信息。 4. 增加或替换 `genarrative-ci` 的精确 `docker://` 映射，然后执行 `docker restart --timeout 660 gitea-runner`。 -5. 重跑真实 PR 的四个 CI job；全部通过且隔离边界复核完成后，才能清理旧镜像。 +5. 用真实 CI 的九个 job 验证新镜像；全部通过且隔离边界复核完成后，才能清理旧镜像。 `docker restart --timeout 660` 只提供容器停止宽限，不是 Runner drain API；rootless DinD 的 supervisor 可能与 runner 同时停止内层 dockerd。重启前必须同时确认 Gitea 没有 `in_progress` run 且内层 `docker ps` 为空，不能依赖该 timeout 等待活跃 job。 @@ -97,9 +97,15 @@ runner 配置保留原 `ubuntu-latest` 映射，`genarrative-ci` 继续映射到 ### Rust 测试组编译对象快照 -`scripts/build-gitea-rust-cache.sh` 在已验证的 job 镜像上生成候选镜像，覆盖 AGC Rust 两条 lane、crates、agent-run smoke、Backend 和 Native shell 的桌面壳测试。Native shell 的 release build smoke 显式清空两个 wrapper，不消费测试快照。固定 sccache `0.18.0` 的 Linux x64 musl 归档并校验 SHA-256，维护者从 origin/master 的确定提交预热编译对象，PR job 没有生成/发布公共快照的权限。 +自动维护由宿主 systemd timer 调用 `scripts/maintain-gitea-rust-cache.py`，只管理 Gitea CI 测试镜像，不修改 Jenkins、生产发布、本地开发或客户端发行构建。六个 Rust job 仅在 master push 中导出本次 CI 新增的 sccache 对象；已命中的继承对象只上传新近使用时间，通过 Gitea 原生 V4 artifact 接口上传；PR 不发布。维护器选择已结束且六组产物完整的最新 master run，校验提交、任务尝试、工具链与来源镜像，与六组实际使用的同一镜像快照合并去重，并按新近使用时间限制快照总容量为 4 GiB，然后从无对象缓存基础镜像组装新镜像，**不重复执行 Cargo 预热编译，也不要求源 run 事先全绿**。缺组、取消或校验失败时保留现役版，不混合不同 run 的对象来假装完整快照。 -基础镜像必须不含 `/opt/genarrative-ci/rust-cache`；脚本在拉取源码、下载工具和预热前执行只读、断网检查，发现已有对象快照就拒绝构建。不能在旧缓存镜像上删除目录再叠加新快照，删除操作不会释放旧镜像层。切换且真实 CI 验证通过后，人工定向清理更旧的缓存镜像与导出归档，保留当前版、一个回滚版及运行中 CI 使用的版本；对象缓存上限不覆盖这些宿主文件，不使用全局 prune。 +切换先通过专属入口阻断新的 FetchTask，确认已转发的领取请求全部收到完整上游响应，并检查入口持久化跟踪的已领取任务全部结束、内层 Docker 没有活动容器。任务终态必须依据 Runner 的执行结束及最终上报协议，不能由容器暂时为空、API 已取消或请求超时推断。有任务即恢复领取并延后，不停止任务；状态未知拒绝切换。维护器只需普通账号的 `write:repository` Token（包括查询、下载及定向删除 artifact），不访问全局 Runner 管理 API。切换后等待使用该 Image ID 的完整真实 master push CI 通过，才允许下一次升级及旧镜像清理；不会自动重跑失败用例或为了验收额外触发整轮 CI。首次接管的历史镜像默认不归自动清理管理。 + +维护状态、凭据、归档和配置备份保存在仓库外。当前版、回滚版、待验证候选、它们的基础镜像及容器引用的镜像均受保护。清理只针对维护器登记的专属 tag、完整 Image ID 和专用目录中的归档；禁止全局 prune。API、构建、验证或空闲检查失败时保留现役镜像与回滚资料，不以失败重跑制造全绿结果。 + +`scripts/build-gitea-rust-cache.sh` 仅用于首次缺少可消费快照时的人工 bootstrap，在已验证的 job 镜像上生成候选镜像，覆盖 AGC Rust 两条 lane、crates、agent-run smoke、Backend 和 Native shell 的桌面壳测试。Native shell 的 release build smoke 显式清空两个 wrapper，不消费测试快照。固定 sccache `0.18.0` 的 Linux x64 musl 归档并校验 SHA-256，维护者从 origin/master 的确定提交预热编译对象，PR job 没有生成/发布公共快照的权限。 + +基础镜像必须不含 `/opt/genarrative-ci/rust-cache`；脚本在拉取源码、下载工具和预热前执行只读、断网检查，发现已有对象快照就拒绝构建。不能在旧缓存镜像上删除目录再叠加新快照，删除操作不会释放旧镜像层。切换且真实 CI 验证通过后，维护器定向清理自己登记的更旧缓存镜像与导出归档，保留当前版、一个回滚版、所需基础镜像及容器引用的版本；接管前的试验镜像/归档仍由维护者确认后人工清理。对象缓存上限不覆盖这些宿主文件，不使用全局 prune，也不清理其它构建的 Docker build cache。 ```bash bash scripts/build-gitea-rust-cache.sh genarrative/gitea-project-ci:20260920.2 genarrative/gitea-project-ci:rust-cache-candidate @@ -108,9 +114,52 @@ bash scripts/gitea-ci-job-image.sh export /仓库外受控路径/ci-rust-cache.t bash scripts/gitea-ci-job-image.sh load-runner genarrative/gitea-project-ci:rust-cache-candidate ``` -预热容器上限为 4 核、12 GiB，移除 capabilities，不挂宿主目录/socket，也不注入 Git/OSS/Jenkins 凭据。源码通过 `git archive` 复制，当前工作区、ignored 文件和 `.git` 不进入容器。最终从原镜像重新组装，仅复制 `/opt/genarrative-ci/rust-cache` 的 sccache、对象和来源元数据，不提交含源码/target 的预热容器；镜像本身的下载缓存与工具链校验保持原样。 +自动维护不调用预热脚本；以下人工 bootstrap 脚本接受第三个参数 `<完整 master SHA>`。脚本确认该 SHA 是抓取到的 master 祖先，源码和 `ci-rust-cache.sh` 均来自该提交；两参数人工调用仍默认使用最新 master。基础镜像的 `revision` 由同一份 build context 清单计算；输入变化时维护器先生成新的无对象缓存基础镜像。Rust 输入指纹保守包含代码、配置、资源、脚本和内嵌 skill，仅排除一般 `docs/` 及几个根说明文档，`docs/openapi/` 始终参与。 -快照由固定 Image ID 分发，每个 job 仅修改容器自己的写时复制层，缓存上限 4 GiB，构建末尾输出实际对象体积，结束后不回传。`ci-rust-cache.sh prepare` 清空继承的 `SCCACHE_*` 远程配置，使用独立配置和 Unix socket；旧镜像、工具链不匹配或限时 wrapper 探测失败时使用直接 rustc，正式编译启用 sccache 的 server IO 错误回退。真实编译/测试失败保留非零退出码。`report` 输出命中统计并停止本 job daemon，分片日志输出独立编译耗时。sccache 0.18.0 的只读模式在 miss 后仍打包再拒绝写入，不能用它宣称零 miss 开销；普通 CI 继续只向容器层写入。 +缓存产物通过六个 `Publish master Rust cache artifact` 步骤发布，daemon 成功停止才导出。源 run 必须结束，且六组为 success/failure 并有完整上传证据；整轮或任一 Rust 组取消、旧 attempt、缺组、混用来源镜像均不采用。上传失败仅告警，不改变测试结论。每组仅导出相对其启动快照的新 key，命中只上报触达时间；宿主按真实 job 日志确认同一来源 Image ID，再复用其对象，不信任产物中的可执行文件。合并校验 tar/zip 路径、对象大小与 SHA256，同 key 内容冲突拒绝发布。sccache 可执行文件取自可信来源镜像，不从 artifact 执行代码。 + +候选已校验、导出并装载内层 Docker 后，可定向删除该 run 已收集的六份上传产物；旧镜像清理仍须等真实 CI 验证。专属缓存 artifact 保留 7 天，宿主也清理超过 7 天且源 run 已结束的遗留项，不删普通构建产物、run 或日志。Gitea 1.26.4 不清理未 finalized 的上传块：上传器使用专属双层编码块标识，宿主从 `artifact_storage_dir/tmp-upload/run--v4/` 定向清理本上传器的普通文件，要求所属仓库 master run 已结束超过 7 天且文件本身也超过 7 天。未知、年轻、符号链接或不属于本上传器的文件受到保护，不改数据库、不删除目录。已完成产物通过 API 删除；未完成上传留下的数据库元数据仍归 Gitea 管理。宿主的未完成下载/合并临时文件仅在登记的私有目录内清理。 + +### 自动维护首次部署与恢复 + +首次部署在本变更合入 master 后进行。维护脚本安装到 `/opt/genarrative-ci-cache/scripts/`；运行状态、源码专用 clone、日志和归档放在 `/var/lib/genarrative-ci-cache/`，token 放在 `/etc/genarrative-ci-cache/api-token`（root 所有、0600），均不进入 Git。复制 `gitea-ci-cache.config.example.json` 为该目录的 `config.json` 并核实仓库、Runner 容器与网关地址。Token 使用具有目标仓库 Actions 读写权限的普通账号，授予 `write:repository` 即可，不需要 `admin`；写操作仅定向删除本维护器的缓存 artifact，不重跑或取消任务。宿主运行账号还须能通过 `clone_url` 拉取专属 clone，API Token 不自动传给 Git。`artifact_storage_dir` 必须指向当前 Gitea 本地 ActionsArtifacts 存储的真实宿主路径（不是容器路径）；按当前 `/data` bind mount 和 `APP_DATA_PATH`，模板为 `/opt/gitea-stack/data/gitea/gitea/actions_artifacts`。部署时由 root 核实路径与 Gitea `[actions.artifacts]`/storage 配置一致，不能指向其它目录；此残块清理实现只支持本地存储，外部对象存储需要另行适配。 + +领取入口由 `gitea-runner-fetch-gate.compose.yml` 启动，使用已验证且含 Python 3 的无对象缓存 CI Image ID（`GITEA_FETCH_GATE_IMAGE`）。它只连接现有 `gitea-actions` 内部网络，不发布宿主端口、不挂 Docker socket；仅转发 `/api/actions/` RPC，控制 socket 位于独立私有目录 `/var/lib/genarrative-ci-cache-gate/`。该目录必须由 root 持有且权限为 0700。普通 job 无法访问控制 socket。部署前先运行下述测试，不直接启用 timer。 + +```bash +python3 -m unittest discover -s scripts -p 'test_gitea_cache_*.py' +install -d -m 700 /etc/genarrative-ci-cache /var/lib/genarrative-ci-cache /var/lib/genarrative-ci-cache-gate +install -d /opt/genarrative-ci-cache/scripts +install -m 755 scripts/maintain-gitea-rust-cache.py scripts/gitea-runner-fetch-gate.py /opt/genarrative-ci-cache/scripts/ +install -m 644 scripts/gitea_cache_snapshot.py scripts/gitea_cache_upload_cleanup.py /opt/genarrative-ci-cache/scripts/ +install -m 600 deploy/container/gitea-ci-cache.config.example.json /etc/genarrative-ci-cache/config.json +# 由维护者写入 write:repository API token，并核实 config.json；不要在终端回显 token。 +# 设置 GITEA_FETCH_GATE_IMAGE 为已验证基础镜像的完整 Image ID 后： +docker compose -f deploy/container/gitea-runner-fetch-gate.compose.yml up -d +``` + +**首次接入或从不跟踪任务的旧网关升级，需要空闲维护窗口**：确认无活跃 CI 且暂停新 CI 触发，再备份现有 runner 配置和注册文件，在 runner 部署的 `GITEA_INSTANCE_URL` 及 `/data/.runner` 的 `address` 中改用 `http://gitea-runner-fetch-gate:8080`，保留其余注册字段。同时在 `/data/config.yaml` 的 `runner.envs` 中设置 `GENARRATIVE_GITEA_REPOSITORY_URL: "http://gitea:3000/GenarrativeAI/Genarrative.git"`，与维护器的 `repository_url` 一致，沿用现有 job 已可达的内部 Git 通道。此专用变量也覆盖旧 PR 的 checkout；不要用同名 GITHUB_SERVER_URL 环境变量代替，Runner 会再次覆盖它。按原流程重启并验证 runner 注册。 + +不能仅改磁盘文件却不让进程加载；维护器还会检查独立 checkout URL，并确认入口实际见到了当前容器本次启动后的 FetchTask 来源 IP。此一次接入不由维护器冒险猜测空闲，也不对运行中 CI 动手。实例 API 根地址仍使用配置中的 HTTPS Gitea 地址，不能指向只支持 runner RPC 的入口。上传脚本从独立 `GENARRATIVE_GITEA_REPOSITORY_URL` 推导真实 Gitea 地址，使用本任务临时凭据调用原生 V4 API；不依赖被网关覆盖的 `GITHUB_SERVER_URL` / `ACTIONS_RUNTIME_URL`。Gitea 1.26.4 的仓库 REST 下载接口只支持 V4，不能换回 V3 上传。宿主下载只跟随同一 HTTPS Gitea origin 的签名重定向，不转发长期 Token；如配置外部对象存储直出，需另行适配下载来源。 + +```bash +# --apply 缺省时只检查 API、runner、master 和入口路径，不修改配置或镜像。 +python3 /opt/genarrative-ci-cache/scripts/maintain-gitea-rust-cache.py --config /etc/genarrative-ci-cache/config.json +install -m 644 deploy/systemd/genarrative-ci-cache.service deploy/systemd/genarrative-ci-cache.timer /etc/systemd/system/ +systemctl daemon-reload +systemctl enable --now genarrative-ci-cache.timer +journalctl -u genarrative-ci-cache.service -n 50 +``` + +timer 在上次执行结束后约 5 分钟再次检查，文件锁防止人工与定时执行重叠。构建日志位于状态目录 `artifacts//build.log`。同一 run 组装失败后不每 5 分钟重复消耗资源；新的完整 master run 到来后自动尝试，也可修复环境后显式运行 `--apply --retry`。切换事务及暂停归属先落状态文件；`ExecStopPost --resume` 恢复本维护器暂停的领取，下次执行再收敛中断的切换。其它人暂停的入口不由维护器擅自恢复；不修改 Gitea 的 Runner disabled 设置。 + +候选切换后的验收读取真实 **master push** 的完整九个 job，要求全部 success、每个 job 均使用目标 Image ID，六个 Rust job 有启用缓存、正命中数和零缓存错误。测试失败、旧 PR 缺 prepare、混用镜像或缺日志均不清理旧版，也不伪造“缓存已验收”。仍保留源代码失败需要修复的原始结果。 + +入口遇到“已发送 FetchTask，但上游响应未完整结束”会持久化 `uncertain` 并拒绝继续领取/自动切换；不会因为客户端的 5 秒超时就认定服务端事务已回滚。其它 RPC 和任务上报仍继续转发。维护者需先核实 Gitea 在途领取事务与该 runner 的任务全部收敛，在维护窗口停止入口，核实并修复它的 `tasks.json` 任务账本及 `uncertain` / `inflight` 标记后再启动并恢复领取。网关按 Gitea 1.26.4 / Runner 2.0.0 的 Connect Protobuf 协议，在最终日志确认及 Runner 执行结束后的最终任务上报确认后才清账；取消响应不等于进程停止，任务 ID 不按超时自动删除。禁止自动删这些标记绕过屏障。停用自动维护先 `systemctl disable --now genarrative-ci-cache.timer`；不要为了停 timer 停止运行中的 CI 容器。已接入的领取入口继续运行，不影响普通 CI。 + +人工 bootstrap 预热容器上限为 4 核、12 GiB，移除 capabilities，不挂宿主目录/socket，也不注入 Git/OSS/Jenkins 凭据。源码通过 `git archive` 复制，当前工作区、ignored 文件和 `.git` 不进入容器。最终从原镜像重新组装，仅复制 `/opt/genarrative-ci/rust-cache` 的 sccache、对象和来源元数据，不提交含源码/target 的预热容器；镜像本身的下载缓存与工具链校验保持原样。 + +快照由固定 Image ID 分发，每个 job 仅修改容器自己的写时复制层，缓存上限 4 GiB，只有 master push 在结束前回传新增对象；PR 不扫描基线、不打包、不回传。`ci-rust-cache.sh prepare` 清空继承的 `SCCACHE_*` 远程配置，使用独立配置和 Unix socket；旧镜像、工具链不匹配或限时 wrapper 探测失败时使用直接 rustc，正式编译启用 sccache 的 server IO 错误回退。真实编译/测试失败保留非零退出码。`report` 输出命中统计并停止本 job daemon，分片日志输出独立编译耗时。sccache 0.18.0 的只读模式在 miss 后仍打包再拒绝写入，不能用它宣称零 miss 开销；普通 CI 继续只向容器层写入。 生成候选不会改变 runner 配置。线上有活跃 CI 时禁止停止 job、重启 runner 或切换标签；只在确认空闲后按上节流程切换固定 Image ID。全组启用须重建完整快照；旧 AGC 单目标快照不能作为其它组预热成功的证据。按组用相同源码、独立干净 target 比较编译命中与耗时；分片测试内容和前置检查不同，不能直接用两条 lane 总耗时推断缓存收益。回滚时可移除各 job 的 prepare/report，或恢复无对象缓存的原基础镜像 ID，均不需要改变 incremental 或测试分片。 diff --git a/deploy/container/gitea-ci-cache.config.example.json b/deploy/container/gitea-ci-cache.config.example.json new file mode 100644 index 000000000..c10600468 --- /dev/null +++ b/deploy/container/gitea-ci-cache.config.example.json @@ -0,0 +1,12 @@ +{ + "api_url": "https://git.genarrative.world/git/api/v1", + "repository": "GenarrativeAI/Genarrative", + "repository_url": "http://gitea:3000/GenarrativeAI/Genarrative.git", + "clone_url": "https://git.genarrative.world/git/GenarrativeAI/Genarrative.git", + "runner_container": "gitea-runner", + "token_file": "/etc/genarrative-ci-cache/api-token", + "state_dir": "/var/lib/genarrative-ci-cache", + "artifact_storage_dir": "/opt/gitea-stack/data/gitea/gitea/actions_artifacts", + "gate_socket": "/var/lib/genarrative-ci-cache-gate/gate.sock", + "gate_url": "http://gitea-runner-fetch-gate:8080" +} diff --git a/deploy/container/gitea-runner-fetch-gate.compose.yml b/deploy/container/gitea-runner-fetch-gate.compose.yml new file mode 100644 index 000000000..f97c433d0 --- /dev/null +++ b/deploy/container/gitea-runner-fetch-gate.compose.yml @@ -0,0 +1,23 @@ +# 只加入 runner 与 Gitea 共用的内部网络，不发布端口，不挂 Docker socket。 +services: + gitea-runner-fetch-gate: + image: ${GITEA_FETCH_GATE_IMAGE:?Set a verified no-object-cache CI image ID with Python 3} + container_name: gitea-runner-fetch-gate + restart: unless-stopped + entrypoint: ['python3', '/app/gitea-runner-fetch-gate.py'] + environment: + GITEA_RUNNER_UPSTREAM: http://gitea:3000 + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + cpus: 0.5 + mem_limit: 256m + pids_limit: 128 + volumes: + - /opt/genarrative-ci-cache/scripts/gitea-runner-fetch-gate.py:/app/gitea-runner-fetch-gate.py:ro + - /var/lib/genarrative-ci-cache-gate:/control + networks: [gitea-actions] + +networks: + gitea-actions: + external: true diff --git a/deploy/systemd/genarrative-ci-cache.service b/deploy/systemd/genarrative-ci-cache.service new file mode 100644 index 000000000..b7e0934b7 --- /dev/null +++ b/deploy/systemd/genarrative-ci-cache.service @@ -0,0 +1,22 @@ +[Unit] +Description=Refresh and retain trusted Gitea CI Rust cache images +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service +ConditionPathExists=/etc/genarrative-ci-cache/config.json + +[Service] +Type=oneshot +User=root +WorkingDirectory=/opt/genarrative-ci-cache +UMask=0077 +ExecStart=/usr/bin/python3 /opt/genarrative-ci-cache/scripts/maintain-gitea-rust-cache.py --config /etc/genarrative-ci-cache/config.json --apply +ExecStopPost=/usr/bin/python3 /opt/genarrative-ci-cache/scripts/maintain-gitea-rust-cache.py --config /etc/genarrative-ci-cache/config.json --resume +TimeoutStartSec=3h +TimeoutStopSec=60s +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/genarrative-ci-cache.timer b/deploy/systemd/genarrative-ci-cache.timer new file mode 100644 index 000000000..417aa54e4 --- /dev/null +++ b/deploy/systemd/genarrative-ci-cache.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check for new Gitea CI compilation inputs + +[Timer] +OnBootSec=5min +OnUnitInactiveSec=5min +RandomizedDelaySec=30s +Unit=genarrative-ci-cache.service + +[Install] +WantedBy=timers.target diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index d37feedb2..8977e7861 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -96,6 +96,10 @@ SpacetimeDB 任务统一先读取 `.codex/skills/genarrative-spacetimedb/SKILL.m ## Gitea CI 依赖闭合 -AGC Rust 两条 lane、crates、smoke、Backend 和桌面壳测试使用镜像内可信 sccache 对象快照；Native shell release step 显式清空双 wrapper，前端/repository checks 不启用。维护者通过 `scripts/build-gitea-rust-cache.sh` 从远端 master 在限额、无宿主挂载的临时容器中按实际 cwd/profile/目标预热全部测试组，仅编译、不执行测试/应用；后端 workspace 与 spacetime-module 保持独立，AGC 的三个 cwd 入口之间清理预热 target，防止 fresh 判断漏产缓存键。最终镜像只追加 sccache、对象和来源元数据，不包含源码或 target。容量上限 4 GiB，不替代宿主旧镜像/归档清理。PR 只写当前容器层、不回传，不开放 Docker API/发布权限；继续禁用 incremental。`ci-rust-cache.sh` 在快照缺失、工具链不符或 wrapper 探测失败时直接编译，并隔离远程缓存配置和 daemon。分片日志记录编译耗时，收尾输出命中统计；两个 lane 的测试和前置检查不同，耗时差不是严格 A/B。线上存在活跃 CI 时不得重启 runner 或切换标签；全组启用前须刷新完整快照并逐组验证，详见开发运维文档。 +Gitea Rust 缓存自动维护由宿主 `genarrative-ci-cache.timer` 收集同一 master push run 六个 Rust job 的原生 V4 缓存产物，不重复执行 Cargo 预热。只传本轮新 key，命中对象只传使用时间；宿主与真实来源镜像对象合并、去重、按新近使用时间裁剪到 4 GiB，从无对象缓存基础镜像重新组装。源 run 不要求全绿，但取消、缺组、旧 attempt、未完成上传或混用来源镜像不得采用。网关暂停新 FetchTask、在途领取结束、持久化任务账本清空且内层活动容器为空才切换，不打断运行中的 CI。首次接入/升级网关需空闲窗口；Token 只需普通仓库 `write:repository`，不查管理员 API。候选装载后清理已收集 artifact，遗留项保留 7 天；真实 master CI 验证后才清理旧镜像，保留当前、一个回滚版、基础镜像及容器引用。部署入口见 `deploy/container/README.md`，合并代码不等于服务启用。 + +修改 Gitea workflow 的 job 显示名称、ID 或缓存导出组时，必须同步维护器的 `JOBS` / `RUST_JOB_IDS`；`test_gitea_cache_maintenance.py` 直接对照实际 workflow 检查全集和导出映射，避免自动刷新或镜像验收因名单漂移长期等待。维护器 `Api.request` 的 `method` 是必填关键字参数，GET 也必须显式指定，不根据 body 推断请求方法。 + +AGC Rust 两条 lane、crates、smoke、Backend 和桌面壳测试使用镜像内可信 sccache 对象快照；Native shell release step 显式清空双 wrapper，前端/repository checks 不启用。仅首次人工 bootstrap 时，维护者通过 `scripts/build-gitea-rust-cache.sh` 从远端 master 在限额、无宿主挂载的临时容器中按实际 cwd/profile/目标预热全部测试组，仅编译、不执行测试/应用；后端 workspace 与 spacetime-module 保持独立，AGC 的三个 cwd 入口之间清理预热 target，防止 fresh 判断漏产缓存键。最终镜像只追加 sccache、对象和来源元数据，不包含源码或 target。容量上限 4 GiB，不替代宿主旧镜像/归档清理。PR 只写当前容器层、不回传，不开放 Docker API/发布权限；继续禁用 incremental。`ci-rust-cache.sh` 在快照缺失、工具链不符或 wrapper 探测失败时直接编译，并隔离远程缓存配置和 daemon。分片日志记录编译耗时，收尾输出命中统计；两个 lane 的测试和前置检查不同，耗时差不是严格 A/B。线上存在活跃 CI 时不得重启 runner 或切换标签；全组启用前须刷新完整快照并逐组验证，详见开发运维文档。 `.gitea/workflows/project-ci.yml` 的客户端门禁拆成 lane 与功能 job，每个 job 只预热自己会构建的那几份依赖：`AI game creator shell Rust lane 1/2`、`lane 2/2` 各自预取一次 AGC 壳 manifest，并顺序运行两片 Rust bin 单测；`AI game creator shell Rust smoke` 同样只预取 AGC 壳 manifest（`agent-run` smoke 会用 `src-tauri/Cargo.toml` spawn `cargo run`），`AI game creator shell Rust crates` 预取 `server-rs/Cargo.toml` 与独立 crate，`Native shell tests` 预取桌面壳与 AGC 壳 manifest，`AI game creator shell web tests` 不触碰 Cargo，不预热。两条 Rust lane、smoke job 与 crates job 只用 cargo 与 node 内建模块，因此不执行 `npm ci`。两个被 `server-rs/Cargo.toml` 排除、且没有提交 `Cargo.lock` 的独立 crate（`agent-runtime-core`、`agent-runtime-orchestration`）只能在 `AI game creator shell Rust crates` 里用不带锁标志的 fetch。AGC 壳的 bin target 单测（约 2466 条）由 `apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs` 编译后按 `--list` 名单分 4 片：每次分片调用用 `--shard-index=` 只跑自己那片，片内保持 `--test-threads=1` 并使用独立 `TMPDIR`；两条 lane 之间并发，lane 内顺序运行两片，避免重复依赖预热和同一容器内多进程争抢。不要改回「一个 job 内多进程并行这几片」——同一容器里它们会争抢共享 `HOME`、target 目录与固定临时路径，实测比整套串行还慢。每个分片调用都会自校验「片并集等于全集且互斥」，因此改分片规则不会静默漏跑。Backend host workspace tests 使用 `cargo test --locked --workspace --exclude spacetime-module --no-fail-fast`，避免 `spacetime-module` 的 `spacetime-types` feature 统一污染普通领域 crate 的 host 测试；随后单独执行 `cargo test --locked -p spacetime-module --no-fail-fast`，由 `spacetime-module/src/active.rs` 在 host 测试构建期间提供仅测试期的 SpacetimeDB ABI 链接支持，使该 crate 的纯单元测试也纳入 Backend 门禁。`spacetime-module` 的 reducer / procedure 运行时行为仍必须通过真实 SpacetimeDB runtime/integration harness 验证，host 链接支持不得被当作运行时替身。Backend 另外执行 `cargo check --locked -p spacetime-module` 验证模块源码。AGC 壳检查还会运行 `platform-llm` 与 `shared-contracts` 的 server-rs workspace 测试，这些命令以及 AGC 壳测试必须带 `--locked`，避免在测试阶段重新解析 registry index；锁文件发生变化时应先更新受信任 CI 镜像缓存，再重跑门禁。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 005d5cd3a..9885a558b 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -6004,5 +6004,7 @@ Cocos Creator 根目录由 `package.json.creator.version` 与普通 `assets/` - 处理：预热使用已核实的 Gitea 路径 `/workspace/GenarrativeAI/Genarrative`，并与分片运行器一样从 AGC `src-tauri` 启动 Cargo；保存 `workspace.txt`，路径不符时回退直接编译。wrapper 放在容器内固定路径，daemon 状态与 Unix socket 仍使用随机私有目录；预热环境与 workflow 的 Cargo 环境由定向契约测试核对。不要为命中率随意增加 `RUSTFLAGS`、改写源码路径或恢复共享可写 target。 - 验证：相同源码、资源上限和独立干净 target 下分别记录无缓存、冷缓存、热缓存的编译耗时和 hit/miss；只有真实热命中有净收益才切换候选镜像。PR 的写入始终留在 job 容器层，公共快照仍由可信维护流程生成。 - 统计：job 私有 daemon 设置 `SCCACHE_IDLE_TIMEOUT=0`，由 `report` 显式停止；最终测试 bin 的不可缓存编译或测试可能超过一分钟，短 idle timeout 会让 daemon 提前退出，结尾查询启动新 daemon 后误报零次请求。容器销毁仍会回收该 job 的全部进程。 -- 磁盘：快照构建拒绝含 `/opt/genarrative-ci/rust-cache` 的基础镜像，始终从无对象缓存的镜像重建；容器内删除旧对象不能释放 Docker 底层。对象缓存容量上限不涵盖宿主旧镜像及导出归档，切换验证后按运维文档人工保留当前版和一个回滚版，同时保护运行中 CI 使用的镜像。 +- 磁盘：快照构建拒绝含 `/opt/genarrative-ci/rust-cache` 的基础镜像，始终从无对象缓存的镜像重建；容器内删除旧对象不能释放 Docker 底层。对象缓存容量上限不涵盖宿主旧镜像及导出归档。自动维护只回收自己预先登记的 Image ID/tag 和专属归档，保留当前、一个回滚版、基础镜像及所有容器引用；内层按 ID 导入的镜像可能没有 tag，不能只按 tag 判断已清理。接管前历史试验版本仍需人工确认。 +- CI 产物清理：Gitea 1.26.4 的仓库 REST 仅列出 finalized/expired V4 artifact，内置到期清理不回收上传中断的 tmp-upload 分块。缓存上传块须带专属标识，宿主只清理目标仓库已结束且超过 7 天的 master run 中同样过期的自有普通文件，未知文件/符号链接保护，不改数据库或全局 prune。Artifact.workflow_run 仅含 ID/SHA，判断过期产物所属事件和状态须再读 run API，不能当作完整 run 使用。 +- 自动切换：Gitea 1.26.4 的 disabled 检查与 FetchTask 事务不原子，Runner 客户端超时不能证明服务端回滚，容器暂时为空也不能证明没有已领取任务。网关必须解析实际 Connect Protobuf/gzip，转发 FetchTask 结果前持久化任务 ID，仅在最终日志及执行清理后的最终 UpdateTask 确认后清账；取消响应不能提前释放。暂停新领取、在途为零、账本为零且内层活动容器为空才可切换，无需全局 Runner admin API。未知协议/响应或崩溃遗留标记停止切换；旧网关缺 active_tasks 不能默认零。首次接入与账本升级须空闲窗口。.runner 的 mtime 不证明地址已加载，应核验真实 FetchTask 来源及本次容器启动时间。 - 扩展：预热所有 Rust 测试组时保留各自 cwd、profile、features 和锁策略；同一临时 target 的 Cargo fresh 不代表不同 cwd 都已生成缓存键，AGC 提示词契约、分片和 smoke 切换入口前清理预热 target。不要把 workspace 与 spacetime-module 合并成一次编译；Native shell release step 清空双 wrapper，避免将测试缓存扩展成发布缓存。当前 sccache 0.18.0 的 READ_ONLY 在 miss 后仍打包产物并产生 cache write error，不适合用来承诺“未命中无开销”。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index cb4500520..2c58e1424 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -334,16 +334,18 @@ master 日常交付必须禁止直接 push，只允许经 PR 在最近一次 Pro #### Rust 测试组的隔离编译缓存 -每次生成快照必须使用不含对象缓存的原始 CI 基础镜像；构建脚本在拉取源码和预热前拒绝已存在 `/opt/genarrative-ci/rust-cache` 的基础镜像，避免重复叠加不可释放的旧对象层。切换并完成真实 CI 验证后，按 `deploy/container/README.md` 的保留规则人工定向清理旧缓存镜像及归档，保护运行中 CI 使用的版本。 +每次生成快照必须使用不含对象缓存的原始 CI 基础镜像，避免叠加不可释放的旧层。宿主自动维护器 `scripts/maintain-gitea-rust-cache.py` 收集完整 master push CI 的六组 V4 增量产物，复用其真实来源镜像对象并合并去重，总容量 4 GiB，不重复执行 Cargo 预热编译。测试失败可收集，取消、缺组、未完成上传、旧 attempt 或混用来源镜像不可发布。候选校验、导出和装载后删除已收集 artifact，遗留项保留 7 天；切换后通过真实 master CI 才定向清理受管旧镜像/归档，保护当前、一个回滚版、基础镜像及容器引用。 AGC Rust 两条 lane、crates、agent-run smoke、Backend 和 Native shell 的桌面壳测试均启用 sccache。Native shell 的 release build smoke 显式清空两个 wrapper，保持发布构建原有 profile/features 与资源 staging；前端和 repository checks 不启用对象缓存。继续设置 `CARGO_INCREMENTAL=0`，不共享 target、不恢复 Actions 可写缓存。可信快照通过已有固定 Image ID 分发：镜像只增加固定版本的 sccache、编译对象和来源元数据，不包含源码、target 或凭据；容器写时复制层承接本 job 的新增对象，job 删除后丢弃，PR 没有 Docker API 或快照发布权限。该权限边界由 runner 基础设施保证，不能仅用 workflow 的分支条件替代。 -维护者在受信任 checkout 中运行 `bash scripts/build-gitea-rust-cache.sh <已验证基础镜像> <候选镜像tag>`。脚本从 origin 获取 master 的确定提交，在无宿主目录挂载、无凭据且有 CPU/内存上限的临时容器中预热各组实际 Cargo 目标，只编译、不执行测试或应用。后端 workspace 仍排除 spacetime-module，再单独预热该模块，禁止为缓存合并 features。AGC 独立 crate 与插件沿用各自 manifest/锁文件策略；未提交锁文件的 crate 仍可能因新版本解析产生 miss。最终镜像从原基础镜像重新组装，只导出 sccache 对象，不提交预热容器。公共快照不接受 PR 上传，也不复用 Jenkins 发布缓存。工具链或系统依赖变化时重新生成快照，缓存命中仍由 sccache 的编译输入校验决定，不能省略 Cargo 构建。 +人工 bootstrap 在没有可消费快照时使用 `bash scripts/build-gitea-rust-cache.sh <已验证基础镜像> <候选镜像tag> [完整master-SHA]`；自动维护不调用它。bootstrap 固定 master 归档，在无宿主挂载、无凭据、有资源限额的容器中只编译预热，不执行测试/应用，Cargo features 和工作目录保持实际 CI 口径。日常更新由 master CI 导出新 key，命中继承对象只传使用时间；PR 不导出、不扫描。宿主验证来源、大小和哈希，不从上传产物执行程序，只从可信来源镜像复制固定 sccache。工具链或基础镜像输入变化时重建无对象缓存基础镜像，缓存工具链必须匹配。公共快照不接受 PR，不复用 Jenkins 发布缓存。 各 Rust job 在编译前执行 `scripts/ci-rust-cache.sh prepare`：检查快照与 rustc 身份，隔离 sccache 配置和 daemon，限时探测真实 wrapper。旧镜像没有快照或探测失败时保留空 wrapper，输出 fallback 原因；缓存故障不得把真实编译/测试失败改成成功，也不允许重跑整个测试组掩盖失败。结束时 `report` 输出命中统计；分片日志单独记录 Cargo 编译耗时。对象缓存上限为 4 GiB，快照构建完成后输出实际体积；最终测试 bin 的链接仍须执行。全组开启时必须生成覆盖全部目标的新快照，不能把旧 AGC 单目标快照当作后端/桌面壳的预热验收。 候选镜像沿用 `gitea-ci-job-image.sh verify/export/load-runner` 验证和装载。现役 CI 正在运行时只允许准备候选镜像，不停止 job、不重启 runner、不切换标签；待确认无运行中的 job 后再按上述镜像更新顺序切换。验收分别记录独立 target 的无缓存、冷缓存和热缓存构建耗时及命中率，并检查失效/故障回退、两容器写入互不影响。未完成真实 CI 验证前不能宣称提速或推广到其它 job。 +自动维护首次部署及故障恢复统一见 `deploy/container/README.md`。timer 约每 5 分钟检查，文件锁串行执行，使用专属 clone/状态/归档目录；只管理 Gitea CI 测试镜像，Native shell release smoke 继续清空双 wrapper。凭据只需普通账号 `write:repository`，读取 run/job/日志/产物并定向删除缓存 artifact，不需要管理员权限。切换依赖 FetchTask 暂停与在途屏障、任务持久账本和内层 Docker 空闲；Gitea 显示 cancelled、disabled 或客户端超时都不能证明执行已经结束。网关依据最终日志与执行收尾后的最终任务上报确认释放任务。首次接入/旧网关升级须等待 CI 自然结束的空闲窗口，不在运行中的 CI 上试切；脚本合并不等于服务已部署。 + Rust 对象 key 包含编译工作目录。预热必须使用已核实的 Gitea checkout 路径 `/workspace/GenarrativeAI/Genarrative`：后端、独立 crate、插件、桌面壳及提示词契约从仓库根目录启动 Cargo，AGC 分片从 `apps/ai-game-creator-shell/src-tauri` 启动，agent-run smoke 从 `apps/ai-game-creator-shell` 启动。切换 AGC 编译入口前清理该临时容器的 AGC target，防止 Cargo 的 fresh 判断跳过新 cwd 所需对象；CI 各 job 的 target 本来就独立。快照保存 `workspace.txt`，job 根路径不符时直接编译。不要仅设置 `SCCACHE_BASEDIRS` 就假定 Rust 可以跨 cwd 命中，也不为缓存改写 `RUSTFLAGS` 或源码路径语义。固定 sccache `0.18.0` 的基础设施错误码 `2` 会由 wrapper 回退执行本次 rustc；其它退出码原样返回，升级 sccache/Rust 时须复核此约定。 sccache `0.18.0` 还会 hash 大部分 `CARGO_*` 环境变量。因此 wrapper 固定在每个容器自己的 `/opt/genarrative-ci/rust-cache/rustc-wrapper`，prepare 重写 launcher 后才启用；daemon 状态与 socket 仍各自随机隔离。预热同步 workflow 的 `CARGO_INCREMENTAL`、`CARGO_HTTP_MULTIPLEXING`、`CARGO_NET_RETRY` 和 `CARGO_TERM_COLOR`，由现有 CI 契约测试验证一致；新增 Cargo 环境配置时须同步评估 cache key，不能只看快照文件是否存在。 diff --git a/scripts/build-gitea-rust-cache.sh b/scripts/build-gitea-rust-cache.sh index 46f87d800..1bd6a22aa 100644 --- a/scripts/build-gitea-rust-cache.sh +++ b/scripts/build-gitea-rust-cache.sh @@ -5,6 +5,15 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" base_ref="${1:?usage: build-gitea-rust-cache.sh }" candidate_tag="${2:?candidate image tag is required}" +requested_source_commit="${3:-}" +if [[ "$#" -gt 3 ]]; then + echo 'usage: build-gitea-rust-cache.sh [master-commit-sha]' >&2 + exit 2 +fi +if [[ -n "${requested_source_commit}" && ! "${requested_source_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo 'master-commit-sha must be a complete 40-character SHA.' >&2 + exit 2 +fi # 与实际 Gitea checkout 路径一致；Rust 对象 key 包含编译 cwd，不能随意换临时根。 workspace=/workspace/GenarrativeAI/Genarrative [[ "${CI:-}" != true ]] || { echo 'Run on the trusted image builder, outside CI jobs.' >&2; exit 1; } @@ -22,7 +31,17 @@ bash "${repo_root}/scripts/gitea-ci-job-image.sh" verify "${base_id}" # 只归档远端 master 的确定提交；不复制当前工作区或本地凭据。 git -C "${repo_root}" fetch --no-tags origin refs/heads/master -source_commit="$(git -C "${repo_root}" rev-parse FETCH_HEAD^{commit})" +master_commit="$(git -C "${repo_root}" rev-parse FETCH_HEAD^{commit})" +if [[ -n "${requested_source_commit}" ]]; then + git -C "${repo_root}" cat-file -e "${requested_source_commit}^{commit}" + if ! git -C "${repo_root}" merge-base --is-ancestor "${requested_source_commit}" "${master_commit}"; then + echo "master-commit-sha is not contained in fetched master: ${requested_source_commit}" >&2 + exit 1 + fi + source_commit="$(git -C "${repo_root}" rev-parse "${requested_source_commit}^{commit}")" +else + source_commit="${master_commit}" +fi work_dir="$(mktemp -d "${TMPDIR:-/tmp}/gitea-rust-cache.XXXXXX")" container_id='' cleanup() { @@ -50,7 +69,7 @@ container_id="$(docker run --detach --cpus=4 --memory=12g --pids-limit=1024 \ docker exec "${container_id}" mkdir -p "${workspace}" /opt/genarrative-ci/rust-cache/objects git -C "${repo_root}" archive "${source_commit}" | docker cp - "${container_id}:${workspace}" docker cp "${work_dir}/snapshot/." "${container_id}:/opt/genarrative-ci/rust-cache/" -docker cp "${repo_root}/scripts/ci-rust-cache.sh" "${container_id}:/tmp/ci-rust-cache.sh" +docker exec "${container_id}" cp "${workspace}/scripts/ci-rust-cache.sh" /tmp/ci-rust-cache.sh docker exec --interactive --workdir "${workspace}" "${container_id}" bash -s <<'WARM' set -euo pipefail rustc -vV > /opt/genarrative-ci/rust-cache/rustc.txt diff --git a/scripts/ci-rust-cache.sh b/scripts/ci-rust-cache.sh index f93aacfc0..c19459da9 100644 --- a/scripts/ci-rust-cache.sh +++ b/scripts/ci-rust-cache.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# 仅消费镜像内的可信快照；所有写入留在当前容器的可写层。 +# 消费镜像内可信快照；写入留在任务容器，master push 可在任务结束后导出。 set -euo pipefail cache_root="${GENARRATIVE_CI_RUST_CACHE_ROOT:-/opt/genarrative-ci/rust-cache}" @@ -22,7 +22,7 @@ configure_local_cache() { case "${1:-}" in prepare) : "${GITHUB_ENV:?GITHUB_ENV is required}" - printf 'RUSTC_WRAPPER=\nCARGO_BUILD_RUSTC_WRAPPER=\nGENARRATIVE_CI_RUST_CACHE_STATE=\n' >> "${GITHUB_ENV}" + printf 'RUSTC_WRAPPER=\nCARGO_BUILD_RUSTC_WRAPPER=\nGENARRATIVE_CI_RUST_CACHE_STATE=\nGENARRATIVE_CI_RUST_CACHE_EXPORT_READY=\n' >> "${GITHUB_ENV}" fallback() { printf '[rust-cache] mode=direct reason=%s\n' "$1"; exit 0; } [[ -x "${cache_binary}" && -d "${cache_root}/objects" && -f "${cache_root}/rustc.txt" && -f "${cache_root}/source-commit.txt" && -f "${cache_root}/workspace.txt" ]] \ || fallback snapshot-unavailable @@ -44,6 +44,10 @@ case "${1:-}" in fallback wrapper-probe-failed fi script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/$(basename "${BASH_SOURCE[0]}")" + if [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_REF:-}" == refs/heads/master ]]; then + # 只记 key/大小/时间，不读取对象正文；PR 没有扫描与打包开销。 + python3 "$(dirname "${script_path}")/export-gitea-rust-cache.py" baseline "${state}/baseline.json" + fi # wrapper 路径也参与 Rust cache key；固定容器内路径，隔离由 job 容器保证。 wrapper_path="${cache_root}/rustc-wrapper" printf '#!/usr/bin/env bash\nexec bash %q "$@"\n' "${script_path}" > "${wrapper_path}" @@ -59,7 +63,13 @@ case "${1:-}" in if [[ -n "${state}" && -d "${state}" ]]; then configure_local_cache timeout --kill-after=2 5 "${cache_binary}" --show-stats || true - timeout --kill-after=2 5 "${cache_binary}" --stop-server >/dev/null 2>&1 || true + # 只有成功停服后才允许读取对象；失败不能把仍在写入的目录发布成完整快照。 + if timeout --kill-after=2 15 "${cache_binary}" --stop-server >/dev/null 2>&1; then + if [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_REF:-}" == refs/heads/master && ! -f "${state}/disabled" && -f "${state}/baseline.json" ]]; then + mv -- "${state}/baseline.json" "${cache_root}/export-baseline.json" + printf 'GENARRATIVE_CI_RUST_CACHE_EXPORT_READY=1\n' >> "${GITHUB_ENV}" + fi + fi rm -rf -- "${state}" else printf '[rust-cache] mode=direct\n' diff --git a/scripts/ci-rust-cache.test.mjs b/scripts/ci-rust-cache.test.mjs index 2b4f4f0ad..84ab6d9f9 100644 --- a/scripts/ci-rust-cache.test.mjs +++ b/scripts/ci-rust-cache.test.mjs @@ -35,7 +35,8 @@ function fixture(t) { `#!/bin/bash set -eu case "$1" in - --stop-server|--show-stats) exit 0 ;; + --stop-server) exit "\${STOP_FAILURE:-0}" ;; + --show-stats) exit 0 ;; esac if [[ "$*" == *-vV ]]; then [[ "\${PROBE_FAILURE:-}" != 1 ]] || exit 1 @@ -215,3 +216,25 @@ linuxTest('lost local cache state still invokes the real compiler', (t) => { const f = fixture(t); assert.equal(f.run(['/bin/bash', '-c', 'exit 43']).status, 43); }); + +linuxTest('only a cleanly stopped master cache can export objects', (t) => { + const master = { + GITHUB_EVENT_NAME: 'push', + GITHUB_REF: 'refs/heads/master', + }; + for (const stopFailure of ['0', '1']) { + const f = fixture(t); + const prepared = f.run(['prepare'], master); + assert.equal(prepared.status, 0, prepared.stderr); + const result = f.run(['report'], { + ...f.preparedEnv(), + ...master, + STOP_FAILURE: stopFailure, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal( + f.preparedEnv().GENARRATIVE_CI_RUST_CACHE_EXPORT_READY, + stopFailure === '0' ? '1' : '', + ); + } +}); diff --git a/scripts/export-gitea-rust-cache.py b/scripts/export-gitea-rust-cache.py new file mode 100644 index 000000000..3ad38e606 --- /dev/null +++ b/scripts/export-gitea-rust-cache.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""只把 master push 的任务内 sccache 对象导出为 Gitea Actions 产物。""" + +import base64 +from datetime import datetime, timedelta, timezone +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile + + +MAX_OBJECT_BYTES = 4 * 1024**3 +CHUNK_BYTES = 8 * 1024**2 +JOBS = { + "ai-game-creator-shell-rust-lane-1", + "ai-game-creator-shell-rust-lane-2", + "ai-game-creator-shell-rust-smoke", + "ai-game-creator-shell-rust-crates", + "backend-tests", + "native-shell-tests", +} + + +def object_path(name): + parts = name.split("/") + return ( + len(parts) == 3 + and re.fullmatch(r"[a-f0-9]{64}", parts[2]) is not None + and parts[0] == parts[2][0] + and parts[1] == parts[2][1] + ) + + +class HashingReader: + def __init__(self, stream): + self.stream = stream + self.digest = hashlib.sha256() + + def read(self, size=-1): + data = self.stream.read(size) + self.digest.update(data) + return data + + +def scan_objects(root): + objects = root / "objects" + if objects.is_symlink() or not objects.is_dir(): + raise ValueError("cache object directory is unavailable") + candidates = [] + for directory, directories, files in os.walk(objects, followlinks=False): + directories[:] = [ + name for name in directories if not (Path(directory) / name).is_symlink() + ] + for filename in files: + path = Path(directory) / filename + relative = path.relative_to(objects).as_posix() + info = path.lstat() + if object_path(relative) and stat.S_ISREG(info.st_mode): + candidates.append((path, relative, info)) + return candidates + + +def save_baseline(root, destination): + # sccache 命中会更新 mtime，不能把时间变化当成内容变化，否则又变成全量上传。 + baseline = { + relative: {"size": info.st_size, "mtime_ns": info.st_mtime_ns} + for _, relative, info in scan_objects(root) + } + destination.write_text(json.dumps(baseline), encoding="utf-8") + + +def pack_snapshot(root, destination, metadata, baseline, limit=MAX_OBJECT_BYTES): + """仅归档新 key；命中的已有对象只传递新近使用时间，不传输对象内容。""" + candidates = scan_objects(root) + candidates.sort(key=lambda item: (-item[2].st_mtime_ns, item[1])) + entries = [] + touched = [] + total = 0 + with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_STORED) as bundle, \ + bundle.open("snapshot.tar", "w", force_zip64=True) as tar_stream, \ + tarfile.open(fileobj=tar_stream, mode="w|", format=tarfile.USTAR_FORMAT) as archive: + for path, relative, info in candidates: + previous = baseline.get(relative) + if previous is not None and previous["size"] == info.st_size: + if previous["mtime_ns"] != info.st_mtime_ns: + touched.append({"path": "objects/" + relative, "mtime_ns": info.st_mtime_ns}) + continue + if info.st_size <= 0 or total + info.st_size > limit: + continue + # report 已成功停止 daemon；打开后再核实对象，避免导出变化中的文件。 + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(descriptor, "rb") as stream: + actual = os.fstat(stream.fileno()) + if not stat.S_ISREG(actual.st_mode) or ( + actual.st_size, actual.st_mtime_ns, actual.st_ino + ) != (info.st_size, info.st_mtime_ns, info.st_ino): + raise ValueError("cache object changed during export") + member = tarfile.TarInfo("objects/" + relative) + member.size = info.st_size + member.mode = 0o644 + member.mtime = int(info.st_mtime) + reader = HashingReader(stream) + archive.addfile(member, reader) + entries.append({ + "path": member.name, + "size": info.st_size, + "sha256": reader.digest.hexdigest(), + "mtime_ns": info.st_mtime_ns, + }) + total += info.st_size + manifest = dict(metadata, schema=1, mode="delta", objects=entries, touched=touched) + data = json.dumps(manifest, ensure_ascii=False, sort_keys=True).encode("utf-8") + member = tarfile.TarInfo("manifest.json") + member.size = len(data) + member.mode = 0o644 + archive.addfile(member, io.BytesIO(data)) + return len(entries), total + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # runtime token 只允许发往明确配置的 Gitea 地址。 + return None + + +def request(method, url, token, data=None, headers=None, attempts=3): + opener = urllib.request.build_opener(NoRedirect()) + for attempt in range(attempts): + req = urllib.request.Request(url, data=data, method=method, headers={ + **({"Authorization": "Bearer " + token} if token else {}), + "Content-Type": "application/json", + **(headers or {}), + }) + try: + with opener.open(req, timeout=120) as response: + body = response.read(1024 * 1024) + return json.loads(body) if body else {} + except urllib.error.HTTPError as error: + if error.code not in (408, 429, 500, 502, 503, 504) or attempt == attempts - 1: + raise RuntimeError(f"artifact {method} failed: HTTP {error.code}") from None + except (urllib.error.URLError, TimeoutError, ConnectionError): + if attempt == attempts - 1: + raise RuntimeError(f"artifact {method} connection failed") from None + time.sleep(2 ** attempt) + + +def artifact_base_url(env): + # .runner.address 可指向仅支持 RPC 的领取网关，不能用 GITHUB_SERVER_URL。 + repository = env["GITHUB_REPOSITORY"] + clone_url = env["GENARRATIVE_GITEA_REPOSITORY_URL"] + suffix = "/" + repository + ".git" + parsed = urllib.parse.urlsplit(clone_url) + if ( + parsed.scheme not in ("http", "https") + or not parsed.netloc + or parsed.username or parsed.password or parsed.query or parsed.fragment + or not parsed.path.endswith(suffix) + ): + raise ValueError("a credential-free HTTP Gitea repository URL is required") + return urllib.parse.urlunsplit(( + parsed.scheme, parsed.netloc, parsed.path[:-len(suffix)], "", "" + )) + + +def upload_snapshot(path, name, run_id, base_url, token): + """Gitea 1.26.4 原生 v4：只有完成全部块并校验 SHA256 后才发布。""" + producer_match = re.fullmatch(r"rust-cache-v1-([a-z0-9-]+)-attempt-(0|[1-9][0-9]*)", name) + if not producer_match or producer_match[1] not in JOBS: + raise ValueError("unexpected cache artifact name") + producer = f"{producer_match[1]}:{producer_match[2]}" + endpoint = f"{base_url}/twirp/github.actions.results.api.v1.ArtifactService" + # Gitea 从任务凭据确定 job，只要求请求中的 run ID 与任务所属 run 一致。 + identity = {"workflowRunBackendId": str(run_id), "name": name} + created = request("POST", endpoint + "/CreateArtifact", token, json.dumps({ + **identity, "version": 4, + # Gitea 将剩余小时向下取整成天；多给一天，服务端才实际保留至少七天。 + "expiresAt": (datetime.now(timezone.utc) + timedelta(days=8)).isoformat(), + }).encode()) + received = urllib.parse.urlsplit(created.get("signedUploadUrl", "")) + expected_path = urllib.parse.urlsplit(endpoint).path + "/UploadArtifact" + if not created.get("ok") or received.path != expected_path or not received.query: + raise ValueError("unexpected artifact upload path") + # 服务可能返回外部 AppURL；保留签名，但只连接配置的内部 Gitea 地址。 + upload_url = endpoint + "/UploadArtifact?" + received.query + size = path.stat().st_size + digest = hashlib.sha256() + blocks = [] + with path.open("rb") as source: + while chunk := source.read(CHUNK_BYTES): + # 双层编码中的内层保留所属任务；宿主只回收带此前缀的过期未完成块。 + block = base64.b64encode( + f"genarrative-rust-cache-v1:{producer}:{len(blocks):08d}".encode() + ).decode() + blocks.append(block) + digest.update(chunk) + request("PUT", upload_url + "&" + urllib.parse.urlencode({ + "comp": "block", "blockid": block, + }), None, chunk, { + "Content-Type": "application/octet-stream", + }) + blocklist = "" + "".join(f"{block}" for block in blocks) + "" + request("PUT", upload_url + "&comp=blocklist", None, blocklist.encode(), { + "Content-Type": "application/xml", + }) + # Finalize 会消耗服务端块列表；响应不确定时不盲目重试、也不宣告成功。 + finalized = request("POST", endpoint + "/FinalizeArtifact", token, json.dumps({ + **identity, "size": str(size), "hash": "sha256:" + digest.hexdigest(), + }).encode(), attempts=1) + if not finalized.get("ok"): + raise ValueError("artifact was not finalized") + + +def export(env): + # 双重限定：手工调用、PR 和其它分支不会扫描、打包或上传对象。 + if env.get("GITHUB_EVENT_NAME") != "push" or env.get("GITHUB_REF") != "refs/heads/master": + return + if env.get("GENARRATIVE_CI_RUST_CACHE_EXPORT_READY") != "1": + print("[rust-cache] export skipped: cache daemon did not stop cleanly") + return + job = env.get("GITHUB_JOB") + if job not in JOBS: + raise ValueError("unexpected Rust cache job") + sha = env["GITHUB_SHA"] + if not re.fullmatch(r"[a-f0-9]{40}", sha): + raise ValueError("a complete source SHA is required") + run_id, attempt = int(env["GITHUB_RUN_ID"]), int(env["GITHUB_RUN_ATTEMPT"]) + if run_id <= 0 or attempt < 0: + raise ValueError("invalid run identity") + token = env.get("ACTIONS_RUNTIME_TOKEN") or env["GENARRATIVE_GITEA_TOKEN"] + base_url = artifact_base_url(env) + root = Path(env.get("GENARRATIVE_CI_RUST_CACHE_ROOT", "/opt/genarrative-ci/rust-cache")) + baseline_path = root / "export-baseline.json" + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + rustc = subprocess.check_output(["rustc", "-vV"], text=True) + workspace = str(Path.cwd().resolve()) + if rustc != (root / "rustc.txt").read_text() or workspace != (root / "workspace.txt").read_text().strip(): + raise ValueError("cache provenance changed after prepare") + metadata = { + "repository": env["GITHUB_REPOSITORY"], + "run_id": run_id, "run_attempt": attempt, "job": job, "source_sha": sha, + "rustc": rustc, "workspace": workspace, + "inherited_source_sha": (root / "source-commit.txt").read_text().strip(), + "sccache_version": subprocess.check_output([str(root / "sccache"), "--version"], text=True).strip(), + } + if (root / "base-image.txt").is_file(): + metadata["base_image"] = (root / "base-image.txt").read_text().strip() + name = f"rust-cache-v1-{job}-attempt-{attempt}" + try: + with tempfile.TemporaryDirectory(prefix="ci-rust-cache-export-") as directory: + path = Path(directory) / "snapshot.zip" + count, size = pack_snapshot(root, path, metadata, baseline) + upload_snapshot(path, name, run_id, base_url, token) + finally: + baseline_path.unlink(missing_ok=True) + print(f"[rust-cache] artifact={name} objects={count} bytes={size} complete=true") + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "baseline": + save_baseline(Path(os.environ.get("GENARRATIVE_CI_RUST_CACHE_ROOT", "/opt/genarrative-ci/rust-cache")), Path(sys.argv[2])) + else: + export(os.environ) diff --git a/scripts/gitea-ci-job-image.sh b/scripts/gitea-ci-job-image.sh index e9e06c8db..b0e3cac4c 100644 --- a/scripts/gitea-ci-job-image.sh +++ b/scripts/gitea-ci-job-image.sh @@ -39,6 +39,7 @@ usage() { cat <<'EOF' 用法： bash scripts/gitea-ci-job-image.sh build + bash scripts/gitea-ci-job-image.sh revision bash scripts/gitea-ci-job-image.sh verify [镜像引用] bash scripts/gitea-ci-job-image.sh load-runner [镜像引用] bash scripts/gitea-ci-job-image.sh export <归档路径> [镜像引用] @@ -65,40 +66,15 @@ verify_image() { command_name="${1:-}" case "${command_name}" in - build) - image_revision="$( + revision) + # 与 build 的 IMAGE_REVISION 使用同一份输入顺序，用于维护器判断基础镜像是否过期。 + ( cd "${repo_root}" - { - sha256sum \ - deploy/container/gitea-ci-job.Dockerfile \ - deploy/container/gitea-ci-job.Dockerfile.dockerignore \ - deploy/container/gitea-ci-checkout.sh \ - package.json \ - package-lock.json \ - apps/admin-web/package.json \ - apps/ai-game-creator-shell/package.json \ - apps/desktop-shell/package.json \ - apps/mobile-shell/package.json \ - apps/preview-deployer-web/package.json \ - packages/image-canvas-core/package.json \ - packages/image-canvas-react/package.json \ - packages/shared/package.json \ - tools/spine-json-export-validator/package.json \ - apps/ai-game-creator-shell/src-tauri/Cargo.toml \ - apps/ai-game-creator-shell/src-tauri/Cargo.lock \ - server-rs/Cargo.toml \ - server-rs/Cargo.lock \ - apps/desktop-shell/src-tauri/Cargo.toml \ - apps/desktop-shell/src-tauri/Cargo.lock - find server-rs/crates plugins/agc-*-editor/native/*-editor-bridge \ - \( -name Cargo.toml -o -path 'plugins/agc-*-editor/native/*-editor-bridge/*' \) \ - -type f -print0 \ - | sort -z \ - | xargs -0 -r sha256sum - } \ - | sha256sum \ - | awk '{ print $1 }' - )" + write_build_context_file_list | xargs -0 -r sha256sum | sha256sum | awk '{ print $1 }' + ) + ;; + build) + image_revision="$(bash "${BASH_SOURCE[0]}" revision)" npm_lock_sha256="$(sha256sum "${repo_root}/package-lock.json")" npm_lock_sha256="${npm_lock_sha256%% *}" server_rust_lock_sha256="$(sha256sum "${repo_root}/server-rs/Cargo.lock")" diff --git a/scripts/gitea-runner-fetch-gate.py b/scripts/gitea-runner-fetch-gate.py new file mode 100644 index 000000000..7f8346e65 --- /dev/null +++ b/scripts/gitea-runner-fetch-gate.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Gitea Runner 领取屏障；控制 socket 仅供宿主缓存维护任务使用。""" + +import http.client +import http.server +import gzip +import io +import json +import os +from pathlib import Path +import socketserver +import threading +import time +import urllib.parse +import zlib + +MAX_BODY = 32 * 1024 * 1024 +HOP_HEADERS = { + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailer", "transfer-encoding", "upgrade", "host", "content-length", +} +RPC_PREFIX = "/api/actions/runner.v1.RunnerService/" + + +def protobuf_fields(data): + """只解码已核对的 actions-proto-go v0.4.1 字段，不引入 protobuf 运行时。""" + fields = {} + offset = 0 + + def varint(): + nonlocal offset + value = 0 + for shift in range(0, 70, 7): + if offset >= len(data): + raise ValueError("truncated protobuf") + byte = data[offset] + offset += 1 + value |= (byte & 127) << shift + if byte < 128: + return value + raise ValueError("invalid protobuf varint") + + while offset < len(data): + tag = varint() + number, wire = tag >> 3, tag & 7 + if not number: + raise ValueError("invalid protobuf tag") + if wire == 0: + value = varint() + elif wire in (1, 2, 5): + length = varint() if wire == 2 else (8 if wire == 1 else 4) + if offset + length > len(data): + raise ValueError("truncated protobuf field") + value = data[offset:offset + length] + offset += length + else: + raise ValueError("unsupported protobuf wire type") + fields.setdefault(number, []).append(value) + return fields + + +def single(fields, number, default=None): + values = fields.get(number, []) + if len(values) > 1: + raise ValueError("ambiguous tracking field") + return values[0] if values else default + + +def rpc_fields(body, headers): + encoding = headers.get("Content-Encoding", "identity").lower() + if encoding == "gzip": + with gzip.GzipFile(fileobj=io.BytesIO(body)) as stream: + body = stream.read(MAX_BODY + 1) + elif encoding != "identity": + raise ValueError("unsupported RPC encoding") + if len(body) > MAX_BODY: + raise ValueError("decoded RPC too large") + if headers.get("Content-Type", "").split(";", 1)[0] != "application/proto": + raise ValueError("task tracking requires Connect protobuf") + return protobuf_fields(body) + + +def task_state(fields): + nested = single(fields, 1) + if not isinstance(nested, bytes): + raise ValueError("missing task state") + state = protobuf_fields(nested) + task_id = positive_id(single(state, 1)) + result = single(state, 2, 0) + if type(result) is not int or result not in range(5): + raise ValueError("unknown task result") + return task_id, result + + +def positive_id(value): + if type(value) is not int or not 0 < value < 2 ** 63: + raise ValueError("invalid task ID") + return str(value) + + +class Gate: + def __init__(self, directory): + self.directory = Path(directory) + self.directory.mkdir(parents=True, exist_ok=True) + self.lock = threading.Lock() + self.inflight = 0 + self.last_fetch_peer = None + self.last_fetch_at = None + self.tasks = {} + self.uncertain = (self.directory / "uncertain").exists() + try: + if (self.directory / "tasks.json").exists(): + tasks = json.loads((self.directory / "tasks.json").read_text()) + if (not isinstance(tasks, dict) or any( + positive_id(int(key)) != key or type(value) is not bool + for key, value in tasks.items())): + raise ValueError("invalid task ledger") + self.tasks = tasks + except (ValueError, TypeError, OSError): + self.uncertain = True + self.mark("uncertain") + if (self.directory / "inflight").exists(): + self.uncertain = True + self.mark("uncertain") + self.paused = (self.directory / "paused").exists() or self.uncertain + + def mark(self, name): + with (self.directory / name).open("w", encoding="ascii") as stream: + stream.write("1\n") + stream.flush() + os.fsync(stream.fileno()) + self.sync_directory() + + def sync_directory(self): + descriptor = os.open(self.directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def snapshot(self): + return {"paused": self.paused, "inflight": self.inflight, + "active_tasks": len(self.tasks), "task_ids": sorted(self.tasks), + "uncertain": self.uncertain, "last_fetch_peer": self.last_fetch_peer, + "last_fetch_at": self.last_fetch_at} + + def save_tasks(self): + temporary = self.directory / "tasks.json.tmp" + with temporary.open("w", encoding="ascii") as stream: + json.dump(self.tasks, stream, sort_keys=True) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self.directory / "tasks.json") + self.sync_directory() + + def fail_closed(self): + with self.lock: + self.uncertain = self.paused = True + self.mark("uncertain") + + def assigned(self, fields): + task = single(fields, 1) + if task is None: + return + if not isinstance(task, bytes): + raise ValueError("invalid fetched task") + task_id = positive_id(single(protobuf_fields(task), 1)) + with self.lock: + self.tasks[task_id] = False + # 必须先持久化，再把领取结果交给 runner；仅保存 ID，不保存 secrets。 + self.save_tasks() + + def reported(self, method, request, response): + if method == "UpdateLog": + task_id = positive_id(single(request, 1)) + index = single(request, 2, 0) + ack = single(response, 1, 0) + no_more = single(request, 4, 0) + if (type(index) is not int or type(ack) is not int + or index < 0 or ack < 0 or no_more not in (0, 1)): + raise ValueError("invalid log acknowledgement") + finalized = no_more == 1 and ack == index + len(request.get(3, [])) + else: + task_id, result = task_state(request) + response_id, response_result = task_state(response) + if response_id != task_id: + raise ValueError("mismatched task acknowledgement") + # 取消响应可能出现在任务执行途中，必须等 runner 自己报告终态。 + finalized = result != 0 and response_result != 0 + output_keys = {single(protobuf_fields(entry), 1, b"") + for entry in request.get(2, [])} + finalized = finalized and output_keys.issubset(set(response.get(2, []))) + with self.lock: + if task_id not in self.tasks: + # 客户端可能未读到已成功发出的终态响应而重试；v2.0.0 仅在 + # executor 清理后的 Close 中发送终态，不为幂等重报增加墓碑账本。 + if method == "UpdateTask" and finalized: + return + raise ValueError("report for untracked task; idle bootstrap required") + if method == "UpdateLog" and finalized: + self.tasks[task_id] = True + self.save_tasks() + elif method == "UpdateTask" and finalized and self.tasks[task_id]: + # act_runner Reporter.Close 在 executor 清理之后先封存日志再报终态。 + del self.tasks[task_id] + self.save_tasks() + + def record_fetch(self, peer): + with self.lock: + self.last_fetch_peer = peer + self.last_fetch_at = time.time() + + def control(self, action): + with self.lock: + if action == "pause": + self.mark("paused") + self.paused = True + elif action == "resume": + if self.uncertain: + return {**self.snapshot(), "error": "upstream completion uncertain; operator recovery required"} + (self.directory / "paused").unlink(missing_ok=True) + self.sync_directory() + self.paused = False + elif action != "status": + return {**self.snapshot(), "error": "unknown action"} + return self.snapshot() + + def enter(self): + with self.lock: + if self.paused or self.uncertain: + return False + # 必须先落盘再转发；崩溃后不能把遗留的领取请求误认为已完成。 + self.mark("inflight") + self.inflight += 1 + return True + + def leave(self, completed): + with self.lock: + self.inflight -= 1 + if not completed: + self.uncertain = self.paused = True + self.mark("uncertain") + if self.inflight == 0 and not self.uncertain: + (self.directory / "inflight").unlink(missing_ok=True) + self.sync_directory() + + +def read_body(stream, headers): + """解码 HTTP 请求；拒绝含糊 framing，不依赖下游连接关闭。""" + encodings = headers.get_all("Transfer-Encoding", []) + lengths = headers.get_all("Content-Length", []) + if encodings and lengths: + raise ValueError("ambiguous request framing") + if len(lengths) > 1 or len(encodings) > 1: + raise ValueError("duplicate request framing") + + def exact(length): + data = stream.read(length) + if len(data) != length: + raise ValueError("incomplete request body") + return data + + if not encodings: + length = int(lengths[0]) if lengths else 0 + if not 0 <= length <= MAX_BODY: + raise ValueError("request too large") + return exact(length) + if encodings[0].strip().lower() != "chunked": + raise ValueError("unsupported transfer encoding") + body = bytearray() + while True: + line = stream.readline(8193) + if len(line) > 8192 or not line.endswith(b"\r\n"): + raise ValueError("invalid chunk header") + length = int(line.split(b";", 1)[0].strip(), 16) + if length < 0 or len(body) + length > MAX_BODY: + raise ValueError("request too large") + if length == 0: + trailer_size = 0 + while True: + line = stream.readline(8193) + trailer_size += len(line) + if trailer_size > 8192 or not line.endswith(b"\r\n"): + raise ValueError("invalid request trailers") + if line == b"\r\n": + return bytes(body) + body.extend(exact(length)) + if exact(2) != b"\r\n": + raise ValueError("invalid chunk terminator") + + +class Proxy(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass # 不输出 RPC 认证头、请求内容或带认证信息的 URL。 + + def reply(self, status, body, headers=()): + delivered = False + try: + self.send_response(status) + excluded = HOP_HEADERS | { + item.strip().lower() for key, value in headers if key.lower() == "connection" + for item in value.split(",") + } + for key, value in headers: + if key.lower() not in excluded: + self.send_header(key, value) + self.send_header("Content-Length", str(len(body))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body) + self.wfile.flush() + delivered = True + except (OSError, ValueError): + pass + self.close_connection = True + return delivered + + def do_POST(self): + path = urllib.parse.urlsplit(self.path) + if (path.scheme or path.netloc or not path.path.startswith("/api/actions/") + or "%" in path.path or any(p in (".", "..") for p in path.path.split("/"))): + self.reply(404, b"runner RPC only\n") + return + try: + body = read_body(self.rfile, self.headers) + except (ValueError, OSError): + self.reply(400, b"invalid request body\n") + return + method = path.path.removeprefix(RPC_PREFIX) if path.path.startswith(RPC_PREFIX) else "" + is_fetch = method == "FetchTask" + if is_fetch: + # 包括暂停时被拒绝的请求；只记录网络来源与时间以证明 daemon 路由。 + self.server.gate.record_fetch(self.client_address[0]) + if not self.server.gate.enter(): + self.reply(503, b"runner maintenance\n") + return + completed = False + connection = None + try: + upstream = self.server.upstream + cls = http.client.HTTPSConnection if upstream.scheme == "https" else http.client.HTTPConnection + connection = cls(upstream.hostname, upstream.port, timeout=None) + try: + connection.connect() + except OSError: + # TCP/TLS 连接阶段尚未发送 RPC，不存在服务端分配事务。 + completed = True + raise + excluded = HOP_HEADERS | { + name.strip().lower() for name in self.headers.get("Connection", "").split(",") + } + headers = {key: value for key, value in self.headers.items() if key.lower() not in excluded} + headers["Content-Length"] = str(len(body)) + connection.request("POST", self.path, body=body, headers=headers) + response = connection.getresponse() + result = bytearray() + oversized = False + # 客户端即使断开，也继续读取上游完整响应，再释放领取屏障。 + while block := response.read(65536): + if len(result) + len(block) <= MAX_BODY and not oversized: + result.extend(block) + else: + oversized = True + result.clear() + if response.length not in (None, 0): + raise http.client.IncompleteRead(bytes(result), response.length) + completed = True + if is_fetch: + try: + if oversized or response.status != 200: + raise ValueError("unknown FetchTask result") + self.server.gate.assigned(rpc_fields(bytes(result), response.headers)) + except (ValueError, TypeError, OSError, EOFError, zlib.error): + self.server.gate.fail_closed() + self.server.gate.leave(True) + is_fetch = False + if oversized: + self.reply(502, b"upstream response too large\n") + else: + # 日志封存先记账再返回，避免 runner 立即发终态时抢先读到旧账本。 + if method == "UpdateLog" and response.status == 200: + self.observe_report(method, body, bytes(result), response.headers) + delivered = self.reply(response.status, bytes(result), response.getheaders()) + if method == "UpdateTask" and response.status == 200 and delivered: + self.observe_report(method, body, bytes(result), response.headers) + except (OSError, http.client.HTTPException, ValueError): + self.reply(502, b"runner upstream unavailable\n") + finally: + if connection is not None: + connection.close() + if is_fetch: + self.server.gate.leave(completed) + + def observe_report(self, method, request, response, headers): + try: + self.server.gate.reported(method, rpc_fields(request, self.headers), + rpc_fields(response, headers)) + except (ValueError, TypeError, OSError, EOFError, zlib.error): + self.server.gate.fail_closed() + + +class Control(socketserver.StreamRequestHandler): + def handle(self): + try: + line = self.rfile.readline(4097) + if len(line) > 4096 or not line.endswith(b"\n"): + raise ValueError("invalid control request") + request = json.loads(line) + result = self.server.gate.control(request["action"]) + except (ValueError, KeyError, TypeError, OSError): + result = {"error": "invalid control request"} + self.wfile.write(json.dumps(result).encode("utf-8") + b"\n") + + +class ControlServer(socketserver.ThreadingUnixStreamServer): + daemon_threads = True + + +def create_proxy(address, upstream, gate): + parsed = urllib.parse.urlsplit(upstream) + if (parsed.scheme not in ("http", "https") or not parsed.hostname + or parsed.username or parsed.password or parsed.path not in ("", "/") + or parsed.query or parsed.fragment): + raise ValueError("upstream must be an HTTP(S) origin") + server = http.server.ThreadingHTTPServer(address, Proxy) + server.upstream = parsed + server.gate = gate + return server + + +def main(): + gate = Gate(os.environ.get("GITEA_GATE_CONTROL_DIR", "/control")) + socket_path = gate.directory / "gate.sock" + socket_path.unlink(missing_ok=True) + control = ControlServer(str(socket_path), Control) + control.gate = gate + os.chmod(socket_path, 0o600) + proxy = create_proxy(("0.0.0.0", 8080), os.environ.get( + "GITEA_RUNNER_UPSTREAM", "http://gitea:3000"), gate) + threading.Thread(target=control.serve_forever, daemon=True).start() + proxy.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/scripts/gitea_cache_snapshot.py b/scripts/gitea_cache_snapshot.py new file mode 100644 index 000000000..46e117e15 --- /dev/null +++ b/scripts/gitea_cache_snapshot.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +"""Validate and merge Gitea CI sccache artifacts into one object snapshot.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import tarfile +import tempfile +from typing import BinaryIO, Mapping, Sequence +import zipfile + + +GIB = 1024 ** 3 +DEFAULT_MAX_COMBINED_BYTES = 4 * GIB +MAX_INPUT_OBJECT_BYTES = 4 * GIB +MAX_MANIFEST_BYTES = 16 * 1024 ** 2 +MAX_OBJECTS = 100_000 +MAX_BASE_OBJECTS = 1_000_000 +MAX_TAR_OVERHEAD_BYTES = 128 * 1024 ** 2 +OBJECT_PATH = re.compile(r"objects/([0-9a-f])/([0-9a-f])/([0-9a-f]{64})\Z") +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +TEMP_PREFIX = ".gitea-cache-snapshot-" + + +class SnapshotError(ValueError): + """The downloaded snapshot does not satisfy the trusted artifact contract.""" + + +@dataclass(frozen=True) +class ArtifactIdentity: + repository: str + run_id: int + run_attempt: int + job: str + source_sha: str + + +@dataclass(frozen=True) +class ArtifactInput: + archive: Path + expected: ArtifactIdentity + + +@dataclass(frozen=True) +class MergeResult: + object_count: int + total_bytes: int + source_sha: str + rustc: str + workspace: str + base_image: str | None + sccache_version: str + + +@dataclass(frozen=True) +class _Object: + path: str + size: int + sha256: str | None + mtime_ns: int + source_index: int | None + + +@dataclass(frozen=True) +class _Touch: + path: str + mtime_ns: int + + +@dataclass(frozen=True) +class _Archive: + input: ArtifactInput + manifest: Mapping[str, object] + objects: tuple[_Object, ...] + touched: tuple[_Touch, ...] + signature: tuple[int, int, int] + + +def _is_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _require_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value or "\0" in value: + raise SnapshotError(f"manifest {field} must be a non-empty string") + return value + + +def _strict_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise SnapshotError(f"manifest contains duplicate JSON key: {key}") + result[key] = value + return result + + +def _archive_signature(path: Path) -> tuple[int, int, int]: + try: + status = path.stat() + except OSError as error: + raise SnapshotError(f"cannot stat artifact archive: {path}") from error + if path.is_symlink() or not path.is_file(): + raise SnapshotError(f"artifact archive must be a regular file: {path}") + return status.st_size, status.st_mtime_ns, status.st_ino + + +def _safe_zip_path(name: str) -> tuple[str, ...]: + if not name or "\0" in name or "\\" in name or name.startswith("/"): + raise SnapshotError(f"unsafe ZIP member path: {name!r}") + parts = PurePosixPath(name.rstrip("/")).parts + if not parts or any(part in {"", ".", ".."} for part in parts): + raise SnapshotError(f"unsafe ZIP member path: {name!r}") + return parts + + +def _snapshot_member(archive: Path) -> tuple[zipfile.ZipFile, zipfile.ZipInfo]: + try: + bundle = zipfile.ZipFile(archive) + except (OSError, zipfile.BadZipFile) as error: + raise SnapshotError(f"invalid artifact ZIP: {archive}") from error + + try: + infos = bundle.infolist() + if len({info.filename for info in infos}) != len(infos): + raise SnapshotError("ZIP contains duplicate members") + files: list[zipfile.ZipInfo] = [] + directories: list[tuple[str, ...]] = [] + for info in infos: + parts = _safe_zip_path(info.filename) + mode = info.external_attr >> 16 + if info.flag_bits & 1: + raise SnapshotError("encrypted ZIP members are not accepted") + if stat.S_ISLNK(mode): + raise SnapshotError("ZIP links are not accepted") + if info.is_dir(): + directories.append(parts) + else: + files.append(info) + if len(files) != 1: + raise SnapshotError("ZIP must contain exactly one snapshot.tar file") + member = files[0] + parts = _safe_zip_path(member.filename) + if not (parts == ("snapshot.tar",) or + (len(parts) == 2 and parts[1] == "snapshot.tar")): + raise SnapshotError("ZIP member must be snapshot.tar or /snapshot.tar") + expected_directories = set() if len(parts) == 1 else {(parts[0],)} + if set(directories) - expected_directories: + raise SnapshotError("ZIP contains unexpected directory members") + maximum_tar_size = ( + MAX_INPUT_OBJECT_BYTES + MAX_MANIFEST_BYTES + MAX_TAR_OVERHEAD_BYTES + ) + if member.file_size > maximum_tar_size: + raise SnapshotError("snapshot.tar exceeds the per-job input bound") + if member.compress_type != zipfile.ZIP_STORED: + raise SnapshotError("snapshot.tar ZIP member must use ZIP_STORED") + return bundle, member + except Exception: + bundle.close() + raise + + +def _tar_stream(archive: Path): + bundle, member = _snapshot_member(archive) + try: + raw = bundle.open(member, "r") + except Exception: + bundle.close() + raise + try: + tar = tarfile.open(fileobj=raw, mode="r|") + except (OSError, tarfile.TarError) as error: + raw.close() + bundle.close() + raise SnapshotError(f"invalid snapshot.tar in {archive}") from error + return bundle, raw, tar + + +def _hash_stream(stream: BinaryIO, expected_size: int) -> tuple[int, str]: + digest = hashlib.sha256() + total = 0 + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > expected_size: + raise SnapshotError("tar member contains more bytes than declared") + digest.update(chunk) + if total != expected_size: + raise SnapshotError("tar member is truncated") + return total, digest.hexdigest() + + +def _validate_object_path(path: str) -> None: + match = OBJECT_PATH.fullmatch(path) + if not match or match[1] != match[3][0] or match[2] != match[3][1]: + raise SnapshotError(f"invalid sccache object path: {path!r}") + + +def _parse_manifest( + contents: bytes, + expected: ArtifactIdentity, + expected_inherited_source_sha: str, +) -> dict[str, object]: + try: + manifest = json.loads(contents, object_pairs_hook=_strict_object) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise SnapshotError("manifest.json is not valid UTF-8 JSON") from error + if not isinstance(manifest, dict): + raise SnapshotError("manifest.json must contain an object") + required = { + "schema", "repository", "run_id", "run_attempt", "job", "source_sha", + "rustc", "workspace", "sccache_version", "mode", "inherited_source_sha", + "objects", "touched", + } + allowed = required | {"base_image"} + if set(manifest) != required and set(manifest) != allowed: + raise SnapshotError("manifest.json has missing or unexpected fields") + if not _is_int(manifest["schema"]) or manifest["schema"] != 1: + raise SnapshotError("manifest schema must be 1") + for field in ("repository", "job", "source_sha", "rustc", "workspace", + "sccache_version", "mode", "inherited_source_sha"): + _require_string(manifest[field], field) + if manifest["mode"] != "delta": + raise SnapshotError("manifest mode must be delta") + if manifest["inherited_source_sha"] != expected_inherited_source_sha: + raise SnapshotError("manifest inherited_source_sha does not match expected source image") + for field in ("run_id", "run_attempt"): + if not _is_int(manifest[field]) or manifest[field] < 1: + raise SnapshotError(f"manifest {field} must be a positive integer") + if "base_image" in manifest: + _require_string(manifest["base_image"], "base_image") + + for field in ("repository", "run_id", "run_attempt", "job", "source_sha"): + if manifest[field] != getattr(expected, field): + raise SnapshotError(f"manifest {field} does not match expected identity") + if not isinstance(manifest["objects"], list): + raise SnapshotError("manifest objects must be a list") + if not isinstance(manifest["touched"], list): + raise SnapshotError("manifest touched must be a list") + return manifest + + +def _manifest_objects(manifest: Mapping[str, object], source_index: int) -> tuple[_Object, ...]: + rows = manifest["objects"] + assert isinstance(rows, list) + if len(rows) > MAX_OBJECTS: + raise SnapshotError("manifest contains too many objects") + objects: list[_Object] = [] + seen: set[str] = set() + for row in rows: + if not isinstance(row, dict) or set(row) != {"path", "size", "sha256", "mtime_ns"}: + raise SnapshotError("manifest object has missing or unexpected fields") + path = row["path"] + checksum = row["sha256"] + if not isinstance(path, str): + raise SnapshotError("manifest object path must be a string") + _validate_object_path(path) + if path in seen: + raise SnapshotError(f"manifest contains duplicate object: {path}") + seen.add(path) + if not _is_int(row["size"]) or row["size"] < 0: + raise SnapshotError(f"manifest object size is invalid: {path}") + if not isinstance(checksum, str) or not SHA256.fullmatch(checksum): + raise SnapshotError(f"manifest object sha256 is invalid: {path}") + if not _is_int(row["mtime_ns"]) or row["mtime_ns"] < 0: + raise SnapshotError(f"manifest object mtime_ns is invalid: {path}") + objects.append(_Object( + path=path, + size=row["size"], + sha256=checksum, + mtime_ns=row["mtime_ns"], + source_index=source_index, + )) + return tuple(objects) + + +def _manifest_touches(manifest: Mapping[str, object]) -> tuple[_Touch, ...]: + rows = manifest["touched"] + assert isinstance(rows, list) + if len(rows) > MAX_OBJECTS: + raise SnapshotError("manifest contains too many touched objects") + touched: list[_Touch] = [] + seen: set[str] = set() + for row in rows: + if not isinstance(row, dict) or set(row) != {"path", "mtime_ns"}: + raise SnapshotError("manifest touched object has missing or unexpected fields") + path = row["path"] + if not isinstance(path, str): + raise SnapshotError("manifest touched path must be a string") + _validate_object_path(path) + if path in seen: + raise SnapshotError(f"manifest contains duplicate touched object: {path}") + seen.add(path) + if not _is_int(row["mtime_ns"]) or row["mtime_ns"] < 0: + raise SnapshotError(f"manifest touched mtime_ns is invalid: {path}") + touched.append(_Touch(path=path, mtime_ns=row["mtime_ns"])) + return tuple(touched) + + +def _validate_archive( + item: ArtifactInput, + source_index: int, + expected_inherited_source_sha: str, +) -> _Archive: + path = Path(item.archive) + signature = _archive_signature(path) + bundle, raw, tar = _tar_stream(path) + actual: dict[str, tuple[int, str]] = {} + manifest_contents: bytes | None = None + total_bytes = 0 + member_count = 0 + try: + try: + for member in tar: + member_count += 1 + if member_count > MAX_OBJECTS + 1: + raise SnapshotError("snapshot.tar contains too many members") + if manifest_contents is not None: + raise SnapshotError("manifest.json must be the last tar member") + if not member.isreg(): + raise SnapshotError(f"tar member must be a regular file: {member.name!r}") + if member.name == "manifest.json": + if member.size < 0 or member.size > MAX_MANIFEST_BYTES: + raise SnapshotError("manifest.json exceeds its size bound") + stream = tar.extractfile(member) + if stream is None: + raise SnapshotError("cannot read manifest.json") + manifest_contents = stream.read(MAX_MANIFEST_BYTES + 1) + if len(manifest_contents) != member.size: + raise SnapshotError("manifest.json is truncated") + continue + _validate_object_path(member.name) + if member.size < 0: + raise SnapshotError(f"tar member has a negative size: {member.name}") + if member.name in actual: + raise SnapshotError(f"snapshot.tar contains duplicate member: {member.name}") + total_bytes += member.size + if total_bytes > MAX_INPUT_OBJECT_BYTES: + raise SnapshotError("snapshot exceeds the per-job object byte bound") + stream = tar.extractfile(member) + if stream is None: + raise SnapshotError(f"cannot read tar member: {member.name}") + actual[member.name] = _hash_stream(stream, member.size) + except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: + raise SnapshotError(f"cannot stream snapshot archive: {path}") from error + finally: + tar.close() + raw.close() + bundle.close() + + if manifest_contents is None: + raise SnapshotError("snapshot.tar is missing final manifest.json") + manifest = _parse_manifest( + manifest_contents, item.expected, expected_inherited_source_sha, + ) + objects = _manifest_objects(manifest, source_index) + touched = _manifest_touches(manifest) + if {obj.path for obj in objects} & {touch.path for touch in touched}: + raise SnapshotError("an object cannot be both uploaded and touched") + described = {obj.path: (obj.size, obj.sha256) for obj in objects} + if actual != described: + raise SnapshotError("manifest object list, sizes, or checksums do not match snapshot.tar") + return _Archive( + input=item, manifest=manifest, objects=objects, touched=touched, + signature=signature, + ) + + +def _consistent_metadata(archives: Sequence[_Archive]) -> tuple[str, str, str, str, str | None]: + first = archives[0].manifest + fields = ( + "source_sha", "rustc", "workspace", "sccache_version", "base_image", + "inherited_source_sha", + ) + expected = tuple(first.get(field) for field in fields) + for archive in archives[1:]: + if tuple(archive.manifest.get(field) for field in fields) != expected: + raise SnapshotError("artifact compiler, workspace, source, or base-image metadata differs") + source_sha, rustc, workspace, sccache_version, base_image, _ = expected + assert isinstance(source_sha, str) + assert isinstance(rustc, str) + assert isinstance(workspace, str) + assert isinstance(sccache_version, str) + assert base_image is None or isinstance(base_image, str) + return source_sha, rustc, workspace, sccache_version, base_image + + +def _validated_base_objects(base_objects: os.PathLike[str] | str) -> tuple[Path, dict[str, _Object]]: + requested = Path(base_objects) + if not requested.is_absolute(): + raise SnapshotError("base_objects must be absolute") + try: + root = requested.resolve(strict=True) + except OSError as error: + raise SnapshotError("base_objects must exist") from error + if requested != root or requested.is_symlink() or not root.is_dir(): + raise SnapshotError("base_objects must be a real canonical directory") + + objects: dict[str, _Object] = {} + for first in root.iterdir(): + if first.is_symlink() or not first.is_dir() or not re.fullmatch(r"[0-9a-f]", first.name): + raise SnapshotError(f"noncanonical entry in base objects: {first}") + for second in first.iterdir(): + if second.is_symlink() or not second.is_dir() or not re.fullmatch(r"[0-9a-f]", second.name): + raise SnapshotError(f"noncanonical entry in base objects: {second}") + for entry in second.iterdir(): + relative = f"objects/{first.name}/{second.name}/{entry.name}" + _validate_object_path(relative) + if entry.is_symlink() or not entry.is_file(): + raise SnapshotError(f"base object must be a regular file: {entry}") + status = entry.stat() + objects[relative] = _Object( + path=relative, + size=status.st_size, + sha256=None, + mtime_ns=status.st_mtime_ns, + source_index=None, + ) + if len(objects) > MAX_BASE_OBJECTS: + raise SnapshotError("base_objects contains too many files") + return root, objects + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + return digest.hexdigest() + digest.update(chunk) + + +def _select_objects( + archives: Sequence[_Archive], + base_root: Path, + base: Mapping[str, _Object], + maximum: int, +) -> tuple[_Object, ...]: + merged = dict(base) + delta_paths = {obj.path for archive in archives for obj in archive.objects} + touched_paths = {touch.path for archive in archives for touch in archive.touched} + overlap = delta_paths & touched_paths + if overlap: + raise SnapshotError(f"object appears as both delta and touched: {min(overlap)}") + for archive in archives: + for touch in archive.touched: + current = merged.get(touch.path) + if current is None or current.source_index is not None: + raise SnapshotError(f"touched object is absent from inherited base: {touch.path}") + if touch.mtime_ns > current.mtime_ns: + merged[touch.path] = _Object( + path=current.path, + size=current.size, + sha256=current.sha256, + mtime_ns=touch.mtime_ns, + source_index=None, + ) + + base_hashes: dict[str, str] = {} + for archive in archives: + for candidate in archive.objects: + current = merged.get(candidate.path) + if current is None: + merged[candidate.path] = candidate + elif current.source_index is None: + if current.size != candidate.size: + raise SnapshotError(f"conflicting content for duplicate object: {candidate.path}") + checksum = base_hashes.get(candidate.path) + if checksum is None: + checksum = _file_sha256( + base_root.joinpath(*PurePosixPath(candidate.path).parts[1:]) + ) + base_hashes[candidate.path] = checksum + if checksum != candidate.sha256: + raise SnapshotError(f"conflicting content for duplicate object: {candidate.path}") + merged[candidate.path] = _Object( + path=current.path, + size=current.size, + sha256=checksum, + mtime_ns=max(current.mtime_ns, candidate.mtime_ns), + source_index=None, + ) + elif (current.size, current.sha256) != (candidate.size, candidate.sha256): + raise SnapshotError(f"conflicting content for duplicate object: {candidate.path}") + elif candidate.mtime_ns > current.mtime_ns: + merged[candidate.path] = _Object( + path=current.path, + size=current.size, + sha256=current.sha256, + mtime_ns=candidate.mtime_ns, + source_index=current.source_index, + ) + + selected: list[_Object] = [] + total = 0 + for candidate in sorted(merged.values(), key=lambda obj: (-obj.mtime_ns, obj.path)): + if candidate.size <= maximum - total: + selected.append(candidate) + total += candidate.size + return tuple(selected) + + +def _copy_selected(archive: _Archive, selected: Mapping[str, _Object], root: Path) -> None: + if _archive_signature(Path(archive.input.archive)) != archive.signature: + raise SnapshotError(f"artifact archive changed while merging: {archive.input.archive}") + bundle, raw, tar = _tar_stream(Path(archive.input.archive)) + remaining = set(selected) + try: + try: + for member in tar: + target_object = selected.get(member.name) + if target_object is None: + continue + if not member.isreg() or member.size != target_object.size: + raise SnapshotError(f"selected object changed while merging: {member.name}") + source = tar.extractfile(member) + if source is None: + raise SnapshotError(f"cannot read selected object: {member.name}") + destination = root.joinpath(*PurePosixPath(member.name).parts) + destination.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + written = 0 + with destination.open("xb") as output: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > target_object.size: + raise SnapshotError(f"selected object grew while merging: {member.name}") + digest.update(chunk) + output.write(chunk) + if written != target_object.size or digest.hexdigest() != target_object.sha256: + raise SnapshotError(f"selected object checksum changed while merging: {member.name}") + os.utime(destination, ns=(target_object.mtime_ns, target_object.mtime_ns)) + remaining.remove(member.name) + except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: + raise SnapshotError(f"cannot copy selected objects from {archive.input.archive}") from error + finally: + tar.close() + raw.close() + bundle.close() + if remaining: + raise SnapshotError(f"selected objects disappeared from {archive.input.archive}") + + +def _copy_base(base_root: Path, selected: Mapping[str, _Object], root: Path) -> None: + for relative, obj in selected.items(): + source = base_root.joinpath(*PurePosixPath(relative).parts[1:]) + if source.is_symlink() or not source.is_file() or source.stat().st_size != obj.size: + raise SnapshotError(f"inherited base object changed while merging: {relative}") + destination = root.joinpath(*PurePosixPath(relative).parts) + destination.parent.mkdir(parents=True, exist_ok=True) + with source.open("rb") as input_stream, destination.open("xb") as output_stream: + shutil.copyfileobj(input_stream, output_stream, length=1024 * 1024) + if destination.stat().st_size != obj.size: + raise SnapshotError(f"inherited base object was truncated while merging: {relative}") + os.utime(destination, ns=(obj.mtime_ns, obj.mtime_ns)) + + +def _write_metadata(path: Path, value: str) -> None: + path.write_text(value if value.endswith("\n") else value + "\n", encoding="utf-8") + + +def _validated_output(output_dir: os.PathLike[str] | str) -> tuple[Path, Path]: + output = Path(output_dir) + if not output.is_absolute(): + raise SnapshotError("output_dir must be absolute") + if output.exists() or output.is_symlink(): + raise SnapshotError("output_dir must not already exist") + try: + parent = output.parent.resolve(strict=True) + except OSError as error: + raise SnapshotError("output_dir parent must already exist") from error + if output.parent.is_symlink() or not parent.is_dir(): + raise SnapshotError("output_dir parent must be a real directory") + if output.parent != parent: + raise SnapshotError("output_dir parent must use its canonical absolute path") + return output, parent + + +def merge_snapshots( + inputs: Sequence[ArtifactInput], + output_dir: os.PathLike[str] | str, + *, + base_objects: os.PathLike[str] | str, + expected_inherited_source_sha: str, + max_combined_bytes: int = DEFAULT_MAX_COMBINED_BYTES, +) -> MergeResult: + """Validate artifacts and atomically create a bounded sccache object snapshot.""" + if not inputs: + raise SnapshotError("at least one artifact is required") + if not _is_int(max_combined_bytes) or max_combined_bytes < 0: + raise SnapshotError("max_combined_bytes must be a non-negative integer") + _require_string(expected_inherited_source_sha, "expected_inherited_source_sha") + output, parent = _validated_output(output_dir) + + base_root, base = _validated_base_objects(base_objects) + archives = tuple( + _validate_archive(item, index, expected_inherited_source_sha) + for index, item in enumerate(inputs) + ) + source_sha, rustc, workspace, sccache_version, base_image = _consistent_metadata(archives) + selected = _select_objects(archives, base_root, base, max_combined_bytes) + by_source: dict[int, dict[str, _Object]] = {} + from_base: dict[str, _Object] = {} + for obj in selected: + if obj.source_index is None: + from_base[obj.path] = obj + else: + by_source.setdefault(obj.source_index, {})[obj.path] = obj + + temporary = Path(tempfile.mkdtemp(prefix=TEMP_PREFIX, dir=parent)) + try: + snapshot = temporary / "snapshot" + (snapshot / "objects").mkdir(parents=True) + _copy_base(base_root, from_base, snapshot) + for source_index, objects in by_source.items(): + _copy_selected(archives[source_index], objects, snapshot) + _write_metadata(snapshot / "rustc.txt", rustc) + _write_metadata(snapshot / "workspace.txt", workspace) + _write_metadata(snapshot / "source-commit.txt", source_sha) + if base_image is not None: + _write_metadata(snapshot / "base-image.txt", base_image) + snapshot.replace(output) + finally: + if temporary.parent != parent or not temporary.name.startswith(TEMP_PREFIX): + raise RuntimeError("refusing to clean an unexpected temporary directory") + shutil.rmtree(temporary, ignore_errors=False) + + return MergeResult( + object_count=len(selected), + total_bytes=sum(obj.size for obj in selected), + source_sha=source_sha, + rustc=rustc, + workspace=workspace, + base_image=base_image, + sccache_version=sccache_version, + ) diff --git a/scripts/gitea_cache_upload_cleanup.py b/scripts/gitea_cache_upload_cleanup.py new file mode 100644 index 000000000..6801d193a --- /dev/null +++ b/scripts/gitea_cache_upload_cleanup.py @@ -0,0 +1,115 @@ +"""定向回收 Gitea 1.26.4 不会自动过期的本仓库缓存上传块。""" + +import base64 +import binascii +import math +from pathlib import Path +import re +import stat + + +JOBS = { + "ai-game-creator-shell-rust-lane-1", + "ai-game-creator-shell-rust-lane-2", + "ai-game-creator-shell-rust-smoke", + "ai-game-creator-shell-rust-crates", + "backend-tests", + "native-shell-tests", +} +CHUNK = re.compile(r"block-([1-9][0-9]*)-([1-9][0-9]*)-(0|[1-9][0-9]*)-([A-Za-z0-9_-]+={0,2})") +BLOCKLIST = re.compile(r"([1-9][0-9]*)-([1-9][0-9]*)-blocklist") +MARKER = re.compile(r"genarrative-rust-cache-v1:([a-z0-9-]+):(0|[1-9][0-9]*):([0-9]{8})") + + +def decode_owner(encoded): + """Gitea 文件名编码为 URL-base64(生产者上传的标准 base64 blockid)。""" + try: + block_id = base64.b64decode(encoded, altchars=b"-_", validate=True) + if base64.urlsafe_b64encode(block_id).decode() != encoded: + return None + marker = base64.b64decode(block_id, validate=True) + if base64.b64encode(marker) != block_id: + return None + match = MARKER.fullmatch(marker.decode("ascii")) + if match and match[1] in JOBS: + return match[1], int(match[2]) + except (binascii.Error, UnicodeError, ValueError): + pass + return None + + +def _unchanged_old_regular(path, previous, cutoff): + try: + current = path.lstat() + except FileNotFoundError: + return False + return ( + stat.S_ISREG(current.st_mode) + and current.st_mtime < cutoff + and (current.st_ino, current.st_size, current.st_mtime_ns) + == (previous.st_ino, previous.st_size, previous.st_mtime_ns) + ) + + +def cleanup_upload_chunks(storage_root: Path, eligible_runs: set[int], cutoff_timestamp: float) -> int: + """调用方先用仓库 API 确认这些 master run 已结束超过保留期限。""" + storage_root = Path(storage_root) + if not storage_root.is_absolute() or storage_root.resolve(strict=True) != storage_root: + raise ValueError("artifact storage root must be an existing absolute real directory") + if not stat.S_ISDIR(storage_root.lstat().st_mode): + raise ValueError("artifact storage root must be a directory") + if not math.isfinite(cutoff_timestamp) or cutoff_timestamp <= 0: + raise ValueError("invalid artifact cleanup cutoff") + if any(type(run_id) is not int or run_id <= 0 for run_id in eligible_runs): + raise ValueError("invalid eligible run ID") + temporary = storage_root / "tmp-upload" + if not temporary.exists() and not temporary.is_symlink(): + return 0 + if temporary.is_symlink() or not temporary.is_dir(): + raise ValueError("artifact temporary path must be a real directory") + + deleted = 0 + for run_id in sorted(eligible_runs): + directory = temporary / f"run-{run_id}-v4" + if directory.is_symlink() or not directory.is_dir(): + continue + groups = {} + unknown = False + for path in directory.iterdir(): + chunk = CHUNK.fullmatch(path.name) + blocklist = BLOCKLIST.fullmatch(path.name) + match = chunk or blocklist + if match and int(match[1]) == run_id: + artifact_id = int(match[2]) + owner = decode_owner(chunk[4]) if chunk else None + owned = owner is not None if chunk else True + else: + # 若仍能识别 artifact ID，仅保护该组；无法识别时保护整个 run。 + prefix = re.match(rf"(?:block-)?{run_id}-([1-9][0-9]*)-", path.name) + if not prefix: + unknown = True + break + artifact_id, owner, owned = int(prefix[1]), None, False + group = groups.setdefault(artifact_id, {"files": [], "owners": set(), "safe": True}) + try: + info = path.lstat() + except FileNotFoundError: + group["safe"] = False + continue + group["safe"] &= owned and stat.S_ISREG(info.st_mode) and info.st_mtime < cutoff_timestamp + if owner: + group["owners"].add(owner) + group["files"].append((path, info)) + if unknown: + continue + for group in groups.values(): + if not group["safe"] or len(group["owners"]) != 1: + continue + files = group["files"] + if not all(_unchanged_old_regular(path, info, cutoff_timestamp) for path, info in files): + continue + # 保留目录，不使用递归删除；没有所属块证明的孤立 blocklist 也不会删除。 + for path, _ in files: + path.unlink() + deleted += 1 + return deleted diff --git a/scripts/maintain-gitea-rust-cache.py b/scripts/maintain-gitea-rust-cache.py new file mode 100644 index 000000000..d4dedde86 --- /dev/null +++ b/scripts/maintain-gitea-rust-cache.py @@ -0,0 +1,865 @@ +#!/usr/bin/env python3 +"""宿主专用的 Rust 缓存维护器；仅使用 Python 标准库，不在 CI job 中运行。""" + +import argparse +import datetime +import hashlib +import json +import os +from pathlib import Path +import re +import socket +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +from gitea_cache_snapshot import ArtifactIdentity, ArtifactInput, merge_snapshots +from gitea_cache_upload_cleanup import cleanup_upload_chunks + + +IMAGE = re.compile(r"sha256:[0-9a-f]{64}\Z") +SHA = re.compile(r"[0-9a-f]{40}\Z") +LABEL = re.compile(r'(?m)^(\s*-\s*[\"\x27]?)genarrative-ci:docker://(sha256:[0-9a-f]{64})([\"\x27]?\s*)$') +JOBS = { + "AI game creator shell Rust lane 1/2", + "AI game creator shell Rust lane 2/2", + "AI game creator shell Rust smoke", + "AI game creator shell Rust crates", + "Backend tests", "Native shell tests", "Frontend tests", + "Repository checks", "AI game creator shell web tests", +} +RUST_JOB_IDS = { + "AI game creator shell Rust lane 1/2": "ai-game-creator-shell-rust-lane-1", + "AI game creator shell Rust lane 2/2": "ai-game-creator-shell-rust-lane-2", + "AI game creator shell Rust smoke": "ai-game-creator-shell-rust-smoke", + "AI game creator shell Rust crates": "ai-game-creator-shell-rust-crates", + "Backend tests": "backend-tests", + "Native shell tests": "native-shell-tests", +} +RUST_JOBS = set(RUST_JOB_IDS) +ARTIFACT_PREFIX = "rust-cache-v1-" +MAX_DOWNLOAD = 4 * 1024 ** 3 + 129 * 1024 ** 2 +EXPORT_STEP = "Publish master Rust cache artifact" + + +def now(): + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def timestamp(value): + return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def log(message): + print(f"[cache-maintenance] {message}", flush=True) + + +def command(*args, cwd=None, data=None, env=None, output=None, timeout=120, combined=False): + result = subprocess.run( + args, cwd=cwd, input=data, text=True, env=env, timeout=timeout, + stdout=output or subprocess.PIPE, + stderr=output or (subprocess.STDOUT if combined else subprocess.PIPE), + ) + if result.returncode: + # 参数、stderr 或 HTTP body 可能含凭据，只把详细构建输出写到私有日志。 + raise RuntimeError(f"{Path(args[0]).name} failed (exit {result.returncode})") + return result.stdout or "" + + +def atomic_json(path, value): + temporary = path.with_suffix(".tmp") + with temporary.open("w", encoding="utf-8") as out: + json.dump(value, out, ensure_ascii=False, indent=2) + out.write("\n") + out.flush() + os.fsync(out.fileno()) + temporary.replace(path) + if os.name == "posix": + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def cache_inputs(tree): + """只排除已知不参与构建的说明文档；docs/openapi 与内嵌 skill 必须参与。""" + rows = [] + for row in (tree.split("\0") if "\0" in tree else tree.splitlines()): + if not row: + continue + path = row.split("\t", 1)[1] + if path.startswith("docs/") and not path.startswith("docs/openapi/"): + continue + if path in {"AGENTS.md", "README.md", "deploy/container/README.md"}: + continue + rows.append(row) + return hashlib.sha256("\n".join(rows).encode()).hexdigest() + + +def configured_image(config): + labels = list(LABEL.finditer(config)) + if len(labels) != 1: + raise RuntimeError("expected exactly one genarrative-ci Image ID label") + return labels[0].group(2) + + +def replace_image(config, old, new): + if not IMAGE.fullmatch(new) or configured_image(config) != old: + raise RuntimeError("runner image changed outside maintenance; refusing overwrite") + return LABEL.sub(lambda match: match[1] + "genarrative-ci:docker://" + new + match[3], config) + + +def fully_passed(run, jobs): + return ( + run.get("status") == "completed" and run.get("conclusion") == "success" + and len(jobs) == len(JOBS) and {job["name"] for job in jobs} == JOBS + and all(job.get("conclusion") == "success" for job in jobs) + ) + + +def verifies_image(job, contents, image, require_cache=True): + if not re.search(r"(?m)^\S+ image: " + re.escape(image) + r"\s*$", contents): + return False + if not require_cache or job["name"] not in RUST_JOBS: + return True + # 旧分支使用了镜像却未 prepare，不算完成缓存上线验收。 + hits = re.search(r"Cache hits\s+(\d+)\s*$", contents, re.M) + errors = [re.search(rf"{name}\s+(\d+)\s*$", contents, re.M) + for name in ("Cache errors", "Cache read errors", "Cache write errors")] + return ("[rust-cache] mode=sccache" in contents and hits is not None + and int(hits[1]) > 0 and all(item and int(item[1]) == 0 for item in errors)) + + +class Api: + def __init__(self, url, token_file): + self.url = url.rstrip("/") + if not self.url.startswith("https://"): + raise ValueError("api_url must use HTTPS") + self.token_file = Path(token_file) + + def request(self, path, *, method, body=None, raw=False): + token = self.token_file.read_text().strip() + if not token or "\n" in token: + raise ValueError("invalid token file") + request = urllib.request.Request( + self.url + "/" + path.lstrip("/"), + data=None if body is None else json.dumps(body).encode(), + method=method, + headers={"Authorization": "token " + token, "Content-Type": "application/json"}, + ) + try: + # 不允许带 Authorization 的请求跟随跨站重定向。 + opener = urllib.request.build_opener(NoRedirect()) + with opener.open(request, timeout=30) as response: + content = response.read().decode() + except urllib.error.HTTPError as error: + error.close() + if method == "DELETE" and error.code == 404: + return None + raise RuntimeError(f"Gitea API HTTP {error.code}") from None + return content if raw else (json.loads(content) if content else None) + + def download(self, path, destination): + """REST V4 archive redirects to a signed URL; never forward the API token.""" + token = self.token_file.read_text().strip() + url = self.url + "/" + path.lstrip("/") + opener = urllib.request.build_opener(NoRedirect()) + request = urllib.request.Request(url, headers={"Authorization": "token " + token}) + try: + response = opener.open(request, timeout=60) + except urllib.error.HTTPError as error: + error.close() + if error.code not in (301, 302, 303, 307, 308): + raise RuntimeError(f"artifact download HTTP {error.code}") from None + target = urllib.parse.urljoin(url, error.headers.get("Location", "")) + parsed, origin = urllib.parse.urlsplit(target), urllib.parse.urlsplit(self.url) + if (parsed.scheme != "https" or parsed.netloc != origin.netloc + or parsed.username or parsed.password or target == url): + raise RuntimeError("artifact redirect must stay on configured HTTPS Gitea origin") from None + response = opener.open(target, timeout=60) + try: + with response, destination.open("wb") as out: + total = 0 + while chunk := response.read(1024 * 1024): + total += len(chunk) + if total > MAX_DOWNLOAD: + raise RuntimeError("artifact exceeds per-job size limit") + out.write(chunk) + except Exception: + destination.unlink(missing_ok=True) + raise + + def pages(self, path, key): + separator = "&" if "?" in path else "?" + page = 1 + while True: + response = self.request(f"{path}{separator}limit=50&page={page}", method="GET") + items = response[key] + yield from items + if len(items) < 50: + return + page += 1 + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +class Maintenance: + def __init__(self, config): + self.config = config + if not Path(config["state_dir"]).is_absolute(): + raise ValueError("state_dir must be a dedicated absolute directory") + self.root = Path(config["state_dir"]).resolve() + if self.root == Path("/"): + raise ValueError("state_dir must be a dedicated absolute directory") + self.repo = self.root / "source" + self.runner = config.get("runner_container", "gitea-runner") + self.api = Api(config["api_url"], config["token_file"]) + self.repo_api = "repos/" + config["repository"] + self.state_path = self.root / "state.json" + self.state = json.loads(self.state_path.read_text()) if self.state_path.exists() else { + "versions": [], "current": None, "rollback": None, "candidate": None, + } + + def save(self): + atomic_json(self.state_path, self.state) + + def docker(self, *args, inner=False, **kwargs): + prefix = ("docker", "exec", self.runner, "docker") if inner else ("docker",) + return command(*prefix, *args, **kwargs) + + def image_info(self, image, inner=False): + return json.loads(self.docker("image", "inspect", image, inner=inner))[0] + + def read_config(self): + return self.docker("exec", self.runner, "cat", "/data/config.yaml") + + def write_config(self, content): + self.docker("exec", "--user", "0", "-i", self.runner, "sh", "-c", + "cat > /data/config.yaml", data=content) + + def version(self, image): + return next(row for row in self.state["versions"] if row["image"] == image) + + def adopt_current(self): + actual = configured_image(self.read_config()) + if self.state["current"]: + if actual != self.state["current"]: + raise RuntimeError("runner configuration differs from recorded current image") + return + info = self.image_info(actual) + labels = info["Config"].get("Labels") or {} + source = labels.get("world.genarrative.ci.rust-cache-source") + base = labels.get("world.genarrative.ci.rust-cache-base", actual) + if source is not None and not SHA.fullmatch(source): + raise RuntimeError("invalid image source SHA") + self.state["versions"].append({ + "image": actual, "source": source, "base": base, "owned": False, + "activated": "1970-01-01T00:00:00+00:00", "verified_run": None, + }) + self.state["current"] = actual + self.save() + + def fetch_source(self): + if self.repo.resolve() != self.repo: + raise RuntimeError("dedicated clone must not be a symlink") + marker = self.repo / ".git" / "genarrative-cache-maintenance" + if not (self.repo / ".git").exists(): + if self.repo.exists(): + raise RuntimeError("source directory already exists without .git") + command("git", "clone", "--no-checkout", self.config["clone_url"], str(self.repo), timeout=900) + marker.write_text("owned source checkout\n") + if not marker.is_file(): + raise RuntimeError("refusing to clean a checkout not created by this maintainer") + if command("git", "remote", "get-url", "origin", cwd=self.repo).strip() != self.config["clone_url"]: + raise RuntimeError("dedicated clone origin differs from configured repository") + command("git", "fetch", "--no-tags", "origin", "refs/heads/master", cwd=self.repo, timeout=900) + sha = command("git", "rev-parse", "FETCH_HEAD^{commit}", cwd=self.repo).strip() + if not SHA.fullmatch(sha): + raise RuntimeError("invalid master SHA") + # 这是维护器专属 clone，不修改开发者工作区；编译期间不再 checkout 新提交。 + command("git", "checkout", "--detach", "--force", sha, cwd=self.repo) + command("git", "clean", "-ffdx", cwd=self.repo) + tree = command("git", "ls-tree", "-rz", sha, cwd=self.repo) + return sha, cache_inputs(tree) + + def build_command(self, log_file, script, *args, env=None): + with log_file.open("a") as out: + command("bash", str(self.repo / "scripts" / script), *args, cwd=self.repo, + env=env, output=out, timeout=7200) + + def master_run(self, run): + return (run.get("path") == "project-ci.yml@refs/heads/master" + and run.get("event") == "push" and run.get("head_branch") == "master" + and SHA.fullmatch(run.get("head_sha", "")) is not None) + + def source_run(self): + """Only complete exports from the latest eligible master run; never mix runs.""" + current = self.version(self.state["current"]) + current_inputs = (cache_inputs(command("git", "ls-tree", "-rz", current["source"], cwd=self.repo)) + if current.get("source") else None) + for run in self.api.pages(self.repo_api + "/actions/runs?branch=master&event=push", "workflow_runs"): + if (not self.master_run(run) or run.get("status") != "completed" + or run.get("conclusion") not in {"success", "failure"} + or run["id"] <= current.get("run_id", 0)): + continue + sha = run["head_sha"] + if sha == current.get("source"): + continue + ancestry = subprocess.run(["git", "merge-base", "--is-ancestor", sha, "FETCH_HEAD"], + cwd=self.repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if ancestry.returncode: + continue + if current.get("source"): + ancestry = subprocess.run(["git", "merge-base", "--is-ancestor", current["source"], sha], + cwd=self.repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if ancestry.returncode: + continue + inputs = cache_inputs(command("git", "ls-tree", "-rz", sha, cwd=self.repo)) + if inputs == current_inputs: + continue + jobs = list(self.api.pages(self.repo_api + f'/actions/runs/{run["id"]}/jobs', "jobs")) + rust_jobs = [job for job in jobs if job["name"] in RUST_JOB_IDS] + if len(rust_jobs) != len(RUST_JOB_IDS) or {job["name"] for job in rust_jobs} != RUST_JOBS: + continue + if any(job.get("status") != "completed" or job.get("conclusion") not in {"success", "failure"} + or job.get("head_sha") != sha + or not any(step.get("name") == EXPORT_STEP and step.get("conclusion") == "success" + for step in job.get("steps", [])) for job in rust_jobs): + continue + artifacts = list(self.api.pages(self.repo_api + f'/actions/runs/{run["id"]}/artifacts', "artifacts")) + selected = [] + images = set() + for job in rust_jobs: + name = f'{ARTIFACT_PREFIX}{RUST_JOB_IDS[job["name"]]}-attempt-{job["run_attempt"]}' + matches = [item for item in artifacts if item["name"] == name and not item.get("expired") + and item.get("workflow_run", {}).get("id") == run["id"]] + if len(matches) != 1: + break + content = self.api.request(self.repo_api + f'/actions/jobs/{job["id"]}/logs', method="GET", raw=True) + if not re.search(re.escape(f"[rust-cache] artifact={name}") + + r" objects=\d+ bytes=\d+ complete=true", content): + break + used = set(re.findall(r"(?m)^\S+ image: (sha256:[0-9a-f]{64})\s*$", content)) + if len(used) != 1: + break + images.update(used) + selected.append({"id": matches[0]["id"], "name": name, + "job": RUST_JOB_IDS[job["name"]], "attempt": job["run_attempt"]}) + if len(selected) != len(RUST_JOB_IDS) or len(images) != 1: + continue + source_image = images.pop() + if source_image not in {row["image"] for row in self.state["versions"]}: + continue + return {"run_id": run["id"], "source": sha, "inputs": inputs, "source_image": source_image, + "exports": selected} + return None + + def build(self, source): + """Assemble existing CI objects; no cargo warm-up or test execution.""" + sha = source["source"] + command("git", "checkout", "--detach", "--force", sha, cwd=self.repo) + command("git", "clean", "-ffdx", cwd=self.repo) + artifact = self.root / "artifacts" / sha + artifact.mkdir(parents=True, exist_ok=True) + build_log = artifact / "build.log" + tag = "genarrative/gitea-project-ci:rust-cache-auto-" + sha + base_tag = "genarrative/gitea-project-ci:base-auto-" + sha + attempt = {**source, "tag": tag, "base_tag": base_tag, "artifact": str(artifact)} + self.state.setdefault("attempts", []).append(attempt) + self.save() + env = {**os.environ, "GENARRATIVE_GITEA_RUNNER_CONTAINER": self.runner} + env.pop("CI", None) + labels = self.image_info(source["source_image"])["Config"].get("Labels") or {} + inherited_source = labels.get("world.genarrative.ci.rust-cache-source") + base = labels.get("world.genarrative.ci.rust-cache-base") + if not IMAGE.fullmatch(base or "") or not SHA.fullmatch(inherited_source or ""): + raise RuntimeError("source image must contain a trusted cache snapshot") + revision = command("bash", "scripts/gitea-ci-job-image.sh", "revision", cwd=self.repo).strip() + base_labels = self.image_info(base)["Config"].get("Labels") or {} + if base_labels.get("com.genarrative.ci.definition-sha256") != revision: + env["GENARRATIVE_GITEA_CI_IMAGE_TAG"] = base_tag + self.build_command(build_log, "gitea-ci-job-image.sh", "build", env=env) + base = self.image_info(base_tag)["Id"] + self.state.setdefault("bases", {})[base] = base_tag + self.save() + # 与旧缓存镜像分离；绝不把 Docker 可写层、源码或 target commit 成镜像。 + self.docker("run", "--rm", "--network", "none", "--read-only", "--cap-drop=ALL", + "--entrypoint", "bash", base, "-c", "test ! -e /opt/genarrative-ci/rust-cache") + with tempfile.TemporaryDirectory(prefix="assemble-", dir=artifact) as temporary: + work = Path(temporary) + inherited = work / "inherited" + inherited.mkdir() + container = self.docker("create", source["source_image"]).strip() + try: + self.docker("cp", container + ":/opt/genarrative-ci/rust-cache/.", str(inherited), timeout=600) + finally: + self.docker("rm", "--volumes", container) + inputs = [] + for export in source["exports"]: + archive = work / (str(export["id"]) + ".zip") + self.api.download(self.repo_api + f'/actions/artifacts/{export["id"]}/zip', archive) + inputs.append(ArtifactInput(archive, ArtifactIdentity( + self.config["repository"], source["run_id"], export["attempt"], export["job"], sha))) + snapshot = work / "snapshot" + merged = merge_snapshots(inputs, snapshot, base_objects=inherited / "objects", + expected_inherited_source_sha=inherited_source) + if merged.sccache_version != "sccache 0.18.0": + raise RuntimeError("unsupported sccache version") + if merged.base_image is not None and merged.base_image != labels["world.genarrative.ci.rust-cache-base"]: + raise RuntimeError("artifact base differs from its actual source image") + rustc = self.docker("run", "--rm", "--network", "none", "--read-only", base, "rustc", "-vV") + if rustc.strip() != merged.rustc.strip(): + raise RuntimeError("artifact toolchain differs from target base image") + if merged.workspace != "/workspace/" + self.config["repository"]: + raise RuntimeError("artifact workspace differs from CI checkout") + shutil.copyfile(inherited / "sccache", snapshot / "sccache") + (snapshot / "sccache").chmod(0o755) + (snapshot / "base-image.txt").write_text(base + "\n") + (work / "Dockerfile").write_text( + f"FROM {base}\nCOPY snapshot/ /opt/genarrative-ci/rust-cache/\n" + f'LABEL world.genarrative.ci.rust-cache-source="{sha}"\n' + f'LABEL world.genarrative.ci.rust-cache-base="{base}"\n') + (work / ".dockerignore").write_text("**\n!Dockerfile\n!snapshot/\n!snapshot/**\n") + with build_log.open("a") as out: + self.docker("build", "--pull=false", "--tag", tag, str(work), output=out, timeout=1800) + self.build_command(build_log, "gitea-ci-job-image.sh", "verify", tag, env=env) + image = self.image_info(tag)["Id"] + self.state["versions"].append({**attempt, "image": image, "base": base, + "owned": True, "verified_run": None}) + self.state["candidate"] = image + self.state["attempts"] = [row for row in self.state["attempts"] if row["source"] != sha] + self.save() + log(f"candidate assembled run={source['run_id']} source={sha} image={image}") + + def stage_candidate(self): + candidate = self.version(self.state["candidate"]) + command("git", "checkout", "--detach", "--force", candidate["source"], cwd=self.repo) + command("git", "clean", "-ffdx", cwd=self.repo) + artifact = Path(candidate["artifact"]) + archive = artifact / "image.tar.zst" + sidecar = archive.with_suffix(".zst.sha256") + if (candidate.get("staged") and archive.is_file() and sidecar.is_file() + and candidate["image"] in self.docker("image", "ls", "--all", "--no-trunc", "--quiet", inner=True).split()): + return + env = {**os.environ, "GENARRATIVE_GITEA_RUNNER_CONTAINER": self.runner} + build_log = artifact / "build.log" + if not sidecar.exists(): + # 只删除登记目录中的未完成导出文件，不覆盖已验证归档。 + archive.unlink(missing_ok=True) + self.build_command(build_log, "gitea-ci-job-image.sh", "export", str(archive), candidate["image"], env=env) + command("sha256sum", "--check", sidecar.name, cwd=artifact, timeout=600) + self.build_command(build_log, "gitea-ci-job-image.sh", "load-runner", candidate["image"], env=env) + if self.image_info(candidate["image"], inner=True)["Id"] != candidate["image"]: + raise RuntimeError("inner runner image mismatch") + candidate["staged"] = True + self.save() + + def idle(self): + gate = self.gate("status") + if type(gate.get("active_tasks")) is not int or gate["active_tasks"] < 0: + raise RuntimeError("gate must support durable task tracking before automatic switching") + # 已领取但尚未建容器、正在收尾上报的任务都由入口跟踪，不查管理员 API。 + return (gate.get("uncertain") is False and gate.get("active_tasks") == 0 + and not self.docker("ps", "-q", "--filter", "status=running", + "--filter", "status=created", "--filter", "status=restarting", + "--filter", "status=paused", inner=True).strip()) + + def verify_current(self): + current = self.version(self.state["current"]) + if current.get("verified_run"): + return True + for run in self.api.pages(self.repo_api + "/actions/runs?status=success&branch=master&event=push", "workflow_runs"): + if run.get("path") != "project-ci.yml@refs/heads/master" or run.get("event") != "push": + continue + sha = run.get("head_sha", "") + if not SHA.fullmatch(sha): + continue + ancestry = subprocess.run(["git", "merge-base", "--is-ancestor", sha, "FETCH_HEAD"], + cwd=self.repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if ancestry.returncode != 0: + continue + if timestamp(run["started_at"]) < timestamp(current["activated"]): + continue + jobs = list(self.api.pages(self.repo_api + f'/actions/runs/{run["id"]}/jobs', "jobs")) + if not fully_passed(run, jobs): + continue + if all(verifies_image(job, self.api.request( + self.repo_api + f'/actions/jobs/{job["id"]}/logs', method="GET", raw=True), current["image"], + require_cache=current.get("source") is not None, + ) for job in jobs): + current["verified_run"] = run["id"] + current["verified_sha"] = sha + self.save() + log(f'current image verified by run={run["id"]}') + return True + log("waiting for a complete successful CI run using current image and cache") + return False + + def protected_images(self): + protected = {self.state[key] for key in ("current", "rollback", "candidate") if self.state.get(key)} + for image in list(protected): + protected.add(self.version(image)["base"]) + # 保护包括已停止容器在内的全部引用；不强制删除被 Docker 引用的镜像。 + for inner in (False, True): + ids = self.docker("ps", "-aq", inner=inner).split() + for container in ids: + protected.add(self.docker("inspect", "--format", "{{.Image}}", container, inner=inner).strip()) + protected.add(configured_image(self.read_config())) + return protected + + def remove_owned_image(self, image, tag, protected): + if image in protected or not IMAGE.fullmatch(image): + return False + for inner in (True, False): + available = self.docker("image", "ls", "--all", "--no-trunc", "--quiet", inner=inner).split() + if image not in available: + continue + info = self.image_info(image, inner=inner) + if not set(info.get("RepoTags") or []).issubset({tag}): + log(f"retain image with additional tags: {image}") + return False + # 不使用 --force；并发创建的容器也由 Docker 的引用检查保护。 + self.docker("image", "rm", image, inner=inner) + if image in self.docker("image", "ls", "--all", "--no-trunc", "--quiet", inner=inner).split(): + raise RuntimeError("managed image is still present after removal") + return True + + def cleanup(self): + if not self.version(self.state["current"]).get("verified_run"): + return + protected = self.protected_images() + for record in list(self.state["versions"]): + if not record.get("owned") or record["image"] in protected: + continue + expected = self.root / "artifacts" / record["source"] + if (not SHA.fullmatch(record["source"]) or Path(record["artifact"]).resolve() != expected + or expected.is_symlink() or expected.parent.is_symlink() + or record["tag"] != "genarrative/gitea-project-ci:rust-cache-auto-" + record["source"]): + raise RuntimeError("archive path outside managed source directory") + if not self.remove_owned_image(record["image"], record["tag"], protected): + continue + # 不递归删除目录，只删除本维护器创建的已知文件。 + for name in ("image.tar.zst", "image.tar.zst.sha256", "build.log"): + (expected / name).unlink(missing_ok=True) + if expected.exists() and not any(expected.iterdir()): + expected.rmdir() + self.state["versions"].remove(record) + self.save() + log(f'removed old managed snapshot {record["image"]}') + for image, tag in list(self.state.get("bases", {}).items()): + used_bases = {row["base"] for row in self.state["versions"]} + if image not in used_bases and self.remove_owned_image(image, tag, protected): + del self.state["bases"][image] + self.save() + + def cleanup_exports(self): + """Only our named artifacts; keep logs/runs and every unrelated artifact.""" + protected_runs = {row["run_id"] for row in self.state.get("attempts", []) if row.get("run_id")} + for row in self.state["versions"]: + if row.get("run_id") and not row.get("staged"): + protected_runs.add(row["run_id"]) + if not row.get("staged"): + continue + for item in row.get("exports", []): + if item.get("deleted"): + continue + self.api.request(self.repo_api + f'/actions/artifacts/{item["id"]}', method="DELETE") + item["deleted"] = True + self.save() + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=7) + # 先收集再删除，避免按页删除让下一页位置前移、漏掉旧产物。 + artifacts = list(self.api.pages(self.repo_api + "/actions/artifacts", "artifacts")) + pattern = re.compile(re.escape(ARTIFACT_PREFIX) + "(" + "|".join(RUST_JOB_IDS.values()) + + r")-attempt-\d+\Z") + runs = {} + for item in artifacts: + run_id = (item.get("workflow_run") or {}).get("id") + if (not pattern.fullmatch(item["name"]) or not run_id or run_id in protected_runs + or timestamp(item["created_at"]) >= cutoff): + continue + # Artifact.workflow_run 在 Gitea 1.26.4 中只有 id/repository_id/head_sha。 + if run_id not in runs: + runs[run_id] = self.api.request(self.repo_api + f"/actions/runs/{run_id}", method="GET") + run = runs[run_id] + if not self.master_run(run) or run.get("status") != "completed": + continue + self.api.request(self.repo_api + f'/actions/artifacts/{item["id"]}', method="DELETE") + + def cleanup_pending_uploads(self): + # Gitea 1.26.4 的过期/DELETE API 不会清理未 finalized 的 V4 分块。 + # 只处理本上传器命名的块，且 run 和文件本身均已过保留期限。 + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=7) + eligible = set() + protected = {row["run_id"] for row in self.state.get("attempts", []) if row.get("run_id")} + for run in self.api.pages(self.repo_api + "/actions/runs?branch=master&event=push", "workflow_runs"): + if (self.master_run(run) and run.get("status") == "completed" + and run["id"] not in protected and run.get("completed_at") + and timestamp(run["completed_at"]) < cutoff): + eligible.add(run["id"]) + removed = cleanup_upload_chunks(Path(self.config["artifact_storage_dir"]), eligible, cutoff.timestamp()) + if removed: + log(f"removed {removed} expired owned upload fragments") + + def tick(self, retry=False): + if not self.recover_switch(): + return + self.adopt_current() + self.fetch_source() + self.recover_builds() + self.cleanup_exports() + self.cleanup_pending_uploads() + if not self.verify_current(): + return + self.cleanup() + if self.state.get("candidate"): + self.stage_candidate() + self.activate() + return + source = self.source_run() + if source is None: + log("waiting for a complete set of master CI cache exports") + return + if not retry and self.state.get("failed_run") == source["run_id"]: + log(f'previous assembly failed at run={source["run_id"]}; waiting for new run or --retry') + return + # 下载、合并和镜像装载也消耗宿主 IO；繁忙时留给 CI，下轮再收集。 + if not self.idle(): + log("CI active; defer refresh") + return + try: + self.build(source) + except Exception: + self.state["failed_run"] = source["run_id"] + self.save() + raise + self.state.pop("failed_run", None) + self.save() + self.stage_candidate() + self.cleanup_exports() + self.activate() + + def recover_builds(self): + # 宕机可能发生在 docker build 完成之后、登记 Image ID 之前。 + # 只接管预先登记的确定性 tag；恢复的候选仍须经过 stage 的 verify/load。 + for attempt in list(self.state.get("attempts", [])): + artifact = self.root / "artifacts" / attempt["source"] + if (not SHA.fullmatch(attempt["source"]) or Path(attempt["artifact"]).resolve() != artifact + or artifact.is_symlink() or artifact.parent.is_symlink()): + raise RuntimeError("interrupted assembly directory is outside managed artifacts") + # 持有维护锁，只有此前中断的组装可能遗留这些私有工作目录。 + for directory in artifact.glob("assemble-*"): + if directory.is_symlink() or not directory.is_dir(): + raise RuntimeError("unexpected interrupted assembly entry") + shutil.rmtree(directory) + for kind in ("base_tag", "tag"): + ids = set(self.docker("image", "ls", "--no-trunc", "--quiet", attempt[kind]).split()) + if not ids: + continue + if len(ids) != 1: + raise RuntimeError("ambiguous managed build tag") + image = ids.pop() + if kind == "base_tag": + self.state.setdefault("bases", {})[image] = attempt[kind] + elif not any(row["image"] == image for row in self.state["versions"]): + labels = self.image_info(image)["Config"].get("Labels") or {} + if labels.get("world.genarrative.ci.rust-cache-source") != attempt["source"]: + raise RuntimeError("interrupted build tag has unexpected source") + self.state["versions"].append({ + **attempt, "image": image, "owned": True, "verified_run": None, + "base": labels["world.genarrative.ci.rust-cache-base"], + }) + if self.state.get("candidate") not in (None, image): + raise RuntimeError("interrupted build conflicts with an existing candidate") + self.state["candidate"] = image + self.state["attempts"].remove(attempt) + self.save() + + def recover_switch(self): + if self.state.get("switch"): + # 不能仅凭配置文件已经替换，就认定内存中的 runner 已加载新映射。 + log("recovering interrupted image switch") + return self.activate() + if self.state.get("pause_owned"): + self.resume() + return True + + def resume(self): + if self.state.get("pause_owned"): + if self.gate("resume").get("paused") is not False: + raise RuntimeError("runner resume could not be confirmed") + self.state.pop("pause_owned", None) + self.save() + + def gate(self, action): + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(10) + client.connect(self.config["gate_socket"]) + client.sendall((json.dumps({"action": action}) + "\n").encode()) + with client.makefile("rb") as stream: + response = json.loads(stream.readline(4096)) + if "error" in response: + raise RuntimeError("runner gate refused command") + return response + + def check_gate_route(self): + checkout = re.search(r"(?m)^\s+GENARRATIVE_GITEA_REPOSITORY_URL:\s*(.+?)\s*$", self.read_config()) + if not checkout or checkout[1].strip("\"'") != self.config["repository_url"]: + raise RuntimeError("runner.envs must pin GENARRATIVE_GITEA_REPOSITORY_URL independently of the RPC gate") + # .runner 含认证材料，只在进程内取 address，不回显原文。 + registration = json.loads(self.docker("exec", self.runner, "cat", "/data/.runner")) + if registration.get("address", "").rstrip("/") != self.config["gate_url"].rstrip("/"): + raise RuntimeError("runner is not configured to use the FetchTask gate") + info = json.loads(self.docker("inspect", self.runner))[0] + started = timestamp(info["State"]["StartedAt"]).timestamp() + addresses = {network["IPAddress"] for network in info["NetworkSettings"]["Networks"].values()} + gate = self.gate("status") + # .runner 会因 label 更新而回写；mtime 不能证明内存中的地址。 + # 要求入口实际见到本次容器启动后、来自它的 FetchTask。 + if gate.get("last_fetch_peer") not in addresses or (gate.get("last_fetch_at") or 0) < started: + raise RuntimeError("gate has not observed FetchTask from this runner startup") + if type(gate.get("active_tasks")) is not int or gate["active_tasks"] < 0: + raise RuntimeError("gate must support durable task tracking before automatic switching") + + def activate(self): + self.check_gate_route() + candidate = self.version(self.state["candidate"]) + config = self.read_config() + gate = self.gate("status") + if gate.get("paused") is not False and not self.state.get("pause_owned"): + log("runner gate paused by operator; defer switch") + return False + if not self.state.get("switch") and not self.idle(): + log("CI active; candidate stays staged") + return False + # 先持久化恢复意图；控制请求超时也可能已生效，ExecStopPost/下次 tick 会恢复。 + self.state["pause_owned"] = True + self.save() + try: + if self.gate("pause").get("paused") is not True: + raise RuntimeError("runner pause could not be confirmed") + # 不用 FetchTask 客户端超时猜测服务端事务是否已经结束。 + # 入口必须完整读完已转发的响应；不确定时拒绝自动重启。 + for _ in range(30): + gate = self.gate("status") + if gate.get("uncertain"): + raise RuntimeError("in-flight FetchTask completion is uncertain; manual gate inspection required") + if gate.get("paused") is not True: + raise RuntimeError("runner gate unexpectedly resumed") + if gate.get("inflight") == 0: + break + time.sleep(1) + else: + log("FetchTask still in flight; defer switch") + return False + if not self.idle(): + log("in-flight task appeared; defer switch without stopping runner") + return False + latest_config = self.read_config() + if latest_config != config: + raise RuntimeError("runner config changed while pausing") + pending = self.state.get("switch") + if pending is None: + old = self.state["current"] + updated = replace_image(config, old, candidate["image"]) + backup = self.root / "backups" / (candidate["source"] + ".yaml") + backup.parent.mkdir(parents=True, exist_ok=True) + backup.write_text(config) + pending = {"old": old, "new": candidate["image"], "backup": str(backup)} + self.state["switch"] = pending + self.save() + else: + actual = configured_image(config) + if actual not in {pending["old"], pending["new"]}: + raise RuntimeError("interrupted switch conflicts with operator configuration") + updated = replace_image(config, actual, pending["new"]) + self.write_config(updated) + if self.read_config() != updated or not self.idle(): + # 配置尚未重启加载，恢复旧文件即可；不干扰意外出现的 job。 + self.write_config(Path(pending["backup"]).read_text()) + self.state.pop("switch", None) + self.save() + log("idle check changed before restart; restored configuration") + return False + self.docker("restart", "--timeout", "660", self.runner, timeout=720) + started = self.docker("inspect", "--format", "{{.State.StartedAt}}", self.runner).strip() + ready = False + for _ in range(30): + try: + info = self.docker("inspect", "--format", "{{.State.Status}}", self.runner).strip() + recent = self.docker("logs", "--since", started, self.runner, combined=True) + ready = (info == "running" and "declare successfully" in recent + and self.image_info(candidate["image"], inner=True)["Id"] == candidate["image"]) + except RuntimeError: + ready = False + if ready: + break + time.sleep(2) + if not ready: + raise RuntimeError("runner registration not confirmed; pending switch retained for recovery") + candidate["activated"] = now() + self.state["rollback"] = pending["old"] + self.state["current"] = pending["new"] + self.state["candidate"] = None + self.state.pop("switch", None) + self.save() + log(f'activated {candidate["image"]}; awaiting real master CI validation') + return True + finally: + # 持久化 switch 保证崩溃后不会错误宣称新镜像已验证。 + self.resume() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--apply", action="store_true", help="执行维护；默认只检查连接和配置") + parser.add_argument("--retry", action="store_true", help="重试同一 master run 上失败的缓存组装") + parser.add_argument("--resume", action="store_true", help="仅恢复本维护器暂停的领取；用于 ExecStopPost") + args = parser.parse_args() + if os.environ.get("CI") == "true": + parser.error("run on the trusted host, outside CI jobs") + os.umask(0o077) + config = json.loads(Path(args.config).read_text()) + maintenance = Maintenance(config) + if not args.apply and not args.resume: + maintenance.api.request(maintenance.repo_api + "/actions/artifacts?limit=1", method="GET") + image = configured_image(maintenance.read_config()) + head = command("git", "ls-remote", config["clone_url"], "refs/heads/master").split()[0] + maintenance.check_gate_route() + cleanup_upload_chunks(Path(config["artifact_storage_dir"]), set(), time.time()) + gate = maintenance.gate("status") + if gate.get("uncertain"): + raise RuntimeError("runner gate needs manual inspection") + log(f'check master={head} image={image} active_tasks={gate.get("active_tasks")}') + return + import fcntl # Linux 宿主；纯逻辑测试仍可在 Windows 上导入。 + maintenance.root.mkdir(parents=True, exist_ok=True) + with (maintenance.root / "maintenance.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + log("another maintenance process holds the lock") + return + # 加锁后重新读取状态，避免另一个进程完成后覆盖旧快照。 + maintenance = Maintenance(config) + if args.resume: + maintenance.resume() + else: + maintenance.tick(retry=args.retry) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + log(f"maintenance failed: {type(error).__name__}: {error}") + sys.exit(1) diff --git a/scripts/project-ci-workflow.test.ts b/scripts/project-ci-workflow.test.ts index 9db340345..7304408dc 100644 --- a/scripts/project-ci-workflow.test.ts +++ b/scripts/project-ci-workflow.test.ts @@ -142,6 +142,38 @@ function backendStepIndex(stepName: string) { } describe('project CI workflow', () => { + it('publishes cache deltas only after Rust reporting on non-cancelled master pushes', () => { + const producers = [ + 'ai-game-creator-shell-rust-lane-1', + 'ai-game-creator-shell-rust-lane-2', + 'ai-game-creator-shell-rust-smoke', + 'ai-game-creator-shell-rust-crates', + 'backend-tests', + 'native-shell-tests', + ]; + for (const job of jobNames) { + if (!producers.includes(job)) { + expect(jobSection(job)).not.toContain('export-gitea-rust-cache.py'); + continue; + } + const publish = stepSection(job, 'Publish master Rust cache artifact'); + expect(publish).toContain( + "if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }}", + ); + expect(publish).toContain('GENARRATIVE_GITEA_TOKEN: ${{ github.token }}'); + expect(publish).toContain('continue-on-error: true'); + expect(publish).toContain( + 'run: python3 scripts/export-gitea-rust-cache.py', + ); + const section = jobSection(job); + expect( + section.indexOf('Publish master Rust cache artifact'), + ).toBeGreaterThan( + section.indexOf('run: bash scripts/ci-rust-cache.sh report'), + ); + } + }); + it('uses isolated compilation caching for every Rust test job', () => { const firstRustSteps = { 'ai-game-creator-shell-rust-lane-1': @@ -331,7 +363,8 @@ describe('project CI workflow', () => { const rustLock = 'apps/ai-game-creator-shell/src-tauri/Cargo.lock'; for (const workspaceManifest of workspaceManifests) { - expect(imageBuildScript.split(workspaceManifest)).toHaveLength(3); + // 构建上下文与 revision 共用一份清单，不重复枚举 manifest。 + expect(imageBuildScript.split(workspaceManifest)).toHaveLength(2); expect(imageDockerignore).toContain(`!${workspaceManifest}`); expect(imageDockerfile).toContain( `COPY ${workspaceManifest} /usr/local/share/genarrative-ci/npm/${workspaceManifest}`, @@ -339,8 +372,8 @@ describe('project CI workflow', () => { } for (const [path, expectedCount] of [ - [rustManifest, 2], - [rustLock, 3], + [rustManifest, 1], + [rustLock, 2], ] as const) { expect(imageBuildScript.split(path)).toHaveLength(expectedCount + 1); expect(imageDockerignore).toContain(`!${path}`); diff --git a/scripts/test_gitea_cache_builder.py b/scripts/test_gitea_cache_builder.py new file mode 100644 index 000000000..1069023d4 --- /dev/null +++ b/scripts/test_gitea_cache_builder.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Targeted regression tests for the trusted Gitea Rust-cache image builder.""" + +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import shlex +import stat +import subprocess +import tempfile +import textwrap +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parent.parent +BUILDER_NAME = "build-gitea-rust-cache.sh" +FIXED_SHA = "a" * 40 +MASTER_SHA = "b" * 40 + + +class GiteaRustCacheBuilderTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + self.root = Path(self.temporary_directory.name) + self.repo = self.root / "repo" + scripts = self.repo / "scripts" + scripts.mkdir(parents=True) + shutil.copy2(REPOSITORY_ROOT / "scripts" / BUILDER_NAME, scripts / BUILDER_NAME) + (scripts / "gitea-ci-job-image.sh").write_text("#!/usr/bin/env bash\nexit 0\n") + (scripts / "ci-rust-cache.sh").write_text("#!/usr/bin/env bash\nexit 0\n") + self.trace = self.root / "trace.log" + self.bin = self.root / "bin" + self.bin.mkdir() + self.write_fake_tools() + self.execution_bin = self.wsl_path(self.bin) + self.wsl_fake_root: str | None = None + if os.name == "nt": + self.wsl_fake_root = f"/tmp/gitea-cache-builder-{self.root.name}" + subprocess.run( + [ + "bash", + "-c", + "rm -rf {root}; mkdir -p {root}/bin; cp {source}/* {root}/bin/; chmod +x {root}/bin/*".format( + root=shlex.quote(self.wsl_fake_root), + source=shlex.quote(self.wsl_path(self.bin)), + ), + ], + check=True, + ) + self.execution_bin = f"{self.wsl_fake_root}/bin" + + def tearDown(self) -> None: + if self.wsl_fake_root is not None: + subprocess.run( + ["bash", "-c", f"rm -rf {shlex.quote(self.wsl_fake_root)}"], + check=False, + ) + self.temporary_directory.cleanup() + + def write_tool(self, name: str, source: str) -> None: + target = self.bin / name + target.write_bytes(textwrap.dedent(source).encode("utf-8")) + target.chmod(target.stat().st_mode | stat.S_IXUSR) + + def write_fake_tools(self) -> None: + self.write_tool( + "git", + """#!/usr/bin/env bash + set -eu + printf 'git:%s\\n' "$*" >> "$TRACE" + if [[ "$1" == '-C' ]]; then shift 2; fi + case "$1" in + fetch|cat-file) exit 0 ;; + merge-base) + [[ "${FAKE_NON_ANCESTOR:-}" != 1 ]] + exit + ;; + rev-parse) + if [[ "$2" == 'FETCH_HEAD^{commit}' ]]; then + printf '%s\\n' "$FAKE_MASTER_SHA" + else + printf '%s\\n' "$FAKE_FIXED_SHA" + fi + ;; + archive) printf 'archived %s\\n' "$2" ;; + *) echo "unexpected git command: $*" >&2; exit 9 ;; + esac + """, + ) + self.write_tool( + "docker", + """#!/usr/bin/env bash + set -eu + # Drain the archive pipeline before logging to avoid concurrent + # appends to the shared trace on Windows/WSL filesystems. + if [[ "$1" == cp && "$2" == '-' ]]; then + cat >/dev/null + fi + printf 'docker:%s\\n' "$*" >> "$TRACE" + if [[ "$1" == image && "$2" == inspect ]]; then + printf 'sha256:%064d\\n' 0 + elif [[ "$1" == run && " $* " == *' --detach '* ]]; then + printf 'fake-container\\n' + elif [[ "$1" == exec && " $* " == *' bash -s '* ]]; then + cat >/dev/null + fi + """, + ) + self.write_tool( + "curl", + """#!/usr/bin/env bash + set -eu + while [[ "$#" -gt 0 ]]; do + if [[ "$1" == --output ]]; then touch "$2"; exit 0; fi + shift + done + exit 9 + """, + ) + self.write_tool("sha256sum", "#!/usr/bin/env bash\ncat >/dev/null\n") + self.write_tool( + "tar", + """#!/usr/bin/env bash + set -eu + while [[ "$#" -gt 0 ]]; do + if [[ "$1" == --directory ]]; then + mkdir -p "$2/sccache-v0.18.0-x86_64-unknown-linux-musl" + printf '#!/usr/bin/env bash\\nexit 0\\n' > "$2/sccache-v0.18.0-x86_64-unknown-linux-musl/sccache" + chmod +x "$2/sccache-v0.18.0-x86_64-unknown-linux-musl/sccache" + exit 0 + fi + shift + done + exit 9 + """, + ) + + @staticmethod + def wsl_path(path: Path) -> str: + value = path.resolve().as_posix() + if len(value) >= 3 and value[1] == ":": + return f"/mnt/{value[0].lower()}{value[2:]}" + return value + + def run_builder(self, *arguments: str, non_ancestor: bool = False) -> subprocess.CompletedProcess[str]: + script_path = self.wsl_path(self.repo / "scripts" / BUILDER_NAME) + assignments = { + "TRACE": self.wsl_path(self.trace), + "FAKE_FIXED_SHA": FIXED_SHA, + "FAKE_MASTER_SHA": MASTER_SHA, + "FAKE_NON_ANCESTOR": "1" if non_ancestor else "", + } + exports = "; ".join( + f"export {name}={shlex.quote(value)}" for name, value in assignments.items() + ) + command = ( + f"unset CI; {exports}; export PATH={shlex.quote(self.execution_bin)}:\"$PATH\"; cd /; " + f"exec bash {shlex.quote(script_path)} " + + " ".join(shlex.quote(argument) for argument in arguments) + ) + return subprocess.run( + ["bash", "-c", command], + env=os.environ, + text=True, + capture_output=True, + check=False, + ) + + def trace_text(self) -> str: + return self.trace.read_bytes().decode("utf-8", errors="replace") + + def test_archives_requested_master_ancestor_and_uses_its_helper(self) -> None: + result = self.run_builder("trusted-base", "candidate", FIXED_SHA) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"snapshot_source={FIXED_SHA}", result.stdout) + trace = self.trace_text() + self.assertIn(f"archive {FIXED_SHA}", trace) + self.assertIn( + "docker:exec fake-container cp " + "/workspace/GenarrativeAI/Genarrative/scripts/ci-rust-cache.sh " + "/tmp/ci-rust-cache.sh", + trace, + ) + + def test_rejects_commit_outside_fetched_master(self) -> None: + result = self.run_builder("trusted-base", "candidate", FIXED_SHA, non_ancestor=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("not contained in fetched master", result.stderr) + self.assertNotIn(" archive ", self.trace_text()) + + def test_rejects_non_full_sha_before_calling_external_tools(self) -> None: + result = self.run_builder("trusted-base", "candidate", "abc123") + self.assertEqual(result.returncode, 2) + self.assertIn("complete 40-character SHA", result.stderr) + self.assertFalse(self.trace.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_gitea_cache_export.py b/scripts/test_gitea_cache_export.py new file mode 100644 index 000000000..0ab36732c --- /dev/null +++ b/scripts/test_gitea_cache_export.py @@ -0,0 +1,205 @@ +"""master 产物的增量归档与 Gitea v4 上传协议回归。""" + +import hashlib +import base64 +from contextlib import redirect_stdout +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +import io +import json +import os +from pathlib import Path +import tarfile +import tempfile +import threading +import unittest +from unittest.mock import patch +import urllib.parse +import zipfile + +from gitea_cache_upload_cleanup import decode_owner + + +spec = importlib.util.spec_from_file_location("cache_export", Path(__file__).with_name("export-gitea-rust-cache.py")) +exporter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(exporter) + + +class ExportTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / "objects").mkdir() + + def object(self, key, value, mtime): + path = self.root / "objects" / key[0] / key[1] / key + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + os.utime(path, ns=(mtime * 10**9, mtime * 10**9)) + return path + + def unpack(self, baseline, limit=1024): + destination = self.root / "snapshot.zip" + result = exporter.pack_snapshot(self.root, destination, {"job": "fixture"}, baseline, limit) + with zipfile.ZipFile(destination) as bundle: + self.assertEqual(bundle.namelist(), ["snapshot.tar"]) + with tarfile.open(fileobj=io.BytesIO(bundle.read("snapshot.tar"))) as archive: + files = {member.name: archive.extractfile(member).read() for member in archive} + return result, json.loads(files.pop("manifest.json")), files + + def test_delta_omits_hit_bytes_preserves_recency_and_exports_new_keys(self): + inherited = self.object("a" * 64, b"old", 10) + baseline_path = self.root / "baseline.json" + exporter.save_baseline(self.root, baseline_path) + baseline = json.loads(baseline_path.read_text()) + # sccache 对命中对象更新 mtime，不能因此重复上传已有对象。 + os.utime(inherited, ns=(20 * 10**9, 20 * 10**9)) + self.object("b" * 64, b"new", 30) + result, manifest, files = self.unpack(baseline) + key = "objects/b/b/" + "b" * 64 + self.assertEqual(result, (1, 3)) + self.assertEqual(files, {key: b"new"}) + self.assertEqual(manifest["mode"], "delta") + self.assertEqual(manifest["touched"], [{"path": "objects/a/a/" + "a" * 64, "mtime_ns": 20 * 10**9}]) + self.assertEqual(manifest["objects"][0]["sha256"], hashlib.sha256(b"new").hexdigest()) + + def test_empty_delta_is_valid_and_changed_size_is_exported(self): + path = self.object("a" * 64, b"old", 10) + baseline = {"a/a/" + "a" * 64: {"size": 3, "mtime_ns": 10 * 10**9}} + self.assertEqual(self.unpack(baseline)[0], (0, 0)) + path.write_bytes(b"replacement") + self.assertEqual(self.unpack(baseline)[0], (1, 11)) + + def test_capacity_prefers_newest_objects_and_excludes_unrelated_files(self): + self.object("a" * 64, b"old", 10) + self.object("b" * 64, b"new", 20) + (self.root / "objects" / "credentials").write_bytes(b"secret") + (self.root / "sccache").write_bytes(b"binary") + result, _, files = self.unpack({}, limit=3) + self.assertEqual(result, (1, 3)) + self.assertEqual(list(files.values()), [b"new"]) + + @unittest.skipUnless(os.name == "posix", "symlink fixture requires POSIX") + def test_symlink_objects_and_subdirectories_are_not_exported(self): + outside = self.root / "secret" + outside.write_bytes(b"secret") + linked = self.object("a" * 64, b"old", 10) + linked.unlink() + linked.symlink_to(outside) + (self.root / "objects" / "b").symlink_to(self.root, target_is_directory=True) + self.assertEqual(self.unpack({})[0], (0, 0)) + + def test_non_master_and_unclean_shutdown_do_not_scan_or_upload(self): + with patch.object(exporter, "scan_objects", side_effect=AssertionError("must not scan")), \ + patch.object(exporter, "upload_snapshot", side_effect=AssertionError("must not upload")): + exporter.export({"GITHUB_EVENT_NAME": "pull_request", "GITHUB_REF": "refs/heads/master"}) + exporter.export({"GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/feature"}) + exporter.export({"GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/master"}) + + def test_internal_repository_route_bypasses_rpc_only_gateway(self): + env = { + "GITHUB_REPOSITORY": "owner/repo", + "GITHUB_SERVER_URL": "http://fetch-gate:8080", + "GENARRATIVE_GITEA_REPOSITORY_URL": "http://gitea:3000/sub/owner/repo.git", + } + self.assertEqual(exporter.artifact_base_url(env), "http://gitea:3000/sub") + env["GENARRATIVE_GITEA_REPOSITORY_URL"] = "http://token@gitea:3000/owner/repo.git" + with self.assertRaises(ValueError): + exporter.artifact_base_url(env) + + def test_native_v4_chunks_finalize_with_verified_size_and_hash(self): + calls = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + calls.append((self.command, self.path, dict(self.headers), body)) + payload = {"ok": True} + if self.path.endswith("CreateArtifact"): + payload["signedUploadUrl"] = "https://external.invalid/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=private&taskID=4&artifactID=8" + else: + payload["artifactId"] = "8" + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps(payload).encode()) + + def do_PUT(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + calls.append((self.command, self.path, dict(self.headers), body)) + self.send_response(201) + self.end_headers() + self.wfile.write(b'"created"') + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + path = self.root / "test.zip" + path.write_bytes(b"123456789") + with patch.object(exporter, "CHUNK_BYTES", 4): + exporter.upload_snapshot(path, "rust-cache-v1-backend-tests-attempt-1", 123, f"http://127.0.0.1:{server.server_port}", "ephemeral") + finally: + server.shutdown() + server.server_close() + thread.join() + self.assertEqual(len(calls), 6) + self.assertEqual(calls[0][3]["workflowRunBackendId"], "123") + remaining = datetime.fromisoformat(calls[0][3]["expiresAt"]) - datetime.now(timezone.utc) + self.assertGreater(remaining.total_seconds(), 7 * 24 * 3600) + self.assertLessEqual(remaining.total_seconds(), 8 * 24 * 3600) + blocks = calls[1:4] + self.assertEqual(b"".join(call[3] for call in blocks), b"123456789") + self.assertEqual(len({urllib.parse.parse_qs(urllib.parse.urlsplit(call[1]).query)["blockid"][0] for call in blocks}), 3) + for call in blocks: + block_id = urllib.parse.parse_qs(urllib.parse.urlsplit(call[1]).query)["blockid"][0] + gitea_filename_suffix = base64.urlsafe_b64encode(block_id.encode()).decode() + self.assertEqual(decode_owner(gitea_filename_suffix), ("backend-tests", 1)) + self.assertTrue(all("Authorization" not in call[2] for call in calls[1:5])) + self.assertEqual(calls[-1][2]["Authorization"], "Bearer ephemeral") + self.assertEqual(calls[-1][3]["size"], "9") + self.assertEqual(calls[-1][3]["hash"], "sha256:" + hashlib.sha256(b"123456789").hexdigest()) + + def test_upload_failure_never_finalizes_an_incomplete_artifact(self): + path = self.root / "test.zip" + path.write_bytes(b"data") + with patch.object(exporter, "request", side_effect=[ + {"ok": True, "signedUploadUrl": "http://gitea/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=x"}, + RuntimeError("upload failed"), + ]) as request: + with self.assertRaisesRegex(RuntimeError, "upload failed"): + exporter.upload_snapshot(path, "rust-cache-v1-backend-tests-attempt-1", 123, "http://gitea", "ephemeral") + self.assertEqual(request.call_count, 2) + + def test_failed_optional_export_never_emits_host_completion_marker(self): + (self.root / "export-baseline.json").write_text("{}") + (self.root / "rustc.txt").write_text("rustc fixture\n") + (self.root / "workspace.txt").write_text(str(Path.cwd().resolve())) + (self.root / "source-commit.txt").write_text("a" * 40) + self.object("b" * 64, b"new", 20) + env = { + "GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/master", + "GENARRATIVE_CI_RUST_CACHE_EXPORT_READY": "1", + "GITHUB_JOB": "backend-tests", "GITHUB_SHA": "b" * 40, + "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "1", + "GENARRATIVE_GITEA_TOKEN": "ephemeral", + "GITHUB_REPOSITORY": "owner/repo", + "GENARRATIVE_GITEA_REPOSITORY_URL": "http://gitea/owner/repo.git", + "GENARRATIVE_CI_RUST_CACHE_ROOT": str(self.root), + } + output = io.StringIO() + with patch.object(exporter.subprocess, "check_output", side_effect=["rustc fixture\n", "sccache 0.18.0\n"]), \ + patch.object(exporter, "upload_snapshot", side_effect=RuntimeError("upload failed")), \ + redirect_stdout(output): + with self.assertRaisesRegex(RuntimeError, "upload failed"): + exporter.export(env) + self.assertNotIn("complete=true", output.getvalue()) + self.assertFalse((self.root / "export-baseline.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_gitea_cache_gate.py b/scripts/test_gitea_cache_gate.py new file mode 100644 index 000000000..ddc3a4a2c --- /dev/null +++ b/scripts/test_gitea_cache_gate.py @@ -0,0 +1,347 @@ +"""使用真实 HTTP/socket 验证领取屏障，不连接线上 Gitea。""" + +import http.client +import http.server +import gzip +import importlib.util +import json +import os +from pathlib import Path +import socket +import tempfile +import threading +import time +import unittest + +if os.name != "posix": + raise unittest.SkipTest("领取屏障使用 Linux Unix socket 和目录 fsync；在 Linux/WSL 运行") + +SPEC = importlib.util.spec_from_file_location( + "gitea_cache_gate", Path(__file__).with_name("gitea-runner-fetch-gate.py")) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) +FETCH = "/api/actions/runner.v1.RunnerService/FetchTask" +UPDATE = "/api/actions/runner.v1.RunnerService/UpdateTask" +LOG = "/api/actions/runner.v1.RunnerService/UpdateLog" +PING = "/api/actions/ping.v1.PingService/Ping" + + +def varint(value): + data = bytearray() + while value > 127: + data.append((value & 127) | 128) + value >>= 7 + data.append(value) + return bytes(data) + + +def field(number, value): + # actions-proto-go v0.4.1: task/state ID=1; result=2; log task_id=1, + # index=2, rows=3, no_more=4; log response ack_index=1. + if isinstance(value, bytes): + return varint(number * 8 + 2) + varint(len(value)) + value + return varint(number * 8) + varint(value) + + +def state(task_id, result=0): + return field(1, field(1, task_id) + field(2, result)) + + +def core_state(state): + return {key: state[key] for key in ("paused", "inflight", "uncertain")} + + +class Upstream(http.server.BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_POST(self): + data = self.rfile.read(int(self.headers.get("Content-Length", 0))) + self.server.requests.append((self.path, data)) + if self.path.endswith("/Redirect"): + self.send_response(302) + self.send_header("Location", "/api/actions/runner.v1.RunnerService/FetchTask") + self.send_header("Content-Length", "0") + self.end_headers() + return + if self.path == FETCH: + self.server.entered.set() + self.server.release.wait(10) + if self.server.truncated and self.path == FETCH: + self.send_response(200) + self.send_header("Content-Length", "100") + self.end_headers() + self.wfile.write(b"incomplete") + self.close_connection = True + return + body = self.server.responses.get(self.path, b"") + if self.path == UPDATE and self.path not in self.server.responses: + body = state(*map(int, MODULE.task_state(MODULE.protobuf_fields(data)))) + if self.path == LOG and self.path not in self.server.responses: + fields = MODULE.protobuf_fields(data) + body = field(1, MODULE.single(fields, 2, 0) + len(fields.get(3, []))) + self.send_response(self.server.statuses.get(self.path, 200)) + self.send_header("Content-Type", self.server.content_type) + if self.server.compressed: + body = gzip.compress(body) + self.send_header("Content-Encoding", "gzip") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class GateTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.gate = MODULE.Gate(self.temp.name) + self.upstream = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + self.upstream.requests = [] + self.upstream.entered = threading.Event() + self.upstream.release = threading.Event() + self.upstream.truncated = False + self.upstream.responses = {} + self.upstream.statuses = {} + self.upstream.content_type = "application/proto" + self.upstream.compressed = False + self.proxy = MODULE.create_proxy(("127.0.0.1", 0), + f"http://127.0.0.1:{self.upstream.server_port}", self.gate) + self.servers = [self.upstream, self.proxy] + for server in self.servers: + threading.Thread(target=server.serve_forever, daemon=True).start() + + def tearDown(self): + self.upstream.release.set() + for server in reversed(self.servers): + server.shutdown() + server.server_close() + self.temp.cleanup() + + def request(self, path, body=b"", chunked=False): + connection = http.client.HTTPConnection("127.0.0.1", self.proxy.server_port, timeout=5) + try: + if chunked: + connection.request("POST", path, body=[body], encode_chunked=True, + headers={"Content-Type": "application/proto"}) + else: + connection.request("POST", path, body=body, + headers={"Content-Type": "application/proto"}) + response = connection.getresponse() + result = response.status, response.read() + return result + finally: + connection.close() + + def wait_for(self, predicate): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + self.fail("condition did not become true") + + def start_fetch(self): + self.result = [] + thread = threading.Thread(target=lambda: self.result.append(self.request(FETCH))) + thread.start() + self.assertTrue(self.upstream.entered.wait(3)) + return thread + + def test_pause_holds_inflight_and_allows_reporting(self): + thread = self.start_fetch() + self.assertEqual(core_state(self.gate.control("pause")), + {"paused": True, "inflight": 1, "uncertain": False}) + self.assertEqual(self.request(FETCH)[0], 503) + self.assertEqual(self.request(PING)[0], 200) + self.assertEqual(sum(path == FETCH for path, _ in self.upstream.requests), 1) + self.upstream.release.set() + thread.join(3) + self.assertFalse(thread.is_alive()) + self.assertEqual(self.result[0][0], 200) + self.assertEqual(self.gate.control("status")["inflight"], 0) + self.assertEqual(self.gate.control("resume")["paused"], False) + self.assertEqual(self.request(FETCH)[0], 200) + + def test_disconnected_client_does_not_release_upstream_request(self): + client = socket.create_connection(self.proxy.server_address, timeout=3) + client.sendall(f"POST {FETCH} HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{{}}".encode()) + self.assertTrue(self.upstream.entered.wait(3)) + client.close() + state = self.gate.control("pause") + self.assertEqual(state["inflight"], 1) + self.upstream.release.set() + self.wait_for(lambda: self.gate.control("status")["inflight"] == 0) + self.assertFalse(self.gate.control("status")["uncertain"]) + + def test_truncated_upstream_latches_uncertainty_across_restart(self): + self.upstream.truncated = True + self.upstream.release.set() + self.assertEqual(self.request(FETCH)[0], 502) + self.wait_for(lambda: self.gate.control("status")["uncertain"]) + self.assertTrue(self.gate.control("resume")["paused"]) + restored = MODULE.Gate(self.temp.name) + self.assertTrue(restored.control("status")["uncertain"]) + self.assertFalse(restored.enter()) + + def test_crashed_inflight_is_not_treated_as_successful_drain(self): + self.assertTrue(self.gate.enter()) + restored = MODULE.Gate(self.temp.name) + self.assertTrue(restored.control("status")["uncertain"]) + self.assertIn("error", restored.control("resume")) + self.gate.leave(True) + + def test_pause_marker_survives_restart(self): + self.gate.control("pause") + restored = MODULE.Gate(self.temp.name) + self.assertEqual(core_state(restored.control("status")), + {"paused": True, "inflight": 0, "uncertain": False}) + restored.control("resume") + self.assertFalse(MODULE.Gate(self.temp.name).control("status")["paused"]) + + def test_chunked_body_is_decoded_and_non_rpc_path_rejected(self): + self.assertEqual(self.request(PING, b"ping", chunked=True)[0], 200) + self.assertEqual(self.upstream.requests[-1], (PING, b"ping")) + self.assertEqual(self.request("/api/v1/repos")[0], 404) + self.assertEqual(self.request("/api/actions/../v1/repos")[0], 404) + + def test_upstream_redirect_is_not_followed(self): + self.assertEqual(self.request("/api/actions/runner.v1.RunnerService/Redirect")[0], 302) + self.assertFalse(self.upstream.entered.is_set()) + self.assertEqual(len(self.upstream.requests), 1) + + def test_connection_refused_before_rpc_does_not_latch_uncertainty(self): + with socket.socket() as unused: + unused.bind(("127.0.0.1", 0)) + port = unused.getsockname()[1] + self.proxy.upstream = MODULE.urllib.parse.urlsplit(f"http://127.0.0.1:{port}") + self.assertEqual(self.request(FETCH)[0], 502) + self.assertEqual(core_state(self.gate.control("status")), + {"paused": False, "inflight": 0, "uncertain": False}) + + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix control socket required") + def test_control_socket_uses_newline_json(self): + path = str(Path(self.temp.name) / "gate.sock") + control = MODULE.ControlServer(path, MODULE.Control) + control.gate = self.gate + self.servers.append(control) + threading.Thread(target=control.serve_forever, daemon=True).start() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(path) + client.sendall(b'{"action":"pause"}\n') + payload = client.makefile("rb").readline() + self.assertEqual(core_state(json.loads(payload)), + {"paused": True, "inflight": 0, "uncertain": False}) + + def test_paused_fetch_records_live_route_and_restart_discards_proof(self): + self.gate.control("pause") + before = time.time() + self.assertEqual(self.request(FETCH)[0], 503) + state = self.gate.control("status") + self.assertEqual(state["last_fetch_peer"], "127.0.0.1") + self.assertGreaterEqual(state["last_fetch_at"], before) + self.assertEqual(self.upstream.requests, []) + self.assertEqual(self.request(PING)[0], 200) + self.assertEqual(self.gate.control("status")["last_fetch_at"], state["last_fetch_at"]) + restarted = MODULE.Gate(self.temp.name).control("status") + self.assertIsNone(restarted["last_fetch_peer"]) + self.assertIsNone(restarted["last_fetch_at"]) + + def fetch_task(self, task_id=42): + self.upstream.responses[FETCH] = field(1, field(1, task_id)) + field(2, 10) + self.upstream.release.set() + self.assertEqual(self.request(FETCH)[0], 200) + self.assertEqual(self.gate.control("status")["task_ids"], [str(task_id)]) + + def finalize_log(self, task_id=42): + self.assertEqual(self.request(LOG, field(1, task_id) + field(2, 3) + + field(3, b"log row") + field(4, 1))[0], 200) + self.wait_for(lambda: self.gate.tasks.get(str(task_id)) is True) + + def test_assigned_task_survives_pause_until_runner_finished_reporting(self): + self.fetch_task() + self.assertEqual(self.gate.control("pause")["active_tasks"], 1) + self.assertEqual(self.request(FETCH)[0], 503) + self.assertEqual(self.request(UPDATE, state(42, 1))[0], 200) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.finalize_log() + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.assertEqual(self.request(UPDATE, state(42, 1))[0], 200) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + self.assertFalse(self.gate.control("status")["uncertain"]) + # 终态响应在客户端超时后重试，不应再次阻断后续任务领取。 + self.request(UPDATE, state(42, 1)) + self.assertFalse(self.gate.control("status")["uncertain"]) + + def test_compressed_fetch_and_task_ledger_survive_restart(self): + self.upstream.compressed = True + self.fetch_task() + restored = MODULE.Gate(self.temp.name) + self.assertEqual(restored.control("status")["task_ids"], ["42"]) + self.assertFalse(restored.control("status")["uncertain"]) + self.proxy.gate = self.gate = restored + self.finalize_log() + self.request(UPDATE, state(42, 2)) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + self.assertEqual(MODULE.Gate(self.temp.name).control("status")["active_tasks"], 0) + + def test_server_cancellation_does_not_retire_executing_task(self): + self.fetch_task() + self.upstream.responses[UPDATE] = state(42, 3) + self.request(UPDATE, state(42)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.finalize_log() + self.request(UPDATE, state(42)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.request(UPDATE, state(42, 3)) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + + def test_partial_final_log_acknowledgement_keeps_task_active(self): + self.fetch_task() + self.upstream.responses[LOG] = field(1, 3) + self.request(LOG, field(1, 42) + field(2, 3) + field(3, b"row") + field(4, 1)) + self.request(UPDATE, state(42, 1)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + del self.upstream.responses[LOG] + self.finalize_log() + self.request(UPDATE, state(42, 1)) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + + def test_final_state_waits_for_success_and_output_acknowledgement(self): + self.fetch_task() + self.finalize_log() + self.upstream.statuses[UPDATE] = 500 + self.request(UPDATE, state(42, 1)) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.upstream.statuses[UPDATE] = 200 + output = field(2, field(1, b"key") + field(2, b"value")) + self.request(UPDATE, state(42, 1) + output) + self.assertEqual(self.gate.control("status")["active_tasks"], 1) + self.upstream.responses[UPDATE] = state(42, 1) + field(2, b"key") + self.request(UPDATE, state(42, 1) + output) + self.wait_for(lambda: self.gate.control("status")["active_tasks"] == 0) + + def test_unknown_or_invalid_fetch_result_prevents_idle(self): + self.upstream.release.set() + self.upstream.responses[FETCH] = b"not protobuf" + self.assertEqual(self.request(FETCH)[0], 200) + self.assertTrue(self.gate.control("status")["uncertain"]) + self.assertTrue(MODULE.Gate(self.temp.name).control("status")["uncertain"]) + + def test_task_report_without_observed_assignment_requires_idle_bootstrap(self): + self.assertEqual(self.request(UPDATE, state(42))[0], 200) + self.wait_for(lambda: self.gate.control("status")["uncertain"]) + self.assertIn("error", self.gate.control("resume")) + + def test_disconnected_fetch_client_still_leaves_assigned_task_busy(self): + self.upstream.responses[FETCH] = field(1, field(1, 42)) + client = socket.create_connection(self.proxy.server_address, timeout=3) + client.sendall(f"POST {FETCH} HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n".encode()) + self.assertTrue(self.upstream.entered.wait(3)) + client.close() + self.gate.control("pause") + self.upstream.release.set() + self.wait_for(lambda: self.gate.control("status")["inflight"] == 0) + self.assertEqual(self.gate.control("status")["task_ids"], ["42"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_gitea_cache_maintenance.py b/scripts/test_gitea_cache_maintenance.py new file mode 100644 index 000000000..00a746b4c --- /dev/null +++ b/scripts/test_gitea_cache_maintenance.py @@ -0,0 +1,638 @@ +#!/usr/bin/env python3 +"""High-risk state-transition tests for the host-only Rust cache maintainer.""" + +from __future__ import annotations + +import importlib.util +import io +import json +from pathlib import Path +import re +import tempfile +import unittest +from unittest.mock import patch + + +SCRIPT = Path(__file__).with_name("maintain-gitea-rust-cache.py") +SPEC = importlib.util.spec_from_file_location("gitea_cache_maintenance", SCRIPT) +assert SPEC and SPEC.loader +maintenance_module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(maintenance_module) + +IMAGE = "sha256:" + "1" * 64 +OLD_IMAGE = "sha256:" + "2" * 64 +BASE_IMAGE = "sha256:" + "3" * 64 +SHA = "a" * 40 + + +def runner_config(image: str) -> str: + return f'runners:\n - "genarrative-ci:docker://{image}"\n' + + +class FakeApi: + def __init__(self): + self.requests: list[str] = [] + + def request(self, path, *, method, body=None, raw=False): + self.requests.append(path) + raise AssertionError(f"unexpected API request: {path}") + + +class GiteaCacheMaintenanceTest(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.root = Path(self.temporary_directory.name) + self.token = self.root / "token" + self.token.write_text("test-token\n", encoding="utf-8") + self.config = { + "state_dir": str((self.root / "state").resolve()), + "api_url": "https://gitea.example.test/api/v1", + "token_file": str(self.token), + "repository": "team/project", + "repository_url": "http://gitea:3000/team/project.git", + "runner_container": "gitea-runner", + "gate_socket": str(self.root / "gate.sock"), + "gate_url": "https://gitea.example.test", + "clone_url": "https://gitea.example.test/team/project.git", + } + + def tearDown(self): + self.temporary_directory.cleanup() + + def test_api_requires_explicit_method_and_preserves_it_with_body(self): + api = maintenance_module.Api(self.config["api_url"], self.token) + with self.assertRaises(TypeError): + api.request("example", body={"value": 1}) + with self.assertRaises(TypeError): + api.request("example") + with patch.object(maintenance_module.urllib.request, "build_opener") as build_opener: + for method in ("GET", "POST", "PUT", "PATCH", "DELETE"): + with self.subTest(method=method): + build_opener.return_value.open.return_value = io.BytesIO(b'{}') + body = {"value": 1} if method in {"POST", "PUT", "PATCH"} else None + self.assertEqual(api.request("example", method=method, body=body), {}) + request = build_opener.return_value.open.call_args.args[0] + self.assertEqual(request.get_method(), method) + self.assertEqual(request.data, None if body is None else json.dumps(body).encode()) + + def test_workflow_jobs_and_cache_producers_match_maintenance_contract(self): + workflow = (SCRIPT.parent.parent / ".gitea/workflows/project-ci.yml").read_text(encoding="utf-8") + # 沿用 workflow 的显式 job/step 格式，枚举实际 job，避免另一份名单漏掉新增项。 + sections = re.split(r"(?m)^ ([\w-]+):\s*$", workflow.split("\njobs:\n", 1)[1]) + jobs, producers = {}, {} + for job_id, section in zip(sections[1::2], sections[2::2]): + names = re.findall(r"(?m)^ name: (.+)$", section) + self.assertEqual(len(names), 1, f"{job_id}: expected one explicit job name") + name = names[0].strip().strip("\"'") + self.assertNotIn(name, jobs, f"duplicate workflow job name: {name}") + jobs[name] = job_id + if "run: python3 scripts/export-gitea-rust-cache.py" in section: + self.assertIn(f" - name: {maintenance_module.EXPORT_STEP}\n", section) + producers[name] = job_id + self.assertEqual(set(jobs), maintenance_module.JOBS, + "workflow job names drifted from maintainer JOBS") + self.assertEqual(producers, maintenance_module.RUST_JOB_IDS, + "workflow cache producer names/IDs drifted from maintainer RUST_JOB_IDS") + + def record(self, image, *, source=SHA, owned=True, verified_run=None, base=BASE_IMAGE): + return { + "image": image, + "tag": ( + "genarrative/gitea-project-ci:rust-cache-auto-" + source + if source is not None else None + ), + "source": source, + "inputs": "inputs", + "base": base, + "owned": owned, + "activated": "2026-01-01T00:00:00+00:00", + "verified_run": verified_run, + "artifact": str(self.root / "state" / "artifacts" / (source or "manual")), + } + + def maintenance(self, state): + state_root = Path(self.config["state_dir"]) + state_root.mkdir(parents=True, exist_ok=True) + (state_root / "state.json").write_text(json.dumps(state), encoding="utf-8") + instance = maintenance_module.Maintenance(self.config) + instance.api = FakeApi() + return instance + + def activation_maintenance(self, *, idle_values, switch=None): + old = self.record(OLD_IMAGE, owned=False, verified_run=1) + candidate = self.record(IMAGE) + state = { + "versions": [old, candidate], + "current": OLD_IMAGE, + "rollback": None, + "candidate": IMAGE, + } + if switch is not None: + state["switch"] = switch + instance = self.maintenance(state) + config = {"value": runner_config(OLD_IMAGE)} + writes: list[str] = [] + restarts: list[tuple] = [] + statuses = 0 + + instance.check_gate_route = lambda: None + instance.read_config = lambda: config["value"] + instance.write_config = lambda value: (writes.append(value), config.__setitem__("value", value)) + instance.image_info = lambda image, inner=False: {"Id": image} + + def docker(*args, inner=False, **kwargs): + if args[:1] == ("restart",): + restarts.append(args) + return "" + if args[:2] == ("inspect", "--format"): + if args[2] == "{{.State.StartedAt}}": + return "2026-01-01T00:00:00+00:00" + if args[2] == "{{.State.Status}}": + return "running" + if args[:1] == ("logs",): + return "declare successfully" + raise AssertionError(f"unexpected docker call: {args}, inner={inner}") + + instance.docker = docker + iterator = iter(idle_values) + instance.idle = lambda: next(iterator) + + def gate(action): + nonlocal statuses + if action == "pause": + return {"paused": True} + if action == "resume": + return {"paused": False} + if action == "status": + statuses += 1 + return {"paused": False} if statuses == 1 else {"paused": True, "inflight": 0} + raise AssertionError(action) + + instance.gate = gate + return instance, writes, restarts + + def test_cache_inputs_keeps_openapi_and_embedded_skill_text(self): + tree = "\n".join( + [ + "100644 blob a\tdocs/guide.md", + "100644 blob b\tdocs/openapi/external.json", + "100644 blob c\t.codex/skills/example/SKILL.md", + "100644 blob d\tREADME.md", + ] + ) + digest = maintenance_module.cache_inputs(tree) + self.assertEqual(digest, maintenance_module.cache_inputs( + tree.replace("guide.md", "【说明】中文文档.md").replace("\n", "\0") + "\0")) + self.assertEqual(digest, maintenance_module.cache_inputs(tree.replace("guide.md", "other.md"))) + self.assertNotEqual(digest, maintenance_module.cache_inputs(tree.replace("external.json", "changed.json"))) + self.assertNotEqual(digest, maintenance_module.cache_inputs(tree.replace("SKILL.md", "PROMPT.md"))) + + def test_gate_route_requires_independent_checkout_and_live_runner_proof(self): + instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None}) + instance.read_config = lambda: runner_config(OLD_IMAGE) + with self.assertRaisesRegex(RuntimeError, "GENARRATIVE_GITEA_REPOSITORY_URL"): + instance.check_gate_route() + instance.read_config = lambda: runner_config(OLD_IMAGE) + ( + ' envs:\n GENARRATIVE_GITEA_REPOSITORY_URL: "http://gitea:3000/team/project.git"\n') + started = "2026-01-01T00:00:00Z" + def docker(*args, **kwargs): + if args[0] == "exec": + return json.dumps({"address": self.config["gate_url"]}) + return json.dumps([{"State": {"StartedAt": started}, "NetworkSettings": { + "Networks": {"internal": {"IPAddress": "10.0.0.2"}}}}]) + instance.docker = docker + proof = {"last_fetch_peer": "10.0.0.2", "last_fetch_at": maintenance_module.timestamp(started).timestamp() + 1, + "active_tasks": 0} + instance.gate = lambda action: proof + instance.check_gate_route() + proof["last_fetch_at"] -= 2 + with self.assertRaisesRegex(RuntimeError, "this runner startup"): + instance.check_gate_route() + proof.update(last_fetch_peer="10.0.0.3", last_fetch_at=9999999999) + with self.assertRaisesRegex(RuntimeError, "this runner startup"): + instance.check_gate_route() + + def test_complete_nine_job_run_requires_real_rust_cache_statistics(self): + run = {"status": "completed", "conclusion": "success"} + jobs = [{"name": name, "conclusion": "success"} for name in maintenance_module.JOBS] + self.assertTrue(maintenance_module.fully_passed(run, jobs)) + self.assertFalse(maintenance_module.fully_passed(run, jobs[:-1])) + good = ( + f"worker image: {IMAGE}\n[rust-cache] mode=sccache\n" + "Cache hits 7\nCache errors 0\nCache read errors 0\nCache write errors 0\n" + ) + self.assertTrue(maintenance_module.verifies_image({"name": "Backend tests"}, good, IMAGE)) + self.assertFalse(maintenance_module.verifies_image({"name": "Backend tests"}, good.replace("Cache hits 7", "Cache hits 0"), IMAGE)) + self.assertFalse(maintenance_module.verifies_image({"name": "Backend tests"}, good.replace("Cache write errors 0", "Cache write errors 1"), IMAGE)) + + def test_fetches_latest_master_without_ci_green_gate(self): + instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None}) + instance.repo.mkdir(parents=True) + (instance.repo / ".git").mkdir() + (instance.repo / ".git" / "genarrative-cache-maintenance").write_text("owned source checkout\n") + calls = [] + + def fake_command(*args, **kwargs): + calls.append(args) + if args[1:4] == ("remote", "get-url", "origin"): + return instance.config["clone_url"] + "\n" + if args[1:3] == ("rev-parse", "FETCH_HEAD^{commit}"): + return SHA + "\n" + if args[1:3] == ("ls-tree", "-rz"): + return "100644 blob x\tdocs/openapi/current.json\n" + return "" + + with patch.object(maintenance_module, "command", fake_command): + self.assertEqual(instance.fetch_source()[0], SHA) + self.assertIn(("git", "remote", "get-url", "origin"), calls) + self.assertIn(("git", "fetch", "--no-tags", "origin", "refs/heads/master"), calls) + self.assertIn(("git", "clean", "-ffdx"), calls) + self.assertFalse(any("runs" in " ".join(call) for call in calls)) + + def test_idle_race_after_pause_restores_without_restart(self): + instance, writes, restarts = self.activation_maintenance(idle_values=[True, False]) + with patch.object(maintenance_module.time, "sleep", lambda _: None): + self.assertFalse(instance.activate()) + self.assertEqual(restarts, []) + self.assertEqual(writes, []) + self.assertEqual(instance.state["current"], OLD_IMAGE) + self.assertEqual(instance.state["candidate"], IMAGE) + self.assertNotIn("pause_owned", instance.state) + + def test_idle_requires_task_ledger_and_all_live_container_states(self): + instance = self.maintenance({"versions": [], "current": None}) + state = {"uncertain": False, "active_tasks": 1} + instance.gate = lambda _: state + calls = [] + instance.docker = lambda *args, **kw: calls.append((args, kw)) or "" + self.assertFalse(instance.idle()) # assigned task before its first container + self.assertEqual(calls, []) + state["active_tasks"] = 0 + self.assertTrue(instance.idle()) + self.assertIn("status=created", calls[-1][0]) + self.assertTrue(calls[-1][1]["inner"]) + instance.docker = lambda *_a, **_k: "created-container\n" + self.assertFalse(instance.idle()) + del state["active_tasks"] + with self.assertRaisesRegex(RuntimeError, "durable task tracking"): + instance.idle() + + def source_fixture(self): + old = self.record(OLD_IMAGE, source="b" * 40, owned=False, verified_run=1) + instance = self.maintenance({"versions": [old], "current": OLD_IMAGE}) + run = {"id": 44, "status": "completed", "conclusion": "failure", "event": "push", + "path": "project-ci.yml@refs/heads/master", "head_branch": "master", "head_sha": SHA} + jobs = [{"id": i, "name": name, "status": "completed", "conclusion": "failure", + "head_sha": SHA, "run_attempt": 1, + "steps": [{"name": maintenance_module.EXPORT_STEP, "conclusion": "success"}]} + for i, name in enumerate(maintenance_module.RUST_JOB_IDS, start=1)] + artifacts = [{"id": job["id"], "name": "rust-cache-v1-" + maintenance_module.RUST_JOB_IDS[job["name"]] + + "-attempt-1", "expired": False, "workflow_run": run} for job in jobs] + class Api: + def pages(self, path, key): + return iter({"workflow_runs": [run], "jobs": jobs, "artifacts": artifacts}[key]) + def request(self, path, *, method, raw=False): + job_id = int(path.split("/jobs/")[1].split("/")[0]) + name = next(item["name"] for item in artifacts if item["id"] == job_id) + return f"worker image: {OLD_IMAGE}\n[rust-cache] artifact={name} objects=1 bytes=10 complete=true\n" + instance.api = Api() + return instance, run, jobs, artifacts + + def select_source(self, instance): + with patch.object(maintenance_module.subprocess, "run", return_value=type("Result", (), {"returncode": 0})()), \ + patch.object(maintenance_module, "command", lambda *args, **kw: "100644 blob " + args[-1] + "\tsrc.rs\0"): + return instance.source_run() + + def test_complete_failed_master_can_supply_cache_but_pr_and_missing_groups_cannot(self): + instance, run, jobs, artifacts = self.source_fixture() + result = self.select_source(instance) + self.assertEqual(result["run_id"], 44) + self.assertEqual(len(result["exports"]), 6) + run["event"] = "pull_request" + self.assertIsNone(self.select_source(instance)) + run["event"] = "push" + artifacts.pop() + self.assertIsNone(self.select_source(instance)) + + def test_stale_attempt_cancelled_job_or_failed_export_cannot_supply_cache(self): + instance, run, jobs, artifacts = self.source_fixture() + run["conclusion"] = "cancelled" + self.assertIsNone(self.select_source(instance)) # all six artifacts may already exist + run["conclusion"] = "failure" + jobs[0]["run_attempt"] = 2 + self.assertIsNone(self.select_source(instance)) + jobs[0]["run_attempt"] = 1 + jobs[0]["conclusion"] = "cancelled" + self.assertIsNone(self.select_source(instance)) + jobs[0]["conclusion"] = "success" + jobs[0]["steps"][0]["conclusion"] = "failure" + self.assertIsNone(self.select_source(instance)) + + def test_exports_using_different_images_are_not_combined(self): + instance, _, _, _ = self.source_fixture() + original = instance.api.request + instance.api.request = lambda path, **kwargs: original(path, **kwargs).replace(OLD_IMAGE, IMAGE) if '/jobs/1/' in path else original(path, **kwargs) + self.assertIsNone(self.select_source(instance)) + + def test_completed_artifact_without_upload_completion_log_is_not_used(self): + instance, _, _, _ = self.source_fixture() + instance.api.request = lambda *_a, **_k: f"worker image: {OLD_IMAGE}\n" + self.assertIsNone(self.select_source(instance)) + + def test_assembly_uses_ci_objects_and_trusted_binary_without_warming_compiler(self): + instance, _, _, _ = self.source_fixture() + source = self.select_source(instance) + shell_calls, docker_calls = [], [] + instance.build_command = lambda log, script, *args, **kw: shell_calls.append((script, args)) + def info(image, inner=False): + return {"Id": IMAGE if image.startswith("genarrative/") else image, "Config": {"Labels": { + "world.genarrative.ci.rust-cache-source": "b" * 40, + "world.genarrative.ci.rust-cache-base": BASE_IMAGE, + "com.genarrative.ci.definition-sha256": "definition", + }}} + instance.image_info = info + def docker(*args, **kw): + docker_calls.append(args) + if args[0] == "create": + return "temporary-copy-container" + if args[0] == "cp": + root = Path(args[-1]) + (root / "objects").mkdir() + (root / "sccache").write_bytes(b"trusted binary") + if args[-2:] == ("rustc", "-vV"): + return "rustc test\n" + return "" + instance.docker = docker + instance.api.download = lambda path, destination: destination.write_bytes(b"download") + def merge(inputs, output, **kwargs): + self.assertEqual(len(inputs), 6) + self.assertEqual(kwargs["expected_inherited_source_sha"], "b" * 40) + output.mkdir() + (output / "objects").mkdir() + return type("Merged", (), {"sccache_version": "sccache 0.18.0", "rustc": "rustc test\n", + "workspace": "/workspace/team/project", "base_image": BASE_IMAGE})() + with patch.object(maintenance_module, "command", return_value="definition\n"), \ + patch.object(maintenance_module, "merge_snapshots", merge): + instance.build(source) + self.assertEqual(shell_calls, [("gitea-ci-job-image.sh", ("verify", "genarrative/gitea-project-ci:rust-cache-auto-" + SHA))]) + self.assertIn(("create", OLD_IMAGE), docker_calls) + self.assertIn(("rm", "--volumes", "temporary-copy-container"), docker_calls) + self.assertEqual(instance.state["candidate"], IMAGE) + self.assertEqual(instance.version(IMAGE)["run_id"], 44) + + def test_export_cleanup_only_deletes_collected_or_expired_owned_master_artifacts(self): + current = self.record(OLD_IMAGE, owned=False, verified_run=1) + current.update(staged=True, run_id=44, exports=[{"id": 1}]) + instance = self.maintenance({"versions": [current], "current": OLD_IMAGE}) + old_run = {"id": 9, "status": "completed", "path": "project-ci.yml@refs/heads/master", + "event": "push", "head_branch": "master", "head_sha": SHA} + artifact = {"id": 2, "name": "rust-cache-v1-backend-tests-attempt-1", + "workflow_run": {"id": 9}, "created_at": "2020-01-01T00:00:00Z"} + rows = [artifact, {**artifact, "id": 3, "name": "release-binary"}, + {**artifact, "id": 4, "workflow_run": {"id": 10}}, + {**artifact, "id": 5, "workflow_run": {"id": 11}}] + deleted = [] + instance.api.pages = lambda *_: iter(rows) + def request(path, *, method): + if method == "DELETE": + return deleted.append((path, method)) + return {9: old_run, 10: {**old_run, "event": "pull_request"}, + 11: {**old_run, "status": "in_progress"}}[int(path.rsplit("/", 1)[1])] + instance.api.request = request + instance.cleanup_exports() + self.assertEqual(deleted, [(instance.repo_api + "/actions/artifacts/1", "DELETE"), + (instance.repo_api + "/actions/artifacts/2", "DELETE")]) + self.assertTrue(instance.version(OLD_IMAGE)["exports"][0]["deleted"]) + + def test_signed_download_drops_token_and_rejects_other_origins(self): + api = maintenance_module.Api(self.config["api_url"], self.token) + destination = self.root / "artifact.zip" + seen = [] + target = ["https://gitea.example.test/api/v1/repos/team/project/actions/artifacts/1/zip/raw?sig=test"] + class Opener: + def open(self, request, timeout): + seen.append(request) + if not isinstance(request, str): + raise maintenance_module.urllib.error.HTTPError(request.full_url, 302, "Found", {"Location": target[0]}, None) + return io.BytesIO(b"archive") + with patch.object(maintenance_module.urllib.request, "build_opener", return_value=Opener()): + api.download("repos/team/project/actions/artifacts/1/zip", destination) + self.assertEqual(destination.read_bytes(), b"archive") + self.assertEqual(seen[0].get_header("Authorization"), "token test-token") + self.assertIsInstance(seen[1], str) # signed URL request has no Authorization header + target[0] = "https://other.example.test/download" + with self.assertRaisesRegex(RuntimeError, "configured HTTPS Gitea origin"): + api.download("repos/team/project/actions/artifacts/1/zip", self.root / "rejected.zip") + self.assertFalse((self.root / "rejected.zip").exists()) + + def test_pending_chunk_cleanup_requires_old_terminal_master_run(self): + instance = self.maintenance({"versions": [], "current": None, "attempts": [{"run_id": 4}]}) + instance.config["artifact_storage_dir"] = str(self.root / "storage") + run = {"id": 1, "status": "completed", "path": "project-ci.yml@refs/heads/master", + "event": "push", "head_branch": "master", "head_sha": SHA, + "completed_at": "2020-01-01T00:00:00Z"} + rows = [run, {**run, "id": 2, "event": "pull_request"}, + {**run, "id": 3, "status": "in_progress"}, {**run, "id": 4}, + {**run, "id": 5, "completed_at": "2999-01-01T00:00:00Z"}] + instance.api.pages = lambda *_: iter(rows) + with patch.object(maintenance_module, "cleanup_upload_chunks", return_value=0) as cleanup: + instance.cleanup_pending_uploads() + self.assertEqual(cleanup.call_args.args[1], {1}) + + def test_uncertain_or_inflight_gate_never_restarts(self): + instance, _, restarts = self.activation_maintenance(idle_values=[True]) + statuses = 0 + + def uncertain_gate(action): + nonlocal statuses + if action == "pause": + return {"paused": True} + if action == "resume": + return {"paused": False} + statuses += 1 + return {"paused": False} if statuses == 1 else {"paused": True, "uncertain": True, "inflight": 1} + + instance.gate = uncertain_gate + with patch.object(maintenance_module.time, "sleep", lambda _: None): + with self.assertRaisesRegex(RuntimeError, "uncertain"): + instance.activate() + self.assertEqual(restarts, []) + self.assertEqual(instance.state["candidate"], IMAGE) + self.assertEqual(instance.state["current"], OLD_IMAGE) + self.assertNotIn("pause_owned", instance.state) + + def test_inflight_gate_defers_without_restart_and_resumes(self): + instance, _, restarts = self.activation_maintenance(idle_values=[True]) + statuses = 0 + + def inflight_gate(action): + nonlocal statuses + if action == "pause": + return {"paused": True} + if action == "resume": + return {"paused": False} + statuses += 1 + return {"paused": False} if statuses == 1 else {"paused": True, "inflight": 1} + + instance.gate = inflight_gate + with patch.object(maintenance_module.time, "sleep", lambda _: None): + self.assertFalse(instance.activate()) + self.assertEqual(restarts, []) + self.assertEqual(instance.state["candidate"], IMAGE) + self.assertEqual(instance.state["current"], OLD_IMAGE) + self.assertNotIn("pause_owned", instance.state) + + def test_interrupted_switch_recovers_current_candidate_and_rollback(self): + backup = self.root / "state" / "backups" / "switch.yaml" + backup.parent.mkdir(parents=True) + backup.write_text(runner_config(OLD_IMAGE), encoding="utf-8") + instance, _, restarts = self.activation_maintenance( + idle_values=[True, True], + switch={"old": OLD_IMAGE, "new": IMAGE, "backup": str(backup)}, + ) + with patch.object(maintenance_module.time, "sleep", lambda _: None): + self.assertTrue(instance.recover_switch()) + self.assertEqual(len(restarts), 1) + self.assertEqual(instance.state["current"], IMAGE) + self.assertIsNone(instance.state["candidate"]) + self.assertEqual(instance.state["rollback"], OLD_IMAGE) + self.assertNotIn("switch", instance.state) + + def test_protected_images_include_state_bases_config_and_container_references(self): + current = self.record(OLD_IMAGE, owned=False, verified_run=1, base=BASE_IMAGE) + candidate = self.record(IMAGE, base=BASE_IMAGE) + container_image = "sha256:" + "4" * 64 + instance = self.maintenance({"versions": [current, candidate], "current": OLD_IMAGE, "rollback": OLD_IMAGE, "candidate": IMAGE}) + instance.read_config = lambda: runner_config(OLD_IMAGE) + + def docker(*args, inner=False, **kwargs): + if args[:2] == ("ps", "-aq"): + return "container-1\n" + if args[:2] == ("inspect", "--format"): + return container_image + "\n" + raise AssertionError(args) + + instance.docker = docker + protected = instance.protected_images() + self.assertTrue({OLD_IMAGE, IMAGE, BASE_IMAGE, container_image}.issubset(protected)) + + def test_cleanup_keeps_manual_images_and_rejects_outside_artifacts(self): + current = self.record(OLD_IMAGE, owned=False, verified_run=1) + manual = self.record(IMAGE, owned=False) + stale = self.record("sha256:" + "5" * 64, source="b" * 40) + stale["artifact"] = str(self.root / "outside") + instance = self.maintenance({"versions": [current, manual, stale], "current": OLD_IMAGE, "rollback": None, "candidate": None}) + instance.protected_images = lambda: set() + instance.remove_owned_image = lambda *_: True + with self.assertRaisesRegex(RuntimeError, "outside"): + instance.cleanup() + self.assertIn(manual, instance.state["versions"]) + self.assertIn(stale, instance.state["versions"]) + + def test_additional_tags_are_never_deleted(self): + instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None}) + removals = [] + tag = "genarrative/gitea-project-ci:rust-cache-auto-" + SHA + instance.image_info = lambda image, inner=False: {"RepoTags": [tag, "operator:keep"]} + instance.docker = lambda *args, inner=False, **kwargs: ( + IMAGE + "\n" if args[:2] == ("image", "ls") else removals.append(args) or "" + ) + self.assertFalse(instance.remove_owned_image(IMAGE, tag, set())) + self.assertEqual(removals, []) + + def test_owned_image_removal_handles_dangling_and_already_absent_images(self): + instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None}) + tag = "genarrative/gitea-project-ci:rust-cache-auto-" + SHA + present = {True: True, False: True} + removals = [] + + def docker(*args, inner=False, **kwargs): + if args[:2] == ("image", "ls"): + return IMAGE + "\n" if present[inner] else "" + if args[:3] == ("image", "rm", IMAGE): + removals.append((args, inner)) + present[inner] = False + return IMAGE + "\n" + raise AssertionError(args) + + instance.docker = docker + instance.image_info = lambda image, inner=False: {"RepoTags": []} + self.assertTrue(instance.remove_owned_image(IMAGE, tag, set())) + self.assertEqual({inner for _, inner in removals}, {True, False}) + calls = [] + instance.docker = lambda *args, inner=False, **kwargs: calls.append((args, inner)) or "" + self.assertTrue(instance.remove_owned_image( + IMAGE, "genarrative/gitea-project-ci:rust-cache-auto-" + SHA, set())) + self.assertFalse(any(args[:3] == ("image", "rm", IMAGE) for args, _ in calls)) + + def test_manual_current_without_cache_stats_can_be_verified(self): + current = self.record(OLD_IMAGE, source=None, owned=False) + instance = self.maintenance({"versions": [current], "current": OLD_IMAGE, "rollback": None, "candidate": None}) + jobs = [ + {"id": index, "name": name, "conclusion": "success"} + for index, name in enumerate(sorted(maintenance_module.JOBS), start=1) + ] + run = { + "id": 44, + "status": "completed", + "conclusion": "success", + "event": "push", + "path": "project-ci.yml@refs/heads/master", + "head_sha": SHA, + "started_at": "2026-01-02T00:00:00Z", + } + + class VerificationApi: + def pages(self, path, key): + if path.endswith("/actions/runs?status=success&branch=master&event=push"): + return iter([run]) + if path.endswith("/actions/runs/44/jobs"): + return iter(jobs) + raise AssertionError(path) + + def request(self, path, *, method, body=None, raw=False): + if not raw or "/actions/jobs/" not in path: + raise AssertionError(path) + return f"runner image: {OLD_IMAGE}\n" + + instance.api = VerificationApi() + with patch.object(maintenance_module.subprocess, "run", return_value=type("Result", (), {"returncode": 0})()): + self.assertTrue(instance.verify_current()) + self.assertEqual(instance.version(OLD_IMAGE)["verified_run"], 44) + self.assertEqual(instance.version(OLD_IMAGE)["verified_sha"], SHA) + + def test_recover_builds_adopts_only_pre_registered_attempt_tags(self): + tag = "genarrative/gitea-project-ci:rust-cache-auto-" + SHA + base_tag = "genarrative/gitea-project-ci:base-auto-" + SHA + artifact = self.root / "state" / "artifacts" / SHA + attempt = {"source": SHA, "inputs": "inputs", "tag": tag, "base_tag": base_tag, "artifact": str(artifact)} + instance = self.maintenance({ + "versions": [], "current": None, "rollback": None, "candidate": None, + "attempts": [attempt], + }) + queries = [] + + def docker(*args, inner=False, **kwargs): + if args[:3] == ("image", "ls", "--no-trunc"): + queries.append(args[-1]) + return BASE_IMAGE + "\n" if args[-1] == base_tag else IMAGE + "\n" + raise AssertionError(args) + + instance.docker = docker + instance.image_info = lambda image, inner=False: { + "Config": {"Labels": { + "world.genarrative.ci.rust-cache-source": SHA, + "world.genarrative.ci.rust-cache-base": BASE_IMAGE, + }} + } + instance.recover_builds() + self.assertEqual(queries, [base_tag, tag]) + self.assertEqual(instance.state["attempts"], []) + self.assertEqual(instance.state["bases"], {BASE_IMAGE: base_tag}) + self.assertEqual(instance.state["versions"][0]["image"], IMAGE) + self.assertTrue(instance.state["versions"][0]["owned"]) + self.assertEqual(instance.state["candidate"], IMAGE) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_gitea_cache_snapshot.py b/scripts/test_gitea_cache_snapshot.py new file mode 100644 index 000000000..135cd903b --- /dev/null +++ b/scripts/test_gitea_cache_snapshot.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Tests for streamed Gitea sccache snapshot validation and merging.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import os +import sys +import tarfile +import tempfile +import unittest +import zipfile + + +SCRIPT = Path(__file__).with_name("gitea_cache_snapshot.py") +SPEC = importlib.util.spec_from_file_location("gitea_cache_snapshot", SCRIPT) +assert SPEC and SPEC.loader +snapshot = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = snapshot +SPEC.loader.exec_module(snapshot) + +REPOSITORY = "team/project" +SOURCE_SHA = "a" * 40 +INHERITED_SOURCE_SHA = "c" * 40 +BASE_IMAGE = "sha256:" + "b" * 64 +RUSTC = "rustc 1.90.0\nbinary: rustc\n" +WORKSPACE = "/workspace/GenarrativeAI/Genarrative" +SCCACHE_VERSION = "0.18.0" + + +def object_path(character: str) -> str: + key = character * 64 + return f"objects/{key[0]}/{key[1]}/{key}" + + +class SnapshotMergeTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + self.root = Path(self.temporary_directory.name).resolve() + self.base = self.root / "base-objects" + self.base.mkdir() + + def tearDown(self) -> None: + self.temporary_directory.cleanup() + + def identity(self, job: str, run_id: int) -> object: + return snapshot.ArtifactIdentity(REPOSITORY, run_id, 1, job, SOURCE_SHA) + + def archive( + self, + name: str, + job: str, + run_id: int, + objects: list[tuple[str, bytes, int]], + *, + prefix: bool = True, + manifest_update=None, + extra_member: tuple[str, bytes, str] | None = None, + touched: list[tuple[str, int]] | None = None, + ) -> object: + rows = [ + { + "path": path, + "size": len(contents), + "sha256": hashlib.sha256(contents).hexdigest(), + "mtime_ns": mtime_ns, + } + for path, contents, mtime_ns in objects + ] + manifest = { + "schema": 1, + "repository": REPOSITORY, + "run_id": run_id, + "run_attempt": 1, + "job": job, + "source_sha": SOURCE_SHA, + "base_image": BASE_IMAGE, + "rustc": RUSTC, + "workspace": WORKSPACE, + "sccache_version": SCCACHE_VERSION, + "mode": "delta", + "inherited_source_sha": INHERITED_SOURCE_SHA, + "objects": rows, + "touched": [ + {"path": path, "mtime_ns": mtime_ns} + for path, mtime_ns in (touched or []) + ], + } + if manifest_update is not None: + manifest_update(manifest) + + tar_bytes = io.BytesIO() + with tarfile.open(fileobj=tar_bytes, mode="w") as bundle: + for path, contents, _ in objects: + info = tarfile.TarInfo(path) + info.size = len(contents) + bundle.addfile(info, io.BytesIO(contents)) + if extra_member is not None: + path, contents, kind = extra_member + info = tarfile.TarInfo(path) + if kind == "symlink": + info.type = tarfile.SYMTYPE + info.linkname = "manifest.json" + bundle.addfile(info) + else: + info.size = len(contents) + bundle.addfile(info, io.BytesIO(contents)) + manifest_bytes = json.dumps(manifest, separators=(",", ":")).encode() + info = tarfile.TarInfo("manifest.json") + info.size = len(manifest_bytes) + bundle.addfile(info, io.BytesIO(manifest_bytes)) + + archive = self.root / f"{name}.zip" + member = f"{name}/snapshot.tar" if prefix else "snapshot.tar" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_STORED) as bundle: + bundle.writestr(member, tar_bytes.getvalue()) + return snapshot.ArtifactInput(archive, self.identity(job, run_id)) + + def base_object(self, path: str, contents: bytes, mtime_ns: int) -> None: + target = self.base.joinpath(*Path(path).parts[1:]) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(contents) + os.utime(target, ns=(mtime_ns, mtime_ns)) + + def merge(self, inputs, output=None, maximum=1024 ** 3): + return snapshot.merge_snapshots( + inputs, + output or self.root / "merged", + base_objects=self.base, + expected_inherited_source_sha=INHERITED_SOURCE_SHA, + max_combined_bytes=maximum, + ) + + def test_merges_groups_deduplicates_newest_mtime_and_prunes_to_bound(self) -> None: + first_path = object_path("a") + second_path = object_path("b") + third_path = object_path("c") + inherited_path = object_path("d") + self.base_object(inherited_path, b"dd", 50) + inputs = [ + self.archive("first", "Backend tests", 41, [ + (first_path, b"aaaa", 100), + (second_path, b"bbbb", 200), + ]), + self.archive("second", "Native shell tests", 41, [ + (first_path, b"aaaa", 300), + (third_path, b"ccc", 250), + ], prefix=False, touched=[(inherited_path, 400)]), + ] + + output = self.root / "merged" + result = self.merge(inputs, output, maximum=9) + + self.assertEqual((result.object_count, result.total_bytes), (3, 9)) + self.assertEqual((output / inherited_path).read_bytes(), b"dd") + self.assertEqual((output / first_path).read_bytes(), b"aaaa") + self.assertEqual((output / third_path).read_bytes(), b"ccc") + self.assertFalse((output / second_path).exists()) + self.assertEqual((output / "rustc.txt").read_text(), RUSTC) + self.assertEqual((output / "workspace.txt").read_text(), WORKSPACE + "\n") + self.assertEqual((output / "source-commit.txt").read_text(), SOURCE_SHA + "\n") + self.assertEqual((output / "base-image.txt").read_text(), BASE_IMAGE + "\n") + self.assertFalse((output / "sccache").exists()) + + def test_rejects_traversal_member_without_creating_output(self) -> None: + item = self.archive( + "traversal", "Backend tests", 42, [], + extra_member=("../outside", b"bad", "file"), + ) + output = self.root / "merged" + with self.assertRaisesRegex(snapshot.SnapshotError, "invalid sccache object path"): + self.merge([item], output) + self.assertFalse(output.exists()) + self.assertFalse((self.root / "outside").exists()) + + def test_rejects_checksum_mismatch(self) -> None: + path = object_path("d") + + def corrupt(manifest): + manifest["objects"][0]["sha256"] = "0" * 64 + + item = self.archive( + "checksum", "Backend tests", 43, [(path, b"content", 1)], + manifest_update=corrupt, + ) + with self.assertRaisesRegex(snapshot.SnapshotError, "checksums do not match"): + self.merge([item]) + + def test_rejects_manifest_identity_mismatch(self) -> None: + item = self.archive( + "identity", "Backend tests", 44, [], + manifest_update=lambda manifest: manifest.update(run_id=999), + ) + with self.assertRaisesRegex(snapshot.SnapshotError, "run_id does not match"): + self.merge([item]) + + def test_rejects_conflicting_duplicate_key(self) -> None: + path = object_path("e") + inputs = [ + self.archive("conflict-one", "Backend tests", 45, [(path, b"one", 1)]), + self.archive("conflict-two", "Native shell tests", 45, [(path, b"two", 2)]), + ] + with self.assertRaisesRegex(snapshot.SnapshotError, "conflicting content"): + self.merge(inputs) + + def test_rejects_delta_that_conflicts_with_inherited_key(self) -> None: + path = object_path("f") + self.base_object(path, b"base", 1) + item = self.archive("base-conflict", "Backend tests", 46, [(path, b"evil", 2)]) + with self.assertRaisesRegex(snapshot.SnapshotError, "conflicting content"): + self.merge([item]) + + def test_accepts_empty_delta(self) -> None: + item = self.archive("empty", "Backend tests", 47, []) + output = self.root / "merged" + result = self.merge([item], output) + self.assertEqual((result.object_count, result.total_bytes), (0, 0)) + self.assertEqual(list((output / "objects").iterdir()), []) + + def test_rejects_noncanonical_base_entry(self) -> None: + (self.base / "unexpected").write_bytes(b"not an object") + item = self.archive("base-path", "Backend tests", 48, []) + with self.assertRaisesRegex(snapshot.SnapshotError, "noncanonical entry"): + self.merge([item]) + + def test_rejects_tar_links(self) -> None: + item = self.archive( + "link", "Backend tests", 49, [], + extra_member=(object_path("9"), b"", "symlink"), + ) + with self.assertRaisesRegex(snapshot.SnapshotError, "regular file"): + self.merge([item]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_gitea_cache_upload_cleanup.py b/scripts/test_gitea_cache_upload_cleanup.py new file mode 100644 index 000000000..6ebe0d200 --- /dev/null +++ b/scripts/test_gitea_cache_upload_cleanup.py @@ -0,0 +1,90 @@ +"""缓存上传残块清理只触碰已确认结束的过期自有文件。""" + +import base64 +import os +from pathlib import Path +import tempfile +import unittest + +from gitea_cache_upload_cleanup import cleanup_upload_chunks, decode_owner + + +class UploadCleanupTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + + def file(self, name, run=12, age=10): + directory = self.root / "tmp-upload" / f"run-{run}-v4" + directory.mkdir(parents=True, exist_ok=True) + path = directory / name + path.write_bytes(b"block data") + os.utime(path, (age, age)) + return path + + def block(self, artifact=34, run=12, age=10, marker="genarrative-rust-cache-v1:backend-tests:1:00000000"): + encoded = base64.urlsafe_b64encode(base64.b64encode(marker.encode())).decode() + return self.file(f"block-{run}-{artifact}-10-{encoded}", run, age) + + def test_old_owned_blocks_and_associated_blocklist_are_deleted(self): + block = self.block() + blocklist = self.file("12-34-blocklist") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 2) + self.assertFalse(block.exists()) + self.assertFalse(blocklist.exists()) + self.assertTrue(block.parent.is_dir()) + + def test_other_runs_artifacts_and_unproven_blocklists_remain(self): + own = self.block() + other_run = self.block(run=13) + other_artifact = self.block(artifact=35, marker="unrelated-upload") + unproven = self.file("12-36-blocklist") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 1) + self.assertFalse(own.exists()) + self.assertTrue(all(path.exists() for path in (other_run, other_artifact, unproven))) + + def test_young_or_unknown_member_protects_entire_artifact_group(self): + own = self.block() + young = self.file("12-34-blocklist", age=20) + other = self.block(artifact=35) + unknown = self.file("block-12-35-invalid-size-unknown") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 0) + self.assertTrue(all(path.exists() for path in (own, young, other, unknown))) + + def test_completely_unknown_file_protects_entire_run(self): + block = self.block() + self.file("unknown") + self.assertEqual(cleanup_upload_chunks(self.root, {12}, 20), 0) + self.assertTrue(block.exists()) + + @unittest.skipUnless(os.name == "posix", "symlink fixture requires POSIX") + def test_symlinks_never_traversed_or_deleted(self): + block = self.block() + outside = self.root / "secret" + outside.write_text("keep") + (block.parent / "12-34-blocklist").symlink_to(outside) + (self.root / "tmp-upload" / "run-13-v4").symlink_to(block.parent, target_is_directory=True) + self.assertEqual(cleanup_upload_chunks(self.root, {12, 13}, 20), 0) + self.assertTrue(block.exists()) + self.assertEqual(outside.read_text(), "keep") + linked = self.root / "root-link" + linked.symlink_to(self.root, target_is_directory=True) + with self.assertRaises(ValueError): + cleanup_upload_chunks(linked, {12}, 20) + + def test_root_missing_or_relative_is_an_error(self): + with self.assertRaises(FileNotFoundError): + cleanup_upload_chunks(self.root / "missing", {12}, 20) + with self.assertRaises(ValueError): + cleanup_upload_chunks(Path("relative"), {12}, 20) + + def test_noncanonical_encoding_and_unknown_job_are_not_owned(self): + for marker in ("genarrative-rust-cache-v1:other-job:1:00000000", "genarrative-rust-cache-v1:backend-tests:01:00000000"): + encoded = base64.urlsafe_b64encode(base64.b64encode(marker.encode())).decode() + self.assertIsNone(decode_owner(encoded)) + self.assertIsNone(decode_owner("not-base64")) + + +if __name__ == "__main__": + unittest.main()