Merge remote-tracking branch 'origin/master' into codex/clear-retired-tables-phase2
Project CI / AI game creator shell Rust smoke (pull_request) Successful in 1m56s
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Successful in 6m55s
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Successful in 8m20s
Project CI / Backend tests (pull_request) Successful in 7m15s
Project CI / Native shell tests (pull_request) Successful in 8m18s
Project CI / Frontend tests (pull_request) Successful in 3m28s
Project CI / AI game creator shell web tests (pull_request) Successful in 3m18s
Project CI / Repository checks (pull_request) Successful in 3m38s
Project CI / AI game creator shell Rust crates (pull_request) Has been cancelled

This commit is contained in:
2026-09-22 19:43:10 +08:00
13 changed files with 580 additions and 64 deletions
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env node
// 持久下载缓存可以保留旧版本;交付给 CI 镜像的快照只带当前 lock 已下载的包。
import fs from 'node:fs/promises';
import { createRequire } from 'node:module';
import path from 'node:path';
import { pipeline } from 'node:stream/promises';
const [npmRoot, lockPath, source, destination] = process.argv.slice(2);
if (!npmRoot || !lockPath || !source || !destination) {
throw new Error(
'usage: export-ci-npm-download-cache.mjs <npm-root> <lock> <source> <destination>',
);
}
const require = createRequire(path.resolve(npmRoot, 'package.json'));
const cacache = require('cacache');
const lock = JSON.parse(await fs.readFile(lockPath, 'utf8'));
const integrities = new Set(
Object.values(lock.packages).flatMap((entry) =>
typeof entry.integrity === 'string' ? [entry.integrity] : [],
),
);
let count = 0;
for await (const entry of cacache.ls.stream(source)) {
if (!integrities.has(entry.integrity)) continue;
await pipeline(
cacache.get.stream(source, entry.key, { integrity: entry.integrity }),
cacache.put.stream(destination, entry.key, {
integrity: entry.integrity,
metadata: entry.metadata,
}),
);
count += 1;
}
console.log(
`[ci-image] exported ${count} npm cache entries for the current lock`,
);
@@ -0,0 +1,83 @@
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { existsSync } from 'node:fs';
import fs from 'node:fs/promises';
import { createRequire } from 'node:module';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { fileURLToPath } from 'node:url';
const npmRoot = [
path.resolve(path.dirname(process.execPath), 'node_modules/npm'),
path.resolve(path.dirname(process.execPath), '../lib/node_modules/npm'),
...(process.env.npm_execpath
? [path.resolve(path.dirname(process.env.npm_execpath), '..')]
: []),
].find((root) => existsSync(path.join(root, 'node_modules/cacache')));
assert.ok(npmRoot, 'tests require the cacache bundled with npm');
const cacache = createRequire(path.join(npmRoot, 'package.json'))('cacache');
const script = fileURLToPath(
new URL('./export-ci-npm-download-cache.mjs', import.meta.url),
);
async function fixture(t) {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ci-npm-snapshot-'));
t.after(() => fs.rm(root, { recursive: true, force: true }));
const source = path.join(root, 'source');
const destination = path.join(root, 'destination');
const lock = path.join(root, 'package-lock.json');
const key =
'make-fetch-happen:request-cache:https://registry.npmjs.org/example/-/example-1.0.0.tgz';
const metadata = {
url: key.slice('make-fetch-happen:request-cache:'.length),
};
const integrity = String(
await cacache.put(source, key, 'current-package', { metadata }),
);
await cacache.put(source, 'old-package', 'unused-old-version');
await fs.writeFile(
lock,
JSON.stringify({ packages: { 'node_modules/example': { integrity } } }),
);
const run = () =>
spawnSync(process.execPath, [script, npmRoot, lock, source, destination], {
encoding: 'utf8',
});
return { source, destination, key, metadata, integrity, run };
}
test('exports only current lock content, preserving npm request metadata for offline use', async (t) => {
const f = await fixture(t);
const result = f.run();
assert.equal(result.status, 0, result.stderr);
assert.deepEqual(Object.keys(await cacache.ls(f.destination)), [f.key]);
const output = await cacache.get(f.destination, f.key);
assert.equal(output.data.toString(), 'current-package');
assert.deepEqual(output.metadata, f.metadata);
// 输出是独立快照;移走持久缓存仍可使用,不依赖挂载、链接或旧 builder。
await fs.rm(f.source, { recursive: true });
assert.equal(
(await cacache.get(f.destination, f.key)).data.toString(),
'current-package',
);
});
test('rejects a corrupted cached package instead of publishing it', async (t) => {
const f = await fixture(t);
const digest = Buffer.from(f.integrity.split('-')[1], 'base64').toString(
'hex',
);
const contentPath = path.join(
f.source,
'content-v2',
'sha512',
digest.slice(0, 2),
digest.slice(2, 4),
digest.slice(4),
);
await fs.writeFile(contentPath, 'corrupted-package');
const result = f.run();
assert.notEqual(result.status, 0);
assert.match(result.stderr, /EINTEGRITY|EBADSIZE/);
});
+47 -3
View File
@@ -6,12 +6,31 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
dockerfile_context_path="deploy/container/gitea-ci-job.Dockerfile"
image_tag="${GENARRATIVE_GITEA_CI_IMAGE_TAG:-genarrative/gitea-project-ci:20260920.2}"
runner_container="${GENARRATIVE_GITEA_RUNNER_CONTAINER:-gitea-runner}"
builder_name="genarrative-ci-images"
prepare_builder() {
if ! docker buildx version >/dev/null 2>&1; then
echo 'Gitea CI image builds require the Docker Buildx plugin; see deploy/container/README.md.' >&2
return 1
fi
if ! docker buildx inspect "${builder_name}" >/dev/null 2>&1; then
docker buildx create --name "${builder_name}" --driver docker-container \
--driver-opt image=moby/buildkit:v0.23.2@sha256:ddd1ca44b21eda906e81ab14a3d467fa6c39cd73b9a39df1196210edcb8db59e \
--buildkitd-config "${repo_root}/deploy/container/gitea-ci-buildkitd.toml"
fi
if [[ "$(docker buildx inspect "${builder_name}" --format '{{.Driver}}')" != docker-container ]]; then
echo "${builder_name} must use the isolated docker-container driver" >&2
return 1
fi
}
write_build_context_file_list() {
printf '%s\0' \
deploy/container/gitea-ci-job.Dockerfile \
deploy/container/gitea-ci-job.Dockerfile.dockerignore \
deploy/container/gitea-ci-buildkitd.toml \
deploy/container/gitea-ci-checkout.sh \
scripts/export-ci-npm-download-cache.mjs \
package.json \
package-lock.json \
apps/admin-web/package.json \
@@ -29,8 +48,9 @@ write_build_context_file_list() {
server-rs/Cargo.lock \
apps/desktop-shell/src-tauri/Cargo.toml \
apps/desktop-shell/src-tauri/Cargo.lock
find server-rs/crates plugins/agc-*-editor/native/*-editor-bridge \
\( -name Cargo.toml -o -path 'plugins/agc-*-editor/native/*-editor-bridge/*' \) \
find server-rs/crates apps/ai-game-creator-shell/src-tauri/vendor \
plugins/agc-*-editor/native/*-editor-bridge \
-name Cargo.toml \
-type f -print0 \
| sort -z
}
@@ -39,6 +59,7 @@ usage() {
cat <<'EOF'
用法:
bash scripts/gitea-ci-job-image.sh build
bash scripts/gitea-ci-job-image.sh seed-downloads <可信 CI 镜像完整 Image ID>
bash scripts/gitea-ci-job-image.sh revision
bash scripts/gitea-ci-job-image.sh verify [镜像引用]
bash scripts/gitea-ci-job-image.sh load-runner [镜像引用]
@@ -66,6 +87,28 @@ verify_image() {
command_name="${1:-}"
case "${command_name}" in
seed-downloads)
# 运维显式指定的可信镜像只贡献下载包,不作为新基础镜像的父层。
seed_image="${2:-}"
[[ "${seed_image}" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo 'seed requires a full trusted Image ID' >&2; exit 2; }
prepare_builder
seed_dir="$(mktemp -d)"
seed_container=""
cleanup_seed() {
if [[ -n "${seed_container}" ]]; then docker rm --volumes "${seed_container}" >/dev/null; fi
rm -rf -- "${seed_dir}"
}
trap cleanup_seed EXIT
seed_container="$(docker create "${seed_image}")"
mkdir -p "${seed_dir}/cargo-cache" "${seed_dir}/cargo-index" "${seed_dir}/npm"
docker cp "${seed_container}:/usr/local/cargo/registry/cache/." "${seed_dir}/cargo-cache/"
docker cp "${seed_container}:/usr/local/cargo/registry/index/." "${seed_dir}/cargo-index/"
docker cp "${seed_container}:/root/.npm/_cacache/." "${seed_dir}/npm/"
docker buildx build --builder "${builder_name}" --progress plain \
--target download-cache-seed --no-cache-filter download-cache-seed \
--build-context "download-seed=${seed_dir}" \
--file "${repo_root}/${dockerfile_context_path}" "${seed_dir}"
;;
revision)
# 与 build 的 IMAGE_REVISION 使用同一份输入顺序,用于维护器判断基础镜像是否过期。
(
@@ -74,6 +117,7 @@ case "${command_name}" in
)
;;
build)
prepare_builder
image_revision="$(bash "${BASH_SOURCE[0]}" revision)"
npm_lock_sha256="$(sha256sum "${repo_root}/package-lock.json")"
npm_lock_sha256="${npm_lock_sha256%% *}"
@@ -87,7 +131,7 @@ case "${command_name}" in
cd "${repo_root}"
write_build_context_file_list \
| tar --null --create --file - --files-from=- \
| docker build \
| docker buildx build --builder "${builder_name}" --load --progress plain \
--pull=false \
--build-arg "IMAGE_REVISION=${image_revision}" \
--build-arg "NPM_LOCK_SHA256=${npm_lock_sha256}" \
+97 -53
View File
@@ -2,6 +2,7 @@
"""宿主专用的 Rust 缓存维护器;仅使用 Python 标准库,不在 CI job 中运行。"""
import argparse
import contextlib
import datetime
import hashlib
import json
@@ -59,6 +60,23 @@ def log(message):
print(f"[cache-maintenance] {message}", flush=True)
@contextlib.contextmanager
def operation(name, *, build_log=None):
"""Record long maintenance stages without exposing command arguments or API data."""
started = time.monotonic()
location = f"; build log={build_log}" if build_log is not None else ""
log(f"{name}: started{location}")
try:
yield
except Exception:
elapsed = time.monotonic() - started
log(f"{name}: failed after {elapsed:.1f}s{location}")
raise
else:
elapsed = time.monotonic() - started
log(f"{name}: completed in {elapsed:.1f}s")
def command(*args, cwd=None, data=None, env=None, output=None, timeout=120, combined=False):
result = subprocess.run(
args, cwd=cwd, input=data, text=True, env=env, timeout=timeout,
@@ -291,10 +309,11 @@ class Maintenance:
tree = command("git", "ls-tree", "-rz", sha, cwd=self.repo)
return sha, cache_inputs(tree)
def build_command(self, log_file, script, *args, env=None):
with log_file.open("a") as out:
command("bash", str(self.repo / "scripts" / script), *args, cwd=self.repo,
env=env, output=out, timeout=7200)
def build_command(self, log_file, script, *args, env=None, description=None):
with operation(description or f"run {script}", build_log=log_file):
with log_file.open("a") as out:
command("bash", str(self.repo / "scripts" / script), *args, cwd=self.repo,
env=env, output=out, timeout=7200)
def master_run(self, run):
return (run.get("path") == "project-ci.yml@refs/heads/master"
@@ -387,13 +406,17 @@ class Maintenance:
base_labels = self.image_info(base)["Config"].get("Labels") or {}
if base_labels.get("com.genarrative.ci.definition-sha256") != revision:
env["GENARRATIVE_GITEA_CI_IMAGE_TAG"] = base_tag
self.build_command(build_log, "gitea-ci-job-image.sh", "build", env=env)
self.build_command(build_log, "gitea-ci-job-image.sh", "build", env=env,
description="rebuild cache base image")
base = self.image_info(base_tag)["Id"]
self.state.setdefault("bases", {})[base] = base_tag
self.save()
else:
log("reuse compatible cache base image")
# 与旧缓存镜像分离;绝不把 Docker 可写层、源码或 target commit 成镜像。
self.docker("run", "--rm", "--network", "none", "--read-only", "--cap-drop=ALL",
"--entrypoint", "bash", base, "-c", "test ! -e /opt/genarrative-ci/rust-cache")
with operation("validate cache base image", build_log=build_log):
self.docker("run", "--rm", "--network", "none", "--read-only", "--cap-drop=ALL",
"--entrypoint", "bash", base, "-c", "test ! -e /opt/genarrative-ci/rust-cache")
with tempfile.TemporaryDirectory(prefix="assemble-", dir=artifact) as temporary:
work = Path(temporary)
inherited = work / "inherited"
@@ -406,21 +429,25 @@ class Maintenance:
inputs = []
for export in source["exports"]:
archive = work / (str(export["id"]) + ".zip")
self.api.download(self.repo_api + f'/actions/artifacts/{export["id"]}/zip', archive)
with operation(f"download cache artifact job={export['job']} attempt={export['attempt']}",
build_log=build_log):
self.api.download(self.repo_api + f'/actions/artifacts/{export["id"]}/zip', archive)
inputs.append(ArtifactInput(archive, ArtifactIdentity(
self.config["repository"], source["run_id"], export["attempt"], export["job"], sha)))
snapshot = work / "snapshot"
merged = merge_snapshots(inputs, snapshot, base_objects=inherited / "objects",
expected_inherited_source_sha=inherited_source)
with operation(f"merge {len(inputs)} cache artifacts", build_log=build_log):
merged = merge_snapshots(inputs, snapshot, base_objects=inherited / "objects",
expected_inherited_source_sha=inherited_source)
if merged.sccache_version != "sccache 0.18.0":
raise RuntimeError("unsupported sccache version")
if merged.base_image is not None and merged.base_image != labels["world.genarrative.ci.rust-cache-base"]:
raise RuntimeError("artifact base differs from its actual source image")
rustc = self.docker("run", "--rm", "--network", "none", "--read-only", base, "rustc", "-vV")
if rustc.strip() != merged.rustc.strip():
raise RuntimeError("artifact toolchain differs from target base image")
if merged.workspace != "/workspace/" + self.config["repository"]:
raise RuntimeError("artifact workspace differs from CI checkout")
with operation("validate merged snapshot against target image", build_log=build_log):
rustc = self.docker("run", "--rm", "--network", "none", "--read-only", base, "rustc", "-vV")
if rustc.strip() != merged.rustc.strip():
raise RuntimeError("artifact toolchain differs from target base image")
if merged.workspace != "/workspace/" + self.config["repository"]:
raise RuntimeError("artifact workspace differs from CI checkout")
shutil.copyfile(inherited / "sccache", snapshot / "sccache")
(snapshot / "sccache").chmod(0o755)
(snapshot / "base-image.txt").write_text(base + "\n")
@@ -429,9 +456,11 @@ class Maintenance:
f'LABEL world.genarrative.ci.rust-cache-source="{sha}"\n'
f'LABEL world.genarrative.ci.rust-cache-base="{base}"\n')
(work / ".dockerignore").write_text("**\n!Dockerfile\n!snapshot/\n!snapshot/**\n")
with build_log.open("a") as out:
self.docker("build", "--pull=false", "--tag", tag, str(work), output=out, timeout=1800)
self.build_command(build_log, "gitea-ci-job-image.sh", "verify", tag, env=env)
with operation("assemble cache candidate image", build_log=build_log):
with build_log.open("a") as out:
self.docker("build", "--pull=false", "--tag", tag, str(work), output=out, timeout=1800)
self.build_command(build_log, "gitea-ci-job-image.sh", "verify", tag, env=env,
description="verify cache candidate image")
image = self.image_info(tag)["Id"]
self.state["versions"].append({**attempt, "image": image, "base": base,
"owned": True, "verified_run": None})
@@ -449,17 +478,22 @@ class Maintenance:
sidecar = archive.with_suffix(".zst.sha256")
if (candidate.get("staged") and archive.is_file() and sidecar.is_file()
and candidate["image"] in self.docker("image", "ls", "--all", "--no-trunc", "--quiet", inner=True).split()):
log("reuse exported and loaded candidate image")
return
env = {**os.environ, "GENARRATIVE_GITEA_RUNNER_CONTAINER": self.runner}
build_log = artifact / "build.log"
if not sidecar.exists():
# 只删除登记目录中的未完成导出文件,不覆盖已验证归档。
archive.unlink(missing_ok=True)
self.build_command(build_log, "gitea-ci-job-image.sh", "export", str(archive), candidate["image"], env=env)
command("sha256sum", "--check", sidecar.name, cwd=artifact, timeout=600)
self.build_command(build_log, "gitea-ci-job-image.sh", "load-runner", candidate["image"], env=env)
if self.image_info(candidate["image"], inner=True)["Id"] != candidate["image"]:
raise RuntimeError("inner runner image mismatch")
self.build_command(build_log, "gitea-ci-job-image.sh", "export", str(archive), candidate["image"], env=env,
description="export cache candidate image")
with operation("validate exported cache candidate image", build_log=build_log):
command("sha256sum", "--check", sidecar.name, cwd=artifact, timeout=600)
self.build_command(build_log, "gitea-ci-job-image.sh", "load-runner", candidate["image"], env=env,
description="load cache candidate image into runner")
with operation("verify loaded cache candidate image", build_log=build_log):
if self.image_info(candidate["image"], inner=True)["Id"] != candidate["image"]:
raise RuntimeError("inner runner image mismatch")
candidate["staged"] = True
self.save()
@@ -473,6 +507,13 @@ class Maintenance:
"--filter", "status=created", "--filter", "status=restarting",
"--filter", "status=paused", inner=True).strip())
def wait_for_idle(self, purpose):
with operation(f"wait for idle runner before {purpose}"):
ready = self.idle()
if not ready:
log(f"runner is busy; defer {purpose}")
return ready
def verify_current(self):
current = self.version(self.state["current"])
if current.get("verified_run"):
@@ -628,12 +669,12 @@ class Maintenance:
if source is None:
log("waiting for a complete set of master CI cache exports")
return
log(f"selected cache source run={source['run_id']} source={source['source']}")
if not retry and self.state.get("failed_run") == source["run_id"]:
log(f'previous assembly failed at run={source["run_id"]}; waiting for new run or --retry')
return
# 下载、合并和镜像装载也消耗宿主 IO;繁忙时留给 CI,下轮再收集。
if not self.idle():
log("CI active; defer refresh")
if not self.wait_for_idle("cache assembly"):
return
try:
self.build(source)
@@ -737,7 +778,7 @@ class Maintenance:
if gate.get("paused") is not False and not self.state.get("pause_owned"):
log("runner gate paused by operator; defer switch")
return False
if not self.state.get("switch") and not self.idle():
if not self.state.get("switch") and not self.wait_for_idle("runner switch"):
log("CI active; candidate stays staged")
return False
# 先持久化恢复意图;控制请求超时也可能已生效,ExecStopPost/下次 tick 会恢复。
@@ -748,19 +789,20 @@ class Maintenance:
raise RuntimeError("runner pause could not be confirmed")
# 不用 FetchTask 客户端超时猜测服务端事务是否已经结束。
# 入口必须完整读完已转发的响应;不确定时拒绝自动重启。
for _ in range(30):
gate = self.gate("status")
if gate.get("uncertain"):
raise RuntimeError("in-flight FetchTask completion is uncertain; manual gate inspection required")
if gate.get("paused") is not True:
raise RuntimeError("runner gate unexpectedly resumed")
if gate.get("inflight") == 0:
break
time.sleep(1)
else:
log("FetchTask still in flight; defer switch")
return False
if not self.idle():
with operation("wait for FetchTask completion before runner switch"):
for _ in range(30):
gate = self.gate("status")
if gate.get("uncertain"):
raise RuntimeError("in-flight FetchTask completion is uncertain; manual gate inspection required")
if gate.get("paused") is not True:
raise RuntimeError("runner gate unexpectedly resumed")
if gate.get("inflight") == 0:
break
time.sleep(1)
else:
log("FetchTask still in flight; defer switch")
return False
if not self.wait_for_idle("runner restart"):
log("in-flight task appeared; defer switch without stopping runner")
return False
latest_config = self.read_config()
@@ -789,22 +831,24 @@ class Maintenance:
self.save()
log("idle check changed before restart; restored configuration")
return False
self.docker("restart", "--timeout", "660", self.runner, timeout=720)
with operation("restart runner with cache candidate image"):
self.docker("restart", "--timeout", "660", self.runner, timeout=720)
started = self.docker("inspect", "--format", "{{.State.StartedAt}}", self.runner).strip()
ready = False
for _ in range(30):
try:
info = self.docker("inspect", "--format", "{{.State.Status}}", self.runner).strip()
recent = self.docker("logs", "--since", started, self.runner, combined=True)
ready = (info == "running" and "declare successfully" in recent
and self.image_info(candidate["image"], inner=True)["Id"] == candidate["image"])
except RuntimeError:
ready = False
if ready:
break
time.sleep(2)
if not ready:
raise RuntimeError("runner registration not confirmed; pending switch retained for recovery")
with operation("wait for runner image registration"):
for _ in range(30):
try:
info = self.docker("inspect", "--format", "{{.State.Status}}", self.runner).strip()
recent = self.docker("logs", "--since", started, self.runner, combined=True)
ready = (info == "running" and "declare successfully" in recent
and self.image_info(candidate["image"], inner=True)["Id"] == candidate["image"])
except RuntimeError:
ready = False
if ready:
break
time.sleep(2)
if not ready:
raise RuntimeError("runner registration not confirmed; pending switch retained for recovery")
candidate["activated"] = now()
self.state["rollback"] = pending["old"]
self.state["current"] = pending["new"]
+33 -3
View File
@@ -379,9 +379,8 @@ describe('project CI workflow', () => {
expect(imageDockerignore).toContain(`!${path}`);
}
// AGC 通过本地 path 依赖引用三个编辑器 bridge crate。镜像预热会对
// AGC manifest 执行 cargo fetch --locked,构建上下文与 dockerignore
// 必须同时放行这些 crate,否则镜像在 cargo fetch 阶段必然失败。
// AGC 通过本地 path 依赖引用三个编辑器 bridge crate。Cargo fetch 只需要
// manifest;完整源码不得进入镜像构建上下文,实际清单闭包由 Python tar 测试核验。
for (const bridgeDir of [
'plugins/agc-cocos-editor/native/cocos-editor-bridge',
'plugins/agc-unity-editor/native/unity-editor-bridge',
@@ -394,6 +393,18 @@ describe('project CI workflow', () => {
`COPY ${bridgeDir} /tmp/genarrative-cargo-cache/${bridgeDir}`,
);
}
expect(imageBuildScript).toContain(
'apps/ai-game-creator-shell/src-tauri/vendor',
);
expect(imageDockerignore).toContain(
'!apps/ai-game-creator-shell/src-tauri/vendor/*/Cargo.toml',
);
expect(imageDockerignore).toContain(
'!plugins/agc-*-editor/native/*-editor-bridge/Cargo.toml',
);
expect(imageDockerfile).toContain(
'COPY apps/ai-game-creator-shell/src-tauri /tmp/genarrative-cargo-cache/apps/ai-game-creator-shell/src-tauri',
);
expect(imageBuildScript).toContain(
'--build-arg "AGC_RUST_LOCK_SHA256=${agc_rust_lock_sha256}"',
@@ -431,6 +442,25 @@ describe('project CI workflow', () => {
expect(imageCheckScript).toContain(
'::warning title=CI dependency cache is partial::',
);
// 下载缓存由 BuildKit 的固定 ID 独占写入,最终镜像只复制受控快照,不继承旧镜像层。
for (const mount of [
'id=genarrative-ci-cargo-cache-v1,target=/usr/local/cargo/registry/cache,sharing=locked',
'id=genarrative-ci-cargo-index-v1,target=/usr/local/cargo/registry/index,sharing=locked',
'id=genarrative-ci-npm-v1,target=/var/cache/genarrative-ci-npm,sharing=locked',
]) {
expect(imageDockerfile).toContain(mount);
}
expect(imageDockerfile).toContain(
'FROM rust-toolchain AS download-cache-seed',
);
expect(imageBuildScript).toContain('--target download-cache-seed');
expect(imageDockerfile).toContain(
'COPY --from=rust-dependency-cache /opt/ci-downloads/registry /usr/local/cargo/registry',
);
expect(imageDockerfile).toContain(
'/var/cache/genarrative-ci-npm/_cacache /root/.npm/_cacache',
);
});
it('copies every workspace manifest before the API image web-builder clean install', () => {
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env python3
"""Regression tests for the minimal trusted Gitea CI download-cache build context."""
from __future__ import annotations
import os
from pathlib import Path
import shutil
import shlex
import stat
import subprocess
import tarfile
import tempfile
import textwrap
import tomllib
import unittest
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
IMAGE_SCRIPT = REPOSITORY_ROOT / "scripts" / "gitea-ci-job-image.sh"
STATIC_CONTEXT_FILES = (
"deploy/container/gitea-ci-job.Dockerfile",
"deploy/container/gitea-ci-job.Dockerfile.dockerignore",
"deploy/container/gitea-ci-buildkitd.toml",
"deploy/container/gitea-ci-checkout.sh",
"scripts/export-ci-npm-download-cache.mjs",
"package.json",
"package-lock.json",
"apps/admin-web/package.json",
"apps/ai-game-creator-shell/package.json",
"apps/desktop-shell/package.json",
"apps/mobile-shell/package.json",
"apps/preview-deployer-web/package.json",
"packages/image-canvas-core/package.json",
"packages/image-canvas-react/package.json",
"packages/shared/package.json",
"tools/spine-json-export-validator/package.json",
"apps/ai-game-creator-shell/src-tauri/Cargo.toml",
"apps/ai-game-creator-shell/src-tauri/Cargo.lock",
"server-rs/Cargo.toml",
"server-rs/Cargo.lock",
"apps/desktop-shell/src-tauri/Cargo.toml",
"apps/desktop-shell/src-tauri/Cargo.lock",
)
class GiteaCiImageContextTest(unittest.TestCase):
def setUp(self) -> None:
self.temporary_directory = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
self.root = Path(self.temporary_directory.name)
self.context_archive = self.root / "context.tar"
self.bin = self.root / "bin"
self.bin.mkdir()
self.write_fake_docker()
self.execution_bin = self.wsl_path(self.bin)
self.wsl_fake_root: str | None = None
if os.name == "nt":
self.wsl_fake_root = f"/tmp/gitea-ci-image-context-{self.root.name}"
subprocess.run(
[
"bash",
"-c",
"rm -rf {root}; mkdir -p {root}/bin; cp {source}/* {root}/bin/; chmod +x {root}/bin/*".format(
root=shlex.quote(self.wsl_fake_root),
source=shlex.quote(self.wsl_path(self.bin)),
),
],
check=True,
)
self.execution_bin = f"{self.wsl_fake_root}/bin"
def tearDown(self) -> None:
if self.wsl_fake_root is not None:
subprocess.run(
["bash", "-c", f"rm -rf {shlex.quote(self.wsl_fake_root)}"], check=False
)
self.temporary_directory.cleanup()
@staticmethod
def wsl_path(path: Path) -> str:
value = path.resolve().as_posix()
if len(value) >= 3 and value[1] == ":":
return f"/mnt/{value[0].lower()}{value[2:]}"
return value
def write_fake_docker(self) -> None:
docker = self.bin / "docker"
docker.write_bytes(textwrap.dedent(
"""#!/usr/bin/env bash
set -eu
if [[ "$1" == buildx && "$2" == version ]]; then exit 0; fi
if [[ "$1" == buildx && "$2" == inspect ]]; then
if [[ " $* " == *" --format "* ]]; then printf 'docker-container\\n'; fi
exit 0
fi
if [[ "$1" == buildx && "$2" == build ]]; then
cat > "$TAR_CAPTURE"
exit 0
fi
if [[ "$1" == image && "$2" == inspect ]]; then
printf 'sha256:%064d\\n' 0
exit 0
fi
if [[ "$1" == run ]]; then exit 0; fi
echo "unexpected docker invocation" >&2
exit 9
"""
).encode("utf-8"))
docker.chmod(docker.stat().st_mode | stat.S_IXUSR)
def run_script(self, script: Path, *arguments: str, capture_context: bool = False) -> subprocess.CompletedProcess[str]:
exports = [f"export PATH={shlex.quote(self.execution_bin)}:\"$PATH\""]
if capture_context:
exports.append(f"export TAR_CAPTURE={shlex.quote(self.wsl_path(self.context_archive))}")
command = "; ".join(exports) + "; cd /; exec bash " + shlex.quote(self.wsl_path(script))
command += " " + " ".join(shlex.quote(argument) for argument in arguments)
return subprocess.run(
["bash", "-c", command], env=os.environ, text=True, capture_output=True, check=False
)
@staticmethod
def dependency_paths(value):
if not isinstance(value, dict):
return
for key, child in value.items():
if key in {"dependencies", "build-dependencies", "dev-dependencies"} and isinstance(child, dict):
for dependency in child.values():
if isinstance(dependency, dict) and isinstance(dependency.get("path"), str):
yield dependency["path"]
yield from GiteaCiImageContextTest.dependency_paths(child)
def test_build_context_contains_all_local_dependency_manifests_and_no_source(self) -> None:
result = self.run_script(IMAGE_SCRIPT, "build", capture_context=True)
self.assertEqual(result.returncode, 0, result.stderr)
with tarfile.open(self.context_archive) as archive:
names = {member.name.removeprefix("./") for member in archive.getmembers() if member.isfile()}
manifests = {Path(name) for name in names if name.endswith("Cargo.toml")}
self.assertTrue(manifests)
expected = set()
for manifest in manifests:
data = tomllib.loads((REPOSITORY_ROOT / manifest).read_text(encoding="utf-8"))
for path in self.dependency_paths(data):
dependency = (REPOSITORY_ROOT / manifest.parent / path).resolve()
try:
cargo_toml = (dependency / "Cargo.toml").relative_to(REPOSITORY_ROOT)
except ValueError:
continue
expected.add(cargo_toml)
self.assertTrue(expected)
self.assertTrue(expected.issubset(manifests), sorted(expected - manifests))
self.assertFalse(any(Path(name).suffix in {".rs", ".c", ".cc", ".cpp", ".h"} for name in names))
self.assertFalse(any("target" in Path(name).parts for name in names))
self.assertFalse(any(
Path(name).name.startswith(".env") or Path(name).suffix in {".pem", ".key"}
for name in names
))
def test_revision_tracks_vendor_manifests_but_ignores_regular_source(self) -> None:
fixture = self.root / "revision-fixture"
script = fixture / "scripts" / "gitea-ci-job-image.sh"
script.parent.mkdir(parents=True)
shutil.copy2(IMAGE_SCRIPT, script)
script.chmod(script.stat().st_mode | stat.S_IXUSR)
for name in STATIC_CONTEXT_FILES:
target = fixture / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text("fixture\n", encoding="utf-8")
vendor_manifest = fixture / "apps/ai-game-creator-shell/src-tauri/vendor/example/Cargo.toml"
vendor_manifest.parent.mkdir(parents=True)
vendor_manifest.write_text('[package]\nname = "example"\nversion = "0.1.0"\n', encoding="utf-8")
bridge_manifest = fixture / "plugins/agc-example-editor/native/example-editor-bridge/Cargo.toml"
bridge_manifest.parent.mkdir(parents=True)
bridge_manifest.write_text('[package]\nname = "bridge"\nversion = "0.1.0"\n', encoding="utf-8")
(fixture / "server-rs/crates").mkdir(parents=True)
first = self.run_script(script, "revision")
self.assertEqual(first.returncode, 0, first.stderr)
source = fixture / "apps/ai-game-creator-shell/src-tauri/src/lib.rs"
source.parent.mkdir(parents=True)
source.write_text("pub fn ignored() {}\n", encoding="utf-8")
self.assertEqual(self.run_script(script, "revision").stdout, first.stdout)
vendor_manifest.write_text('[package]\nname = "example"\nversion = "0.2.0"\n', encoding="utf-8")
changed = self.run_script(script, "revision")
self.assertEqual(changed.returncode, 0, changed.stderr)
self.assertNotEqual(changed.stdout, first.stdout)
if __name__ == "__main__":
unittest.main()
+12
View File
@@ -75,6 +75,18 @@ class GiteaCacheMaintenanceTest(unittest.TestCase):
self.assertEqual(request.get_method(), method)
self.assertEqual(request.data, None if body is None else json.dumps(body).encode())
def test_operation_logs_duration_and_build_log_without_exception_secrets(self):
build_log = self.root / "artifacts" / SHA / "build.log"
with patch.object(maintenance_module.time, "monotonic", side_effect=[10.0, 13.25]), \
patch("builtins.print") as printed:
with self.assertRaisesRegex(RuntimeError, "test failure"):
with maintenance_module.operation("merge cache artifacts", build_log=build_log):
raise RuntimeError("test failure")
messages = [call.args[0] for call in printed.call_args_list]
self.assertEqual(messages[0], f"[cache-maintenance] merge cache artifacts: started; build log={build_log}")
self.assertEqual(messages[1],
f"[cache-maintenance] merge cache artifacts: failed after 3.2s; build log={build_log}")
def test_workflow_jobs_and_cache_producers_match_maintenance_contract(self):
workflow = (SCRIPT.parent.parent / ".gitea/workflows/project-ci.yml").read_text(encoding="utf-8")
# 沿用 workflow 的显式 job/step 格式,枚举实际 job,避免另一份名单漏掉新增项。