实现 Gitea CI 编译缓存自动刷新与定向清理
基于固定 master 提交构建全新缓存镜像，并在成功验证后保留当前版和回滚版。 增加任务领取网关，在不打断运行中 CI 的前提下切换镜像。 补充宿主维护服务、部署说明、共享记忆和定向行为测试。 修复构建测试在 Windows 与 WSL 下并发写入日志的竞态。
This commit is contained in:
@@ -5,6 +5,15 @@ set -euo pipefail
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
base_ref="${1:?usage: build-gitea-rust-cache.sh <verified-base-image> <candidate-tag>}"
|
||||
candidate_tag="${2:?candidate image tag is required}"
|
||||
requested_source_commit="${3:-}"
|
||||
if [[ "$#" -gt 3 ]]; then
|
||||
echo 'usage: build-gitea-rust-cache.sh <verified-base-image> <candidate-tag> [master-commit-sha]' >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ -n "${requested_source_commit}" && ! "${requested_source_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
||||
echo 'master-commit-sha must be a complete 40-character SHA.' >&2
|
||||
exit 2
|
||||
fi
|
||||
# 与实际 Gitea checkout 路径一致；Rust 对象 key 包含编译 cwd，不能随意换临时根。
|
||||
workspace=/workspace/GenarrativeAI/Genarrative
|
||||
[[ "${CI:-}" != true ]] || { echo 'Run on the trusted image builder, outside CI jobs.' >&2; exit 1; }
|
||||
@@ -22,7 +31,17 @@ bash "${repo_root}/scripts/gitea-ci-job-image.sh" verify "${base_id}"
|
||||
|
||||
# 只归档远端 master 的确定提交；不复制当前工作区或本地凭据。
|
||||
git -C "${repo_root}" fetch --no-tags origin refs/heads/master
|
||||
source_commit="$(git -C "${repo_root}" rev-parse FETCH_HEAD^{commit})"
|
||||
master_commit="$(git -C "${repo_root}" rev-parse FETCH_HEAD^{commit})"
|
||||
if [[ -n "${requested_source_commit}" ]]; then
|
||||
git -C "${repo_root}" cat-file -e "${requested_source_commit}^{commit}"
|
||||
if ! git -C "${repo_root}" merge-base --is-ancestor "${requested_source_commit}" "${master_commit}"; then
|
||||
echo "master-commit-sha is not contained in fetched master: ${requested_source_commit}" >&2
|
||||
exit 1
|
||||
fi
|
||||
source_commit="$(git -C "${repo_root}" rev-parse "${requested_source_commit}^{commit}")"
|
||||
else
|
||||
source_commit="${master_commit}"
|
||||
fi
|
||||
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/gitea-rust-cache.XXXXXX")"
|
||||
container_id=''
|
||||
cleanup() {
|
||||
@@ -50,7 +69,7 @@ container_id="$(docker run --detach --cpus=4 --memory=12g --pids-limit=1024 \
|
||||
docker exec "${container_id}" mkdir -p "${workspace}" /opt/genarrative-ci/rust-cache/objects
|
||||
git -C "${repo_root}" archive "${source_commit}" | docker cp - "${container_id}:${workspace}"
|
||||
docker cp "${work_dir}/snapshot/." "${container_id}:/opt/genarrative-ci/rust-cache/"
|
||||
docker cp "${repo_root}/scripts/ci-rust-cache.sh" "${container_id}:/tmp/ci-rust-cache.sh"
|
||||
docker exec "${container_id}" cp "${workspace}/scripts/ci-rust-cache.sh" /tmp/ci-rust-cache.sh
|
||||
docker exec --interactive --workdir "${workspace}" "${container_id}" bash -s <<'WARM'
|
||||
set -euo pipefail
|
||||
rustc -vV > /opt/genarrative-ci/rust-cache/rustc.txt
|
||||
|
||||
@@ -39,6 +39,7 @@ usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/gitea-ci-job-image.sh build
|
||||
bash scripts/gitea-ci-job-image.sh revision
|
||||
bash scripts/gitea-ci-job-image.sh verify [镜像引用]
|
||||
bash scripts/gitea-ci-job-image.sh load-runner [镜像引用]
|
||||
bash scripts/gitea-ci-job-image.sh export <归档路径> [镜像引用]
|
||||
@@ -65,40 +66,15 @@ verify_image() {
|
||||
|
||||
command_name="${1:-}"
|
||||
case "${command_name}" in
|
||||
build)
|
||||
image_revision="$(
|
||||
revision)
|
||||
# 与 build 的 IMAGE_REVISION 使用同一份输入顺序，用于维护器判断基础镜像是否过期。
|
||||
(
|
||||
cd "${repo_root}"
|
||||
{
|
||||
sha256sum \
|
||||
deploy/container/gitea-ci-job.Dockerfile \
|
||||
deploy/container/gitea-ci-job.Dockerfile.dockerignore \
|
||||
deploy/container/gitea-ci-checkout.sh \
|
||||
package.json \
|
||||
package-lock.json \
|
||||
apps/admin-web/package.json \
|
||||
apps/ai-game-creator-shell/package.json \
|
||||
apps/desktop-shell/package.json \
|
||||
apps/mobile-shell/package.json \
|
||||
apps/preview-deployer-web/package.json \
|
||||
packages/image-canvas-core/package.json \
|
||||
packages/image-canvas-react/package.json \
|
||||
packages/shared/package.json \
|
||||
tools/spine-json-export-validator/package.json \
|
||||
apps/ai-game-creator-shell/src-tauri/Cargo.toml \
|
||||
apps/ai-game-creator-shell/src-tauri/Cargo.lock \
|
||||
server-rs/Cargo.toml \
|
||||
server-rs/Cargo.lock \
|
||||
apps/desktop-shell/src-tauri/Cargo.toml \
|
||||
apps/desktop-shell/src-tauri/Cargo.lock
|
||||
find server-rs/crates plugins/agc-*-editor/native/*-editor-bridge \
|
||||
\( -name Cargo.toml -o -path 'plugins/agc-*-editor/native/*-editor-bridge/*' \) \
|
||||
-type f -print0 \
|
||||
| sort -z \
|
||||
| xargs -0 -r sha256sum
|
||||
} \
|
||||
| sha256sum \
|
||||
| awk '{ print $1 }'
|
||||
)"
|
||||
write_build_context_file_list | xargs -0 -r sha256sum | sha256sum | awk '{ print $1 }'
|
||||
)
|
||||
;;
|
||||
build)
|
||||
image_revision="$(bash "${BASH_SOURCE[0]}" revision)"
|
||||
npm_lock_sha256="$(sha256sum "${repo_root}/package-lock.json")"
|
||||
npm_lock_sha256="${npm_lock_sha256%% *}"
|
||||
server_rust_lock_sha256="$(sha256sum "${repo_root}/server-rs/Cargo.lock")"
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gitea Runner 领取屏障；控制 socket 仅供宿主缓存维护任务使用。"""
|
||||
|
||||
import http.client
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
MAX_BODY = 32 * 1024 * 1024
|
||||
HOP_HEADERS = {
|
||||
"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
|
||||
"te", "trailer", "transfer-encoding", "upgrade", "host", "content-length",
|
||||
}
|
||||
|
||||
|
||||
class Gate:
|
||||
def __init__(self, directory):
|
||||
self.directory = Path(directory)
|
||||
self.directory.mkdir(parents=True, exist_ok=True)
|
||||
self.lock = threading.Lock()
|
||||
self.inflight = 0
|
||||
self.last_fetch_peer = None
|
||||
self.last_fetch_at = None
|
||||
self.uncertain = (self.directory / "uncertain").exists()
|
||||
if (self.directory / "inflight").exists():
|
||||
self.uncertain = True
|
||||
self.mark("uncertain")
|
||||
self.paused = (self.directory / "paused").exists() or self.uncertain
|
||||
|
||||
def mark(self, name):
|
||||
with (self.directory / name).open("w", encoding="ascii") as stream:
|
||||
stream.write("1\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
self.sync_directory()
|
||||
|
||||
def sync_directory(self):
|
||||
descriptor = os.open(self.directory, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def snapshot(self):
|
||||
return {"paused": self.paused, "inflight": self.inflight,
|
||||
"uncertain": self.uncertain, "last_fetch_peer": self.last_fetch_peer,
|
||||
"last_fetch_at": self.last_fetch_at}
|
||||
|
||||
def record_fetch(self, peer):
|
||||
with self.lock:
|
||||
self.last_fetch_peer = peer
|
||||
self.last_fetch_at = time.time()
|
||||
|
||||
def control(self, action):
|
||||
with self.lock:
|
||||
if action == "pause":
|
||||
self.mark("paused")
|
||||
self.paused = True
|
||||
elif action == "resume":
|
||||
if self.uncertain:
|
||||
return {**self.snapshot(), "error": "upstream completion uncertain; operator recovery required"}
|
||||
(self.directory / "paused").unlink(missing_ok=True)
|
||||
self.sync_directory()
|
||||
self.paused = False
|
||||
elif action != "status":
|
||||
return {**self.snapshot(), "error": "unknown action"}
|
||||
return self.snapshot()
|
||||
|
||||
def enter(self):
|
||||
with self.lock:
|
||||
if self.paused or self.uncertain:
|
||||
return False
|
||||
# 必须先落盘再转发；崩溃后不能把遗留的领取请求误认为已完成。
|
||||
self.mark("inflight")
|
||||
self.inflight += 1
|
||||
return True
|
||||
|
||||
def leave(self, completed):
|
||||
with self.lock:
|
||||
self.inflight -= 1
|
||||
if not completed:
|
||||
self.uncertain = self.paused = True
|
||||
self.mark("uncertain")
|
||||
if self.inflight == 0 and not self.uncertain:
|
||||
(self.directory / "inflight").unlink(missing_ok=True)
|
||||
self.sync_directory()
|
||||
|
||||
|
||||
def read_body(stream, headers):
|
||||
"""解码 HTTP 请求；拒绝含糊 framing，不依赖下游连接关闭。"""
|
||||
encodings = headers.get_all("Transfer-Encoding", [])
|
||||
lengths = headers.get_all("Content-Length", [])
|
||||
if encodings and lengths:
|
||||
raise ValueError("ambiguous request framing")
|
||||
if len(lengths) > 1 or len(encodings) > 1:
|
||||
raise ValueError("duplicate request framing")
|
||||
|
||||
def exact(length):
|
||||
data = stream.read(length)
|
||||
if len(data) != length:
|
||||
raise ValueError("incomplete request body")
|
||||
return data
|
||||
|
||||
if not encodings:
|
||||
length = int(lengths[0]) if lengths else 0
|
||||
if not 0 <= length <= MAX_BODY:
|
||||
raise ValueError("request too large")
|
||||
return exact(length)
|
||||
if encodings[0].strip().lower() != "chunked":
|
||||
raise ValueError("unsupported transfer encoding")
|
||||
body = bytearray()
|
||||
while True:
|
||||
line = stream.readline(8193)
|
||||
if len(line) > 8192 or not line.endswith(b"\r\n"):
|
||||
raise ValueError("invalid chunk header")
|
||||
length = int(line.split(b";", 1)[0].strip(), 16)
|
||||
if length < 0 or len(body) + length > MAX_BODY:
|
||||
raise ValueError("request too large")
|
||||
if length == 0:
|
||||
trailer_size = 0
|
||||
while True:
|
||||
line = stream.readline(8193)
|
||||
trailer_size += len(line)
|
||||
if trailer_size > 8192 or not line.endswith(b"\r\n"):
|
||||
raise ValueError("invalid request trailers")
|
||||
if line == b"\r\n":
|
||||
return bytes(body)
|
||||
body.extend(exact(length))
|
||||
if exact(2) != b"\r\n":
|
||||
raise ValueError("invalid chunk terminator")
|
||||
|
||||
|
||||
class Proxy(http.server.BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass # 不输出 RPC 认证头、请求内容或带认证信息的 URL。
|
||||
|
||||
def reply(self, status, body, headers=()):
|
||||
try:
|
||||
self.send_response(status)
|
||||
excluded = HOP_HEADERS | {
|
||||
item.strip().lower() for key, value in headers if key.lower() == "connection"
|
||||
for item in value.split(",")
|
||||
}
|
||||
for key, value in headers:
|
||||
if key.lower() not in excluded:
|
||||
self.send_header(key, value)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
self.close_connection = True
|
||||
|
||||
def do_POST(self):
|
||||
path = urllib.parse.urlsplit(self.path)
|
||||
if (path.scheme or path.netloc or not path.path.startswith("/api/actions/")
|
||||
or "%" in path.path or any(p in (".", "..") for p in path.path.split("/"))):
|
||||
self.reply(404, b"runner RPC only\n")
|
||||
return
|
||||
try:
|
||||
body = read_body(self.rfile, self.headers)
|
||||
except (ValueError, OSError):
|
||||
self.reply(400, b"invalid request body\n")
|
||||
return
|
||||
is_fetch = path.path.endswith("/FetchTask")
|
||||
if is_fetch:
|
||||
# 包括暂停时被拒绝的请求；只记录网络来源与时间以证明 daemon 路由。
|
||||
self.server.gate.record_fetch(self.client_address[0])
|
||||
if not self.server.gate.enter():
|
||||
self.reply(503, b"runner maintenance\n")
|
||||
return
|
||||
completed = False
|
||||
connection = None
|
||||
try:
|
||||
upstream = self.server.upstream
|
||||
cls = http.client.HTTPSConnection if upstream.scheme == "https" else http.client.HTTPConnection
|
||||
connection = cls(upstream.hostname, upstream.port, timeout=None)
|
||||
try:
|
||||
connection.connect()
|
||||
except OSError:
|
||||
# TCP/TLS 连接阶段尚未发送 RPC，不存在服务端分配事务。
|
||||
completed = True
|
||||
raise
|
||||
excluded = HOP_HEADERS | {
|
||||
name.strip().lower() for name in self.headers.get("Connection", "").split(",")
|
||||
}
|
||||
headers = {key: value for key, value in self.headers.items() if key.lower() not in excluded}
|
||||
headers["Content-Length"] = str(len(body))
|
||||
connection.request("POST", self.path, body=body, headers=headers)
|
||||
response = connection.getresponse()
|
||||
result = bytearray()
|
||||
oversized = False
|
||||
# 客户端即使断开，也继续读取上游完整响应，再释放领取屏障。
|
||||
while block := response.read(65536):
|
||||
if len(result) + len(block) <= MAX_BODY and not oversized:
|
||||
result.extend(block)
|
||||
else:
|
||||
oversized = True
|
||||
result.clear()
|
||||
if response.length not in (None, 0):
|
||||
raise http.client.IncompleteRead(bytes(result), response.length)
|
||||
completed = True
|
||||
if is_fetch:
|
||||
self.server.gate.leave(True)
|
||||
is_fetch = False
|
||||
if oversized:
|
||||
self.reply(502, b"upstream response too large\n")
|
||||
else:
|
||||
self.reply(response.status, bytes(result), response.getheaders())
|
||||
except (OSError, http.client.HTTPException, ValueError):
|
||||
self.reply(502, b"runner upstream unavailable\n")
|
||||
finally:
|
||||
if connection is not None:
|
||||
connection.close()
|
||||
if is_fetch:
|
||||
self.server.gate.leave(completed)
|
||||
|
||||
|
||||
class Control(socketserver.StreamRequestHandler):
|
||||
def handle(self):
|
||||
try:
|
||||
line = self.rfile.readline(4097)
|
||||
if len(line) > 4096 or not line.endswith(b"\n"):
|
||||
raise ValueError("invalid control request")
|
||||
request = json.loads(line)
|
||||
result = self.server.gate.control(request["action"])
|
||||
except (ValueError, KeyError, TypeError, OSError):
|
||||
result = {"error": "invalid control request"}
|
||||
self.wfile.write(json.dumps(result).encode("utf-8") + b"\n")
|
||||
|
||||
|
||||
class ControlServer(socketserver.ThreadingUnixStreamServer):
|
||||
daemon_threads = True
|
||||
|
||||
|
||||
def create_proxy(address, upstream, gate):
|
||||
parsed = urllib.parse.urlsplit(upstream)
|
||||
if (parsed.scheme not in ("http", "https") or not parsed.hostname
|
||||
or parsed.username or parsed.password or parsed.path not in ("", "/")
|
||||
or parsed.query or parsed.fragment):
|
||||
raise ValueError("upstream must be an HTTP(S) origin")
|
||||
server = http.server.ThreadingHTTPServer(address, Proxy)
|
||||
server.upstream = parsed
|
||||
server.gate = gate
|
||||
return server
|
||||
|
||||
|
||||
def main():
|
||||
gate = Gate(os.environ.get("GITEA_GATE_CONTROL_DIR", "/control"))
|
||||
socket_path = gate.directory / "gate.sock"
|
||||
socket_path.unlink(missing_ok=True)
|
||||
control = ControlServer(str(socket_path), Control)
|
||||
control.gate = gate
|
||||
os.chmod(socket_path, 0o600)
|
||||
proxy = create_proxy(("0.0.0.0", 8080), os.environ.get(
|
||||
"GITEA_RUNNER_UPSTREAM", "http://gitea:3000"), gate)
|
||||
threading.Thread(target=control.serve_forever, daemon=True).start()
|
||||
proxy.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -331,7 +331,8 @@ describe('project CI workflow', () => {
|
||||
const rustLock = 'apps/ai-game-creator-shell/src-tauri/Cargo.lock';
|
||||
|
||||
for (const workspaceManifest of workspaceManifests) {
|
||||
expect(imageBuildScript.split(workspaceManifest)).toHaveLength(3);
|
||||
// 构建上下文与 revision 共用一份清单，不重复枚举 manifest。
|
||||
expect(imageBuildScript.split(workspaceManifest)).toHaveLength(2);
|
||||
expect(imageDockerignore).toContain(`!${workspaceManifest}`);
|
||||
expect(imageDockerfile).toContain(
|
||||
`COPY ${workspaceManifest} /usr/local/share/genarrative-ci/npm/${workspaceManifest}`,
|
||||
@@ -339,8 +340,8 @@ describe('project CI workflow', () => {
|
||||
}
|
||||
|
||||
for (const [path, expectedCount] of [
|
||||
[rustManifest, 2],
|
||||
[rustLock, 3],
|
||||
[rustManifest, 1],
|
||||
[rustLock, 2],
|
||||
] as const) {
|
||||
expect(imageBuildScript.split(path)).toHaveLength(expectedCount + 1);
|
||||
expect(imageDockerignore).toContain(`!${path}`);
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Targeted regression tests for the trusted Gitea Rust-cache image builder."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import shlex
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
|
||||
BUILDER_NAME = "build-gitea-rust-cache.sh"
|
||||
FIXED_SHA = "a" * 40
|
||||
MASTER_SHA = "b" * 40
|
||||
|
||||
|
||||
class GiteaRustCacheBuilderTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary_directory = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
||||
self.root = Path(self.temporary_directory.name)
|
||||
self.repo = self.root / "repo"
|
||||
scripts = self.repo / "scripts"
|
||||
scripts.mkdir(parents=True)
|
||||
shutil.copy2(REPOSITORY_ROOT / "scripts" / BUILDER_NAME, scripts / BUILDER_NAME)
|
||||
(scripts / "gitea-ci-job-image.sh").write_text("#!/usr/bin/env bash\nexit 0\n")
|
||||
(scripts / "ci-rust-cache.sh").write_text("#!/usr/bin/env bash\nexit 0\n")
|
||||
self.trace = self.root / "trace.log"
|
||||
self.bin = self.root / "bin"
|
||||
self.bin.mkdir()
|
||||
self.write_fake_tools()
|
||||
self.execution_bin = self.wsl_path(self.bin)
|
||||
self.wsl_fake_root: str | None = None
|
||||
if os.name == "nt":
|
||||
self.wsl_fake_root = f"/tmp/gitea-cache-builder-{self.root.name}"
|
||||
subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
"rm -rf {root}; mkdir -p {root}/bin; cp {source}/* {root}/bin/; chmod +x {root}/bin/*".format(
|
||||
root=shlex.quote(self.wsl_fake_root),
|
||||
source=shlex.quote(self.wsl_path(self.bin)),
|
||||
),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
self.execution_bin = f"{self.wsl_fake_root}/bin"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
if self.wsl_fake_root is not None:
|
||||
subprocess.run(
|
||||
["bash", "-c", f"rm -rf {shlex.quote(self.wsl_fake_root)}"],
|
||||
check=False,
|
||||
)
|
||||
self.temporary_directory.cleanup()
|
||||
|
||||
def write_tool(self, name: str, source: str) -> None:
|
||||
target = self.bin / name
|
||||
target.write_bytes(textwrap.dedent(source).encode("utf-8"))
|
||||
target.chmod(target.stat().st_mode | stat.S_IXUSR)
|
||||
|
||||
def write_fake_tools(self) -> None:
|
||||
self.write_tool(
|
||||
"git",
|
||||
"""#!/usr/bin/env bash
|
||||
set -eu
|
||||
printf 'git:%s\\n' "$*" >> "$TRACE"
|
||||
if [[ "$1" == '-C' ]]; then shift 2; fi
|
||||
case "$1" in
|
||||
fetch|cat-file) exit 0 ;;
|
||||
merge-base)
|
||||
[[ "${FAKE_NON_ANCESTOR:-}" != 1 ]]
|
||||
exit
|
||||
;;
|
||||
rev-parse)
|
||||
if [[ "$2" == 'FETCH_HEAD^{commit}' ]]; then
|
||||
printf '%s\\n' "$FAKE_MASTER_SHA"
|
||||
else
|
||||
printf '%s\\n' "$FAKE_FIXED_SHA"
|
||||
fi
|
||||
;;
|
||||
archive) printf 'archived %s\\n' "$2" ;;
|
||||
*) echo "unexpected git command: $*" >&2; exit 9 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
self.write_tool(
|
||||
"docker",
|
||||
"""#!/usr/bin/env bash
|
||||
set -eu
|
||||
# Drain the archive pipeline before logging to avoid concurrent
|
||||
# appends to the shared trace on Windows/WSL filesystems.
|
||||
if [[ "$1" == cp && "$2" == '-' ]]; then
|
||||
cat >/dev/null
|
||||
fi
|
||||
printf 'docker:%s\\n' "$*" >> "$TRACE"
|
||||
if [[ "$1" == image && "$2" == inspect ]]; then
|
||||
printf 'sha256:%064d\\n' 0
|
||||
elif [[ "$1" == run && " $* " == *' --detach '* ]]; then
|
||||
printf 'fake-container\\n'
|
||||
elif [[ "$1" == exec && " $* " == *' bash -s '* ]]; then
|
||||
cat >/dev/null
|
||||
fi
|
||||
""",
|
||||
)
|
||||
self.write_tool(
|
||||
"curl",
|
||||
"""#!/usr/bin/env bash
|
||||
set -eu
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
if [[ "$1" == --output ]]; then touch "$2"; exit 0; fi
|
||||
shift
|
||||
done
|
||||
exit 9
|
||||
""",
|
||||
)
|
||||
self.write_tool("sha256sum", "#!/usr/bin/env bash\ncat >/dev/null\n")
|
||||
self.write_tool(
|
||||
"tar",
|
||||
"""#!/usr/bin/env bash
|
||||
set -eu
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
if [[ "$1" == --directory ]]; then
|
||||
mkdir -p "$2/sccache-v0.18.0-x86_64-unknown-linux-musl"
|
||||
printf '#!/usr/bin/env bash\\nexit 0\\n' > "$2/sccache-v0.18.0-x86_64-unknown-linux-musl/sccache"
|
||||
chmod +x "$2/sccache-v0.18.0-x86_64-unknown-linux-musl/sccache"
|
||||
exit 0
|
||||
fi
|
||||
shift
|
||||
done
|
||||
exit 9
|
||||
""",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def wsl_path(path: Path) -> str:
|
||||
value = path.resolve().as_posix()
|
||||
if len(value) >= 3 and value[1] == ":":
|
||||
return f"/mnt/{value[0].lower()}{value[2:]}"
|
||||
return value
|
||||
|
||||
def run_builder(self, *arguments: str, non_ancestor: bool = False) -> subprocess.CompletedProcess[str]:
|
||||
script_path = self.wsl_path(self.repo / "scripts" / BUILDER_NAME)
|
||||
assignments = {
|
||||
"TRACE": self.wsl_path(self.trace),
|
||||
"FAKE_FIXED_SHA": FIXED_SHA,
|
||||
"FAKE_MASTER_SHA": MASTER_SHA,
|
||||
"FAKE_NON_ANCESTOR": "1" if non_ancestor else "",
|
||||
}
|
||||
exports = "; ".join(
|
||||
f"export {name}={shlex.quote(value)}" for name, value in assignments.items()
|
||||
)
|
||||
command = (
|
||||
f"unset CI; {exports}; export PATH={shlex.quote(self.execution_bin)}:\"$PATH\"; cd /; "
|
||||
f"exec bash {shlex.quote(script_path)} "
|
||||
+ " ".join(shlex.quote(argument) for argument in arguments)
|
||||
)
|
||||
return subprocess.run(
|
||||
["bash", "-c", command],
|
||||
env=os.environ,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def trace_text(self) -> str:
|
||||
return self.trace.read_bytes().decode("utf-8", errors="replace")
|
||||
|
||||
def test_archives_requested_master_ancestor_and_uses_its_helper(self) -> None:
|
||||
result = self.run_builder("trusted-base", "candidate", FIXED_SHA)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn(f"snapshot_source={FIXED_SHA}", result.stdout)
|
||||
trace = self.trace_text()
|
||||
self.assertIn(f"archive {FIXED_SHA}", trace)
|
||||
self.assertIn(
|
||||
"docker:exec fake-container cp "
|
||||
"/workspace/GenarrativeAI/Genarrative/scripts/ci-rust-cache.sh "
|
||||
"/tmp/ci-rust-cache.sh",
|
||||
trace,
|
||||
)
|
||||
|
||||
def test_rejects_commit_outside_fetched_master(self) -> None:
|
||||
result = self.run_builder("trusted-base", "candidate", FIXED_SHA, non_ancestor=True)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("not contained in fetched master", result.stderr)
|
||||
self.assertNotIn(" archive ", self.trace_text())
|
||||
|
||||
def test_rejects_non_full_sha_before_calling_external_tools(self) -> None:
|
||||
result = self.run_builder("trusted-base", "candidate", "abc123")
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn("complete 40-character SHA", result.stderr)
|
||||
self.assertFalse(self.trace.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,211 @@
|
||||
"""使用真实 HTTP/socket 验证领取屏障，不连接线上 Gitea。"""
|
||||
|
||||
import http.client
|
||||
import http.server
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
if os.name != "posix":
|
||||
raise unittest.SkipTest("领取屏障使用 Linux Unix socket 和目录 fsync；在 Linux/WSL 运行")
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"gitea_cache_gate", Path(__file__).with_name("gitea-runner-fetch-gate.py"))
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
FETCH = "/api/actions/runner.v1.RunnerService/FetchTask"
|
||||
UPDATE = "/api/actions/runner.v1.RunnerService/UpdateTask"
|
||||
|
||||
|
||||
def core_state(state):
|
||||
return {key: state[key] for key in ("paused", "inflight", "uncertain")}
|
||||
|
||||
|
||||
class Upstream(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
data = self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||
self.server.requests.append((self.path, data))
|
||||
if self.path.endswith("/Redirect"):
|
||||
self.send_response(302)
|
||||
self.send_header("Location", "/api/actions/runner.v1.RunnerService/FetchTask")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
if self.path == FETCH:
|
||||
self.server.entered.set()
|
||||
self.server.release.wait(10)
|
||||
if self.server.truncated and self.path == FETCH:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", "100")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"incomplete")
|
||||
self.close_connection = True
|
||||
return
|
||||
body = b'{"task":null}'
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
|
||||
class GateTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.gate = MODULE.Gate(self.temp.name)
|
||||
self.upstream = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
|
||||
self.upstream.requests = []
|
||||
self.upstream.entered = threading.Event()
|
||||
self.upstream.release = threading.Event()
|
||||
self.upstream.truncated = False
|
||||
self.proxy = MODULE.create_proxy(("127.0.0.1", 0),
|
||||
f"http://127.0.0.1:{self.upstream.server_port}", self.gate)
|
||||
self.servers = [self.upstream, self.proxy]
|
||||
for server in self.servers:
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
|
||||
def tearDown(self):
|
||||
self.upstream.release.set()
|
||||
for server in reversed(self.servers):
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
self.temp.cleanup()
|
||||
|
||||
def request(self, path, body=b"{}", chunked=False):
|
||||
connection = http.client.HTTPConnection("127.0.0.1", self.proxy.server_port, timeout=5)
|
||||
try:
|
||||
if chunked:
|
||||
connection.request("POST", path, body=[body], encode_chunked=True)
|
||||
else:
|
||||
connection.request("POST", path, body=body)
|
||||
response = connection.getresponse()
|
||||
result = response.status, response.read()
|
||||
return result
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def wait_for(self, predicate):
|
||||
deadline = time.monotonic() + 3
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(0.01)
|
||||
self.fail("condition did not become true")
|
||||
|
||||
def start_fetch(self):
|
||||
self.result = []
|
||||
thread = threading.Thread(target=lambda: self.result.append(self.request(FETCH)))
|
||||
thread.start()
|
||||
self.assertTrue(self.upstream.entered.wait(3))
|
||||
return thread
|
||||
|
||||
def test_pause_holds_inflight_and_allows_reporting(self):
|
||||
thread = self.start_fetch()
|
||||
self.assertEqual(core_state(self.gate.control("pause")),
|
||||
{"paused": True, "inflight": 1, "uncertain": False})
|
||||
self.assertEqual(self.request(FETCH)[0], 503)
|
||||
self.assertEqual(self.request(UPDATE)[0], 200)
|
||||
self.assertEqual(sum(path == FETCH for path, _ in self.upstream.requests), 1)
|
||||
self.upstream.release.set()
|
||||
thread.join(3)
|
||||
self.assertFalse(thread.is_alive())
|
||||
self.assertEqual(self.result[0][0], 200)
|
||||
self.assertEqual(self.gate.control("status")["inflight"], 0)
|
||||
self.assertEqual(self.gate.control("resume")["paused"], False)
|
||||
self.assertEqual(self.request(FETCH)[0], 200)
|
||||
|
||||
def test_disconnected_client_does_not_release_upstream_request(self):
|
||||
client = socket.create_connection(self.proxy.server_address, timeout=3)
|
||||
client.sendall(f"POST {FETCH} HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{{}}".encode())
|
||||
self.assertTrue(self.upstream.entered.wait(3))
|
||||
client.close()
|
||||
state = self.gate.control("pause")
|
||||
self.assertEqual(state["inflight"], 1)
|
||||
self.upstream.release.set()
|
||||
self.wait_for(lambda: self.gate.control("status")["inflight"] == 0)
|
||||
self.assertFalse(self.gate.control("status")["uncertain"])
|
||||
|
||||
def test_truncated_upstream_latches_uncertainty_across_restart(self):
|
||||
self.upstream.truncated = True
|
||||
self.upstream.release.set()
|
||||
self.assertEqual(self.request(FETCH)[0], 502)
|
||||
self.wait_for(lambda: self.gate.control("status")["uncertain"])
|
||||
self.assertTrue(self.gate.control("resume")["paused"])
|
||||
restored = MODULE.Gate(self.temp.name)
|
||||
self.assertTrue(restored.control("status")["uncertain"])
|
||||
self.assertFalse(restored.enter())
|
||||
|
||||
def test_crashed_inflight_is_not_treated_as_successful_drain(self):
|
||||
self.assertTrue(self.gate.enter())
|
||||
restored = MODULE.Gate(self.temp.name)
|
||||
self.assertTrue(restored.control("status")["uncertain"])
|
||||
self.assertIn("error", restored.control("resume"))
|
||||
self.gate.leave(True)
|
||||
|
||||
def test_pause_marker_survives_restart(self):
|
||||
self.gate.control("pause")
|
||||
restored = MODULE.Gate(self.temp.name)
|
||||
self.assertEqual(core_state(restored.control("status")),
|
||||
{"paused": True, "inflight": 0, "uncertain": False})
|
||||
restored.control("resume")
|
||||
self.assertFalse(MODULE.Gate(self.temp.name).control("status")["paused"])
|
||||
|
||||
def test_chunked_body_is_decoded_and_non_rpc_path_rejected(self):
|
||||
self.assertEqual(self.request(UPDATE, b'{"state":"running"}', chunked=True)[0], 200)
|
||||
self.assertEqual(self.upstream.requests[-1], (UPDATE, b'{"state":"running"}'))
|
||||
self.assertEqual(self.request("/api/v1/repos")[0], 404)
|
||||
self.assertEqual(self.request("/api/actions/../v1/repos")[0], 404)
|
||||
|
||||
def test_upstream_redirect_is_not_followed(self):
|
||||
self.assertEqual(self.request("/api/actions/runner.v1.RunnerService/Redirect")[0], 302)
|
||||
self.assertFalse(self.upstream.entered.is_set())
|
||||
self.assertEqual(len(self.upstream.requests), 1)
|
||||
|
||||
def test_connection_refused_before_rpc_does_not_latch_uncertainty(self):
|
||||
with socket.socket() as unused:
|
||||
unused.bind(("127.0.0.1", 0))
|
||||
port = unused.getsockname()[1]
|
||||
self.proxy.upstream = MODULE.urllib.parse.urlsplit(f"http://127.0.0.1:{port}")
|
||||
self.assertEqual(self.request(FETCH)[0], 502)
|
||||
self.assertEqual(core_state(self.gate.control("status")),
|
||||
{"paused": False, "inflight": 0, "uncertain": False})
|
||||
|
||||
@unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix control socket required")
|
||||
def test_control_socket_uses_newline_json(self):
|
||||
path = str(Path(self.temp.name) / "gate.sock")
|
||||
control = MODULE.ControlServer(path, MODULE.Control)
|
||||
control.gate = self.gate
|
||||
self.servers.append(control)
|
||||
threading.Thread(target=control.serve_forever, daemon=True).start()
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
|
||||
client.connect(path)
|
||||
client.sendall(b'{"action":"pause"}\n')
|
||||
payload = client.makefile("rb").readline()
|
||||
self.assertEqual(core_state(json.loads(payload)),
|
||||
{"paused": True, "inflight": 0, "uncertain": False})
|
||||
|
||||
def test_paused_fetch_records_live_route_and_restart_discards_proof(self):
|
||||
self.gate.control("pause")
|
||||
before = time.time()
|
||||
self.assertEqual(self.request(FETCH)[0], 503)
|
||||
state = self.gate.control("status")
|
||||
self.assertEqual(state["last_fetch_peer"], "127.0.0.1")
|
||||
self.assertGreaterEqual(state["last_fetch_at"], before)
|
||||
self.assertEqual(self.upstream.requests, [])
|
||||
self.assertEqual(self.request(UPDATE)[0], 200)
|
||||
self.assertEqual(self.gate.control("status")["last_fetch_at"], state["last_fetch_at"])
|
||||
restarted = MODULE.Gate(self.temp.name).control("status")
|
||||
self.assertIsNone(restarted["last_fetch_peer"])
|
||||
self.assertIsNone(restarted["last_fetch_at"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,424 @@
|
||||
#!/usr/bin/env python3
|
||||
"""High-risk state-transition tests for the host-only Rust cache maintainer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("maintain-gitea-rust-cache.py")
|
||||
SPEC = importlib.util.spec_from_file_location("gitea_cache_maintenance", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
maintenance_module = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(maintenance_module)
|
||||
|
||||
IMAGE = "sha256:" + "1" * 64
|
||||
OLD_IMAGE = "sha256:" + "2" * 64
|
||||
BASE_IMAGE = "sha256:" + "3" * 64
|
||||
SHA = "a" * 40
|
||||
|
||||
|
||||
def runner_config(image: str) -> str:
|
||||
return f'runners:\n - "genarrative-ci:docker://{image}"\n'
|
||||
|
||||
|
||||
class FakeApi:
|
||||
def __init__(self, disabled: bool = False):
|
||||
self.disabled = disabled
|
||||
self.requests: list[str] = []
|
||||
|
||||
def request(self, path, body=None, raw=False):
|
||||
self.requests.append(path)
|
||||
if path.startswith("admin/actions/runners/"):
|
||||
return {"disabled": self.disabled, "busy": False}
|
||||
raise AssertionError(f"unexpected API request: {path}")
|
||||
|
||||
|
||||
class GiteaCacheMaintenanceTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary_directory.name)
|
||||
self.token = self.root / "token"
|
||||
self.token.write_text("test-token\n", encoding="utf-8")
|
||||
self.config = {
|
||||
"state_dir": str((self.root / "state").resolve()),
|
||||
"api_url": "https://gitea.example.test/api/v1",
|
||||
"token_file": str(self.token),
|
||||
"repository": "team/project",
|
||||
"repository_url": "http://gitea:3000/team/project.git",
|
||||
"runner_api_path": "admin/actions/runners/1",
|
||||
"runner_container": "gitea-runner",
|
||||
"gate_socket": str(self.root / "gate.sock"),
|
||||
"gate_url": "https://gitea.example.test",
|
||||
"clone_url": "https://gitea.example.test/team/project.git",
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
self.temporary_directory.cleanup()
|
||||
|
||||
def record(self, image, *, source=SHA, owned=True, verified_run=None, base=BASE_IMAGE):
|
||||
return {
|
||||
"image": image,
|
||||
"tag": (
|
||||
"genarrative/gitea-project-ci:rust-cache-auto-" + source
|
||||
if source is not None else None
|
||||
),
|
||||
"source": source,
|
||||
"inputs": "inputs",
|
||||
"base": base,
|
||||
"owned": owned,
|
||||
"activated": "2026-01-01T00:00:00+00:00",
|
||||
"verified_run": verified_run,
|
||||
"artifact": str(self.root / "state" / "artifacts" / (source or "manual")),
|
||||
}
|
||||
|
||||
def maintenance(self, state):
|
||||
state_root = Path(self.config["state_dir"])
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
(state_root / "state.json").write_text(json.dumps(state), encoding="utf-8")
|
||||
instance = maintenance_module.Maintenance(self.config)
|
||||
instance.api = FakeApi()
|
||||
return instance
|
||||
|
||||
def activation_maintenance(self, *, idle_values, switch=None):
|
||||
old = self.record(OLD_IMAGE, owned=False, verified_run=1)
|
||||
candidate = self.record(IMAGE)
|
||||
state = {
|
||||
"versions": [old, candidate],
|
||||
"current": OLD_IMAGE,
|
||||
"rollback": None,
|
||||
"candidate": IMAGE,
|
||||
}
|
||||
if switch is not None:
|
||||
state["switch"] = switch
|
||||
instance = self.maintenance(state)
|
||||
config = {"value": runner_config(OLD_IMAGE)}
|
||||
writes: list[str] = []
|
||||
restarts: list[tuple] = []
|
||||
statuses = 0
|
||||
|
||||
instance.check_gate_route = lambda: None
|
||||
instance.read_config = lambda: config["value"]
|
||||
instance.write_config = lambda value: (writes.append(value), config.__setitem__("value", value))
|
||||
instance.image_info = lambda image, inner=False: {"Id": image}
|
||||
|
||||
def docker(*args, inner=False, **kwargs):
|
||||
if args[:1] == ("restart",):
|
||||
restarts.append(args)
|
||||
return ""
|
||||
if args[:2] == ("inspect", "--format"):
|
||||
if args[2] == "{{.State.StartedAt}}":
|
||||
return "2026-01-01T00:00:00+00:00"
|
||||
if args[2] == "{{.State.Status}}":
|
||||
return "running"
|
||||
if args[:1] == ("logs",):
|
||||
return "declare successfully"
|
||||
raise AssertionError(f"unexpected docker call: {args}, inner={inner}")
|
||||
|
||||
instance.docker = docker
|
||||
iterator = iter(idle_values)
|
||||
instance.idle = lambda: next(iterator)
|
||||
|
||||
def gate(action):
|
||||
nonlocal statuses
|
||||
if action == "pause":
|
||||
return {"paused": True}
|
||||
if action == "resume":
|
||||
return {"paused": False}
|
||||
if action == "status":
|
||||
statuses += 1
|
||||
return {"paused": False} if statuses == 1 else {"paused": True, "inflight": 0}
|
||||
raise AssertionError(action)
|
||||
|
||||
instance.gate = gate
|
||||
return instance, writes, restarts
|
||||
|
||||
def test_cache_inputs_keeps_openapi_and_embedded_skill_text(self):
|
||||
tree = "\n".join(
|
||||
[
|
||||
"100644 blob a\tdocs/guide.md",
|
||||
"100644 blob b\tdocs/openapi/external.json",
|
||||
"100644 blob c\t.codex/skills/example/SKILL.md",
|
||||
"100644 blob d\tREADME.md",
|
||||
]
|
||||
)
|
||||
digest = maintenance_module.cache_inputs(tree)
|
||||
self.assertEqual(digest, maintenance_module.cache_inputs(
|
||||
tree.replace("guide.md", "【说明】中文文档.md").replace("\n", "\0") + "\0"))
|
||||
self.assertEqual(digest, maintenance_module.cache_inputs(tree.replace("guide.md", "other.md")))
|
||||
self.assertNotEqual(digest, maintenance_module.cache_inputs(tree.replace("external.json", "changed.json")))
|
||||
self.assertNotEqual(digest, maintenance_module.cache_inputs(tree.replace("SKILL.md", "PROMPT.md")))
|
||||
|
||||
def test_gate_route_requires_independent_checkout_and_live_runner_proof(self):
|
||||
instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None})
|
||||
instance.read_config = lambda: runner_config(OLD_IMAGE)
|
||||
with self.assertRaisesRegex(RuntimeError, "GENARRATIVE_GITEA_REPOSITORY_URL"):
|
||||
instance.check_gate_route()
|
||||
instance.read_config = lambda: runner_config(OLD_IMAGE) + (
|
||||
' envs:\n GENARRATIVE_GITEA_REPOSITORY_URL: "http://gitea:3000/team/project.git"\n')
|
||||
started = "2026-01-01T00:00:00Z"
|
||||
def docker(*args, **kwargs):
|
||||
if args[0] == "exec":
|
||||
return json.dumps({"address": self.config["gate_url"]})
|
||||
return json.dumps([{"State": {"StartedAt": started}, "NetworkSettings": {
|
||||
"Networks": {"internal": {"IPAddress": "10.0.0.2"}}}}])
|
||||
instance.docker = docker
|
||||
proof = {"last_fetch_peer": "10.0.0.2", "last_fetch_at": maintenance_module.timestamp(started).timestamp() + 1}
|
||||
instance.gate = lambda action: proof
|
||||
instance.check_gate_route()
|
||||
proof["last_fetch_at"] -= 2
|
||||
with self.assertRaisesRegex(RuntimeError, "this runner startup"):
|
||||
instance.check_gate_route()
|
||||
proof.update(last_fetch_peer="10.0.0.3", last_fetch_at=9999999999)
|
||||
with self.assertRaisesRegex(RuntimeError, "this runner startup"):
|
||||
instance.check_gate_route()
|
||||
|
||||
def test_complete_nine_job_run_requires_real_rust_cache_statistics(self):
|
||||
run = {"status": "completed", "conclusion": "success"}
|
||||
jobs = [{"name": name, "conclusion": "success"} for name in maintenance_module.JOBS]
|
||||
self.assertTrue(maintenance_module.fully_passed(run, jobs))
|
||||
self.assertFalse(maintenance_module.fully_passed(run, jobs[:-1]))
|
||||
good = (
|
||||
f"worker image: {IMAGE}\n[rust-cache] mode=sccache\n"
|
||||
"Cache hits 7\nCache errors 0\nCache read errors 0\nCache write errors 0\n"
|
||||
)
|
||||
self.assertTrue(maintenance_module.verifies_image({"name": "Backend tests"}, good, IMAGE))
|
||||
self.assertFalse(maintenance_module.verifies_image({"name": "Backend tests"}, good.replace("Cache hits 7", "Cache hits 0"), IMAGE))
|
||||
self.assertFalse(maintenance_module.verifies_image({"name": "Backend tests"}, good.replace("Cache write errors 0", "Cache write errors 1"), IMAGE))
|
||||
|
||||
def test_fetches_latest_master_without_ci_green_gate(self):
|
||||
instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None})
|
||||
instance.repo.mkdir(parents=True)
|
||||
(instance.repo / ".git").mkdir()
|
||||
(instance.repo / ".git" / "genarrative-cache-maintenance").write_text("owned source checkout\n")
|
||||
calls = []
|
||||
|
||||
def fake_command(*args, **kwargs):
|
||||
calls.append(args)
|
||||
if args[1:4] == ("remote", "get-url", "origin"):
|
||||
return instance.config["clone_url"] + "\n"
|
||||
if args[1:3] == ("rev-parse", "FETCH_HEAD^{commit}"):
|
||||
return SHA + "\n"
|
||||
if args[1:3] == ("ls-tree", "-rz"):
|
||||
return "100644 blob x\tdocs/openapi/current.json\n"
|
||||
return ""
|
||||
|
||||
with patch.object(maintenance_module, "command", fake_command):
|
||||
self.assertEqual(instance.fetch_source()[0], SHA)
|
||||
self.assertIn(("git", "remote", "get-url", "origin"), calls)
|
||||
self.assertIn(("git", "fetch", "--no-tags", "origin", "refs/heads/master"), calls)
|
||||
self.assertIn(("git", "clean", "-ffdx"), calls)
|
||||
self.assertFalse(any("runs" in " ".join(call) for call in calls))
|
||||
|
||||
def test_idle_race_after_pause_restores_without_restart(self):
|
||||
instance, writes, restarts = self.activation_maintenance(idle_values=[True, False])
|
||||
with patch.object(maintenance_module.time, "sleep", lambda _: None):
|
||||
self.assertFalse(instance.activate())
|
||||
self.assertEqual(restarts, [])
|
||||
self.assertEqual(writes, [])
|
||||
self.assertEqual(instance.state["current"], OLD_IMAGE)
|
||||
self.assertEqual(instance.state["candidate"], IMAGE)
|
||||
self.assertNotIn("pause_owned", instance.state)
|
||||
|
||||
def test_uncertain_or_inflight_gate_never_restarts(self):
|
||||
instance, _, restarts = self.activation_maintenance(idle_values=[True])
|
||||
statuses = 0
|
||||
|
||||
def uncertain_gate(action):
|
||||
nonlocal statuses
|
||||
if action == "pause":
|
||||
return {"paused": True}
|
||||
if action == "resume":
|
||||
return {"paused": False}
|
||||
statuses += 1
|
||||
return {"paused": False} if statuses == 1 else {"paused": True, "uncertain": True, "inflight": 1}
|
||||
|
||||
instance.gate = uncertain_gate
|
||||
with patch.object(maintenance_module.time, "sleep", lambda _: None):
|
||||
with self.assertRaisesRegex(RuntimeError, "uncertain"):
|
||||
instance.activate()
|
||||
self.assertEqual(restarts, [])
|
||||
self.assertEqual(instance.state["candidate"], IMAGE)
|
||||
self.assertEqual(instance.state["current"], OLD_IMAGE)
|
||||
self.assertNotIn("pause_owned", instance.state)
|
||||
|
||||
def test_inflight_gate_defers_without_restart_and_resumes(self):
|
||||
instance, _, restarts = self.activation_maintenance(idle_values=[True])
|
||||
statuses = 0
|
||||
|
||||
def inflight_gate(action):
|
||||
nonlocal statuses
|
||||
if action == "pause":
|
||||
return {"paused": True}
|
||||
if action == "resume":
|
||||
return {"paused": False}
|
||||
statuses += 1
|
||||
return {"paused": False} if statuses == 1 else {"paused": True, "inflight": 1}
|
||||
|
||||
instance.gate = inflight_gate
|
||||
with patch.object(maintenance_module.time, "sleep", lambda _: None):
|
||||
self.assertFalse(instance.activate())
|
||||
self.assertEqual(restarts, [])
|
||||
self.assertEqual(instance.state["candidate"], IMAGE)
|
||||
self.assertEqual(instance.state["current"], OLD_IMAGE)
|
||||
self.assertNotIn("pause_owned", instance.state)
|
||||
|
||||
def test_interrupted_switch_recovers_current_candidate_and_rollback(self):
|
||||
backup = self.root / "state" / "backups" / "switch.yaml"
|
||||
backup.parent.mkdir(parents=True)
|
||||
backup.write_text(runner_config(OLD_IMAGE), encoding="utf-8")
|
||||
instance, _, restarts = self.activation_maintenance(
|
||||
idle_values=[True, True],
|
||||
switch={"old": OLD_IMAGE, "new": IMAGE, "backup": str(backup)},
|
||||
)
|
||||
with patch.object(maintenance_module.time, "sleep", lambda _: None):
|
||||
self.assertTrue(instance.recover_switch())
|
||||
self.assertEqual(len(restarts), 1)
|
||||
self.assertEqual(instance.state["current"], IMAGE)
|
||||
self.assertIsNone(instance.state["candidate"])
|
||||
self.assertEqual(instance.state["rollback"], OLD_IMAGE)
|
||||
self.assertNotIn("switch", instance.state)
|
||||
|
||||
def test_protected_images_include_state_bases_config_and_container_references(self):
|
||||
current = self.record(OLD_IMAGE, owned=False, verified_run=1, base=BASE_IMAGE)
|
||||
candidate = self.record(IMAGE, base=BASE_IMAGE)
|
||||
container_image = "sha256:" + "4" * 64
|
||||
instance = self.maintenance({"versions": [current, candidate], "current": OLD_IMAGE, "rollback": OLD_IMAGE, "candidate": IMAGE})
|
||||
instance.read_config = lambda: runner_config(OLD_IMAGE)
|
||||
|
||||
def docker(*args, inner=False, **kwargs):
|
||||
if args[:2] == ("ps", "-aq"):
|
||||
return "container-1\n"
|
||||
if args[:2] == ("inspect", "--format"):
|
||||
return container_image + "\n"
|
||||
raise AssertionError(args)
|
||||
|
||||
instance.docker = docker
|
||||
protected = instance.protected_images()
|
||||
self.assertTrue({OLD_IMAGE, IMAGE, BASE_IMAGE, container_image}.issubset(protected))
|
||||
|
||||
def test_cleanup_keeps_manual_images_and_rejects_outside_artifacts(self):
|
||||
current = self.record(OLD_IMAGE, owned=False, verified_run=1)
|
||||
manual = self.record(IMAGE, owned=False)
|
||||
stale = self.record("sha256:" + "5" * 64, source="b" * 40)
|
||||
stale["artifact"] = str(self.root / "outside")
|
||||
instance = self.maintenance({"versions": [current, manual, stale], "current": OLD_IMAGE, "rollback": None, "candidate": None})
|
||||
instance.protected_images = lambda: set()
|
||||
instance.remove_owned_image = lambda *_: True
|
||||
with self.assertRaisesRegex(RuntimeError, "outside"):
|
||||
instance.cleanup()
|
||||
self.assertIn(manual, instance.state["versions"])
|
||||
self.assertIn(stale, instance.state["versions"])
|
||||
|
||||
def test_additional_tags_are_never_deleted(self):
|
||||
instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None})
|
||||
removals = []
|
||||
tag = "genarrative/gitea-project-ci:rust-cache-auto-" + SHA
|
||||
instance.image_info = lambda image, inner=False: {"RepoTags": [tag, "operator:keep"]}
|
||||
instance.docker = lambda *args, inner=False, **kwargs: (
|
||||
IMAGE + "\n" if args[:2] == ("image", "ls") else removals.append(args) or ""
|
||||
)
|
||||
self.assertFalse(instance.remove_owned_image(IMAGE, tag, set()))
|
||||
self.assertEqual(removals, [])
|
||||
|
||||
def test_owned_image_removal_handles_dangling_and_already_absent_images(self):
|
||||
instance = self.maintenance({"versions": [], "current": None, "rollback": None, "candidate": None})
|
||||
tag = "genarrative/gitea-project-ci:rust-cache-auto-" + SHA
|
||||
present = {True: True, False: True}
|
||||
removals = []
|
||||
|
||||
def docker(*args, inner=False, **kwargs):
|
||||
if args[:2] == ("image", "ls"):
|
||||
return IMAGE + "\n" if present[inner] else ""
|
||||
if args[:3] == ("image", "rm", IMAGE):
|
||||
removals.append((args, inner))
|
||||
present[inner] = False
|
||||
return IMAGE + "\n"
|
||||
raise AssertionError(args)
|
||||
|
||||
instance.docker = docker
|
||||
instance.image_info = lambda image, inner=False: {"RepoTags": []}
|
||||
self.assertTrue(instance.remove_owned_image(IMAGE, tag, set()))
|
||||
self.assertEqual({inner for _, inner in removals}, {True, False})
|
||||
calls = []
|
||||
instance.docker = lambda *args, inner=False, **kwargs: calls.append((args, inner)) or ""
|
||||
self.assertTrue(instance.remove_owned_image(
|
||||
IMAGE, "genarrative/gitea-project-ci:rust-cache-auto-" + SHA, set()))
|
||||
self.assertFalse(any(args[:3] == ("image", "rm", IMAGE) for args, _ in calls))
|
||||
|
||||
def test_manual_current_without_cache_stats_can_be_verified(self):
|
||||
current = self.record(OLD_IMAGE, source=None, owned=False)
|
||||
instance = self.maintenance({"versions": [current], "current": OLD_IMAGE, "rollback": None, "candidate": None})
|
||||
jobs = [
|
||||
{"id": index, "name": name, "conclusion": "success"}
|
||||
for index, name in enumerate(sorted(maintenance_module.JOBS), start=1)
|
||||
]
|
||||
run = {
|
||||
"id": 44,
|
||||
"status": "completed",
|
||||
"conclusion": "success",
|
||||
"event": "push",
|
||||
"path": "project-ci.yml@refs/heads/master",
|
||||
"head_sha": SHA,
|
||||
"started_at": "2026-01-02T00:00:00Z",
|
||||
}
|
||||
|
||||
class VerificationApi:
|
||||
def pages(self, path, key):
|
||||
if path.endswith("/actions/runs?status=success&branch=master&event=push"):
|
||||
return iter([run])
|
||||
if path.endswith("/actions/runs/44/jobs"):
|
||||
return iter(jobs)
|
||||
raise AssertionError(path)
|
||||
|
||||
def request(self, path, body=None, raw=False):
|
||||
if not raw or "/actions/jobs/" not in path:
|
||||
raise AssertionError(path)
|
||||
return f"runner image: {OLD_IMAGE}\n"
|
||||
|
||||
instance.api = VerificationApi()
|
||||
with patch.object(maintenance_module.subprocess, "run", return_value=type("Result", (), {"returncode": 0})()):
|
||||
self.assertTrue(instance.verify_current())
|
||||
self.assertEqual(instance.version(OLD_IMAGE)["verified_run"], 44)
|
||||
self.assertEqual(instance.version(OLD_IMAGE)["verified_sha"], SHA)
|
||||
|
||||
def test_recover_builds_adopts_only_pre_registered_attempt_tags(self):
|
||||
tag = "genarrative/gitea-project-ci:rust-cache-auto-" + SHA
|
||||
base_tag = "genarrative/gitea-project-ci:base-auto-" + SHA
|
||||
artifact = self.root / "state" / "artifacts" / SHA
|
||||
attempt = {"source": SHA, "inputs": "inputs", "tag": tag, "base_tag": base_tag, "artifact": str(artifact)}
|
||||
instance = self.maintenance({
|
||||
"versions": [], "current": None, "rollback": None, "candidate": None,
|
||||
"attempts": [attempt],
|
||||
})
|
||||
queries = []
|
||||
|
||||
def docker(*args, inner=False, **kwargs):
|
||||
if args[:3] == ("image", "ls", "--no-trunc"):
|
||||
queries.append(args[-1])
|
||||
return BASE_IMAGE + "\n" if args[-1] == base_tag else IMAGE + "\n"
|
||||
raise AssertionError(args)
|
||||
|
||||
instance.docker = docker
|
||||
instance.image_info = lambda image, inner=False: {
|
||||
"Config": {"Labels": {
|
||||
"world.genarrative.ci.rust-cache-source": SHA,
|
||||
"world.genarrative.ci.rust-cache-base": BASE_IMAGE,
|
||||
}}
|
||||
}
|
||||
instance.recover_builds()
|
||||
self.assertEqual(queries, [base_tag, tag])
|
||||
self.assertEqual(instance.state["attempts"], [])
|
||||
self.assertEqual(instance.state["bases"], {BASE_IMAGE: base_tag})
|
||||
self.assertEqual(instance.state["versions"][0]["image"], IMAGE)
|
||||
self.assertTrue(instance.state["versions"][0]["owned"])
|
||||
self.assertEqual(instance.state["candidate"], IMAGE)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user