lhk229
e3c2c1b732
基于固定 master 提交构建全新缓存镜像，并在成功验证后保留当前版和回滚版。 增加任务领取网关，在不打断运行中 CI 的前提下切换镜像。 补充宿主维护服务、部署说明、共享记忆和定向行为测试。 修复构建测试在 Windows 与 WSL 下并发写入日志的竞态。
651 lines
31 KiB
Python
651 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
"""宿主专用的 Rust 缓存维护器；仅使用 Python 标准库，不在 CI job 中运行。"""
|
|
|
|
import argparse
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
IMAGE = re.compile(r"sha256:[0-9a-f]{64}\Z")
|
|
SHA = re.compile(r"[0-9a-f]{40}\Z")
|
|
LABEL = re.compile(r'(?m)^(\s*-\s*[\"\x27]?)genarrative-ci:docker://(sha256:[0-9a-f]{64})([\"\x27]?\s*)$')
|
|
JOBS = {
|
|
"AI game creator shell Rust lane 1/2",
|
|
"AI game creator shell Rust lane 2/2",
|
|
"AI game creator shell Rust smoke",
|
|
"AI game creator shell Rust crates",
|
|
"Backend tests", "Native shell tests", "Frontend tests",
|
|
"Repository checks", "AI game creator shell web tests",
|
|
}
|
|
RUST_JOBS = {name for name in JOBS if "Rust" in name or name in {"Backend tests", "Native shell tests"}}
|
|
|
|
|
|
def now():
|
|
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
|
|
|
|
def timestamp(value):
|
|
return datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
|
|
|
|
def log(message):
|
|
print(f"[cache-maintenance] {message}", flush=True)
|
|
|
|
|
|
def command(*args, cwd=None, data=None, env=None, output=None, timeout=120, combined=False):
|
|
result = subprocess.run(
|
|
args, cwd=cwd, input=data, text=True, env=env, timeout=timeout,
|
|
stdout=output or subprocess.PIPE,
|
|
stderr=output or (subprocess.STDOUT if combined else subprocess.PIPE),
|
|
)
|
|
if result.returncode:
|
|
# 参数、stderr 或 HTTP body 可能含凭据，只把详细构建输出写到私有日志。
|
|
raise RuntimeError(f"{Path(args[0]).name} failed (exit {result.returncode})")
|
|
return result.stdout or ""
|
|
|
|
|
|
def atomic_json(path, value):
|
|
temporary = path.with_suffix(".tmp")
|
|
with temporary.open("w", encoding="utf-8") as out:
|
|
json.dump(value, out, ensure_ascii=False, indent=2)
|
|
out.write("\n")
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
temporary.replace(path)
|
|
if os.name == "posix":
|
|
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(directory)
|
|
finally:
|
|
os.close(directory)
|
|
|
|
|
|
def cache_inputs(tree):
|
|
"""只排除已知不参与构建的说明文档；docs/openapi 与内嵌 skill 必须参与。"""
|
|
rows = []
|
|
for row in (tree.split("\0") if "\0" in tree else tree.splitlines()):
|
|
if not row:
|
|
continue
|
|
path = row.split("\t", 1)[1]
|
|
if path.startswith("docs/") and not path.startswith("docs/openapi/"):
|
|
continue
|
|
if path in {"AGENTS.md", "README.md", "deploy/container/README.md"}:
|
|
continue
|
|
rows.append(row)
|
|
return hashlib.sha256("\n".join(rows).encode()).hexdigest()
|
|
|
|
|
|
def configured_image(config):
|
|
labels = list(LABEL.finditer(config))
|
|
if len(labels) != 1:
|
|
raise RuntimeError("expected exactly one genarrative-ci Image ID label")
|
|
return labels[0].group(2)
|
|
|
|
|
|
def replace_image(config, old, new):
|
|
if not IMAGE.fullmatch(new) or configured_image(config) != old:
|
|
raise RuntimeError("runner image changed outside maintenance; refusing overwrite")
|
|
return LABEL.sub(lambda match: match[1] + "genarrative-ci:docker://" + new + match[3], config)
|
|
|
|
|
|
def fully_passed(run, jobs):
|
|
return (
|
|
run.get("status") == "completed" and run.get("conclusion") == "success"
|
|
and len(jobs) == len(JOBS) and {job["name"] for job in jobs} == JOBS
|
|
and all(job.get("conclusion") == "success" for job in jobs)
|
|
)
|
|
|
|
|
|
def verifies_image(job, contents, image, require_cache=True):
|
|
if not re.search(r"(?m)^\S+ image: " + re.escape(image) + r"\s*$", contents):
|
|
return False
|
|
if not require_cache or job["name"] not in RUST_JOBS:
|
|
return True
|
|
# 旧分支使用了镜像却未 prepare，不算完成缓存上线验收。
|
|
hits = re.search(r"Cache hits\s+(\d+)\s*$", contents, re.M)
|
|
errors = [re.search(rf"{name}\s+(\d+)\s*$", contents, re.M)
|
|
for name in ("Cache errors", "Cache read errors", "Cache write errors")]
|
|
return ("[rust-cache] mode=sccache" in contents and hits is not None
|
|
and int(hits[1]) > 0 and all(item and int(item[1]) == 0 for item in errors))
|
|
|
|
|
|
class Api:
|
|
def __init__(self, url, token_file):
|
|
self.url = url.rstrip("/")
|
|
if not self.url.startswith("https://"):
|
|
raise ValueError("api_url must use HTTPS")
|
|
self.token_file = Path(token_file)
|
|
|
|
def request(self, path, body=None, raw=False):
|
|
token = self.token_file.read_text().strip()
|
|
if not token or "\n" in token:
|
|
raise ValueError("invalid token file")
|
|
request = urllib.request.Request(
|
|
self.url + "/" + path.lstrip("/"),
|
|
data=None if body is None else json.dumps(body).encode(),
|
|
method="GET" if body is None else "PATCH",
|
|
headers={"Authorization": "token " + token, "Content-Type": "application/json"},
|
|
)
|
|
try:
|
|
# 不允许带 Authorization 的请求跟随跨站重定向。
|
|
opener = urllib.request.build_opener(NoRedirect())
|
|
with opener.open(request, timeout=30) as response:
|
|
content = response.read().decode()
|
|
except urllib.error.HTTPError as error:
|
|
raise RuntimeError(f"Gitea API HTTP {error.code}") from None
|
|
return content if raw else json.loads(content)
|
|
|
|
def pages(self, path, key):
|
|
separator = "&" if "?" in path else "?"
|
|
page = 1
|
|
while True:
|
|
response = self.request(f"{path}{separator}limit=50&page={page}")
|
|
items = response[key]
|
|
yield from items
|
|
if len(items) < 50:
|
|
return
|
|
page += 1
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
|
|
class Maintenance:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
if not Path(config["state_dir"]).is_absolute():
|
|
raise ValueError("state_dir must be a dedicated absolute directory")
|
|
self.root = Path(config["state_dir"]).resolve()
|
|
if self.root == Path("/"):
|
|
raise ValueError("state_dir must be a dedicated absolute directory")
|
|
self.repo = self.root / "source"
|
|
self.runner = config.get("runner_container", "gitea-runner")
|
|
self.api = Api(config["api_url"], config["token_file"])
|
|
self.repo_api = "repos/" + config["repository"]
|
|
self.runner_api = config["runner_api_path"].strip("/")
|
|
if not re.fullmatch(r"admin/actions/runners/\d+", self.runner_api):
|
|
raise ValueError("this service requires the global runner admin endpoint")
|
|
self.state_path = self.root / "state.json"
|
|
self.state = json.loads(self.state_path.read_text()) if self.state_path.exists() else {
|
|
"versions": [], "current": None, "rollback": None, "candidate": None,
|
|
}
|
|
|
|
def save(self):
|
|
atomic_json(self.state_path, self.state)
|
|
|
|
def docker(self, *args, inner=False, **kwargs):
|
|
prefix = ("docker", "exec", self.runner, "docker") if inner else ("docker",)
|
|
return command(*prefix, *args, **kwargs)
|
|
|
|
def image_info(self, image, inner=False):
|
|
return json.loads(self.docker("image", "inspect", image, inner=inner))[0]
|
|
|
|
def read_config(self):
|
|
return self.docker("exec", self.runner, "cat", "/data/config.yaml")
|
|
|
|
def write_config(self, content):
|
|
self.docker("exec", "--user", "0", "-i", self.runner, "sh", "-c",
|
|
"cat > /data/config.yaml", data=content)
|
|
|
|
def version(self, image):
|
|
return next(row for row in self.state["versions"] if row["image"] == image)
|
|
|
|
def adopt_current(self):
|
|
actual = configured_image(self.read_config())
|
|
if self.state["current"]:
|
|
if actual != self.state["current"]:
|
|
raise RuntimeError("runner configuration differs from recorded current image")
|
|
return
|
|
info = self.image_info(actual)
|
|
labels = info["Config"].get("Labels") or {}
|
|
source = labels.get("world.genarrative.ci.rust-cache-source")
|
|
base = labels.get("world.genarrative.ci.rust-cache-base", actual)
|
|
if source is not None and not SHA.fullmatch(source):
|
|
raise RuntimeError("invalid image source SHA")
|
|
self.state["versions"].append({
|
|
"image": actual, "source": source, "base": base, "owned": False,
|
|
"activated": "1970-01-01T00:00:00+00:00", "verified_run": None,
|
|
})
|
|
self.state["current"] = actual
|
|
self.save()
|
|
|
|
def fetch_source(self):
|
|
if self.repo.resolve() != self.repo:
|
|
raise RuntimeError("dedicated clone must not be a symlink")
|
|
marker = self.repo / ".git" / "genarrative-cache-maintenance"
|
|
if not (self.repo / ".git").exists():
|
|
if self.repo.exists():
|
|
raise RuntimeError("source directory already exists without .git")
|
|
command("git", "clone", "--no-checkout", self.config["clone_url"], str(self.repo), timeout=900)
|
|
marker.write_text("owned source checkout\n")
|
|
if not marker.is_file():
|
|
raise RuntimeError("refusing to clean a checkout not created by this maintainer")
|
|
if command("git", "remote", "get-url", "origin", cwd=self.repo).strip() != self.config["clone_url"]:
|
|
raise RuntimeError("dedicated clone origin differs from configured repository")
|
|
command("git", "fetch", "--no-tags", "origin", "refs/heads/master", cwd=self.repo, timeout=900)
|
|
sha = command("git", "rev-parse", "FETCH_HEAD^{commit}", cwd=self.repo).strip()
|
|
if not SHA.fullmatch(sha):
|
|
raise RuntimeError("invalid master SHA")
|
|
# 这是维护器专属 clone，不修改开发者工作区；编译期间不再 checkout 新提交。
|
|
command("git", "checkout", "--detach", "--force", sha, cwd=self.repo)
|
|
command("git", "clean", "-ffdx", cwd=self.repo)
|
|
tree = command("git", "ls-tree", "-rz", sha, cwd=self.repo)
|
|
return sha, cache_inputs(tree)
|
|
|
|
def build_command(self, log_file, script, *args, env=None):
|
|
with log_file.open("a") as out:
|
|
command("bash", str(self.repo / "scripts" / script), *args, cwd=self.repo,
|
|
env=env, output=out, timeout=7200)
|
|
|
|
def build(self, sha, inputs):
|
|
artifact = self.root / "artifacts" / sha
|
|
artifact.mkdir(parents=True, exist_ok=True)
|
|
build_log = artifact / "build.log"
|
|
tag = "genarrative/gitea-project-ci:rust-cache-auto-" + sha
|
|
base_tag = "genarrative/gitea-project-ci:base-auto-" + sha
|
|
self.state.setdefault("attempts", []).append({
|
|
"source": sha, "inputs": inputs, "tag": tag,
|
|
"base_tag": base_tag, "artifact": str(artifact),
|
|
})
|
|
self.save()
|
|
env = {**os.environ, "GENARRATIVE_GITEA_RUNNER_CONTAINER": self.runner}
|
|
env.pop("CI", None)
|
|
current = self.version(self.state["current"])
|
|
base = current["base"]
|
|
revision = command("bash", "scripts/gitea-ci-job-image.sh", "revision", cwd=self.repo).strip()
|
|
labels = self.image_info(base)["Config"].get("Labels") or {}
|
|
if labels.get("com.genarrative.ci.definition-sha256") != revision:
|
|
env["GENARRATIVE_GITEA_CI_IMAGE_TAG"] = base_tag
|
|
self.build_command(build_log, "gitea-ci-job-image.sh", "build", env=env)
|
|
base = self.image_info(base_tag)["Id"]
|
|
self.state.setdefault("bases", {})[base] = base_tag
|
|
self.save()
|
|
self.build_command(build_log, "build-gitea-rust-cache.sh", base, tag, sha, env=env)
|
|
image = self.image_info(tag)["Id"]
|
|
record = {"image": image, "tag": tag, "source": sha, "inputs": inputs,
|
|
"base": base, "owned": True, "verified_run": None,
|
|
"artifact": str(artifact)}
|
|
# 先登记再导出；进程中断后下次可以复用候选，不留下无归属成功镜像。
|
|
self.state["versions"].append(record)
|
|
self.state["candidate"] = image
|
|
self.state["attempts"] = [row for row in self.state["attempts"] if row["source"] != sha]
|
|
self.save()
|
|
log(f"candidate built source={sha} image={image}")
|
|
|
|
def stage_candidate(self):
|
|
candidate = self.version(self.state["candidate"])
|
|
command("git", "checkout", "--detach", "--force", candidate["source"], cwd=self.repo)
|
|
command("git", "clean", "-ffdx", cwd=self.repo)
|
|
artifact = Path(candidate["artifact"])
|
|
archive = artifact / "image.tar.zst"
|
|
sidecar = archive.with_suffix(".zst.sha256")
|
|
if (candidate.get("staged") and archive.is_file() and sidecar.is_file()
|
|
and candidate["image"] in self.docker("image", "ls", "--all", "--no-trunc", "--quiet", inner=True).split()):
|
|
return
|
|
env = {**os.environ, "GENARRATIVE_GITEA_RUNNER_CONTAINER": self.runner}
|
|
build_log = artifact / "build.log"
|
|
if not sidecar.exists():
|
|
# 只删除登记目录中的未完成导出文件，不覆盖已验证归档。
|
|
archive.unlink(missing_ok=True)
|
|
self.build_command(build_log, "gitea-ci-job-image.sh", "export", str(archive), candidate["image"], env=env)
|
|
command("sha256sum", "--check", sidecar.name, cwd=artifact, timeout=600)
|
|
self.build_command(build_log, "gitea-ci-job-image.sh", "load-runner", candidate["image"], env=env)
|
|
if self.image_info(candidate["image"], inner=True)["Id"] != candidate["image"]:
|
|
raise RuntimeError("inner runner image mismatch")
|
|
candidate["staged"] = True
|
|
self.save()
|
|
|
|
def idle(self):
|
|
runner = self.api.request(self.runner_api)
|
|
active = self.api.request("admin/actions/runs?status=in_progress&limit=1")
|
|
return (runner.get("busy") is False and active["total_count"] == 0
|
|
and not self.docker("ps", "-q", inner=True).strip())
|
|
|
|
def verify_current(self):
|
|
current = self.version(self.state["current"])
|
|
if current.get("verified_run"):
|
|
return True
|
|
for run in self.api.pages(self.repo_api + "/actions/runs?status=success&branch=master&event=push", "workflow_runs"):
|
|
if run.get("path") != "project-ci.yml@refs/heads/master" or run.get("event") != "push":
|
|
continue
|
|
sha = run.get("head_sha", "")
|
|
if not SHA.fullmatch(sha):
|
|
continue
|
|
ancestry = subprocess.run(["git", "merge-base", "--is-ancestor", sha, "FETCH_HEAD"],
|
|
cwd=self.repo, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
if ancestry.returncode != 0:
|
|
continue
|
|
if timestamp(run["started_at"]) < timestamp(current["activated"]):
|
|
continue
|
|
jobs = list(self.api.pages(self.repo_api + f'/actions/runs/{run["id"]}/jobs', "jobs"))
|
|
if not fully_passed(run, jobs):
|
|
continue
|
|
if all(verifies_image(job, self.api.request(
|
|
self.repo_api + f'/actions/jobs/{job["id"]}/logs', raw=True), current["image"],
|
|
require_cache=current.get("source") is not None,
|
|
) for job in jobs):
|
|
current["verified_run"] = run["id"]
|
|
current["verified_sha"] = sha
|
|
self.save()
|
|
log(f'current image verified by run={run["id"]}')
|
|
return True
|
|
log("waiting for a complete successful CI run using current image and cache")
|
|
return False
|
|
|
|
def protected_images(self):
|
|
protected = {self.state[key] for key in ("current", "rollback", "candidate") if self.state.get(key)}
|
|
for image in list(protected):
|
|
protected.add(self.version(image)["base"])
|
|
# 保护包括已停止容器在内的全部引用；不强制删除被 Docker 引用的镜像。
|
|
for inner in (False, True):
|
|
ids = self.docker("ps", "-aq", inner=inner).split()
|
|
for container in ids:
|
|
protected.add(self.docker("inspect", "--format", "{{.Image}}", container, inner=inner).strip())
|
|
protected.add(configured_image(self.read_config()))
|
|
return protected
|
|
|
|
def remove_owned_image(self, image, tag, protected):
|
|
if image in protected or not IMAGE.fullmatch(image):
|
|
return False
|
|
for inner in (True, False):
|
|
available = self.docker("image", "ls", "--all", "--no-trunc", "--quiet", inner=inner).split()
|
|
if image not in available:
|
|
continue
|
|
info = self.image_info(image, inner=inner)
|
|
if not set(info.get("RepoTags") or []).issubset({tag}):
|
|
log(f"retain image with additional tags: {image}")
|
|
return False
|
|
# 不使用 --force；并发创建的容器也由 Docker 的引用检查保护。
|
|
self.docker("image", "rm", image, inner=inner)
|
|
if image in self.docker("image", "ls", "--all", "--no-trunc", "--quiet", inner=inner).split():
|
|
raise RuntimeError("managed image is still present after removal")
|
|
return True
|
|
|
|
def cleanup(self):
|
|
if not self.version(self.state["current"]).get("verified_run"):
|
|
return
|
|
protected = self.protected_images()
|
|
for record in list(self.state["versions"]):
|
|
if not record.get("owned") or record["image"] in protected:
|
|
continue
|
|
expected = self.root / "artifacts" / record["source"]
|
|
if (not SHA.fullmatch(record["source"]) or Path(record["artifact"]).resolve() != expected
|
|
or expected.is_symlink() or expected.parent.is_symlink()
|
|
or record["tag"] != "genarrative/gitea-project-ci:rust-cache-auto-" + record["source"]):
|
|
raise RuntimeError("archive path outside managed source directory")
|
|
if not self.remove_owned_image(record["image"], record["tag"], protected):
|
|
continue
|
|
# 不递归删除目录，只删除本维护器创建的已知文件。
|
|
for name in ("image.tar.zst", "image.tar.zst.sha256", "build.log"):
|
|
(expected / name).unlink(missing_ok=True)
|
|
if expected.exists() and not any(expected.iterdir()):
|
|
expected.rmdir()
|
|
self.state["versions"].remove(record)
|
|
self.save()
|
|
log(f'removed old managed snapshot {record["image"]}')
|
|
for image, tag in list(self.state.get("bases", {}).items()):
|
|
used_bases = {row["base"] for row in self.state["versions"]}
|
|
if image not in used_bases and self.remove_owned_image(image, tag, protected):
|
|
del self.state["bases"][image]
|
|
self.save()
|
|
|
|
def tick(self, retry=False):
|
|
if not self.recover_switch():
|
|
return
|
|
self.adopt_current()
|
|
sha, inputs = self.fetch_source()
|
|
self.recover_builds()
|
|
if not self.verify_current():
|
|
return
|
|
self.cleanup()
|
|
if self.state.get("candidate"):
|
|
self.stage_candidate()
|
|
self.activate()
|
|
return
|
|
current = self.version(self.state["current"])
|
|
if "inputs" not in current and current.get("source"):
|
|
current["inputs"] = cache_inputs(command("git", "ls-tree", "-rz", current["source"], cwd=self.repo))
|
|
self.save()
|
|
if current.get("inputs") == inputs:
|
|
log(f"unchanged compilation inputs at master={sha}")
|
|
return
|
|
if not retry and self.state.get("failed_source") == sha:
|
|
log(f"previous build failed at {sha}; waiting for new master or --retry")
|
|
return
|
|
# 新的 CI 波峰不额外争抢预热资源，等下一次维护周期。
|
|
if not self.idle():
|
|
log("CI active; defer refresh")
|
|
return
|
|
try:
|
|
self.build(sha, inputs)
|
|
except Exception:
|
|
self.state["failed_source"] = sha
|
|
self.save()
|
|
raise
|
|
self.state.pop("failed_source", None)
|
|
self.save()
|
|
self.stage_candidate()
|
|
self.activate()
|
|
|
|
def recover_builds(self):
|
|
# 宕机可能发生在 docker build 完成之后、登记 Image ID 之前。
|
|
# 只接管预先登记的确定性 tag；恢复的候选仍须经过 stage 的 verify/load。
|
|
for attempt in list(self.state.get("attempts", [])):
|
|
for kind in ("base_tag", "tag"):
|
|
ids = set(self.docker("image", "ls", "--no-trunc", "--quiet", attempt[kind]).split())
|
|
if not ids:
|
|
continue
|
|
if len(ids) != 1:
|
|
raise RuntimeError("ambiguous managed build tag")
|
|
image = ids.pop()
|
|
if kind == "base_tag":
|
|
self.state.setdefault("bases", {})[image] = attempt[kind]
|
|
elif not any(row["image"] == image for row in self.state["versions"]):
|
|
labels = self.image_info(image)["Config"].get("Labels") or {}
|
|
if labels.get("world.genarrative.ci.rust-cache-source") != attempt["source"]:
|
|
raise RuntimeError("interrupted build tag has unexpected source")
|
|
self.state["versions"].append({
|
|
**attempt, "image": image, "owned": True, "verified_run": None,
|
|
"base": labels["world.genarrative.ci.rust-cache-base"],
|
|
})
|
|
if self.state.get("candidate") not in (None, image):
|
|
raise RuntimeError("interrupted build conflicts with an existing candidate")
|
|
self.state["candidate"] = image
|
|
self.state["attempts"].remove(attempt)
|
|
self.save()
|
|
|
|
def recover_switch(self):
|
|
if self.state.get("switch"):
|
|
# 不能仅凭配置文件已经替换，就认定内存中的 runner 已加载新映射。
|
|
log("recovering interrupted image switch")
|
|
return self.activate()
|
|
if self.state.get("pause_owned"):
|
|
self.resume()
|
|
return True
|
|
|
|
def resume(self):
|
|
if self.state.get("pause_owned"):
|
|
if self.gate("resume").get("paused") is not False:
|
|
raise RuntimeError("runner resume could not be confirmed")
|
|
self.state.pop("pause_owned", None)
|
|
self.save()
|
|
|
|
def gate(self, action):
|
|
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
|
|
client.settimeout(10)
|
|
client.connect(self.config["gate_socket"])
|
|
client.sendall((json.dumps({"action": action}) + "\n").encode())
|
|
with client.makefile("rb") as stream:
|
|
response = json.loads(stream.readline(4096))
|
|
if "error" in response:
|
|
raise RuntimeError("runner gate refused command")
|
|
return response
|
|
|
|
def check_gate_route(self):
|
|
checkout = re.search(r"(?m)^\s+GENARRATIVE_GITEA_REPOSITORY_URL:\s*(.+?)\s*$", self.read_config())
|
|
if not checkout or checkout[1].strip("\"'") != self.config["repository_url"]:
|
|
raise RuntimeError("runner.envs must pin GENARRATIVE_GITEA_REPOSITORY_URL independently of the RPC gate")
|
|
# .runner 含认证材料，只在进程内取 address，不回显原文。
|
|
registration = json.loads(self.docker("exec", self.runner, "cat", "/data/.runner"))
|
|
if registration.get("address", "").rstrip("/") != self.config["gate_url"].rstrip("/"):
|
|
raise RuntimeError("runner is not configured to use the FetchTask gate")
|
|
info = json.loads(self.docker("inspect", self.runner))[0]
|
|
started = timestamp(info["State"]["StartedAt"]).timestamp()
|
|
addresses = {network["IPAddress"] for network in info["NetworkSettings"]["Networks"].values()}
|
|
gate = self.gate("status")
|
|
# .runner 会因 label 更新而回写；mtime 不能证明内存中的地址。
|
|
# 要求入口实际见到本次容器启动后、来自它的 FetchTask。
|
|
if gate.get("last_fetch_peer") not in addresses or gate.get("last_fetch_at", 0) < started:
|
|
raise RuntimeError("gate has not observed FetchTask from this runner startup")
|
|
|
|
def activate(self):
|
|
self.check_gate_route()
|
|
candidate = self.version(self.state["candidate"])
|
|
runner = self.api.request(self.runner_api)
|
|
if runner.get("disabled") is not False:
|
|
log("runner disabled by operator; defer switch")
|
|
return False
|
|
config = self.read_config()
|
|
gate = self.gate("status")
|
|
if gate.get("paused") is not False and not self.state.get("pause_owned"):
|
|
log("runner gate paused by operator; defer switch")
|
|
return False
|
|
if not self.state.get("switch") and not self.idle():
|
|
log("CI active; candidate stays staged")
|
|
return False
|
|
# 先持久化恢复意图；控制请求超时也可能已生效，ExecStopPost/下次 tick 会恢复。
|
|
self.state["pause_owned"] = True
|
|
self.save()
|
|
try:
|
|
if self.gate("pause").get("paused") is not True:
|
|
raise RuntimeError("runner pause could not be confirmed")
|
|
# 不用 FetchTask 客户端超时猜测服务端事务是否已经结束。
|
|
# 入口必须完整读完已转发的响应；不确定时拒绝自动重启。
|
|
for _ in range(30):
|
|
gate = self.gate("status")
|
|
if gate.get("uncertain"):
|
|
raise RuntimeError("in-flight FetchTask completion is uncertain; manual gate inspection required")
|
|
if gate.get("paused") is not True:
|
|
raise RuntimeError("runner gate unexpectedly resumed")
|
|
if gate.get("inflight") == 0:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
log("FetchTask still in flight; defer switch")
|
|
return False
|
|
if not self.idle():
|
|
log("in-flight task appeared; defer switch without stopping runner")
|
|
return False
|
|
latest_config = self.read_config()
|
|
if latest_config != config:
|
|
raise RuntimeError("runner config changed while pausing")
|
|
pending = self.state.get("switch")
|
|
if pending is None:
|
|
old = self.state["current"]
|
|
updated = replace_image(config, old, candidate["image"])
|
|
backup = self.root / "backups" / (candidate["source"] + ".yaml")
|
|
backup.parent.mkdir(parents=True, exist_ok=True)
|
|
backup.write_text(config)
|
|
pending = {"old": old, "new": candidate["image"], "backup": str(backup)}
|
|
self.state["switch"] = pending
|
|
self.save()
|
|
else:
|
|
actual = configured_image(config)
|
|
if actual not in {pending["old"], pending["new"]}:
|
|
raise RuntimeError("interrupted switch conflicts with operator configuration")
|
|
updated = replace_image(config, actual, pending["new"])
|
|
self.write_config(updated)
|
|
if self.read_config() != updated or not self.idle():
|
|
# 配置尚未重启加载，恢复旧文件即可；不干扰意外出现的 job。
|
|
self.write_config(Path(pending["backup"]).read_text())
|
|
self.state.pop("switch", None)
|
|
self.save()
|
|
log("idle check changed before restart; restored configuration")
|
|
return False
|
|
self.docker("restart", "--timeout", "660", self.runner, timeout=720)
|
|
started = self.docker("inspect", "--format", "{{.State.StartedAt}}", self.runner).strip()
|
|
ready = False
|
|
for _ in range(30):
|
|
try:
|
|
info = self.docker("inspect", "--format", "{{.State.Status}}", self.runner).strip()
|
|
recent = self.docker("logs", "--since", started, self.runner, combined=True)
|
|
ready = (info == "running" and "declare successfully" in recent
|
|
and self.image_info(candidate["image"], inner=True)["Id"] == candidate["image"])
|
|
except RuntimeError:
|
|
ready = False
|
|
if ready:
|
|
break
|
|
time.sleep(2)
|
|
if not ready:
|
|
raise RuntimeError("runner registration not confirmed; pending switch retained for recovery")
|
|
candidate["activated"] = now()
|
|
self.state["rollback"] = pending["old"]
|
|
self.state["current"] = pending["new"]
|
|
self.state["candidate"] = None
|
|
self.state.pop("switch", None)
|
|
self.save()
|
|
log(f'activated {candidate["image"]}; awaiting real master CI validation')
|
|
return True
|
|
finally:
|
|
# 持久化 switch 保证崩溃后不会错误宣称新镜像已验证。
|
|
self.resume()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--config", required=True)
|
|
parser.add_argument("--apply", action="store_true", help="执行维护；默认只检查连接和配置")
|
|
parser.add_argument("--retry", action="store_true", help="重试同一 master 上失败的构建")
|
|
parser.add_argument("--resume", action="store_true", help="仅恢复本维护器暂停的领取；用于 ExecStopPost")
|
|
args = parser.parse_args()
|
|
if os.environ.get("CI") == "true":
|
|
parser.error("run on the trusted host, outside CI jobs")
|
|
os.umask(0o077)
|
|
config = json.loads(Path(args.config).read_text())
|
|
maintenance = Maintenance(config)
|
|
if not args.apply and not args.resume:
|
|
runner = maintenance.api.request(maintenance.runner_api)
|
|
image = configured_image(maintenance.read_config())
|
|
head = command("git", "ls-remote", config["clone_url"], "refs/heads/master").split()[0]
|
|
maintenance.check_gate_route()
|
|
gate = maintenance.gate("status")
|
|
if gate.get("uncertain"):
|
|
raise RuntimeError("runner gate needs manual inspection")
|
|
log(f'check master={head} image={image} runner_disabled={runner["disabled"]} busy={runner["busy"]}')
|
|
return
|
|
import fcntl # Linux 宿主；纯逻辑测试仍可在 Windows 上导入。
|
|
maintenance.root.mkdir(parents=True, exist_ok=True)
|
|
with (maintenance.root / "maintenance.lock").open("a") as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
log("another maintenance process holds the lock")
|
|
return
|
|
# 加锁后重新读取状态，避免另一个进程完成后覆盖旧快照。
|
|
maintenance = Maintenance(config)
|
|
if args.resume:
|
|
maintenance.resume()
|
|
else:
|
|
maintenance.tick(retry=args.retry)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
log(f"maintenance failed: {type(error).__name__}: {error}")
|
|
sys.exit(1)
|