lhk229
e3c2c1b732
基于固定 master 提交构建全新缓存镜像，并在成功验证后保留当前版和回滚版。 增加任务领取网关，在不打断运行中 CI 的前提下切换镜像。 补充宿主维护服务、部署说明、共享记忆和定向行为测试。 修复构建测试在 Windows 与 WSL 下并发写入日志的竞态。
271 lines
10 KiB
Python
271 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Gitea Runner 领取屏障；控制 socket 仅供宿主缓存维护任务使用。"""
|
|
|
|
import http.client
|
|
import http.server
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socketserver
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
|
|
MAX_BODY = 32 * 1024 * 1024
|
|
HOP_HEADERS = {
|
|
"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
|
|
"te", "trailer", "transfer-encoding", "upgrade", "host", "content-length",
|
|
}
|
|
|
|
|
|
class Gate:
|
|
def __init__(self, directory):
|
|
self.directory = Path(directory)
|
|
self.directory.mkdir(parents=True, exist_ok=True)
|
|
self.lock = threading.Lock()
|
|
self.inflight = 0
|
|
self.last_fetch_peer = None
|
|
self.last_fetch_at = None
|
|
self.uncertain = (self.directory / "uncertain").exists()
|
|
if (self.directory / "inflight").exists():
|
|
self.uncertain = True
|
|
self.mark("uncertain")
|
|
self.paused = (self.directory / "paused").exists() or self.uncertain
|
|
|
|
def mark(self, name):
|
|
with (self.directory / name).open("w", encoding="ascii") as stream:
|
|
stream.write("1\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
self.sync_directory()
|
|
|
|
def sync_directory(self):
|
|
descriptor = os.open(self.directory, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
def snapshot(self):
|
|
return {"paused": self.paused, "inflight": self.inflight,
|
|
"uncertain": self.uncertain, "last_fetch_peer": self.last_fetch_peer,
|
|
"last_fetch_at": self.last_fetch_at}
|
|
|
|
def record_fetch(self, peer):
|
|
with self.lock:
|
|
self.last_fetch_peer = peer
|
|
self.last_fetch_at = time.time()
|
|
|
|
def control(self, action):
|
|
with self.lock:
|
|
if action == "pause":
|
|
self.mark("paused")
|
|
self.paused = True
|
|
elif action == "resume":
|
|
if self.uncertain:
|
|
return {**self.snapshot(), "error": "upstream completion uncertain; operator recovery required"}
|
|
(self.directory / "paused").unlink(missing_ok=True)
|
|
self.sync_directory()
|
|
self.paused = False
|
|
elif action != "status":
|
|
return {**self.snapshot(), "error": "unknown action"}
|
|
return self.snapshot()
|
|
|
|
def enter(self):
|
|
with self.lock:
|
|
if self.paused or self.uncertain:
|
|
return False
|
|
# 必须先落盘再转发；崩溃后不能把遗留的领取请求误认为已完成。
|
|
self.mark("inflight")
|
|
self.inflight += 1
|
|
return True
|
|
|
|
def leave(self, completed):
|
|
with self.lock:
|
|
self.inflight -= 1
|
|
if not completed:
|
|
self.uncertain = self.paused = True
|
|
self.mark("uncertain")
|
|
if self.inflight == 0 and not self.uncertain:
|
|
(self.directory / "inflight").unlink(missing_ok=True)
|
|
self.sync_directory()
|
|
|
|
|
|
def read_body(stream, headers):
|
|
"""解码 HTTP 请求；拒绝含糊 framing，不依赖下游连接关闭。"""
|
|
encodings = headers.get_all("Transfer-Encoding", [])
|
|
lengths = headers.get_all("Content-Length", [])
|
|
if encodings and lengths:
|
|
raise ValueError("ambiguous request framing")
|
|
if len(lengths) > 1 or len(encodings) > 1:
|
|
raise ValueError("duplicate request framing")
|
|
|
|
def exact(length):
|
|
data = stream.read(length)
|
|
if len(data) != length:
|
|
raise ValueError("incomplete request body")
|
|
return data
|
|
|
|
if not encodings:
|
|
length = int(lengths[0]) if lengths else 0
|
|
if not 0 <= length <= MAX_BODY:
|
|
raise ValueError("request too large")
|
|
return exact(length)
|
|
if encodings[0].strip().lower() != "chunked":
|
|
raise ValueError("unsupported transfer encoding")
|
|
body = bytearray()
|
|
while True:
|
|
line = stream.readline(8193)
|
|
if len(line) > 8192 or not line.endswith(b"\r\n"):
|
|
raise ValueError("invalid chunk header")
|
|
length = int(line.split(b";", 1)[0].strip(), 16)
|
|
if length < 0 or len(body) + length > MAX_BODY:
|
|
raise ValueError("request too large")
|
|
if length == 0:
|
|
trailer_size = 0
|
|
while True:
|
|
line = stream.readline(8193)
|
|
trailer_size += len(line)
|
|
if trailer_size > 8192 or not line.endswith(b"\r\n"):
|
|
raise ValueError("invalid request trailers")
|
|
if line == b"\r\n":
|
|
return bytes(body)
|
|
body.extend(exact(length))
|
|
if exact(2) != b"\r\n":
|
|
raise ValueError("invalid chunk terminator")
|
|
|
|
|
|
class Proxy(http.server.BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
def log_message(self, *_args):
|
|
pass # 不输出 RPC 认证头、请求内容或带认证信息的 URL。
|
|
|
|
def reply(self, status, body, headers=()):
|
|
try:
|
|
self.send_response(status)
|
|
excluded = HOP_HEADERS | {
|
|
item.strip().lower() for key, value in headers if key.lower() == "connection"
|
|
for item in value.split(",")
|
|
}
|
|
for key, value in headers:
|
|
if key.lower() not in excluded:
|
|
self.send_header(key, value)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("Connection", "close")
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
except (OSError, ValueError):
|
|
pass
|
|
self.close_connection = True
|
|
|
|
def do_POST(self):
|
|
path = urllib.parse.urlsplit(self.path)
|
|
if (path.scheme or path.netloc or not path.path.startswith("/api/actions/")
|
|
or "%" in path.path or any(p in (".", "..") for p in path.path.split("/"))):
|
|
self.reply(404, b"runner RPC only\n")
|
|
return
|
|
try:
|
|
body = read_body(self.rfile, self.headers)
|
|
except (ValueError, OSError):
|
|
self.reply(400, b"invalid request body\n")
|
|
return
|
|
is_fetch = path.path.endswith("/FetchTask")
|
|
if is_fetch:
|
|
# 包括暂停时被拒绝的请求；只记录网络来源与时间以证明 daemon 路由。
|
|
self.server.gate.record_fetch(self.client_address[0])
|
|
if not self.server.gate.enter():
|
|
self.reply(503, b"runner maintenance\n")
|
|
return
|
|
completed = False
|
|
connection = None
|
|
try:
|
|
upstream = self.server.upstream
|
|
cls = http.client.HTTPSConnection if upstream.scheme == "https" else http.client.HTTPConnection
|
|
connection = cls(upstream.hostname, upstream.port, timeout=None)
|
|
try:
|
|
connection.connect()
|
|
except OSError:
|
|
# TCP/TLS 连接阶段尚未发送 RPC，不存在服务端分配事务。
|
|
completed = True
|
|
raise
|
|
excluded = HOP_HEADERS | {
|
|
name.strip().lower() for name in self.headers.get("Connection", "").split(",")
|
|
}
|
|
headers = {key: value for key, value in self.headers.items() if key.lower() not in excluded}
|
|
headers["Content-Length"] = str(len(body))
|
|
connection.request("POST", self.path, body=body, headers=headers)
|
|
response = connection.getresponse()
|
|
result = bytearray()
|
|
oversized = False
|
|
# 客户端即使断开，也继续读取上游完整响应，再释放领取屏障。
|
|
while block := response.read(65536):
|
|
if len(result) + len(block) <= MAX_BODY and not oversized:
|
|
result.extend(block)
|
|
else:
|
|
oversized = True
|
|
result.clear()
|
|
if response.length not in (None, 0):
|
|
raise http.client.IncompleteRead(bytes(result), response.length)
|
|
completed = True
|
|
if is_fetch:
|
|
self.server.gate.leave(True)
|
|
is_fetch = False
|
|
if oversized:
|
|
self.reply(502, b"upstream response too large\n")
|
|
else:
|
|
self.reply(response.status, bytes(result), response.getheaders())
|
|
except (OSError, http.client.HTTPException, ValueError):
|
|
self.reply(502, b"runner upstream unavailable\n")
|
|
finally:
|
|
if connection is not None:
|
|
connection.close()
|
|
if is_fetch:
|
|
self.server.gate.leave(completed)
|
|
|
|
|
|
class Control(socketserver.StreamRequestHandler):
|
|
def handle(self):
|
|
try:
|
|
line = self.rfile.readline(4097)
|
|
if len(line) > 4096 or not line.endswith(b"\n"):
|
|
raise ValueError("invalid control request")
|
|
request = json.loads(line)
|
|
result = self.server.gate.control(request["action"])
|
|
except (ValueError, KeyError, TypeError, OSError):
|
|
result = {"error": "invalid control request"}
|
|
self.wfile.write(json.dumps(result).encode("utf-8") + b"\n")
|
|
|
|
|
|
class ControlServer(socketserver.ThreadingUnixStreamServer):
|
|
daemon_threads = True
|
|
|
|
|
|
def create_proxy(address, upstream, gate):
|
|
parsed = urllib.parse.urlsplit(upstream)
|
|
if (parsed.scheme not in ("http", "https") or not parsed.hostname
|
|
or parsed.username or parsed.password or parsed.path not in ("", "/")
|
|
or parsed.query or parsed.fragment):
|
|
raise ValueError("upstream must be an HTTP(S) origin")
|
|
server = http.server.ThreadingHTTPServer(address, Proxy)
|
|
server.upstream = parsed
|
|
server.gate = gate
|
|
return server
|
|
|
|
|
|
def main():
|
|
gate = Gate(os.environ.get("GITEA_GATE_CONTROL_DIR", "/control"))
|
|
socket_path = gate.directory / "gate.sock"
|
|
socket_path.unlink(missing_ok=True)
|
|
control = ControlServer(str(socket_path), Control)
|
|
control.gate = gate
|
|
os.chmod(socket_path, 0o600)
|
|
proxy = create_proxy(("0.0.0.0", 8080), os.environ.get(
|
|
"GITEA_RUNNER_UPSTREAM", "http://gitea:3000"), gate)
|
|
threading.Thread(target=control.serve_forever, daemon=True).start()
|
|
proxy.serve_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|