CI缓存自动维护与清理 (#462)

Reviewed-on: https://git.genarrative.world/git/GenarrativeAI/Genarrative/pulls/462

scripts/gitea_cache_upload_cleanup.py

@@ -0,0 +1,115 @@
+"""定向回收 Gitea 1.26.4 不会自动过期的本仓库缓存上传块。"""
+
+import base64
+import binascii
+import math
+from pathlib import Path
+import re
+import stat
+
+
+JOBS = {
+    "ai-game-creator-shell-rust-lane-1",
+    "ai-game-creator-shell-rust-lane-2",
+    "ai-game-creator-shell-rust-smoke",
+    "ai-game-creator-shell-rust-crates",
+    "backend-tests",
+    "native-shell-tests",
+}
+CHUNK = re.compile(r"block-([1-9][0-9]*)-([1-9][0-9]*)-(0|[1-9][0-9]*)-([A-Za-z0-9_-]+={0,2})")
+BLOCKLIST = re.compile(r"([1-9][0-9]*)-([1-9][0-9]*)-blocklist")
+MARKER = re.compile(r"genarrative-rust-cache-v1:([a-z0-9-]+):(0|[1-9][0-9]*):([0-9]{8})")
+
+
+def decode_owner(encoded):
+    """Gitea 文件名编码为 URL-base64(生产者上传的标准 base64 blockid)。"""
+    try:
+        block_id = base64.b64decode(encoded, altchars=b"-_", validate=True)
+        if base64.urlsafe_b64encode(block_id).decode() != encoded:
+            return None
+        marker = base64.b64decode(block_id, validate=True)
+        if base64.b64encode(marker) != block_id:
+            return None
+        match = MARKER.fullmatch(marker.decode("ascii"))
+        if match and match[1] in JOBS:
+            return match[1], int(match[2])
+    except (binascii.Error, UnicodeError, ValueError):
+        pass
+    return None
+
+
+def _unchanged_old_regular(path, previous, cutoff):
+    try:
+        current = path.lstat()
+    except FileNotFoundError:
+        return False
+    return (
+        stat.S_ISREG(current.st_mode)
+        and current.st_mtime < cutoff
+        and (current.st_ino, current.st_size, current.st_mtime_ns)
+        == (previous.st_ino, previous.st_size, previous.st_mtime_ns)
+    )
+
+
+def cleanup_upload_chunks(storage_root: Path, eligible_runs: set[int], cutoff_timestamp: float) -> int:
+    """调用方先用仓库 API 确认这些 master run 已结束超过保留期限。"""
+    storage_root = Path(storage_root)
+    if not storage_root.is_absolute() or storage_root.resolve(strict=True) != storage_root:
+        raise ValueError("artifact storage root must be an existing absolute real directory")
+    if not stat.S_ISDIR(storage_root.lstat().st_mode):
+        raise ValueError("artifact storage root must be a directory")
+    if not math.isfinite(cutoff_timestamp) or cutoff_timestamp <= 0:
+        raise ValueError("invalid artifact cleanup cutoff")
+    if any(type(run_id) is not int or run_id <= 0 for run_id in eligible_runs):
+        raise ValueError("invalid eligible run ID")
+    temporary = storage_root / "tmp-upload"
+    if not temporary.exists() and not temporary.is_symlink():
+        return 0
+    if temporary.is_symlink() or not temporary.is_dir():
+        raise ValueError("artifact temporary path must be a real directory")
+
+    deleted = 0
+    for run_id in sorted(eligible_runs):
+        directory = temporary / f"run-{run_id}-v4"
+        if directory.is_symlink() or not directory.is_dir():
+            continue
+        groups = {}
+        unknown = False
+        for path in directory.iterdir():
+            chunk = CHUNK.fullmatch(path.name)
+            blocklist = BLOCKLIST.fullmatch(path.name)
+            match = chunk or blocklist
+            if match and int(match[1]) == run_id:
+                artifact_id = int(match[2])
+                owner = decode_owner(chunk[4]) if chunk else None
+                owned = owner is not None if chunk else True
+            else:
+                # 若仍能识别 artifact ID，仅保护该组；无法识别时保护整个 run。
+                prefix = re.match(rf"(?:block-)?{run_id}-([1-9][0-9]*)-", path.name)
+                if not prefix:
+                    unknown = True
+                    break
+                artifact_id, owner, owned = int(prefix[1]), None, False
+            group = groups.setdefault(artifact_id, {"files": [], "owners": set(), "safe": True})
+            try:
+                info = path.lstat()
+            except FileNotFoundError:
+                group["safe"] = False
+                continue
+            group["safe"] &= owned and stat.S_ISREG(info.st_mode) and info.st_mtime < cutoff_timestamp
+            if owner:
+                group["owners"].add(owner)
+            group["files"].append((path, info))
+        if unknown:
+            continue
+        for group in groups.values():
+            if not group["safe"] or len(group["owners"]) != 1:
+                continue
+            files = group["files"]
+            if not all(_unchanged_old_regular(path, info, cutoff_timestamp) for path, info in files):
+                continue
+            # 保留目录，不使用递归删除；没有所属块证明的孤立 blocklist 也不会删除。
+            for path, _ in files:
+                path.unlink()
+                deleted += 1
+    return deleted