Genarrative/scripts/gitea_cache_upload_cleanup.py

"""定向回收 Gitea 1.26.4 不会自动过期的本仓库缓存上传块。"""

import base64
import binascii
import math
from pathlib import Path
import re
import stat


JOBS = {
    "ai-game-creator-shell-rust-lane-1",
    "ai-game-creator-shell-rust-lane-2",
    "ai-game-creator-shell-rust-crates",
    "backend-tests",
    "native-shell-tests",
}
CHUNK = re.compile(r"block-([1-9][0-9]*)-([1-9][0-9]*)-(0|[1-9][0-9]*)-([A-Za-z0-9_-]+={0,2})")
BLOCKLIST = re.compile(r"([1-9][0-9]*)-([1-9][0-9]*)-blocklist")
MARKER = re.compile(r"genarrative-rust-cache-v1:([a-z0-9-]+):(0|[1-9][0-9]*):([0-9]{8})")


def decode_owner(encoded):
    """Gitea 文件名编码为 URL-base64(生产者上传的标准 base64 blockid)。"""
    try:
        block_id = base64.b64decode(encoded, altchars=b"-_", validate=True)
        if base64.urlsafe_b64encode(block_id).decode() != encoded:
            return None
        marker = base64.b64decode(block_id, validate=True)
        if base64.b64encode(marker) != block_id:
            return None
        match = MARKER.fullmatch(marker.decode("ascii"))
        if match and match[1] in JOBS:
            return match[1], int(match[2])
    except (binascii.Error, UnicodeError, ValueError):
        pass
    return None


def _unchanged_old_regular(path, previous, cutoff):
    try:
        current = path.lstat()
    except FileNotFoundError:
        return False
    return (
        stat.S_ISREG(current.st_mode)
        and current.st_mtime < cutoff
        and (current.st_ino, current.st_size, current.st_mtime_ns)
        == (previous.st_ino, previous.st_size, previous.st_mtime_ns)
    )


def cleanup_upload_chunks(storage_root: Path, eligible_runs: set[int], cutoff_timestamp: float) -> int:
    """调用方先用仓库 API 确认这些 master run 已结束超过保留期限。"""
    storage_root = Path(storage_root)
    if not storage_root.is_absolute() or storage_root.resolve(strict=True) != storage_root:
        raise ValueError("artifact storage root must be an existing absolute real directory")
    if not stat.S_ISDIR(storage_root.lstat().st_mode):
        raise ValueError("artifact storage root must be a directory")
    if not math.isfinite(cutoff_timestamp) or cutoff_timestamp <= 0:
        raise ValueError("invalid artifact cleanup cutoff")
    if any(type(run_id) is not int or run_id <= 0 for run_id in eligible_runs):
        raise ValueError("invalid eligible run ID")
    temporary = storage_root / "tmp-upload"
    if not temporary.exists() and not temporary.is_symlink():
        return 0
    if temporary.is_symlink() or not temporary.is_dir():
        raise ValueError("artifact temporary path must be a real directory")

    deleted = 0
    for run_id in sorted(eligible_runs):
        directory = temporary / f"run-{run_id}-v4"
        if directory.is_symlink() or not directory.is_dir():
            continue
        groups = {}
        unknown = False
        for path in directory.iterdir():
            chunk = CHUNK.fullmatch(path.name)
            blocklist = BLOCKLIST.fullmatch(path.name)
            match = chunk or blocklist
            if match and int(match[1]) == run_id:
                artifact_id = int(match[2])
                owner = decode_owner(chunk[4]) if chunk else None
                owned = owner is not None if chunk else True
            else:
                # 若仍能识别 artifact ID，仅保护该组；无法识别时保护整个 run。
                prefix = re.match(rf"(?:block-)?{run_id}-([1-9][0-9]*)-", path.name)
                if not prefix:
                    unknown = True
                    break
                artifact_id, owner, owned = int(prefix[1]), None, False
            group = groups.setdefault(artifact_id, {"files": [], "owners": set(), "safe": True})
            try:
                info = path.lstat()
            except FileNotFoundError:
                group["safe"] = False
                continue
            group["safe"] &= owned and stat.S_ISREG(info.st_mode) and info.st_mtime < cutoff_timestamp
            if owner:
                group["owners"].add(owner)
            group["files"].append((path, info))
        if unknown:
            continue
        for group in groups.values():
            if not group["safe"] or len(group["owners"]) != 1:
                continue
            files = group["files"]
            if not all(_unchanged_old_regular(path, info, cutoff_timestamp) for path, info in files):
                continue
            # 保留目录，不使用递归删除；没有所属块证明的孤立 blocklist 也不会删除。
            for path, _ in files:
                path.unlink()
                deleted += 1
    return deleted