m/market labResearch, with a record.
Source on GitHub ↗

sync_public_source.py

1"""Preview a source archive against the public checkout; --apply copies reviewed files.23No commits, branches, tags or network calls are created. Existing public checkouts must4be clean Git worktrees with the manifest from the previous sync. First sync requires an5empty directory (an existing .git directory is allowed). An uncommitted review directory6can be updated with --previous-archive after byte comparison. Only previously managed7files can be deleted. A working-tree archive additionally requires --allow-candidate.8"""910import argparse11import json12from pathlib import Path13import subprocess14import tarfile1516from build_source_release import PROVENANCE, public_path1718MANIFEST = '.market-lab-sync.json'192021def read_candidate(path, allow_candidate):22    """Read regular public files only; do not extract archive paths onto disk."""23    files = {}24    roots = set()25    with tarfile.open(path, 'r:gz') as archive:26        for item in archive:27            root, separator, name = item.name.partition('/')28            roots.add(root)29            if not separator or not item.isfile() or not (public_path(name) or name == PROVENANCE):30                raise ValueError('Unexpected archive member: ' + item.name)31            if name in files:32                raise ValueError('Duplicate archive member: ' + name)33            files[name] = (archive.extractfile(item).read(), bool(item.mode & 0o111))34    if len(roots) != 1 or not next(iter(roots)).startswith('market-lab-'):35        raise ValueError('Expected one market-lab archive root')36    provenance = json.loads(files[PROVENANCE][0])37    kind = provenance['source_kind']38    if kind not in ('commit', 'working-tree-candidate'):39        raise ValueError('Unknown source provenance')40    if kind != 'commit' and not allow_candidate:41        raise ValueError(42            'Uncommitted candidate requires --allow-candidate; it is not release approval'43        )44    for required in ('README.md', 'LICENSE', 'NOTICE', 'sim_platform/__init__.py'):45        if required not in files:46            raise ValueError('Required public source missing: ' + required)47    return files, provenance484950def safe_target(destination, name):51    target = destination / name52    for part in (target, *target.parents):53        if part.is_symlink():54            raise ValueError('Destination contains a symlink: ' + str(part))55        if part != target and part.exists() and not part.is_dir():56            raise ValueError('Destination parent is not a directory: ' + str(part))57        if part == destination:58            break59    return target606162def review_copy_files(destination, previous_archive):63    """Allow a review-copy update only when every file still matches its saved archive."""64    files, provenance = read_candidate(previous_archive, True)65    manifest = safe_target(destination, MANIFEST)66    expected = {'source': provenance, 'files': sorted(files)}67    if not manifest.is_file() or json.loads(manifest.read_text()) != expected:68        raise ValueError('Previous archive does not match the review-copy manifest')69    actual = {p.relative_to(destination).as_posix() for p in destination.rglob('*') if p.is_file()}70    if actual != set(files) | {MANIFEST}:71        raise ValueError('Review copy has added or missing files; resolve local changes first')72    for name, (content, executable) in files.items():73        target = safe_target(destination, name)74        if target.read_bytes() != content or bool(target.stat().st_mode & 0o111) != executable:75            raise ValueError('Review copy has local changes: ' + name)76    return set(files)777879def previous_files(destination, previous_archive=None):80    """Require a recoverable, clean public checkout before replacing existing content."""81    entries = [p for p in destination.iterdir() if p.name != '.git'] if destination.exists() else []82    if not entries:83        return set()84    if not (destination / '.git').exists():85        if previous_archive is not None:86            return review_copy_files(destination, previous_archive)87        raise ValueError('Nonempty destination must be a clean Git checkout')88    if previous_archive is not None:89        raise ValueError('Use clean Git state, not --previous-archive, for an existing repository')90    root = subprocess.run(91        ['git', '-C', str(destination), 'rev-parse', '--show-toplevel'],92        check=True,93        capture_output=True,94        text=True,95    ).stdout.strip()96    if Path(root).resolve() != destination.resolve():97        raise ValueError('Destination must be the public repository root')98    status = subprocess.run(99        ['git', '-C', str(destination), 'status', '--porcelain', '--untracked-files=all'],100        check=True,101        capture_output=True,102    ).stdout103    if status:104        raise ValueError('Public checkout has local changes; commit or resolve them before syncing')105    manifest = safe_target(destination, MANIFEST)106    if not manifest.is_file():107        raise ValueError(108            'Existing checkout has no sync manifest; review it manually before adoption'109        )110    names = json.loads(manifest.read_text())['files']111    if not isinstance(names, list) or any(112        not isinstance(n, str) or not (public_path(n) or n == PROVENANCE) for n in names113    ):114        raise ValueError('Invalid managed file list')115    return set(names)116117118def plan_sync(destination, files, previous_archive=None):119    if any(p.is_symlink() for p in (destination, *destination.parents)):120        raise ValueError('Destination must not be a symlink')121    previous = previous_files(destination, previous_archive)122    plan = {'added': [], 'modified': [], 'deleted': sorted(previous - files.keys())}123    for name, (content, executable) in sorted(files.items()):124        target = safe_target(destination, name)125        if target.exists():126            if name not in previous or not target.is_file():127                raise ValueError('Refusing to overwrite unmanaged path: ' + name)128            if target.read_bytes() != content or bool(target.stat().st_mode & 0o111) != executable:129                plan['modified'].append(name)130        else:131            plan['added'].append(name)132    for name in plan['deleted']:133        if not safe_target(destination, name).is_file():134            raise ValueError('Managed deletion is not a regular file: ' + name)135    return plan136137138def apply_sync(destination, files, provenance, plan):139    """Change only planned paths; restore their original contents if a write fails."""140    changed = plan['added'] + plan['modified'] + plan['deleted'] + [MANIFEST]141    original = {}142    destination.mkdir(parents=True, exist_ok=True)143    for name in changed:144        path = safe_target(destination, name)145        original[name] = (path.read_bytes(), path.stat().st_mode) if path.exists() else None146    try:147        for name in plan['added'] + plan['modified']:148            path = safe_target(destination, name)149            path.parent.mkdir(parents=True, exist_ok=True)150            content, executable = files[name]151            path.write_bytes(content)152            path.chmod(0o755 if executable else 0o644)153        for name in plan['deleted']:154            safe_target(destination, name).unlink()155        manifest = {'source': provenance, 'files': sorted(files)}156        (destination / MANIFEST).write_text(json.dumps(manifest, indent=2) + '\n')157    except Exception:158        for name, saved in original.items():159            path = destination / name160            if saved is None:161                path.unlink(missing_ok=True)162            else:163                path.write_bytes(saved[0])164                path.chmod(saved[1])165        raise166    # Remove only empty ancestors of managed deletions; leave unrelated folders alone.167    for name in plan['deleted']:168        parent = (destination / name).parent169        while parent != destination:170            try:171                parent.rmdir()172            except OSError:173                break174            parent = parent.parent175176177def sync(archive, destination, *, apply=False, allow_candidate=False, previous_archive=None):178    files, provenance = read_candidate(archive, allow_candidate)179    plan = plan_sync(destination, files, previous_archive)180    print(json.dumps({'source': provenance, 'changes': plan, 'applied': apply}, indent=2))181    if apply:182        apply_sync(destination, files, provenance, plan)183    return plan184185186if __name__ == '__main__':187    parser = argparse.ArgumentParser(description=__doc__)188    parser.add_argument('archive', type=Path)189    parser.add_argument('destination', type=Path)190    parser.add_argument(191        '--previous-archive',192        type=Path,193        help='For a non-Git review copy only: require byte equality with its previous archive',194    )195    parser.add_argument(196        '--apply', action='store_true', help='Apply the preview to local files only'197    )198    parser.add_argument(199        '--allow-candidate',200        action='store_true',201        help='Explicitly permit an uncommitted review candidate',202    )203    args = parser.parse_args()204    sync(205        args.archive,206        args.destination.absolute(),207        apply=args.apply,208        allow_candidate=args.allow_candidate,209        previous_archive=args.previous_archive,210    )