m/market labResearch, with a record.
Source on GitHub ↗

check_documentation.py

1"""Check documentation language and local links without network access.23Run from any directory. Historical internal archives are excluded. Use4--include-internal to check the active development documents as well.5Canonical guides and API docstrings use English; explicitly named Cantonese6user-guide translations retain their language and receive the same link checks.7"""89import argparse10import ast11from pathlib import Path12import re13from urllib.parse import unquote, urlsplit1415ROOT = Path(__file__).resolve().parents[1]16HAN = re.compile(r'[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]')17LINK = re.compile(r'!?\[[^\]\n]*\]\(\s*(<[^>]+>|[^\s)]+)(?:\s+"[^"]*")?\s*\)')18ROOTS = ('docs', 'sim_platform', 'src', 'include', 'examples', 'tests', 'tools')19EXCLUDED_DIRECTORIES = frozenset(('node_modules', '__pycache__', '.venv', 'build'))202122def current_pages(root: Path, include_internal: bool) -> list[Path]:23    """Find maintained Markdown; generated output and historical records stay outside it."""24    pages = set(root.glob('*.md'))25    for name in ROOTS + (('internal',) if include_internal else ()):26        for page in (root / name).rglob('*.md'):27            relative = page.relative_to(root)28            if not (29                relative.as_posix().startswith('internal/archive/')30                or EXCLUDED_DIRECTORIES.intersection(relative.parts[:-1])31            ):32                pages.add(page)33    return sorted(pages)343536def prose_lines(text: str):37    """Skip fenced examples when interpreting Markdown headings and links."""38    fence = None39    for number, line in enumerate(text.splitlines(), 1):40        marker = re.match(r'^\s*(`{3,}|~{3,})', line)41        if marker:42            kind = marker[1][0]43            if fence is None:44                fence = kind45            elif fence == kind:46                fence = None47        elif fence is None:48            yield number, line495051def anchors(text: str) -> set[str]:52    """Recognize ordinary GitHub heading anchors and explicit HTML IDs."""53    result, counts = set(), {}54    for _, line in prose_lines(text):55        result.update(re.findall(r'\bid=["\']([^"\']+)["\']', line))56        heading = re.match(r'^#{1,6}\s+(.+?)\s*#*\s*$', line)57        if heading:58            slug = re.sub(r'[^\w\- ]', '', heading[1].lower()).replace(' ', '-')59            count = counts.get(slug, 0)60            counts[slug] = count + 161            result.add(slug + ('-' + str(count) if count else ''))62    return result636465def check_page(page: Path, root: Path) -> list[str]:66    text = page.read_text(encoding='utf-8')67    name = page.relative_to(root).as_posix()68    translated = re.fullmatch(r'docs/user-guide/[^/]+-zh-hk\.md', name) is not None69    errors = [70        f'{name}:{line}: write current documentation in English'71        for line, value in enumerate(text.splitlines(), 1)72        if not translated and HAN.search(value)73    ]74    for line, value in prose_lines(text):75        for match in LINK.finditer(value):76            link = urlsplit(match[1].strip('<>'))77            if link.scheme or link.netloc:78                continue79            target = (page.parent / unquote(link.path)).resolve() if link.path else page80            if not target.is_relative_to(root.resolve()):81                errors.append(f'{name}:{line}: local link leaves repository: {match[1]}')82            elif not target.exists():83                errors.append(f'{name}:{line}: missing local target: {match[1]}')84            elif not name.startswith('internal/') and target.is_relative_to(root / 'internal'):85                errors.append(f'{name}:{line}: public guide depends on internal documentation')86            elif link.fragment and target.suffix == '.md':87                if unquote(link.fragment) not in anchors(target.read_text(encoding='utf-8')):88                    errors.append(f'{name}:{line}: missing heading: {match[1]}')89    return errors909192def check_docstrings(root: Path) -> list[str]:93    errors = []94    for source in (root / 'sim_platform').rglob('*.py'):95        tree = ast.parse(source.read_text(encoding='utf-8'))96        for node in ast.walk(tree):97            if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):98                if HAN.search(ast.get_docstring(node) or ''):99                    errors.append(100                        f'{source.relative_to(root)}:{getattr(node, "lineno", 1)}: '101                        'write module and API documentation in English'102                    )103    return errors104105106def check(root: Path, *, include_internal: bool = False) -> tuple[list[Path], list[str]]:107    root = root.resolve()108    pages = current_pages(root, include_internal)109    errors = [error for page in pages for error in check_page(page, root)]110    errors.extend(check_docstrings(root))111    return pages, errors112113114def main() -> int:115    parser = argparse.ArgumentParser(description=__doc__)116    parser.add_argument('--include-internal', action='store_true')117    args = parser.parse_args()118    pages, errors = check(ROOT, include_internal=args.include_internal)119    if errors:120        print('\n'.join(errors))121        return 1122    print(123        f'Checked {len(pages)} guides, language rules, local links/headings and Python docstrings.'124    )125    return 0126127128if __name__ == '__main__':129    raise SystemExit(main())