weblate/scripts/generate-cjk-regexp.py

#!/usr/bin/env python

# Copyright © yhëhtozr <conlang2012@outlook.com>
#
# SPDX-License-Identifier: GPL-3.0-or-later

"""
Generate optimized regexp pattern to match CJK characters.

Character ranges should be made as contiguous as possible for maximum performance.

Used in weblate/trans/util.py
"""

from __future__ import annotations

import argparse
import re
import urllib.request
from pathlib import Path

# renovate-unicode-version
UNICODE_VERSION = "18.0.0"
UNICODE_BLOCKS_URL = "https://www.unicode.org/Public/{version}/ucd/Blocks.txt"
GENERATED_MARKER = "# Generated by scripts/generate-cjk-regexp.py"
UTIL_PATH = Path(__file__).resolve().parents[1] / "weblate" / "trans" / "util.py"
BLOCK_RE = re.compile(r"^([0-9A-F]+)\.\.([0-9A-F]+);\s+(.+)$")

CJK_BLOCKS = {
    "CJK Unified Ideographs",
    "CJK Unified Ideographs Extension A",
    # 'CJK Unified Ideographs Extension B', # commented out in favor of catch-all Planes 2-3 match
    # 'CJK Unified Ideographs Extension C', # commented out in favor of catch-all Planes 2-3 match
    # 'CJK Unified Ideographs Extension D', # commented out in favor of catch-all Planes 2-3 match
    # 'CJK Unified Ideographs Extension E', # commented out in favor of catch-all Planes 2-3 match
    # 'CJK Unified Ideographs Extension F', # commented out in favor of catch-all Planes 2-3 match
    # 'CJK Unified Ideographs Extension G', # commented out in favor of catch-all Planes 2-3 match
    # 'CJK Unified Ideographs Extension H', # commented out in favor of catch-all Planes 2-3 match
    # 'CJK Unified Ideographs Extension I', # commented out in favor of catch-all Planes 2-3 match
    "CJK Compatibility",
    "CJK Compatibility Forms",
    "CJK Compatibility Ideographs",
    # 'CJK Compatibility Ideographs Supplement', # commented out in favor of catch-all Planes 2-3 match
    "CJK Radicals Supplement",
    "CJK Strokes",
    "CJK Symbols and Punctuation",
    "Hiragana",
    "Katakana",
    "Katakana Phonetic Extensions",
    "Kana Extended-A",
    "Kana Extended-B",
    "Kana Supplement",
    "Small Kana Extension",
    "Hangul Jamo",
    "Hangul Compatibility Jamo",
    "Hangul Jamo Extended-A",
    "Hangul Jamo Extended-B",
    "Hangul Syllables",
    "Halfwidth and Fullwidth Forms",
    "Enclosed CJK Letters and Months",
    "Enclosed Ideographic Supplement",
    "Kangxi Radicals",
    "Ideographic Description Characters",
    "Kanbun",
    "Yijing Hexagram Symbols",  # not strictly necessary but for the sake of range continuity
    "Bopomofo",
    "Bopomofo Extended",
}


def parse_blocks(data: str) -> list[tuple[int, int, str]]:
    blocks: list[tuple[int, int, str]] = []
    for line in data.splitlines():
        line = line.partition("#")[0].strip()
        if not line:
            continue
        match = BLOCK_RE.fullmatch(line)
        if match is None:
            msg = f"Could not parse Unicode block line: {line!r}"
            raise ValueError(msg)
        start, end, name = match.groups()
        blocks.append((int(start, 16), int(end, 16), name))
    return blocks


def fetch_blocks(version: str) -> str:
    url = UNICODE_BLOCKS_URL.format(version=version)
    with urllib.request.urlopen(url, timeout=30) as response:  # ruff: ignore[suspicious-url-open-usage]
        return response.read().decode("utf-8")


def load_blocks(version: str, blocks_file: Path | None) -> list[tuple[int, int, str]]:
    if blocks_file is None:
        return parse_blocks(fetch_blocks(version))
    return parse_blocks(blocks_file.read_text(encoding="utf-8"))


def merge_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
    merged: list[tuple[int, int]] = []
    prev: tuple[int, int] | None = None
    for current in sorted(ranges):
        if prev is None:
            prev = current
        elif prev[1] == current[0] - 1:
            prev = (prev[0], current[1])
        else:
            merged.append(prev)
            prev = current
    if prev is not None:
        merged.append(prev)
    return merged


def hexchar(codepoint: int) -> str:
    if codepoint > 0xFFFF:
        return rf"\U{codepoint:08x}"
    return rf"\u{codepoint:04x}"


def build_pattern(blocks: list[tuple[int, int, str]]) -> str:
    seen = set()
    ranges = []
    for start, end, name in blocks:
        if name in CJK_BLOCKS:
            seen.add(name)
            ranges.append((start, end))

    missing = CJK_BLOCKS - seen
    if missing:
        msg = f"Missing Unicode blocks: {', '.join(sorted(missing))}"
        raise ValueError(msg)

    cjkmerged = merge_ranges(ranges)
    return rf"([{''.join(f'{hexchar(start)}-{hexchar(end)}' for start, end in cjkmerged)}\U00020000-\U0003FFFF]+)"


def format_stdout(pattern: str) -> str:
    return f'CJK_PATTERN = re.compile(r"{pattern}")'


def format_util(pattern: str) -> str:
    return f'{GENERATED_MARKER}\nCJK_PATTERN = re.compile(\n    r"{pattern}"\n)'


def update_util(pattern: str, util_path: Path = UTIL_PATH) -> None:
    source = util_path.read_text(encoding="utf-8")
    generated = format_util(pattern)
    # re.sub replacement templates only need literal backslashes escaped.
    literal_generated = generated.replace("\\", "\\\\")
    replacement_re = re.compile(
        rf"{re.escape(GENERATED_MARKER)}\nCJK_PATTERN = re\.compile\(\n    r\".+?\"\n\)",
        re.DOTALL,
    )
    updated, count = replacement_re.subn(literal_generated, source, count=1)
    if count != 1:
        msg = f"Could not find generated CJK_PATTERN in {util_path}"
        raise ValueError(msg)
    util_path.write_text(updated, encoding="utf-8")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--blocks-file",
        type=Path,
        help="Use a local Unicode Blocks.txt file instead of downloading it.",
    )
    parser.add_argument(
        "--update",
        action="store_true",
        help="Update the generated CJK_PATTERN in weblate/trans/util.py.",
    )
    parser.add_argument(
        "--version",
        default=UNICODE_VERSION,
        help="Unicode version to download when --blocks-file is not used.",
    )
    args = parser.parse_args()

    pattern = build_pattern(load_blocks(args.version, args.blocks_file))
    if args.update:
        update_util(pattern)
    else:
        print(format_stdout(pattern))


if __name__ == "__main__":
    main()