scripts/update_cddl.py

#!/usr/bin/env python
"""Update the pinned CDDL spec files downloaded from w3c/webref.

The WebDriver BiDi (and related) CDDL grammars are not published as an npm
package; they are extracted from the edited specs and committed to the
``ed/cddl`` directory of https://github.com/w3c/webref . We pin a single
webref commit plus a sha256 for the ``-all`` (union) CDDL file of each
protocol in that directory (see ``common/webref_cddl.bzl``) so the Bazel
build fetches them reproducibly. The per-end ``-local``/``-remote`` splits are
skipped: generation merges the union, so only the ``-all`` files are consumed.

This script repoints that pin at the tip of webref's "main" branch (the
continuous reffy extraction; the "curated" branch is a separate published
lineage with no shared history, so main keeps pin-to-pin diffs auditable),
refreshes every hash, and picks up files that upstream has added or removed.
It also refreshes two companion pins at the same time so they stay in lockstep:
the per-spec ``dfns`` indexes (same webref commit), and the rendered core spec
HTML pinned from ``w3c/webdriver-bidi``'s ``gh-pages`` branch (its prose section
anchors are the source of the readable ``#type-``/``#command-`` spec links, and
that branch is a separate repo, so it is resolved separately). It regenerates:

  - ``_COMMIT``, ``_CDDL_FILES``, ``_DFNS_FILES``, and the ``_BIDI_SPEC_HTML_*``
    pins in ``common/webref_cddl.bzl``
  - the matching ``use_repo(...)`` list for the extension in ``MODULE.bazel``

Nothing is written unless the consumed webref content changed, so both commit pins record
where the CDDL content last changed rather than the latest tip. The rendered spec is repinned
in lockstep with the CDDL, since it only annotates types the grammar defines.
"""

import hashlib
import json
import os
import re
from pathlib import Path

import urllib3

http = urllib3.PoolManager()
root_dir = Path(os.path.realpath(__file__)).parent.parent

REPO = "w3c/webref"
BRANCH = "main"
CDDL_PATH = "ed/cddl"
DFNS_PATH = "ed/dfns"
API_HEADERS = {"Accept": "application/vnd.github+json", "User-Agent": "selenium-update-cddl"}

# The webref dfns index for each spec merged into the BiDi schema (a deliberate subset
# of everything webref publishes — only these feed generation). Each is (repo_name,
# dfns_filename); the sha256 is refreshed from the pinned webref commit. Keep in sync
# with the javascript/ and py/ BUILD merge lists.
MERGED_DFNS = [
    ("webdriver_bidi_dfns", "webdriver-bidi.json"),
    ("permissions_dfns", "permissions.json"),
    ("prefetch_dfns", "prefetch.json"),
    ("ua_client_hints_dfns", "ua-client-hints.json"),
    ("web_bluetooth_dfns", "web-bluetooth.json"),
]

# The rendered core spec is pinned separately: it lives in w3c/webdriver-bidi's
# "gh-pages" branch (which commits the built HTML), not in webref. Its prose section
# anchors are the only source of the readable `#type-`/`#command-` links.
BIDI_SPEC_REPO = "w3c/webdriver-bidi"
BIDI_SPEC_BRANCH = "gh-pages"
BIDI_SPEC_FILE = "index.html"
BIDI_SPEC_REPO_NAME = "webdriver_bidi_spec_html"

BZL_FILE = root_dir / "common" / "webref_cddl.bzl"
MODULE_FILE = root_dir / "MODULE.bazel"


def list_cddl_files(commit):
    r = http.request(
        "GET",
        f"https://api.github.com/repos/{REPO}/contents/{CDDL_PATH}?ref={commit}",
        headers=API_HEADERS,
    )
    if r.status != 200:
        raise RuntimeError(f"Failed to list {CDDL_PATH} at {commit}: HTTP {r.status}")
    entries = json.loads(r.data)
    # Only the "-all" union of each protocol is consumed; the local/remote splits
    # feed nothing (BiDi generation merges the union), so they are not pinned.
    return sorted(e["name"] for e in entries if e["type"] == "file" and e["name"].endswith("-all.cddl"))


def repo_name(filename):
    """Derive the Bazel repo name from a CDDL filename.

    ``at-driver-all.cddl`` -> ``at_driver_all_cddl``
    """
    return filename[: -len(".cddl")].replace("-", "_") + "_cddl"


def sha256_of_url(url):
    r = http.request("GET", url)
    if r.status != 200:
        raise RuntimeError(f"Failed to download {url}: HTTP {r.status}")
    return hashlib.sha256(r.data).hexdigest()


def sha256_of(commit, filename):
    return sha256_of_url(f"https://raw.githubusercontent.com/{REPO}/{commit}/{CDDL_PATH}/{filename}")


def build_entries(commit, filenames):
    return [(repo_name(name), name, sha256_of(commit, name)) for name in filenames]


def build_dfns_entries(commit):
    """(repo, dfns_filename, sha256) for each merged spec, hashed at the webref commit."""
    return [
        (name, filename, sha256_of_url(f"https://raw.githubusercontent.com/{REPO}/{commit}/{DFNS_PATH}/{filename}"))
        for name, filename in MERGED_DFNS
    ]


def resolve_bidi_spec(branch):
    """Resolve the w3c/webdriver-bidi gh-pages tip and hash its rendered index.html."""
    commit = resolve_commit_for(BIDI_SPEC_REPO, branch)
    url = f"https://raw.githubusercontent.com/{BIDI_SPEC_REPO}/{commit}/{BIDI_SPEC_FILE}"
    return commit, sha256_of_url(url)


def resolve_commit_for(repo, branch):
    r = http.request("GET", f"https://api.github.com/repos/{repo}/commits/{branch}", headers=API_HEADERS)
    if r.status != 200:
        raise RuntimeError(f"Failed to resolve {repo}@{branch}: HTTP {r.status}")
    return json.loads(r.data)["sha"]


def existing_repo_names(content):
    return set(re.findall(r'\(\s*"([a-z0-9_]+)"\s*,\s*"[^"]+\.cddl"', content))


def render_files(var, entries):
    lines = [f"{var} = ["]
    for name, filename, sha256 in entries:
        lines.append(f'    ("{name}", "{filename}", "{sha256}"),')
    lines.append("]")
    return "\n".join(lines)


def sub_once(content, pattern, replacement, where):
    content, n = re.subn(pattern, replacement, content, flags=re.S)
    if n != 1:
        raise RuntimeError(f"Expected exactly one {where} in {BZL_FILE.name}, found {n}")
    return content


def current_bidi_pin(content):
    commit = re.search(r'_BIDI_SPEC_HTML_COMMIT = "([0-9a-f]+)"', content).group(1)
    sha256 = re.search(r'_BIDI_SPEC_HTML_SHA256 = "([0-9a-f]+)"', content).group(1)
    return commit, sha256


def drop_commit(content):
    """Content with the webref commit blanked, so a pin bump alone does not read as a change."""
    return re.sub(r'(?<![A-Z_])_COMMIT = "[0-9a-f]+"', "", content)


def update_pin(content, commit, cddl_entries, dfns_entries, bidi_commit, bidi_sha256):
    # Anchor so this does not also match the tail of `_BIDI_SPEC_HTML_COMMIT = "…"`.
    content = sub_once(content, r'(?<![A-Z_])_COMMIT = "[0-9a-f]+"', f'_COMMIT = "{commit}"', "_COMMIT assignment")
    content = sub_once(
        content, r"_CDDL_FILES = \[.*?\n\]", lambda _: render_files("_CDDL_FILES", cddl_entries), "_CDDL_FILES block"
    )
    content = sub_once(
        content, r"_DFNS_FILES = \[.*?\n\]", lambda _: render_files("_DFNS_FILES", dfns_entries), "_DFNS_FILES block"
    )
    content = sub_once(
        content,
        r'_BIDI_SPEC_HTML_COMMIT = "[0-9a-f]+"',
        f'_BIDI_SPEC_HTML_COMMIT = "{bidi_commit}"',
        "_BIDI_SPEC_HTML_COMMIT assignment",
    )
    content = sub_once(
        content,
        r'_BIDI_SPEC_HTML_SHA256 = "[0-9a-f]+"',
        f'_BIDI_SPEC_HTML_SHA256 = "{bidi_sha256}"',
        "_BIDI_SPEC_HTML_SHA256 assignment",
    )

    return content


def update_module(repo_names):
    content = MODULE_FILE.read_text()
    repo_list = "\n".join(f'    "{name}",' for name in sorted(repo_names))
    new_block = f"use_repo(\n    webref_cddl_extension,\n{repo_list}\n)"
    content, count = re.subn(
        r"use_repo\(\n    webref_cddl_extension,\n.*?\n\)",
        new_block,
        content,
        flags=re.S,
    )
    if count != 1:
        raise RuntimeError(f"Expected exactly one webref_cddl_extension use_repo block, found {count}")
    MODULE_FILE.write_text(content)


def main():
    commit = resolve_commit_for(REPO, BRANCH)
    print(f"Pinning {REPO}@{commit}")

    old = BZL_FILE.read_text()
    before = existing_repo_names(old)

    filenames = list_cddl_files(commit)
    print(f"Found {len(filenames)} CDDL files in {CDDL_PATH}")
    cddl_entries = build_entries(commit, filenames)

    dfns_entries = build_dfns_entries(commit)
    print(f"Refreshed {len(dfns_entries)} dfns indexes in {DFNS_PATH}")

    # Probe with the pin already in the file so a gh-pages rebuild cannot open the gate.
    webref_only = update_pin(old, commit, cddl_entries, dfns_entries, *current_bidi_pin(old))
    if drop_commit(webref_only) == drop_commit(old):
        print("No pinned spec content changed; leaving the pins at their current commits.")
        return

    bidi_commit, bidi_sha256 = resolve_bidi_spec(BIDI_SPEC_BRANCH)
    print(f"Pinning {BIDI_SPEC_REPO}@{bidi_commit} ({BIDI_SPEC_FILE})")

    BZL_FILE.write_text(update_pin(old, commit, cddl_entries, dfns_entries, bidi_commit, bidi_sha256))

    cddl_names = {name for name, _, _ in cddl_entries}
    repo_names = cddl_names | {name for name, _, _ in dfns_entries} | {BIDI_SPEC_REPO_NAME}
    update_module(repo_names)

    for name in sorted(cddl_names - before):
        print(f"  added: {name}")
    for name in sorted(before - cddl_names):
        print(f"  removed: {name}")
    print(f"Updated {BZL_FILE.relative_to(root_dir)} and {MODULE_FILE.relative_to(root_dir)}")


if __name__ == "__main__":
    main()