From 1031408a88672b94dcbe1d1e528fbe2fd2883c26 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 13:47:56 -0700 Subject: [PATCH 1/9] ADFA-3305: Delete documentation rows that nothing reads drop_unread_rows.py deletes four groups of Content rows, with their chunk rows: the javadoc Use pages and module graphs (4,956 rows), pdf.js's CMaps (169), pdf.js's no-WebAssembly fallbacks and scripting sandbox (5), and the x/ placeholder pages (1,238). For the sandbox it also sets enableScripting's default to false in p/web/viewer.mjs, the only way the viewer loads it. Before writing, it refuses with the list and exit 1 if any kept text row, template, tooltip button, tooltip text or Bookshelf entry refers to a row it would delete; if a row to delete is not what its group holds; if a stored PDF has a font pdf.js would fetch a CMap for; if the viewer edit cannot be made; or if a kept row cannot be read. Three references are expected and allowed: the worker's two fallback imports and the viewer's sandbox path. Each group alone, on the 2026-10-18 database after ADFA-6627, packed with rebuild_database.py --pack Content,Tooltips --page-size 4096, saves (file / brotli -q 11 -w 24 bytes): javadoc-use 5,042,176 / 4,330,908; pdfjs-cmaps 860,160 / 851,736; pdfjs-extras 339,968 / 346,809; x-placeholders 122,880 / 103,920. In headless Chrome, all 15 stored PDFs show the same page count, text and text layer in pdf.js before and after, and none requests a deleted file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- CLAUDE.md | 4 +- scripts/drop_unread_rows.py | 556 ++++++++++++++++++++++++++++++ scripts/test_drop_unread_rows.py | 563 +++++++++++++++++++++++++++++++ 3 files changed, 1122 insertions(+), 1 deletion(-) create mode 100644 scripts/drop_unread_rows.py create mode 100644 scripts/test_drop_unread_rows.py diff --git a/CLAUDE.md b/CLAUDE.md index 73e32deb..123777e4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -169,7 +169,9 @@ uses. Any future Android/Java tooltip work should target the normalized `Tooltip - **`scripts/`** — the original CLI toolbox. Live/current: `DocumentationDatabase.py` (Content ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), - `create_empty_database.py`, `list_database_documents.py`. **Deprecated/dead** (target the + `create_empty_database.py`, `list_database_documents.py`, `drop_unread_rows.py` (deletes Content + rows nothing reads - javadoc Use pages, pdf.js CMaps, pdf.js fallbacks and scripting sandbox, x/ + placeholders - after checking that nothing kept refers to them). **Deprecated/dead** (target the removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, `load_android_data.py`. diff --git a/scripts/drop_unread_rows.py b/scripts/drop_unread_rows.py new file mode 100644 index 00000000..606a81ae --- /dev/null +++ b/scripts/drop_unread_rows.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "scour", +# "cairosvg", +# ] +# /// +""" +drop_unread_rows.py + +Deletes Content rows that nothing in the app or the documentation reads, after +checking that no row it keeps refers to one of them. + +The rows, in four groups (pick some with --only; the default is all four): + +- javadoc-use: the javadoc Use pages (j/html/api/**/class-use/*.html and + *package-use.html) and the module graphs (j/html/api/**/*module-graph.svg). + Only other Use pages link to a Use page, and no page names a module graph. + 4,896 + 60 rows. +- pdfjs-cmaps: pdf.js's predefined CMaps (p/web/cmaps/, 168 .bcmap files and + their LICENSE). pdf.js fetches one by name only for a font that needs it; see + "PDF check" below. +- pdfjs-extras: pdf.js's no-WebAssembly fallbacks (openjpeg_nowasm_fallback.js, + jbig2_nowasm_fallback.js), which its worker imports only when WebAssembly + cannot start, and its scripting sandbox (pdf.sandbox.mjs, quickjs-eval.js, + quickjs-eval.wasm), which the viewer loads only with enableScripting on. The + step also sets enableScripting's default to false in p/web/viewer.mjs, so the + viewer never asks for the sandbox: with it off, the viewer passes no + scripting manager to its PDFViewer, which is the only caller of the one that + imports pdf.sandbox.mjs. The generic viewer never writes the pdfjs.preferences + entry that could turn it back on. +- x-placeholders: the x/ pages, each "Placeholder for XML layout attribute + ...". XML tooltips link to a/ pages instead. 1,238 rows. x.html and x.pdf + are not in this group. + +A row's chunk continuations ("-1", "-2", ...) go with it. + +Measured on the 2026-10-18 database after ADFA-6627, each group alone, packed +with rebuild_database.py --pack Content,Tooltips --page-size 4096, as bytes +saved in the file and in its brotli -q 11 -w 24: javadoc-use 5,042,176 and +4,330,908; pdfjs-cmaps 860,160 and 851,736; pdfjs-extras 339,968 and 346,809; +x-placeholders 122,880 and 103,920. Repacking alone moves the .br by up to +about 0.2 MB, so the last figure is within that. + +Before deleting anything the step checks, and refuses with the list and exit 1 +when any check fails, writing nothing: + +- References. Every kept text row (HTML, CSS, JS, SVG, JSON, XML, Markdown and + plain text, decoded, chunks joined), every Templates row, every TooltipButtons + uri, the Tooltips text and the Bookshelf entries are read for links: HTML and + SVG attributes, srcset, CSS url() and @import, quoted file names in scripts, + the string values of a templated row's JSON, viewer.html?file= parameters and + http://localhost:6174/ URLs. A link that resolves (relative to its row, or + from the root) to a deleted row is refused. So is a quoted file name that + resolves to no row at all but is the name of a deleted row and of no kept + one, which is how a script that builds its URLs names a file. Three such + references are expected and allowed; ALLOWED below says why each is dead. +- Content. Each x/ row must be a placeholder page, each javadoc-use HTML row a + "Uses of" page and each module graph an SVG, and p/web/cmaps/ may hold only + .bcmap files and LICENSE, so a real page that lands on one of these paths is + never deleted. +- PDF check (pdfjs-cmaps). pdf.js 6.1.200 fetches a CMap file for: a font whose + /Encoding names a predefined CMap other than Identity-H or Identity-V; an + embedded CMap that uses one (usecmap); a CID font of the Adobe GB1, CNS1, + Japan1 or Korea1 collection (Adobe--UCS2, unless the font has its + own ToUnicode map); and a font with one of ten Chinese names (GBK-EUC-H). + Every stored PDF is read, its Flate streams decompressed, and any sign of + these refuses the run. The collection check ignores ToUnicode, so it refuses + more than pdf.js needs, never less. It cannot see PDFs that a plugin installs + on the phone. +- The viewer edit (pdfjs-extras). p/web/viewer.mjs must hold exactly one + enableScripting default; if it holds none or more, nothing is written. +- A kept text row that cannot be decoded cannot be checked, so it refuses too. + +Runs in one transaction (rolled back on any error), backs the database up +first, and VACUUMs afterwards. --dry-run does all the checks and reports what +would change without writing. A second run finds nothing to do and exits 0. + +Pipeline position: after every step that writes page content or links, so the +reference check sees the final pages, and before rebuild_database.py. Deleting +rows does not call for a dictionary re-mint. + + uv run scripts/drop_unread_rows.py documentation.db --dry-run + uv run scripts/drop_unread_rows.py documentation.db [--only GROUP ...] [--workers N] [--verbose] + +Recovering: the timestamped backup taken before the first write is the whole +undo. +""" +from __future__ import annotations + +import argparse +import html +import json +import posixpath +import re +import sqlite3 +import sys +import zlib +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import unquote + +from optimize_db_media import ( + BrotliCodec, backup_database, delete_media_row, human, is_continuation_path, load_dictionary, + owned_fragment_paths, reassemble, +) +from update_media_references import replace_row + +# Content types read for references. Everything else is media. +SCANNED_TYPES = ("text/html", "text/css", "text/javascript", "image/svg+xml", "application/json", + "application/xml", "text/markdown", "text/plain", "text", "text/text", + "application/x-typescript") + +VIEWER = "p/web/viewer.mjs" +PDFJS_EXTRAS = ("p/web/wasm/openjpeg_nowasm_fallback.js", "p/web/wasm/jbig2_nowasm_fallback.js", + "p/web/wasm/quickjs-eval.wasm", "p/web/wasm/quickjs-eval.js", "p/build/pdf.sandbox.mjs") +PLACEHOLDER = b"Placeholder for XML layout attribute" + +# References to rows this step deletes that nothing follows, as (referring row, +# deleted row) -> why. Each is printed when found, so it stays visible. +ALLOWED = { + ("p/build/pdf.worker.mjs", "p/web/wasm/openjpeg_nowasm_fallback.js"): + "the worker imports it only when WebAssembly cannot start", + ("p/build/pdf.worker.mjs", "p/web/wasm/jbig2_nowasm_fallback.js"): + "the worker imports it only when WebAssembly cannot start", + ("p/web/viewer.mjs", "p/build/pdf.sandbox.mjs"): + "the viewer imports it only with enableScripting on, and this step turns that off", +} + +# The enableScripting entry of the viewer's default options, minified or not. +_SCRIPTING_DEFAULT = re.compile(rb"(\benableScripting\s*:\s*\{\s*value\s*:\s*)(true|false|!0|!1)\b") + +_ATTR = re.compile( + r"""(?`]+))""", re.I) +_SRCSET = re.compile(r"""(?]{1,400}?\.(?:html?|m?js|css|png|gif|webp|svg|webm|mp4|json|pdf|wasm|bcmap|pfb|ttf|""" + r"""otf|woff2?|icc|ico|md|xml|txt|jpe?g|ogv|mov|peb))(?:[?#][^"'`\s]*)?["'`]""", re.I) +_META_REFRESH = re.compile(r"\s*\d+\s*;\s*url\s*=\s*", re.I) +_FILE_PARAM = re.compile(r"[?&]file=([^&#]+)") +_SCHEME = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*:") +LOCAL_ORIGINS = ("http://localhost:6174/", "http://127.0.0.1:6174/") + +# What pdf.js 6.1.200 reads to decide it needs a CMap file; see the docstring. +_PDF_NAME = rb"/((?:[^\s/<>\[\]()%{}#]|#[0-9A-Fa-f]{2})+)" +_PDF_ENCODING = re.compile(rb"/(?:Encoding|UseCMap)\s*" + _PDF_NAME) +_PDF_USECMAP = re.compile(_PDF_NAME + rb"\s+usecmap\b") +_PDF_ORDERING = re.compile(rb"/Ordering\s*(?:\(([^)]*)\)|<([0-9A-Fa-f\s]*)>)") +_PDF_BASEFONT = re.compile(rb"/BaseFont\s*" + _PDF_NAME) +_PDF_STREAM = re.compile(rb"stream(?:\r\n|\n|\r)") +CJK_ORDERINGS = (b"GB1", b"CNS1", b"Japan1", b"Korea1") +# pdf.js's chineseFontNames, as the bytes it compares a font name with. +CHINESE_FONT_NAMES = {name.encode("latin-1") for name in ( + "\u00cb\u00ce\u00cc\u00e5", "\u00ba\u00da\u00cc\u00e5", "\u00bf\u00ac\u00cc\u00e5", "\u00b7\u00c2\u00cb\u00ce", + "\u00bf\u00ac\u00cc\u00e5_GB2312", "\u00b7\u00c2\u00cb\u00ce_GB2312", "\u00c1\u00a5\u00ca\u00e9", + "\u00d0\u00c2\u00cb\u00ce", "\u00b7\u00c2\u00cb\u00ce\u00cc\u00e5", "\u00d0\u00a1\u00b1\u00ea\u00cb\u00ce")} +IDENTITY_CMAPS = {"Identity-H", "Identity-V"} + + +def _javadoc_use(path: str) -> bool: + if not path.startswith("j/html/api/"): + return False + directory, name = posixpath.split(path) + return ((directory.endswith("/class-use") and name.endswith(".html")) or name.endswith("package-use.html") + or name.endswith("module-graph.svg")) + + +def _check_javadoc_use(path: str, raw: bytes): + if path.endswith(".svg"): + return None if b"Uses of " in raw else "is not a javadoc Use page" + + +def _check_cmap(path: str, raw: bytes): + name = posixpath.basename(path) + return None if name.endswith(".bcmap") or name == "LICENSE" else "is not a CMap or its LICENSE" + + +def _check_placeholder(path: str, raw: bytes): + return None if PLACEHOLDER in raw else "is not a placeholder page" + + +# name -> (what it holds, which rows it selects, a check on each row's decoded bytes) +GROUPS = { + "javadoc-use": ("javadoc Use pages and module graphs", _javadoc_use, _check_javadoc_use), + "pdfjs-cmaps": ("pdf.js CMaps", lambda path: path.startswith("p/web/cmaps/"), _check_cmap), + "pdfjs-extras": ("pdf.js no-WebAssembly fallbacks and scripting sandbox", lambda path: path in PDFJS_EXTRAS, + None), + "x-placeholders": ("x/ placeholder pages", lambda path: path.startswith("x/"), _check_placeholder), +} + + +def turn_scripting_off(viewer: bytes) -> tuple: + """(viewer with enableScripting's default false, whether that changed it), or (None, why not).""" + found = list(_SCRIPTING_DEFAULT.finditer(viewer)) + if len(found) != 1: + return None, f"holds {len(found)} enableScripting defaults, not 1" + m = found[0] + if m.group(2) in (b"false", b"!1"): + return viewer, False + return viewer[:m.start(2)] + b"false" + viewer[m.end(2):], True + + +def link_candidates(text: str, is_json: bool) -> set: + """Every string in text that could be a link to a stored file.""" + found = set() + blobs = [text] + if is_json: + try: + strings = [] + _json_strings(json.loads(text), strings) + except ValueError: + pass # not JSON after all; read it as text + else: + blobs = strings + found.update(s for s in strings if len(s) < 400 and "<" not in s) + for blob in blobs: + for m in _ATTR.finditer(blob): + value = next(g for g in m.groups() if g is not None) + if "&" in value: + value = html.unescape(value) + refresh = _META_REFRESH.match(value) + found.add(value[refresh.end():] if refresh else value) + for m in _SRCSET.finditer(blob): + for part in (m.group(1) or m.group(2) or "").split(","): + if part.strip(): + found.add(html.unescape(part.split()[0])) + for rx in (_CSS_URL, _CSS_IMPORT): + for m in rx.finditer(blob): + found.add(next((g for g in m.groups() if g is not None), "")) + for m in _QUOTED.finditer(blob): + found.add(m.group(1)) + found.discard("") + return found + + +def _json_strings(value, out: list) -> None: + if isinstance(value, str): + out.append(value) + elif isinstance(value, dict): + for item in value.values(): + _json_strings(item, out) + elif isinstance(value, list): + for item in value: + _json_strings(item, out) + + +def is_external(ref: str) -> bool: + """Whether ref points off this server: another scheme, or a host of its own.""" + ref = ref.strip() + if ref.lower().startswith(LOCAL_ORIGINS): + return False + return ref.startswith("//") or bool(_SCHEME.match(ref)) + + +def link_paths(ref: str, base_dirs, nested: bool = False) -> list: + """The stored paths ref could name, read from each of base_dirs (from the root if it starts with "/").""" + ref = ref.strip() + out = [] + param = _FILE_PARAM.search(ref) + if param and not nested: + out.extend(link_paths(unquote(param.group(1)), base_dirs, nested=True)) + lowered = ref.lower() + for origin in LOCAL_ORIGINS: + if lowered.startswith(origin): + ref = "/" + ref[len(origin):] + break + if not ref or ref.startswith(("#", "//")) or _SCHEME.match(ref): + return out + ref = ref.split("#", 1)[0].split("?", 1)[0] + for candidate in {ref, unquote(ref)}: + if not candidate: + continue + rooted = candidate.startswith("/") + for base in ("",) if rooted else base_dirs: + path = posixpath.normpath(candidate.lstrip("/") if rooted else posixpath.join(base, candidate)) + if path not in (".", "..") and not path.startswith("../"): + out.append(path) + return out + + +class References: + """Finds the references a text makes to rows being deleted.""" + + def __init__(self, stored: set, deleted: set): + self.stored = stored + self.deleted = deleted + kept_names = {posixpath.basename(p) for p in stored - deleted} + self.only_deleted = defaultdict(list) + for path in deleted: + name = posixpath.basename(path) + if name not in kept_names: + self.only_deleted[name].append(path) + + def targets(self, ref: str, base_dirs) -> list: + """[(deleted path, how)] for one reference.""" + out, resolved = [], False + for path in link_paths(ref, base_dirs): + for variant in (path, path + ".html", path + "/index.html"): + if variant in self.stored: + resolved = True + if variant in self.deleted: + out.append((variant, "links to")) + break + if not resolved and not is_external(ref): + name = unquote(posixpath.basename(ref.strip().split("#", 1)[0].split("?", 1)[0])) + out.extend((target, "names") for target in self.only_deleted.get(name, ())) + return out + + def in_text(self, text: str, base_dirs, is_json: bool = False) -> set: + found = set() + for ref in link_candidates(text, is_json): + found.update(self.targets(ref, base_dirs)) + return found + + +def _unescape_name(name: bytes) -> bytes: + return re.sub(rb"#([0-9A-Fa-f]{2})", lambda m: bytes([int(m.group(1), 16)]), name) + + +def pdf_parts(data: bytes): + """The PDF's bytes, then each of its streams that Flate-decodes.""" + yield data + for m in _PDF_STREAM.finditer(data): + end = data.find(b"endstream", m.end()) + if end < 0: + break + try: + yield zlib.decompressobj().decompress(data[m.end():end]) + except zlib.error: + continue + + +def cmap_needs(data: bytes, cmap_names: set) -> list: + """Why pdf.js would fetch a CMap file for this PDF, if it would; see the docstring.""" + needs = set() + for part in pdf_parts(data): + for m in list(_PDF_ENCODING.finditer(part)) + list(_PDF_USECMAP.finditer(part)): + name = _unescape_name(m.group(1)).decode("latin-1") + if name in cmap_names: + needs.add(f"uses the predefined CMap {name}") + for m in _PDF_ORDERING.finditer(part): + ordering = m.group(1) if m.group(1) is not None else bytes.fromhex(m.group(2).decode().replace(" ", "")) + if ordering.strip() in CJK_ORDERINGS: + needs.add(f"has a CID font of the Adobe {ordering.strip().decode()} collection") + for m in _PDF_BASEFONT.finditer(part): + name = _unescape_name(m.group(1)) + if name in CHINESE_FONT_NAMES or name.split(b"+", 1)[-1] in CHINESE_FONT_NAMES: + needs.add("has a font pdf.js reads as Chinese (GBK-EUC-H)") + return sorted(needs) + + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + groups = cfg["groups"] or list(GROUPS) + + conn = sqlite3.connect(db_path) + codec = None + try: + codec = BrotliCodec(load_dictionary(conn)) + rows = {path: (row_id, length, mime, compression, template_id) + for row_id, path, length, mime, compression, template_id in conn.execute( + "SELECT c.id, c.path, LENGTH(c.content), ct.value, ct.compression, c.templateId " + "FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id ORDER BY c.path")} + lengths = {path: row[1] for path, row in rows.items()} + bases = [path for path in rows if not is_continuation_path(lengths, path)] + + def stored_bytes(path: str) -> bytes: + """A row's stored bytes, chunks joined. Main thread only: it queries.""" + first = conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0] + return reassemble(conn, path, first) + + doomed = {} # base row -> its group + for group in groups: + doomed.update((path, group) for path in bases if GROUPS[group][1](path)) + deleted = {} # every row deleted, chunk rows included -> its group + for path, group in doomed.items(): + deleted[path] = group + deleted.update((fragment, group) for fragment in owned_fragment_paths(conn, path, lengths[path])) + + problems = [] + viewer_new, viewer_changed = None, False + if "pdfjs-extras" in groups: + if VIEWER in rows: + viewer_new, viewer_changed = turn_scripting_off(codec.decompress(stored_bytes(VIEWER), + rows[VIEWER][3])) + if viewer_new is None: + problems.append(f"{VIEWER} {viewer_changed}, so scripting cannot be turned off") + viewer_changed = False + elif any(group == "pdfjs-extras" for group in doomed.values()): + problems.append(f"{VIEWER} is not stored, so this step cannot tell that scripting is off") + + if not doomed and not viewer_changed and not problems: + print("Nothing to do: no row in " + ", ".join(groups) + + (", and scripting is already off." if "pdfjs-extras" in groups else ".")) + return 0 + + def decode(item): + path, full = item + try: + if path == VIEWER and viewer_new is not None: + return path, viewer_new, None + return path, codec.decompress(full, rows[path][3]), None + except Exception as exc: # noqa: BLE001 - reported, and the run refused + return path, None, str(exc) + + print(f"Checking the {len(doomed)} row(s) to delete...") + checked = [(path, stored_bytes(path)) for path in sorted(doomed) if GROUPS[doomed[path]][2] is not None] + with ThreadPoolExecutor(max_workers=cfg["workers"]) as pool: + for path, raw, error in pool.map(decode, checked): + problem = f"could not be read: {error}" if raw is None else GROUPS[doomed[path]][2](path, raw) + if problem: + problems.append(f"{path} {problem}") + checked = None + + refs = References(set(rows), set(deleted)) + found = set() # (referring row, deleted row, how) + kept = [path for path in bases if path not in deleted and rows[path][2] in SCANNED_TYPES] + print(f"Reading {len(kept)} kept text row(s) for references...") + texts = [(path, stored_bytes(path)) for path in kept] + + def scan(item): + path, raw, error = decode(item) + if raw is None: + return path, None, error + text = raw.decode("utf-8", errors="replace") + return path, refs.in_text(text, (posixpath.dirname(path),), rows[path][4] != 0), None + + with ThreadPoolExecutor(max_workers=cfg["workers"]) as pool: + for path, targets, error in pool.map(scan, texts): + if targets is None: + problems.append(f"{path} could not be read, so its references were not checked: {error}") + else: + found.update((path, target, how) for target, how in targets) + texts = None + + # A template renders at the paths of the rows that use it. + dirs_by_template = defaultdict(set) + for path, row in rows.items(): + if row[4]: + dirs_by_template[row[4]].add(posixpath.dirname(path)) + for template_id, name, content in conn.execute("SELECT id, name, content FROM Templates"): + dirs = ("",) + tuple(sorted(dirs_by_template[template_id])) + text = bytes(content).decode("utf-8", errors="replace") + found.update((f"template {name}", target, how) for target, how in refs.in_text(text, dirs)) + for tooltip_id, uri in conn.execute("SELECT tooltipId, uri FROM TooltipButtons WHERE uri IS NOT NULL"): + found.update((f"tooltip {tooltip_id} button", target, how) for target, how in refs.targets(uri, ("",))) + for tooltip_id, summary, detail in conn.execute("SELECT id, summary, detail FROM Tooltips"): + found.update((f"tooltip {tooltip_id} text", target, how) + for target, how in refs.in_text(f"{summary}\n{detail}", ("",))) + by_id = {rows[path][0]: path for path in deleted} + for (content_id,) in conn.execute("SELECT contentID FROM Bookshelf"): + if content_id in by_id: + found.add(("Bookshelf", by_id[content_id], "lists")) + + allowed = sorted({(src, dst) for src, dst, _how in found if (src, dst) in ALLOWED}) + problems.extend(f"{src} {how} {dst}" for src, dst, how in sorted(found) if (src, dst) not in ALLOWED) + + cmap_names = {posixpath.basename(path)[:-len(".bcmap")] for path, group in doomed.items() + if group == "pdfjs-cmaps" and path.endswith(".bcmap")} - IDENTITY_CMAPS + if cmap_names: + pdfs = [path for path in bases if path not in deleted and rows[path][2] == "application/pdf"] + for path, raw, error in map(decode, [(path, stored_bytes(path)) for path in pdfs]): + needs = [f"could not be read: {error}"] if raw is None else cmap_needs(raw, cmap_names) + problems.extend(f"{path} {need}" for need in needs) + print(f"Read {len(pdfs)} PDF(s) for fonts that need a CMap.") + + for src, dst in allowed: + print(f" allowed: {src} refers to {dst}; {ALLOWED[(src, dst)]}") + if problems: + for problem in problems: + print(f" error: {problem}", file=sys.stderr) + print(f"error: {len(problems)} problem(s) found; nothing was changed.", file=sys.stderr) + return 1 + + freed_total = sum(lengths[path] for path in deleted) + for group in groups: + count = sum(1 for g in doomed.values() if g == group) + chunks = sum(1 for path, g in deleted.items() if g == group and path not in doomed) + freed = sum(lengths[path] for path, g in deleted.items() if g == group) + print(f" {group}: {count} row(s) of {GROUPS[group][0]}" + + (f" and {chunks} chunk row(s)" if chunks else "") + f", {human(freed)} stored byte(s)") + if viewer_changed: + print(f" {VIEWER}: enableScripting default set to false") + + if cfg["dry_run"]: + print("Dry run: nothing written.") + else: + print(f"Backing up {db_path} ...") + print(f"Backup written to {backup_database(db_path)}") + conn.execute("BEGIN") + try: + for path in sorted(doomed): + delete_media_row(conn, path, lengths[path]) + if cfg["verbose"]: + print(f" [DEL] {path}") + if viewer_changed: + language_id, content_type_id, template_id = conn.execute( + "SELECT languageID, contentTypeID, templateId FROM Content WHERE path = ?", + (VIEWER,)).fetchone() + replace_row(conn, VIEWER, lengths[VIEWER], codec.compress(viewer_new, rows[VIEWER][3]), + language_id, content_type_id, template_id, lambda m: print(m, file=sys.stderr)) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + if codec is not None: + codec.close() + conn.close() + + if not cfg["dry_run"]: + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + + verb = "would delete" if cfg["dry_run"] else "deleted" + print(f"\nDone. {verb} {len(deleted)} row(s), {human(freed_total)} stored byte(s)" + + ("; scripting turned off in the viewer." if viewer_changed else ".")) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to edit in place, e.g. documentation.db") + p.add_argument("--dry-run", action="store_true", + help="Do all the checks and report what would change, without writing or backing up") + p.add_argument("--only", action="append", choices=list(GROUPS), dest="groups", + help="Delete only this group; repeat for more (default: all of them)") + p.add_argument("--workers", type=int, default=8, + help="Threads used to decompress and read rows (default: 8)") + p.add_argument("--verbose", action="store_true", help="Log every row deleted") + return p + + +def main() -> None: + args = build_parser().parse_args() + sys.exit(run({"db_path": args.db_path, "dry_run": args.dry_run, "groups": args.groups, + "workers": args.workers, "verbose": args.verbose})) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py new file mode 100644 index 00000000..f29d21b1 --- /dev/null +++ b/scripts/test_drop_unread_rows.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_drop_unread_rows.py + +Covers drop_unread_rows.py: that each group deletes its rows, their chunk rows +and nothing else; that scripting is turned off in the viewer; and that every +check refuses the run, writing nothing, when a kept row, template, tooltip or +Bookshelf entry refers to a deleted row, when a row to delete is not what its +group expects, when a stored PDF needs a CMap, when the viewer cannot be +edited, or when a kept row cannot be read. A run is all or nothing, a dry run +writes nothing, and a second run changes nothing. + +Run it either way - as a plain script: + + uv run scripts/test_drop_unread_rows.py + +or under pytest: + + uv run --with brotli --with Pillow --with pytest pytest scripts/test_drop_unread_rows.py + +The dictionary test needs the `brotli` CLI on PATH, as the tool itself does. +""" +from __future__ import annotations + +import contextlib +import io +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import traceback +import zlib +from pathlib import Path + +import brotli + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import drop_unread_rows as tool # noqa: E402 + +CHUNK_SIZE = 1024 * 1024 + +TYPES = {"text/css": (1, "brotli"), "image/svg+xml": (2, "brotli"), "text/javascript": (6, "brotli"), + "application/json": (8, "brotli"), "text/html": (12, "brotli"), "text": (13, "none"), + "application/pdf": (14, "brotli"), "application/wasm": (19, "brotli"), + "application/octet-stream": (27, "brotli")} + +VIEWER_MJS = (b'const defaultOptions={enableScripting:{value:true,kind:OptionKind.VIEWER+OptionKind.PREFERENCE},' + b'wasmUrl:{value:"../web/wasm/"}};defaultOptions.sandboxBundleSrc={value:"../build/pdf.sandbox.mjs"};' + b'scriptingManager:AppOptions.get("enableScripting")&&pdfScriptingManager') +VIEWER_MJS_OFF = VIEWER_MJS.replace(b"enableScripting:{value:true", b"enableScripting:{value:false") +WORKER_MJS = (b'class JpxImage{_filename="openjpeg.wasm";_noWasmFilename="openjpeg_nowasm_fallback.js"}' + b'class JBig2{_filename="jbig2.wasm";_noWasmFilename="jbig2_nowasm_fallback.js"}') +SANDBOX_MJS = b'import("./quickjs-eval.js");fetch("../web/wasm/quickjs-eval.wasm")' +CLASS_PAGE = b'BB' +USE_PAGE = (b'Uses of Class a.BB' + b'package') +PACKAGE_USE = b'Uses of Package aB' +MODULE_GRAPH = b'' + + +def _placeholder(name: str) -> bytes: + return (b"Placeholder for %s" + b"Placeholder for XML layout attribute %s" % (name.encode(), name.encode())) + + +def _pdf(*objects: bytes) -> bytes: + return (b"%PDF-1.7\n" + b"".join(b"%d 0 obj\n%s\nendobj\n" % (i + 1, o) for i, o in enumerate(objects)) + + b"%%EOF\n") + + +def _flate(body: bytes) -> bytes: + data = zlib.compress(body) + return b"<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream" % (len(data), data) + + +PLAIN_PDF = _pdf(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>", + b"<< /Type /Font /Subtype /Type0 /BaseFont /ABCDEF+Noto /Encoding /Identity-H >>", + b"<< /Registry (Adobe) /Ordering (Identity) /Supplement 0 >>", + _flate(b"BT /F1 12 Tf (Hello) Tj ET")) + +# What run() deletes from _base_rows() with every group on. +DELETED = { + "j/html/api/m/a/class-use/B.html", "j/html/api/m/a/package-use.html", "j/html/api/m/module-graph.svg", + "p/web/cmaps/90ms-RKSJ-H.bcmap", "p/web/cmaps/Adobe-Japan1-UCS2.bcmap", "p/web/cmaps/LICENSE", + "p/web/wasm/openjpeg_nowasm_fallback.js", "p/web/wasm/jbig2_nowasm_fallback.js", + "p/web/wasm/quickjs-eval.wasm", "p/web/wasm/quickjs-eval.js", "p/build/pdf.sandbox.mjs", + "x/a.html", "x/b.html", +} + + +def _base_rows() -> dict: + """path -> (content type, decoded bytes [, templateId [, stored as they are]]) for a database holding a row + of every group, the pdf.js files that refer to them, and kept pages.""" + return { + "i/index.html": ("text/html", b'xguide'), + "j/html/api/m/a/B.html": ("text/html", CLASS_PAGE), + "j/html/api/m/a/class-use/B.html": ("text/html", USE_PAGE), + "j/html/api/m/a/package-use.html": ("text/html", PACKAGE_USE), + "j/html/api/m/module-graph.svg": ("image/svg+xml", MODULE_GRAPH), + "j/html/api/script.js": ("text/javascript", b'var page = "B.html"; var tree = "package-tree.html";'), + "j/html/api/m/a/package-tree.html": ("text/html", b"

tree

"), + "p/web/cmaps/90ms-RKSJ-H.bcmap": ("application/octet-stream", b"\xe0\x01cmap"), + "p/web/cmaps/Adobe-Japan1-UCS2.bcmap": ("application/octet-stream", b"\xe0\x02cmap"), + "p/web/cmaps/LICENSE": ("text", b"license text"), + "p/web/wasm/openjpeg.wasm": ("application/wasm", b"\x00asm\x01"), + "p/web/wasm/openjpeg_nowasm_fallback.js": ("text/javascript", b"export default function(){}"), + "p/web/wasm/jbig2_nowasm_fallback.js": ("text/javascript", b"export default function(){ }"), + "p/web/wasm/quickjs-eval.wasm": ("application/wasm", b"\x00asm\x02"), + "p/web/wasm/quickjs-eval.js": ("text/javascript", b'fetch("quickjs-eval.wasm")'), + "p/build/pdf.sandbox.mjs": ("text/javascript", SANDBOX_MJS), + "p/build/pdf.worker.mjs": ("text/javascript", WORKER_MJS), + "p/web/viewer.mjs": ("text/javascript", VIEWER_MJS), + "p/web/viewer.html": ("text/html", b''), + "x/a.html": ("text/html", _placeholder("a")), + "x/b.html": ("text/html", _placeholder("b")), + "x.html": ("text", b"

Hello, App Dev for All

"), + "x.pdf": ("application/pdf", PLAIN_PDF), + "d/guide.pdf": ("application/pdf", PLAIN_PDF + b" "), + } + + +def _compress(raw: bytes, dictionary: bytes = None) -> bytes: + """Stored bytes for a brotli row, at quality 5, so a row the tool re-compressed (at 11) looks different.""" + if dictionary is None: + return brotli.compress(raw, quality=5) + with tempfile.TemporaryDirectory() as directory: + dict_path = Path(directory) / "dictionary.bin" + dict_path.write_bytes(dictionary) + return subprocess.run(["brotli", "-D", str(dict_path), "-q", "5", "-c"], input=raw, + stdout=subprocess.PIPE, check=True).stdout + + +def _decompress(blob: bytes, dictionary: bytes = None) -> bytes: + if dictionary is None: + return brotli.decompress(blob) + with tempfile.TemporaryDirectory() as directory: + dict_path = Path(directory) / "dictionary.bin" + dict_path.write_bytes(dictionary) + return subprocess.run(["brotli", "-D", str(dict_path), "-d", "-c"], input=blob, + stdout=subprocess.PIPE, check=True).stdout + + +def _database(tmp_path: Path, rows: dict, dictionary: bytes = None, templates: dict = None, + buttons: list = None, tooltips: list = None) -> Path: + """A miniature documentation database with the production schema's triggers. `templates` maps + id -> (name, body), `buttons` lists (tooltipId, uri) and `tooltips` (id, summary, detail).""" + tmp_path.mkdir(parents=True, exist_ok=True) + db_path = tmp_path / "documentation.db" + conn = sqlite3.connect(db_path) + conn.executescript( + "CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL);" + "CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL," + " content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE(path));" + "CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL," + " UNIQUE(name));" + "CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL);" + "CREATE TABLE Tooltips (id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," + " summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag));" + "CREATE TABLE TooltipButtons (tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT);" + "CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT ''," + " bookCategoryID INTEGER, UNIQUE(title, bookCategoryId));" + "CREATE TRIGGER AddBook AFTER INSERT ON Content WHEN NEW.path LIKE '%.pdf' BEGIN" + " INSERT INTO Bookshelf (contentID, title) VALUES (NEW.id, CURRENT_TIMESTAMP || NEW.id); END;" + "CREATE TRIGGER DeleteBook AFTER DELETE ON Content WHEN OLD.path LIKE '%.pdf' BEGIN" + " DELETE FROM Bookshelf WHERE contentID = OLD.id; END;" + "INSERT INTO TooltipCategories VALUES (1, 'ide');" + ) + conn.executemany("INSERT INTO ContentTypes (id, value, compression) VALUES (?, ?, ?)", + [(type_id, value, compression) for value, (type_id, compression) in TYPES.items()]) + if dictionary is not None: + conn.execute("CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY, data BLOB NOT NULL)") + conn.execute("INSERT INTO CompressionDictionary (id, data) VALUES (1, ?)", (dictionary,)) + for path, (content_type, raw, *extra) in rows.items(): + template_id, as_is = (list(extra) + [0, False][len(extra):])[:2] + type_id, compression = TYPES[content_type] + stored = raw if as_is or compression == "none" else _compress(raw, dictionary) + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, ?, ?)", + (path, stored, type_id, template_id)) + for template_id, (name, body) in (templates or {}).items(): + conn.execute("INSERT INTO Templates (id, name, content) VALUES (?, ?, ?)", (template_id, name, body)) + for tooltip_id, summary, detail in tooltips or []: + conn.execute("INSERT INTO Tooltips (id, categoryId, tag, summary, detail) VALUES (?, 1, ?, ?, ?)", + (tooltip_id, f"tag{tooltip_id}", summary, detail)) + for tooltip_id, uri in buttons or []: + conn.execute("INSERT INTO TooltipButtons VALUES (?, 1, 'View full documentation', ?)", (tooltip_id, uri)) + conn.commit() + conn.close() + return db_path + + +def _stored(db_path: Path) -> dict: + conn = sqlite3.connect(db_path) + try: + return {row[0]: row[1:] for row in conn.execute( + "SELECT path, languageID, content, contentTypeID, templateId FROM Content")} + finally: + conn.close() + + +def _table(db_path: Path, sql: str) -> list: + conn = sqlite3.connect(db_path) + try: + return sorted(conn.execute(sql).fetchall()) + finally: + conn.close() + + +def _run(db_path: Path, **overrides) -> tuple: + """(exit code, stdout, stderr) of one run of the tool.""" + cfg = {"db_path": db_path, "dry_run": False, "groups": None, "workers": 2, "verbose": False} + cfg.update(overrides) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = tool.run(cfg) + return code, out.getvalue(), err.getvalue() + + +def _refused(tmp_path: Path, rows: dict, expected: str, **database) -> None: + """Asserts that a run over rows exits 1 naming `expected` and leaves the database and directory as they were.""" + db = _database(tmp_path, rows, **database) + before = _stored(db) + code, _out, err = _run(db) + assert code == 1, f"exit {code}, expected a refusal naming {expected!r}" + assert expected in err, err + assert _stored(db) == before + assert len(list(tmp_path.iterdir())) == 1, "a refused run wrote a backup" + + +def _skip(reason: str) -> None: + try: + import pytest + except ImportError: + raise _Skipped(reason) from None + pytest.skip(reason) + + +class _Skipped(Exception): + pass + + +def test_every_group_deletes_its_rows_and_nothing_else(tmp_path: Path): + rows = _base_rows() + db = _database(tmp_path, rows) + before = _stored(db) + code, out, err = _run(db) + after = _stored(db) + assert code == 0, err + assert set(before) - set(after) == DELETED + for path in after: + if path != tool.VIEWER: + assert after[path] == before[path], path + assert _decompress(after[tool.VIEWER][1]) == VIEWER_MJS_OFF + assert after[tool.VIEWER][0] == before[tool.VIEWER][0] and after[tool.VIEWER][2:] == before[tool.VIEWER][2:] + assert "javadoc-use: 3 row(s)" in out and "pdfjs-cmaps: 3 row(s)" in out + assert "pdfjs-extras: 5 row(s)" in out and "x-placeholders: 2 row(s)" in out + for src, dst in tool.ALLOWED: + assert f"allowed: {src} refers to {dst}" in out + + +def test_only_the_named_groups_run(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + code, out, err = _run(db, groups=["x-placeholders", "javadoc-use"]) + after = _stored(db) + assert code == 0, err + assert set(before) - set(after) == {p for p in DELETED if p.startswith(("x/", "j/"))} + assert after[tool.VIEWER] == before[tool.VIEWER] + assert "pdfjs" not in out + + +def test_the_viewer_is_edited_in_either_form(): + unminified = b"const defaultOptions = {\n enableScripting: {\n value: true,\n kind: OptionKind.VIEWER\n }};" + new, changed = tool.turn_scripting_off(unminified) + assert changed and new == unminified.replace(b"value: true", b"value: false") + new, changed = tool.turn_scripting_off(b"enableScripting:{value:!0,kind:1}") + assert changed and new == b"enableScripting:{value:false,kind:1}" + assert tool.turn_scripting_off(VIEWER_MJS_OFF) == (VIEWER_MJS_OFF, False) + + +def test_a_viewer_without_exactly_one_scripting_default_refuses(tmp_path: Path): + for i, viewer in enumerate((b"const defaultOptions={};", VIEWER_MJS + b";" + VIEWER_MJS)): + rows = _base_rows() + rows[tool.VIEWER] = ("text/javascript", viewer) + _refused(tmp_path / str(i), rows, "so scripting cannot be turned off") + + +def test_a_missing_viewer_refuses_while_the_sandbox_is_stored(tmp_path: Path): + rows = _base_rows() + del rows[tool.VIEWER] + del rows["p/web/viewer.html"] + _refused(tmp_path, rows, "cannot tell that scripting is off") + + +def test_a_database_without_pdfjs_needs_no_viewer(tmp_path: Path): + rows = {path: row for path, row in _base_rows().items() if not path.startswith("p/")} + db = _database(tmp_path, rows) + code, _out, err = _run(db) + assert code == 0, err + + +def test_each_kind_of_link_to_a_deleted_row_refuses(tmp_path: Path): + cases = [ + ("i/a.html", "text/html", b'use', + "links to j/html/api/m/a/class-use/B.html"), + ("i/b.html", "text/html", b"", "links to x/a.html"), + ("i/c.html", "text/html", b'', "links to x/b.html"), + ("i/d.html", "text/html", b'', "links to x/a.html"), + ("i/e.css", "text/css", b".a { background: url( ../p/web/cmaps/LICENSE ) }", "links to p/web/cmaps/LICENSE"), + ("i/f.css", "text/css", b"@import '../p/web/cmaps/LICENSE';", "links to p/web/cmaps/LICENSE"), + ("i/g.svg", "image/svg+xml", b'', + "links to j/html/api/m/module-graph.svg"), + ("i/h.js", "text/javascript", b'load("../p/web/wasm/quickjs-eval.js")', "links to p/web/wasm/quickjs-eval.js"), + ("k/i.js", "text/javascript", b'const name = `${base}/package-use.html`;', + "names j/html/api/m/a/package-use.html"), + ("i/j.html", "text/html", b'a', "links to x/a.html"), + ("i/k.html", "text/html", b'b', "links to x/b.html"), + ("i/l.html", "text/html", b'pdf', "links to x/a.html"), + ("i/l2.html", "text/html", b'pdf', + "links to x/b.html"), + ("i/m.html", "text/html", b'b', "links to x/b.html"), + ("i/n.html", "text/html", b'a', "links to x/a.html"), + ("k/o.html", "text/html", b'{"link": "../x/b.html", "body": "a"}', + "k/o.html links to x/a.html"), + ("k/p.html", "text/html", b'{"next": "../x/b.html"}', "k/p.html links to x/b.html"), + ] + for i, (path, content_type, body, expected) in enumerate(cases): + rows = _base_rows() + rows[path] = (content_type, body, 7) if path.startswith("k/") and path.endswith(".html") else \ + (content_type, body) + _refused(tmp_path / str(i), rows, f"{path} {expected}" if not expected.startswith(path) else expected) + + +def test_a_reference_the_allowlist_names_only_for_another_row_still_refuses(tmp_path: Path): + rows = _base_rows() + rows["p/web/other.mjs"] = ("text/javascript", b'import("../build/pdf.sandbox.mjs")') + _refused(tmp_path, rows, "p/web/other.mjs links to p/build/pdf.sandbox.mjs") + + +def test_a_template_link_refuses(tmp_path: Path): + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + _refused(tmp_path, rows, "template page.peb links to x/a.html", + templates={1: ("page.peb", b'{{ title }}')}) + + +def test_a_tooltip_button_refuses(tmp_path: Path): + _refused(tmp_path, _base_rows(), "tooltip 3 button links to j/html/api/m/a/package-use.html", + tooltips=[(3, "s", "")], buttons=[(3, "i/index.html"), (3, "j/html/api/m/a/package-use.html#x")]) + + +def test_a_link_in_tooltip_text_refuses(tmp_path: Path): + _refused(tmp_path, _base_rows(), "tooltip 4 text links to x/b.html", + tooltips=[(4, 'See b.', "")]) + + +def test_a_bookshelf_entry_refuses(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + conn = sqlite3.connect(db) + conn.execute("INSERT INTO Bookshelf (contentID, title) SELECT id, 'placeholder book' FROM Content " + "WHERE path = 'x/a.html'") + conn.commit() + conn.close() + before, shelf = _stored(db), _table(db, "SELECT * FROM Bookshelf") + code, _out, err = _run(db) + assert code == 1 and "Bookshelf lists x/a.html" in err, err + assert _stored(db) == before and _table(db, "SELECT * FROM Bookshelf") == shelf + + +def test_links_to_kept_rows_are_not_refused(tmp_path: Path): + # B.html is kept and class-use/B.html shares only its file name; package-use.html on another host is not + # this database's. + rows = _base_rows() + rows["i/q.html"] = ("text/html", b'B' + b'elsewhere') + db = _database(tmp_path, rows, buttons=[(1, "j/html/api/m/a/B.html")]) + code, _out, err = _run(db) + assert code == 0, err + + +def test_a_row_to_delete_that_is_not_what_its_group_holds_refuses(tmp_path: Path): + cases = [ + ("x/c.html", ("text/html", b"

A real page

"), "x/c.html is not a placeholder page"), + ("j/html/api/m/b/class-use/C.html", ("text/html", b"C"), + "j/html/api/m/b/class-use/C.html is not a javadoc Use page"), + ("j/html/api/n/module-graph.svg", ("image/svg+xml", b"

not svg

"), + "j/html/api/n/module-graph.svg is not an SVG"), + ("p/web/cmaps/notes.html", ("text/html", b"

notes

"), "p/web/cmaps/notes.html is not a CMap"), + ] + for i, (path, row, expected) in enumerate(cases): + rows = _base_rows() + rows[path] = row + _refused(tmp_path / str(i), rows, expected) + + +def test_a_pdf_that_needs_a_cmap_refuses(tmp_path: Path): + cases = [ + (_pdf(b"<< /Type /Font /Subtype /Type0 /BaseFont /MS-Gothic /Encoding /90ms-RKSJ-H >>"), + "uses the predefined CMap 90ms-RKSJ-H"), + (_pdf(_flate(b"/CIDInit /ProcSet findresource begin /90ms-RKSJ-H usecmap begincmap endcmap")), + "uses the predefined CMap 90ms-RKSJ-H"), + (_pdf(_flate(b"<< /Type /Font /Subtype /Type0 /Encoding /Adobe-Japan1-UCS2 >>")), + "uses the predefined CMap Adobe-Japan1-UCS2"), + (_pdf(b"<< /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >>"), + "has a CID font of the Adobe Japan1 collection"), + (_pdf(b"<< /Registry (Adobe) /Ordering <4B 6F72656131> /Supplement 2 >>"), + "has a CID font of the Adobe Korea1 collection"), + (_pdf(b"<< /Type /Font /Subtype /TrueType /BaseFont /#CB#CE#CC#E5 /Encoding /WinAnsiEncoding >>"), + "has a font pdf.js reads as Chinese"), + ] + for i, (pdf, expected) in enumerate(cases): + rows = _base_rows() + rows["d/cjk.pdf"] = ("application/pdf", pdf) + _refused(tmp_path / str(i), rows, f"d/cjk.pdf {expected}") + + +def test_the_pdf_check_runs_only_with_the_cmaps(tmp_path: Path): + rows = _base_rows() + rows["d/cjk.pdf"] = ("application/pdf", _pdf(b"<< /Registry (Adobe) /Ordering (Japan1) >>")) + db = _database(tmp_path, rows) + code, _out, err = _run(db, groups=["x-placeholders"]) + assert code == 0, err + + +def test_an_unreadable_kept_row_refuses(tmp_path: Path): + rows = _base_rows() + rows["i/broken.html"] = ("text/html", b"not brotli at all", 0, True) + _refused(tmp_path, rows, "i/broken.html could not be read, so its references were not checked") + + +def test_a_chunked_row_goes_with_its_base_and_others_stay(tmp_path: Path): + import random + rng = random.Random(7) + noise = bytes(rng.getrandbits(8) for _ in range(CHUNK_SIZE + 5000)) + rows = _base_rows() + # Stored as they are: a deleted placeholder spanning two rows, a kept page spanning two, and a page + # that only looks like a chunk row because its base is short. + page = brotli.compress(_placeholder("big") + b"", quality=0) + assert len(page) > CHUNK_SIZE + rows["x/big.html"] = ("text/html", page[:CHUNK_SIZE], 0, True) + rows["x/big.html-1"] = ("text/html", page[CHUNK_SIZE:], 0, True) + kept = brotli.compress(b"

" + noise.hex().encode() + b"

", quality=0) + rows["i/big.html"] = ("text/html", kept[:CHUNK_SIZE], 0, True) + rows["i/big.html-1"] = ("text/html", kept[CHUNK_SIZE:], 0, True) + rows["x/a.html-1"] = ("text/html", _placeholder("a-1")) + db = _database(tmp_path, rows) + code, out, err = _run(db) + after = _stored(db) + assert code == 0, err + assert not {"x/big.html", "x/big.html-1", "x/a.html-1"} & set(after) + assert {"i/big.html", "i/big.html-1"} <= set(after) + assert "x-placeholders: 4 row(s) of x/ placeholder pages and 1 chunk row(s)" in out + + +def test_dry_run_writes_nothing(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + code, out, _err = _run(db, dry_run=True) + assert code == 0 and "would delete 13 row(s)" in out + assert _stored(db) == before + assert len(list(tmp_path.iterdir())) == 1 + + +def test_a_second_run_changes_nothing(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + assert _run(db)[0] == 0 + once = _stored(db) + files = sorted(tmp_path.iterdir()) + code, out, _err = _run(db) + assert code == 0 and out.startswith("Nothing to do") + assert _stored(db) == once and sorted(tmp_path.iterdir()) == files + + +def test_the_backup_holds_the_database_as_it_was(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + code, out, _err = _run(db) + backup = Path(re.search(r"Backup written to (.+)", out).group(1).strip()) + assert code == 0 and backup.is_file() + assert _stored(backup) == before + + +def test_an_error_while_writing_rolls_everything_back(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + real = tool.delete_media_row + calls = [] + + def failing(conn, path, length): + calls.append(path) + if len(calls) == 3: + raise RuntimeError("disk full") + return real(conn, path, length) + + tool.delete_media_row = failing + try: + try: + _run(db) + except RuntimeError: + pass + else: + raise AssertionError("the failure did not propagate") + finally: + tool.delete_media_row = real + assert len(calls) == 3 and _stored(db) == before + + +def test_a_dictionary_database_is_read_and_written_through_the_dictionary(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + dictionary = VIEWER_MJS * 4 + USE_PAGE * 4 + db = _database(tmp_path, _base_rows(), dictionary=dictionary) + code, _out, err = _run(db) + assert code == 0, err + assert _decompress(_stored(db)[tool.VIEWER][1], dictionary) == VIEWER_MJS_OFF + + +def test_a_missing_database_is_an_error(tmp_path: Path): + code, _out, err = _run(tmp_path / "absent.db") + assert code == 1 and "does not exist" in err + + +def _main() -> int: + """Runs the tests without pytest installed as a runner (`uv run scripts/test_drop_unread_rows.py`).""" + failures = [] + for name, function in sorted(globals().items()): + if not name.startswith("test_") or not callable(function): + continue + with tempfile.TemporaryDirectory() as directory: + needs_tmp = "tmp_path" in function.__code__.co_varnames[:function.__code__.co_argcount] + try: + function(Path(directory)) if needs_tmp else function() + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: # noqa: BLE001 - a failure is the result here + if type(exc).__name__ in ("Skipped", "_Skipped"): + print(f"skip {name}: {exc}") + continue + frame = traceback.extract_tb(exc.__traceback__)[-1] + failures.append(f"{name} (line {frame.lineno}: {frame.line}): {type(exc).__name__}: {exc}") + print(f"FAIL {name}", file=sys.stderr) + else: + print(f"ok {name}") + print() + print(f"{len(failures)} failure(s)" if failures else "all tests passed") + for failure in failures: + print(f" {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(_main()) From 87b8b627f9e4405f1f0e2feaf2259087a3accf61 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 13:47:56 -0700 Subject: [PATCH 2/9] ADFA-3305: Empty tooltip details that repeat their summary clear_repeated_tooltip_detail.py sets Tooltips.detail to '' where it is exactly the summary: 1,605 tooltips (1,592 Kotlin, 13 Java), 151,274 bytes. All of them have buttons, so See More stays, and its popup lists the links without repeating the summary above them. The first popup is unchanged. Packed, the file is 151,552 bytes smaller; the .br change (10,049 bytes) is within what repacking alone moves it. With drop_unread_rows.py, all five changes applied in order and packed save 6,520,832 file bytes and 5,635,912 brotli -q 11 -w 24 bytes (110,895,104 -> 104,374,272; 87,091,418 -> 81,455,506). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- CLAUDE.md | 3 +- scripts/clear_repeated_tooltip_detail.py | 126 +++++++++++ scripts/test_clear_repeated_tooltip_detail.py | 204 ++++++++++++++++++ 3 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 scripts/clear_repeated_tooltip_detail.py create mode 100644 scripts/test_clear_repeated_tooltip_detail.py diff --git a/CLAUDE.md b/CLAUDE.md index 123777e4..b319b090 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -171,7 +171,8 @@ uses. Any future Android/Java tooltip work should target the normalized `Tooltip `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), `create_empty_database.py`, `list_database_documents.py`, `drop_unread_rows.py` (deletes Content rows nothing reads - javadoc Use pages, pdf.js CMaps, pdf.js fallbacks and scripting sandbox, x/ - placeholders - after checking that nothing kept refers to them). **Deprecated/dead** (target the + placeholders - after checking that nothing kept refers to them), `clear_repeated_tooltip_detail.py` + (empties a tooltip detail that repeats its summary). **Deprecated/dead** (target the removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, `load_android_data.py`. diff --git a/scripts/clear_repeated_tooltip_detail.py b/scripts/clear_repeated_tooltip_detail.py new file mode 100644 index 00000000..54aeec1e --- /dev/null +++ b/scripts/clear_repeated_tooltip_detail.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +""" +clear_repeated_tooltip_detail.py + +Empties Tooltips.detail where it is the same text as Tooltips.summary. + +In the 2026-10-18 database 1,605 tooltips (1,592 Kotlin, 13 Java) repeat their +summary as their detail, 151,274 bytes in all; 25,285 others already have an +empty detail. + +What the user sees differently (ToolTipManager.kt, the only reader of detail): +- The first popup shows the summary, as before. +- "See More" shows when the detail is not blank or the tooltip has buttons. + For a tooltip with buttons it still shows, and the second popup now lists + just the buttons' links instead of the summary again, a blank line, then the + links. All 1,605 tooltips in that database have buttons. +- A tooltip with no buttons loses "See More", whose popup held only the + summary again (and the sponsor button every second popup shows). This step + counts such tooltips and says so. +- The info dialog and the feedback e-mail show an empty Detail. + +The comparison is exact (Python string equality), whatever collation the +schema gives the columns. Tooltips are not Brotli-compressed, so the shared +dictionary does not need re-minting. + +Measured on the 2026-10-18 database after ADFA-6627, packed with +rebuild_database.py --pack Content,Tooltips --page-size 4096: the file is +151,552 bytes smaller; its brotli -q 11 -w 24 is 10,049 bytes smaller, which is +within the 0.2 MB that repacking alone moves it. + +Runs in one transaction (rolled back on any error), backs the database up +first, and VACUUMs afterwards. --dry-run reports what would change without +writing. A second run finds nothing to do and exits 0. + +Pipeline position: after every step that writes Tooltips, before +rebuild_database.py. + + uv run scripts/clear_repeated_tooltip_detail.py documentation.db [--dry-run] [--verbose] +""" +from __future__ import annotations + +import argparse +import sqlite3 +import sys +from collections import Counter +from pathlib import Path + +from optimize_db_media import backup_database, human + + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + + conn = sqlite3.connect(db_path) + try: + columns = {row[1] for row in conn.execute("PRAGMA table_info(Tooltips)")} + if not {"id", "categoryId", "tag", "summary", "detail"} <= columns: + print("error: this database has no Tooltips table with id, categoryId, tag, summary and detail", + file=sys.stderr) + return 1 + rows = conn.execute( + "SELECT T.id, T.tag, T.summary, T.detail, COALESCE(TC.category, '?'), " + "EXISTS (SELECT 1 FROM TooltipButtons B WHERE B.tooltipId = T.id) " + "FROM Tooltips T LEFT JOIN TooltipCategories TC ON TC.id = T.categoryId ORDER BY T.id").fetchall() + repeated = [row for row in rows if isinstance(row[3], str) and row[3] != "" and row[3] == row[2]] + saved = sum(len(row[3].encode("utf-8")) for row in repeated) + by_category = Counter(row[4] for row in repeated) + no_buttons = [row for row in repeated if not row[5]] + + if not repeated: + print(f"Nothing to do: none of {len(rows)} tooltip(s) repeats its summary as its detail.") + return 0 + print(f"{len(repeated)} of {len(rows)} tooltip(s) repeat their summary as their detail " + f"({', '.join(f'{category} {count}' for category, count in sorted(by_category.items()))}); " + f"emptying it saves {human(saved)} byte(s).") + if no_buttons: + print(f" {len(no_buttons)} of them have no buttons, so their tooltip loses its See More link, " + "which led only to the same text again.") + + if cfg["dry_run"]: + print("Dry run: nothing written.") + return 0 + print(f"Backing up {db_path} ...") + print(f"Backup written to {backup_database(db_path)}") + conn.execute("BEGIN") + try: + for tooltip_id, tag, _summary, _detail, category, _buttons in repeated: + conn.execute("UPDATE Tooltips SET detail = '' WHERE id = ?", (tooltip_id,)) + if cfg["verbose"]: + print(f" [CLEAR] {category} {tag}") + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + print(f"\nDone. emptied the detail of {len(repeated)} tooltip(s), saving {human(saved)} byte(s).") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to edit in place, e.g. documentation.db") + p.add_argument("--dry-run", action="store_true", + help="Report what would change, without writing or backing up") + p.add_argument("--verbose", action="store_true", help="Log every tooltip changed") + return p + + +def main() -> None: + args = build_parser().parse_args() + sys.exit(run({"db_path": args.db_path, "dry_run": args.dry_run, "verbose": args.verbose})) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_clear_repeated_tooltip_detail.py b/scripts/test_clear_repeated_tooltip_detail.py new file mode 100644 index 00000000..4daaddfb --- /dev/null +++ b/scripts/test_clear_repeated_tooltip_detail.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_clear_repeated_tooltip_detail.py + +Covers clear_repeated_tooltip_detail.py: that it empties a detail only when it +is exactly its summary, whatever collation the schema gives the columns; that +it says how many such tooltips have no buttons; and that a run is all or +nothing, a dry run writes nothing, and a second run changes nothing. + +Run it either way - as a plain script: + + uv run scripts/test_clear_repeated_tooltip_detail.py + +or under pytest: + + uv run --with brotli --with Pillow --with pytest pytest scripts/test_clear_repeated_tooltip_detail.py +""" +from __future__ import annotations + +import contextlib +import io +import re +import sqlite3 +import sys +import tempfile +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import clear_repeated_tooltip_detail as tool # noqa: E402 + +# (id, category, summary, detail, buttons) +TOOLTIPS = [ + (1, 4, "Returns the size.", "Returns the size.", 1), + (2, 4, "Adds an element.", "Adds an element, if there is room.", 1), + (3, 4, "Removes it.", "", 1), + (4, 3, "Splits a string.", "Splits a string.", 2), + (5, 4, "Trims it.", "Trims it. ", 1), + (6, 4, "Counts.", "counts.", 1), + (7, 1, "Opens the file.", "Opens the file.", 0), + (8, 4, "", "", 1), +] + + +def _database(tmp_path: Path, tooltips=TOOLTIPS, collate: str = "") -> Path: + db_path = tmp_path / "documentation.db" + conn = sqlite3.connect(db_path) + conn.executescript( + "CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL);" + "INSERT INTO TooltipCategories VALUES (1, 'ide'), (3, 'java'), (4, 'kotlin');" + f"CREATE TABLE Tooltips (id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," + f" summary TEXT NOT NULL {collate}, detail TEXT NOT NULL {collate}, UNIQUE (categoryId, tag));" + "CREATE TABLE TooltipButtons (tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT);" + ) + for tooltip_id, category, summary, detail, buttons in tooltips: + conn.execute("INSERT INTO Tooltips VALUES (?, ?, ?, ?, ?)", (tooltip_id, category, f"tag{tooltip_id}", + summary, detail)) + for number in range(buttons): + conn.execute("INSERT INTO TooltipButtons VALUES (?, ?, 'View full documentation', 'a/x.html')", + (tooltip_id, number + 1)) + conn.commit() + conn.close() + return db_path + + +def _details(db_path: Path) -> dict: + conn = sqlite3.connect(db_path) + try: + return {row[0]: row[1:] for row in conn.execute("SELECT id, summary, detail FROM Tooltips")} + finally: + conn.close() + + +def _run(db_path: Path, **overrides) -> tuple: + cfg = {"db_path": db_path, "dry_run": False, "verbose": False} + cfg.update(overrides) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = tool.run(cfg) + return code, out.getvalue(), err.getvalue() + + +def test_only_a_detail_that_is_exactly_its_summary_is_emptied(tmp_path: Path): + db = _database(tmp_path) + before = _details(db) + code, out, err = _run(db) + after = _details(db) + assert code == 0, err + assert {i for i in before if after[i] != before[i]} == {1, 4, 7} + for i in (1, 4, 7): + assert after[i] == (before[i][0], "") + assert "3 of 8 tooltip(s)" in out and "ide 1, java 1, kotlin 1" in out + assert "saves 48 byte(s)" in out + + +def test_a_case_insensitive_schema_does_not_widen_the_match(tmp_path: Path): + db = _database(tmp_path, collate="COLLATE NOCASE") + code, _out, err = _run(db) + assert code == 0, err + assert _details(db)[6] == ("Counts.", "counts.") + + +def test_a_detail_that_is_not_text_is_left_alone(tmp_path: Path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + conn.execute("UPDATE Tooltips SET summary = CAST(summary AS BLOB), detail = CAST(detail AS BLOB) WHERE id = 1") + conn.commit() + conn.close() + code, out, err = _run(db) + assert code == 0, err + assert _details(db)[1] == (b"Returns the size.", b"Returns the size.") and "2 of 8 tooltip(s)" in out + + +def test_tooltips_without_buttons_are_counted(tmp_path: Path): + code, out, _err = _run(_database(tmp_path)) + assert code == 0 and "1 of them have no buttons, so their tooltip loses its See More link" in out + + +def test_dry_run_writes_nothing(tmp_path: Path): + db = _database(tmp_path) + before = _details(db) + code, out, _err = _run(db, dry_run=True) + assert code == 0 and "3 of 8 tooltip(s)" in out and "Dry run" in out + assert _details(db) == before and len(list(tmp_path.iterdir())) == 1 + + +def test_a_second_run_changes_nothing(tmp_path: Path): + db = _database(tmp_path) + assert _run(db)[0] == 0 + once, files = _details(db), sorted(tmp_path.iterdir()) + code, out, _err = _run(db) + assert code == 0 and out.startswith("Nothing to do") + assert _details(db) == once and sorted(tmp_path.iterdir()) == files + + +def test_the_backup_holds_the_database_as_it_was(tmp_path: Path): + db = _database(tmp_path) + before = _details(db) + code, out, _err = _run(db) + backup = Path(re.search(r"Backup written to (.+)", out).group(1).strip()) + assert code == 0 and _details(backup) == before + + +def test_an_error_while_writing_rolls_everything_back(tmp_path: Path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + conn.execute("CREATE TRIGGER fail BEFORE UPDATE ON Tooltips WHEN OLD.id = 4 BEGIN SELECT RAISE(ABORT, 'disk full');" + " END") + conn.commit() + conn.close() + before = _details(db) + try: + _run(db) + except sqlite3.DatabaseError: + pass + else: + raise AssertionError("the failure did not propagate") + assert _details(db) == before + + +def test_a_database_without_tooltips_is_an_error(tmp_path: Path): + db = tmp_path / "documentation.db" + sqlite3.connect(db).close() + code, _out, err = _run(db) + assert code == 1 and "no Tooltips table" in err + code, _out, err = _run(tmp_path / "absent.db") + assert code == 1 and "does not exist" in err + + +def _main() -> int: + """Runs the tests without pytest (`uv run scripts/test_clear_repeated_tooltip_detail.py`).""" + failures = [] + for name, function in sorted(globals().items()): + if not name.startswith("test_") or not callable(function): + continue + with tempfile.TemporaryDirectory() as directory: + try: + function(Path(directory)) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: # noqa: BLE001 - a failure is the result here + frame = traceback.extract_tb(exc.__traceback__)[-1] + failures.append(f"{name} (line {frame.lineno}: {frame.line}): {type(exc).__name__}: {exc}") + print(f"FAIL {name}", file=sys.stderr) + else: + print(f"ok {name}") + print() + print(f"{len(failures)} failure(s)" if failures else "all tests passed") + for failure in failures: + print(f" {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(_main()) From 238a1d4d14196edc07dc88e0a406393804807613 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 21:37:55 -0700 Subject: [PATCH 3/9] ADFA-3305: Read tooltip links in either schema; place the tooltip step before the migration drop_unread_rows.py read TooltipButtons.uri and Tooltips.summary/detail, which migrate_tooltip_schema.py removes, so after the migration it scanned for a minute and then crashed (no such column: uri). It now reads either shape, detected before the scan, and refuses tables in neither shape before reading anything. clear_repeated_tooltip_detail.py said it runs after every step that writes Tooltips, which would put it after the migration. It now says it runs before migrate_tooltip_schema.py and refuses a migrated database by name. PR #46 review finding R46-01. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/clear_repeated_tooltip_detail.py | 11 ++- scripts/drop_unread_rows.py | 46 ++++++++-- scripts/test_clear_repeated_tooltip_detail.py | 23 +++++ scripts/test_drop_unread_rows.py | 91 +++++++++++++++---- 4 files changed, 146 insertions(+), 25 deletions(-) diff --git a/scripts/clear_repeated_tooltip_detail.py b/scripts/clear_repeated_tooltip_detail.py index 54aeec1e..1fc2f278 100644 --- a/scripts/clear_repeated_tooltip_detail.py +++ b/scripts/clear_repeated_tooltip_detail.py @@ -32,8 +32,11 @@ first, and VACUUMs afterwards. --dry-run reports what would change without writing. A second run finds nothing to do and exits 0. -Pipeline position: after every step that writes Tooltips, before -rebuild_database.py. +Pipeline position: after every step that writes tooltip text, and before +migrate_tooltip_schema.py, which moves summary and detail out of Tooltips into +TooltipTexts (documentation schema 3). This step reads only the schema-2 shape +and refuses a migrated database, writing nothing. rebuild_database.py comes +after both. uv run scripts/clear_repeated_tooltip_detail.py documentation.db [--dry-run] [--verbose] """ @@ -57,6 +60,10 @@ def run(cfg: dict) -> int: conn = sqlite3.connect(db_path) try: columns = {row[1] for row in conn.execute("PRAGMA table_info(Tooltips)")} + if conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'TooltipTexts'").fetchone(): + print("error: this database's tooltip text is in TooltipTexts (documentation schema 3); run this step " + "before migrate_tooltip_schema.py", file=sys.stderr) + return 1 if not {"id", "categoryId", "tag", "summary", "detail"} <= columns: print("error: this database has no Tooltips table with id, categoryId, tag, summary and detail", file=sys.stderr) diff --git a/scripts/drop_unread_rows.py b/scripts/drop_unread_rows.py index 606a81ae..5c11986a 100644 --- a/scripts/drop_unread_rows.py +++ b/scripts/drop_unread_rows.py @@ -49,8 +49,12 @@ when any check fails, writing nothing: - References. Every kept text row (HTML, CSS, JS, SVG, JSON, XML, Markdown and - plain text, decoded, chunks joined), every Templates row, every TooltipButtons - uri, the Tooltips text and the Bookshelf entries are read for links: HTML and + plain text, decoded, chunks joined), every Templates row, every tooltip + button's uri, every tooltip's text and the Bookshelf entries are read for + links. The tooltip tables are read in either documentation schema: 2 + (TooltipButtons.uri, Tooltips.summary and detail) or 3, after + migrate_tooltip_schema.py (TooltipButtonUris, TooltipTexts); tables in + neither shape refuse the run before anything is read. Links are: HTML and SVG attributes, srcset, CSS url() and @import, quoted file names in scripts, the string values of a templated row's JSON, viewer.html?file= parameters and http://localhost:6174/ URLs. A link that resolves (relative to its row, or @@ -79,9 +83,11 @@ first, and VACUUMs afterwards. --dry-run does all the checks and reports what would change without writing. A second run finds nothing to do and exits 0. -Pipeline position: after every step that writes page content or links, so the -reference check sees the final pages, and before rebuild_database.py. Deleting -rows does not call for a dictionary re-mint. +Pipeline position: after every step that writes page content, links or +tooltips, so the reference check sees the final pages, and before +rebuild_database.py. It may run before or after migrate_tooltip_schema.py, +since it reads both tooltip shapes. Deleting rows does not call for a +dictionary re-mint. uv run scripts/drop_unread_rows.py documentation.db --dry-run uv run scripts/drop_unread_rows.py documentation.db [--only GROUP ...] [--workers N] [--verbose] @@ -357,6 +363,25 @@ def cmap_needs(data: bytes, cmap_names: set) -> list: return sorted(needs) +def tooltip_queries(conn): + """(buttons query, texts query) for the database's tooltip tables, schema 2 or 3, or None for any other + shape. The buttons query yields (tooltipId, uri), the texts query (id, summary, detail).""" + def columns(table): + return {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} + + tooltips, buttons, texts, uris = (columns(t) for t in ("Tooltips", "TooltipButtons", "TooltipTexts", + "TooltipButtonUris")) + if {"id", "summary", "detail"} <= tooltips and {"tooltipId", "uri"} <= buttons and not texts and not uris: + return ("SELECT tooltipId, uri FROM TooltipButtons WHERE uri IS NOT NULL", + "SELECT id, summary, detail FROM Tooltips") + if ({"id", "categoryId", "tag"} <= tooltips and not {"summary", "detail"} & tooltips + and {"tooltipId", "uriId"} <= buttons and {"id", "summary", "detail"} <= texts and {"id", "uri"} <= uris): + return ("SELECT B.tooltipId, U.uri FROM TooltipButtons B JOIN TooltipButtonUris U ON U.id = B.uriId " + "WHERE U.uri IS NOT NULL", + "SELECT id, summary, detail FROM TooltipTexts") + return None + + def run(cfg: dict) -> int: db_path = cfg["db_path"] if not db_path.is_file(): @@ -367,6 +392,13 @@ def run(cfg: dict) -> int: conn = sqlite3.connect(db_path) codec = None try: + queries = tooltip_queries(conn) + if queries is None: + print("error: the tooltip tables are in neither the schema-2 shape (Tooltips.summary and detail, " + "TooltipButtons.uri) nor the schema-3 shape (TooltipTexts, TooltipButtonUris), so their links " + "cannot be read; nothing was changed.", file=sys.stderr) + return 1 + buttons_query, texts_query = queries codec = BrotliCodec(load_dictionary(conn)) rows = {path: (row_id, length, mime, compression, template_id) for row_id, path, length, mime, compression, template_id in conn.execute( @@ -453,9 +485,9 @@ def scan(item): dirs = ("",) + tuple(sorted(dirs_by_template[template_id])) text = bytes(content).decode("utf-8", errors="replace") found.update((f"template {name}", target, how) for target, how in refs.in_text(text, dirs)) - for tooltip_id, uri in conn.execute("SELECT tooltipId, uri FROM TooltipButtons WHERE uri IS NOT NULL"): + for tooltip_id, uri in conn.execute(buttons_query): found.update((f"tooltip {tooltip_id} button", target, how) for target, how in refs.targets(uri, ("",))) - for tooltip_id, summary, detail in conn.execute("SELECT id, summary, detail FROM Tooltips"): + for tooltip_id, summary, detail in conn.execute(texts_query): found.update((f"tooltip {tooltip_id} text", target, how) for target, how in refs.in_text(f"{summary}\n{detail}", ("",))) by_id = {rows[path][0]: path for path in deleted} diff --git a/scripts/test_clear_repeated_tooltip_detail.py b/scripts/test_clear_repeated_tooltip_detail.py index 4daaddfb..ca5bc5e1 100644 --- a/scripts/test_clear_repeated_tooltip_detail.py +++ b/scripts/test_clear_repeated_tooltip_detail.py @@ -167,6 +167,29 @@ def test_an_error_while_writing_rolls_everything_back(tmp_path: Path): assert _details(db) == before +def test_a_migrated_database_is_refused_by_name(tmp_path: Path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + conn.executescript("CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY, summary TEXT, detail TEXT);" + "INSERT INTO TooltipTexts SELECT id, summary, detail FROM Tooltips;" + "CREATE TABLE T2 (id INTEGER, categoryId INTEGER, tag TEXT);" + "INSERT INTO T2 SELECT id, categoryId, tag FROM Tooltips;" + "DROP TABLE Tooltips; ALTER TABLE T2 RENAME TO Tooltips;") + conn.close() + code, out, err = _run(db) + assert code == 1 and "run this step before migrate_tooltip_schema.py" in err, err + assert sorted(p.name for p in tmp_path.iterdir()) == ["documentation.db"] + # Even with the old columns still on Tooltips, a TooltipTexts means the text lives there. + (tmp_path / "half").mkdir() + db2 = _database(tmp_path / "half") + conn = sqlite3.connect(db2) + conn.execute("CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY, summary TEXT, detail TEXT)") + conn.close() + before = _details(db2) + code, _out, err = _run(db2) + assert code == 1 and "migrate_tooltip_schema.py" in err and _details(db2) == before + + def test_a_database_without_tooltips_is_an_error(tmp_path: Path): db = tmp_path / "documentation.db" sqlite3.connect(db).close() diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py index f29d21b1..4de4f5dc 100644 --- a/scripts/test_drop_unread_rows.py +++ b/scripts/test_drop_unread_rows.py @@ -152,10 +152,27 @@ def _decompress(blob: bytes, dictionary: bytes = None) -> bytes: stdout=subprocess.PIPE, check=True).stdout +TOOLTIP_TABLES = { + 2: "CREATE TABLE Tooltips (id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," + " summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag));" + "CREATE TABLE TooltipButtons (tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT);", + # What migrate_tooltip_schema.py leaves (documentation schema 3). + 3: "CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, detail TEXT NOT NULL);" + "CREATE TABLE Tooltips (id INTEGER NOT NULL, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," + " PRIMARY KEY (categoryId, tag)) WITHOUT ROWID;" + "CREATE TABLE TooltipButtonDescriptions (id INTEGER PRIMARY KEY, description TEXT NOT NULL);" + "CREATE TABLE TooltipButtonUris (id INTEGER PRIMARY KEY, uri TEXT NOT NULL);" + "CREATE TABLE TooltipButtons (tooltipId INTEGER NOT NULL, buttonNumberId INTEGER NOT NULL," + " descriptionId INTEGER NOT NULL, uriId INTEGER NOT NULL, PRIMARY KEY (tooltipId, buttonNumberId)) WITHOUT ROWID;" + "INSERT INTO TooltipButtonDescriptions VALUES (1, 'View full documentation');", +} + + def _database(tmp_path: Path, rows: dict, dictionary: bytes = None, templates: dict = None, - buttons: list = None, tooltips: list = None) -> Path: - """A miniature documentation database with the production schema's triggers. `templates` maps - id -> (name, body), `buttons` lists (tooltipId, uri) and `tooltips` (id, summary, detail).""" + buttons: list = None, tooltips: list = None, schema: int = 2) -> Path: + """A miniature documentation database with the production schema's triggers and the tooltip tables of + documentation schema 2 or 3. `templates` maps id -> (name, body), `buttons` lists (tooltipId, uri) and + `tooltips` (id, summary, detail).""" tmp_path.mkdir(parents=True, exist_ok=True) db_path = tmp_path / "documentation.db" conn = sqlite3.connect(db_path) @@ -166,9 +183,7 @@ def _database(tmp_path: Path, rows: dict, dictionary: bytes = None, templates: d "CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL," " UNIQUE(name));" "CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL);" - "CREATE TABLE Tooltips (id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," - " summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag));" - "CREATE TABLE TooltipButtons (tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT);" + + TOOLTIP_TABLES[schema] + "CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT ''," " bookCategoryID INTEGER, UNIQUE(title, bookCategoryId));" "CREATE TRIGGER AddBook AFTER INSERT ON Content WHEN NEW.path LIKE '%.pdf' BEGIN" @@ -191,10 +206,19 @@ def _database(tmp_path: Path, rows: dict, dictionary: bytes = None, templates: d for template_id, (name, body) in (templates or {}).items(): conn.execute("INSERT INTO Templates (id, name, content) VALUES (?, ?, ?)", (template_id, name, body)) for tooltip_id, summary, detail in tooltips or []: - conn.execute("INSERT INTO Tooltips (id, categoryId, tag, summary, detail) VALUES (?, 1, ?, ?, ?)", - (tooltip_id, f"tag{tooltip_id}", summary, detail)) - for tooltip_id, uri in buttons or []: - conn.execute("INSERT INTO TooltipButtons VALUES (?, 1, 'View full documentation', ?)", (tooltip_id, uri)) + if schema == 2: + conn.execute("INSERT INTO Tooltips (id, categoryId, tag, summary, detail) VALUES (?, 1, ?, ?, ?)", + (tooltip_id, f"tag{tooltip_id}", summary, detail)) + else: + conn.execute("INSERT INTO TooltipTexts (id, summary, detail) VALUES (?, ?, ?)", (tooltip_id, summary, detail)) + conn.execute("INSERT INTO Tooltips (id, categoryId, tag) VALUES (?, 1, ?)", (tooltip_id, f"tag{tooltip_id}")) + for number, (tooltip_id, uri) in enumerate(buttons or [], start=1): + if schema == 2: + conn.execute("INSERT INTO TooltipButtons VALUES (?, ?, 'View full documentation', ?)", + (tooltip_id, number, uri)) + else: + conn.execute("INSERT INTO TooltipButtonUris (id, uri) VALUES (?, ?)", (number, uri)) + conn.execute("INSERT INTO TooltipButtons VALUES (?, ?, 1, ?)", (tooltip_id, number, number)) conn.commit() conn.close() return db_path @@ -355,14 +379,49 @@ def test_a_template_link_refuses(tmp_path: Path): templates={1: ("page.peb", b'{{ title }}')}) -def test_a_tooltip_button_refuses(tmp_path: Path): - _refused(tmp_path, _base_rows(), "tooltip 3 button links to j/html/api/m/a/package-use.html", - tooltips=[(3, "s", "")], buttons=[(3, "i/index.html"), (3, "j/html/api/m/a/package-use.html#x")]) +def test_a_tooltip_button_refuses_in_either_schema(tmp_path: Path): + for schema in (2, 3): + _refused(tmp_path / str(schema), _base_rows(), "tooltip 3 button links to j/html/api/m/a/package-use.html", + tooltips=[(3, "s", "")], buttons=[(3, "i/index.html"), (3, "j/html/api/m/a/package-use.html#x")], + schema=schema) + +def test_a_link_in_tooltip_text_refuses_in_either_schema(tmp_path: Path): + for schema in (2, 3): + _refused(tmp_path / f"s{schema}", _base_rows(), "tooltip 4 text links to x/b.html", + tooltips=[(4, 'See b.', "")], schema=schema) + _refused(tmp_path / f"d{schema}", _base_rows(), "tooltip 5 text links to x/a.html", + tooltips=[(5, "s", 'a')], schema=schema) -def test_a_link_in_tooltip_text_refuses(tmp_path: Path): - _refused(tmp_path, _base_rows(), "tooltip 4 text links to x/b.html", - tooltips=[(4, 'See b.', "")]) + +def test_a_schema_3_database_deletes_the_same_rows(tmp_path: Path): + db = _database(tmp_path, _base_rows(), tooltips=[(1, "s", "d")], buttons=[(1, "j/html/api/m/a/B.html")], schema=3) + before = _stored(db) + code, _out, err = _run(db) + assert code == 0, err + assert set(before) - set(_stored(db)) == DELETED + + +def test_tooltip_tables_in_neither_shape_refuse_before_the_scan(tmp_path: Path): + cases = [ + # Half migrated: the schema-2 tables beside a TooltipTexts. + (2, "CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY, summary TEXT, detail TEXT);"), + # Schema 3 without its uri table. + (3, "DROP TABLE TooltipButtonUris;"), + # Tooltips without their text anywhere. + (2, "CREATE TABLE T2 (id INTEGER, categoryId INTEGER, tag TEXT); DROP TABLE Tooltips;" + " ALTER TABLE T2 RENAME TO Tooltips;"), + ] + for i, (schema, change) in enumerate(cases): + db = _database(tmp_path / str(i), _base_rows(), schema=schema) + conn = sqlite3.connect(db) + conn.executescript(change) + conn.close() + before = _stored(db) + code, out, err = _run(db) + assert code == 1 and "in neither the schema-2 shape" in err, (i, err) + assert "Reading" not in out and "Checking" not in out, (i, out) + assert _stored(db) == before and len(list((tmp_path / str(i)).iterdir())) == 1 def test_a_bookshelf_entry_refuses(tmp_path: Path): From cf65e44938ff0161193044776b6ae2069cb1d74f Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 21:38:26 -0700 Subject: [PATCH 4/9] ADFA-3305: Declare the tooltip step's dependencies for uv run clear_repeated_tooltip_detail.py had no PEP 723 block, but it imports optimize_db_media, which imports brotli and PIL at module level, so the documented `uv run scripts/clear_repeated_tooltip_detail.py` failed with ModuleNotFoundError. The test file declared both itself, which hid it; a test now checks the script's own block. The plain-script test runner also calls tests that take no directory. PR #46 review finding R46-02. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/clear_repeated_tooltip_detail.py | 7 +++++++ scripts/test_clear_repeated_tooltip_detail.py | 18 +++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/clear_repeated_tooltip_detail.py b/scripts/clear_repeated_tooltip_detail.py index 1fc2f278..5e91119d 100644 --- a/scripts/clear_repeated_tooltip_detail.py +++ b/scripts/clear_repeated_tooltip_detail.py @@ -1,4 +1,11 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# ] +# /// """ clear_repeated_tooltip_detail.py diff --git a/scripts/test_clear_repeated_tooltip_detail.py b/scripts/test_clear_repeated_tooltip_detail.py index ca5bc5e1..f51adfd4 100644 --- a/scripts/test_clear_repeated_tooltip_detail.py +++ b/scripts/test_clear_repeated_tooltip_detail.py @@ -190,6 +190,21 @@ def test_a_migrated_database_is_refused_by_name(tmp_path: Path): assert code == 1 and "migrate_tooltip_schema.py" in err and _details(db2) == before +def test_the_script_declares_what_its_imports_need(): + # `uv run scripts/clear_repeated_tooltip_detail.py` installs only what its PEP 723 block lists, and + # optimize_db_media, which it imports, imports brotli and PIL at module level. This test file declares both + # itself, so without this check a missing block shows up only when the step is run. + script = Path(tool.__file__).read_text() + block = re.search(r"^# /// script\n(.*?)^# ///$", script, re.S | re.M) + assert block, "no PEP 723 script block" + declared = set(re.findall(r'"([A-Za-z0-9_.-]+)', block.group(1).split("dependencies", 1)[-1])) + imported = Path(tool.__file__).with_name("optimize_db_media.py").read_text() + needed = {"brotli": r"^import brotli\b", "Pillow": r"^from PIL import"} + for package, pattern in needed.items(): + assert re.search(pattern, imported, re.M), f"optimize_db_media no longer imports {package}; update this test" + assert package in declared, f"{package} is not declared" + + def test_a_database_without_tooltips_is_an_error(tmp_path: Path): db = tmp_path / "documentation.db" sqlite3.connect(db).close() @@ -206,8 +221,9 @@ def _main() -> int: if not name.startswith("test_") or not callable(function): continue with tempfile.TemporaryDirectory() as directory: + needs_tmp = "tmp_path" in function.__code__.co_varnames[:function.__code__.co_argcount] try: - function(Path(directory)) + function(Path(directory)) if needs_tmp else function() except (KeyboardInterrupt, SystemExit): raise except BaseException as exc: # noqa: BLE001 - a failure is the result here From dd4424fb739a82203136ba74ebf35f89d9d5d24c Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 21:39:18 -0700 Subject: [PATCH 5/9] ADFA-3305: Read FontName as well as BaseFont for pdf.js's Chinese-font rule pdf.js 6.1.200 takes a font's name from its FontDescriptor's /FontName when that differs from /BaseFont, and that name decides whether it fetches the GBK-EUC-H CMap. The check read /BaseFont only, as a name. It now reads both keys, each as a name, a literal string (escapes and nested parentheses) or a hex string, and refuses if either is one of the ten names. PR #46 review finding R46-03. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/drop_unread_rows.py | 70 ++++++++++++++++++++++++++++++-- scripts/test_drop_unread_rows.py | 14 +++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/scripts/drop_unread_rows.py b/scripts/drop_unread_rows.py index 5c11986a..279a0d63 100644 --- a/scripts/drop_unread_rows.py +++ b/scripts/drop_unread_rows.py @@ -70,7 +70,9 @@ /Encoding names a predefined CMap other than Identity-H or Identity-V; an embedded CMap that uses one (usecmap); a CID font of the Adobe GB1, CNS1, Japan1 or Korea1 collection (Adobe--UCS2, unless the font has its - own ToUnicode map); and a font with one of ten Chinese names (GBK-EUC-H). + own ToUnicode map); and a font with one of ten Chinese names (GBK-EUC-H), + read from both /BaseFont and the FontDescriptor's /FontName, which pdf.js + prefers when the two differ, as a name or a string. Every stored PDF is read, its Flate streams decompressed, and any sign of these refuses the run. The collection check ignores ToUnicode, so it refuses more than pdf.js needs, never less. It cannot see PDFs that a plugin installs @@ -159,7 +161,10 @@ _PDF_ENCODING = re.compile(rb"/(?:Encoding|UseCMap)\s*" + _PDF_NAME) _PDF_USECMAP = re.compile(_PDF_NAME + rb"\s+usecmap\b") _PDF_ORDERING = re.compile(rb"/Ordering\s*(?:\(([^)]*)\)|<([0-9A-Fa-f\s]*)>)") -_PDF_BASEFONT = re.compile(rb"/BaseFont\s*" + _PDF_NAME) +_PDF_FONT_NAME_KEY = re.compile(rb"/(?:BaseFont|FontName)\s*") +_PDF_NAME_AT = re.compile(_PDF_NAME) +_PDF_HEX_AT = re.compile(rb"<([0-9A-Fa-f\s]*)>") +_PDF_ESCAPES = {ord("n"): b"\n", ord("r"): b"\r", ord("t"): b"\t", ord("b"): b"\b", ord("f"): b"\f"} _PDF_STREAM = re.compile(rb"stream(?:\r\n|\n|\r)") CJK_ORDERINGS = (b"GB1", b"CNS1", b"Japan1", b"Korea1") # pdf.js's chineseFontNames, as the bytes it compares a font name with. @@ -331,6 +336,64 @@ def _unescape_name(name: bytes) -> bytes: return re.sub(rb"#([0-9A-Fa-f]{2})", lambda m: bytes([int(m.group(1), 16)]), name) +def _literal_string(data: bytes, start: int): + """The bytes of the PDF literal string whose "(" is at start (escapes decoded, nested parentheses kept), + or None if it does not end.""" + out, depth, i = bytearray(), 0, start + 1 + while i < len(data): + c = data[i] + if c == 0x5C: # backslash + i += 1 + if i >= len(data): + return None + e = data[i] + if e in _PDF_ESCAPES: + out += _PDF_ESCAPES[e] + elif 0x30 <= e <= 0x37: + digits = re.match(rb"[0-7]{1,3}", data[i:i + 3]).group(0) + out.append(int(digits, 8) & 0xFF) + i += len(digits) - 1 + elif e == 0x0D: # a line continuation, \r or \r\n + if data[i + 1:i + 2] == b"\n": + i += 1 + elif e != 0x0A: + out.append(e) + elif c == 0x28: + depth += 1 + out.append(c) + elif c == 0x29: + if depth == 0: + return bytes(out) + depth -= 1 + out.append(c) + else: + out.append(c) + i += 1 + return None + + +def font_names(part: bytes): + """Each /BaseFont and /FontName value in part, as the bytes pdf.js compares: a name (#xx decoded), a + literal string or a hex string. pdf.js takes the FontDescriptor's FontName when it differs from the + BaseFont, so both are read.""" + for m in _PDF_FONT_NAME_KEY.finditer(part): + i = m.end() + lead = part[i:i + 1] + if lead == b"/": + name = _PDF_NAME_AT.match(part, i) + if name: + yield _unescape_name(name.group(1)) + elif lead == b"(": + text = _literal_string(part, i) + if text is not None: + yield text + elif lead == b"<" and part[i + 1:i + 2] != b"<": + hexed = _PDF_HEX_AT.match(part, i) + if hexed: + digits = re.sub(rb"\s", b"", hexed.group(1)) + yield bytes.fromhex((digits + b"0" * (len(digits) % 2)).decode()) + + def pdf_parts(data: bytes): """The PDF's bytes, then each of its streams that Flate-decodes.""" yield data @@ -356,8 +419,7 @@ def cmap_needs(data: bytes, cmap_names: set) -> list: ordering = m.group(1) if m.group(1) is not None else bytes.fromhex(m.group(2).decode().replace(" ", "")) if ordering.strip() in CJK_ORDERINGS: needs.add(f"has a CID font of the Adobe {ordering.strip().decode()} collection") - for m in _PDF_BASEFONT.finditer(part): - name = _unescape_name(m.group(1)) + for name in font_names(part): if name in CHINESE_FONT_NAMES or name.split(b"+", 1)[-1] in CHINESE_FONT_NAMES: needs.add("has a font pdf.js reads as Chinese (GBK-EUC-H)") return sorted(needs) diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py index 4de4f5dc..9992c994 100644 --- a/scripts/test_drop_unread_rows.py +++ b/scripts/test_drop_unread_rows.py @@ -477,6 +477,14 @@ def test_a_pdf_that_needs_a_cmap_refuses(tmp_path: Path): "has a CID font of the Adobe Korea1 collection"), (_pdf(b"<< /Type /Font /Subtype /TrueType /BaseFont /#CB#CE#CC#E5 /Encoding /WinAnsiEncoding >>"), "has a font pdf.js reads as Chinese"), + # pdf.js takes the FontDescriptor's FontName when it differs from the BaseFont. + (_pdf(b"<< /Type /Font /Subtype /TrueType /BaseFont /SimSunX /FontDescriptor 2 0 R" + b" /Encoding /WinAnsiEncoding >>", b"<< /Type /FontDescriptor /FontName /#CB#CE#CC#E5 >>"), + "has a font pdf.js reads as Chinese"), + (_pdf(b"<< /Type /FontDescriptor /FontName (\\313\\316\\314\\345) >>"), "has a font pdf.js reads as Chinese"), + (_pdf(b"<< /Type /FontDescriptor /FontName >>"), "has a font pdf.js reads as Chinese"), + (_pdf(_flate(b"<< /Type /FontDescriptor /FontName (ABCDEF+\xcb\xce\xcc\xe5) >>")), + "has a font pdf.js reads as Chinese"), ] for i, (pdf, expected) in enumerate(cases): rows = _base_rows() @@ -484,6 +492,12 @@ def test_a_pdf_that_needs_a_cmap_refuses(tmp_path: Path): _refused(tmp_path / str(i), rows, f"d/cjk.pdf {expected}") +def test_font_names_are_read_as_pdf_js_reads_them(): + names = list(tool.font_names(b"/BaseFont /A#42C /FontName (x\\(y\\)z(n)\\101\\\nq\\\\) /FontName<4142 4>" + b" /FontName <>")) + assert names == [b"ABC", b"x(y)z(n)Aq\\", b"AB@"] + + def test_the_pdf_check_runs_only_with_the_cmaps(tmp_path: Path): rows = _base_rows() rows["d/cjk.pdf"] = ("application/pdf", _pdf(b"<< /Registry (Adobe) /Ordering (Japan1) >>")) From 20c7b37a5a2b6f300c2f4661036a5817197ccce2 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 21:41:28 -0700 Subject: [PATCH 6/9] ADFA-3305: Decode PDF filter chains, and refuse what cannot be read The CMap check inflated each stream with zlib and silently skipped any that failed: an object stream filtered [/ASCII85Decode /FlateDecode], every stream of an encrypted PDF, an LZW stream. A font hidden in one was neither seen nor refused, though the docstring promised that any sign of a CMap need refuses. It also read the "stream" inside "endstream" as the start of a stream. Each stream is now found after its dictionary's ">>", bounded by a direct /Length when one fits, and decoded through its filters (FlateDecode, ASCII85Decode, ASCIIHexDecode, RunLengthDecode, in any chain). Image and cross-reference streams are skipped, since neither can hold a font. An encrypted PDF, or a stream with any other filter, a predictor or corrupt data, refuses the run by name. All 15 stored PDFs decode fully and need no CMap. PR #46 review finding R46-04. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/drop_unread_rows.py | 109 ++++++++++++++++++++++++++----- scripts/test_drop_unread_rows.py | 48 ++++++++++++++ 2 files changed, 142 insertions(+), 15 deletions(-) diff --git a/scripts/drop_unread_rows.py b/scripts/drop_unread_rows.py index 279a0d63..6fe9fe6c 100644 --- a/scripts/drop_unread_rows.py +++ b/scripts/drop_unread_rows.py @@ -73,10 +73,14 @@ own ToUnicode map); and a font with one of ten Chinese names (GBK-EUC-H), read from both /BaseFont and the FontDescriptor's /FontName, which pdf.js prefers when the two differ, as a name or a string. - Every stored PDF is read, its Flate streams decompressed, and any sign of - these refuses the run. The collection check ignores ToUnicode, so it refuses - more than pdf.js needs, never less. It cannot see PDFs that a plugin installs - on the phone. + Every stored PDF is read and each stream decoded through its filters + (FlateDecode, ASCII85Decode, ASCIIHexDecode and RunLengthDecode, in any + chain), skipping image and cross-reference streams, which cannot hold a font; + any sign of these refuses the run. So does an encrypted PDF, or a stream it + cannot decode (another filter, a predictor, corrupt data), since its fonts + cannot be read. The collection check ignores ToUnicode, so it refuses more + than pdf.js needs, never less. It cannot see PDFs that a plugin installs on + the phone. - The viewer edit (pdfjs-extras). p/web/viewer.mjs must hold exactly one enableScripting default; if it holds none or more, nothing is written. - A kept text row that cannot be decoded cannot be checked, so it refuses too. @@ -100,6 +104,8 @@ from __future__ import annotations import argparse +import base64 +import bisect import html import json import posixpath @@ -165,7 +171,17 @@ _PDF_NAME_AT = re.compile(_PDF_NAME) _PDF_HEX_AT = re.compile(rb"<([0-9A-Fa-f\s]*)>") _PDF_ESCAPES = {ord("n"): b"\n", ord("r"): b"\r", ord("t"): b"\t", ord("b"): b"\b", ord("f"): b"\f"} -_PDF_STREAM = re.compile(rb"stream(?:\r\n|\n|\r)") +# A stream starts after its dictionary's ">>"; matching that keeps "endstream" from reading as a start. +_PDF_STREAM = re.compile(rb">>\s*stream(?:\r\n|\n|\r)") +_PDF_OBJ = re.compile(rb"\d+\s+\d+\s+obj\b") +_PDF_LENGTH = re.compile(rb"/Length\s+(\d+)\b(?!\s+\d+\s+R)") +_PDF_FILTER = re.compile(rb"/Filter\s*(\[[^\]]*\]|" + _PDF_NAME + rb")") +_PDF_PREDICTOR = re.compile(rb"/Predictor\s+(\d+)") +_PDF_IMAGE = re.compile(rb"/Subtype\s*/Image\b") +_PDF_XREF = re.compile(rb"/Type\s*/XRef\b") +_PDF_ENCRYPT = re.compile(rb"/Encrypt\s*(?:\d+\s+\d+\s+R|<<)") +# Filters that only ever hold image data, so their streams cannot hold a font. +IMAGE_FILTERS = {"DCTDecode", "DCT", "JPXDecode", "CCITTFaxDecode", "CCF", "JBIG2Decode"} CJK_ORDERINGS = (b"GB1", b"CNS1", b"Japan1", b"Korea1") # pdf.js's chineseFontNames, as the bytes it compares a font name with. CHINESE_FONT_NAMES = {name.encode("latin-1") for name in ( @@ -394,23 +410,86 @@ def font_names(part: bytes): yield bytes.fromhex((digits + b"0" * (len(digits) % 2)).decode()) -def pdf_parts(data: bytes): - """The PDF's bytes, then each of its streams that Flate-decodes.""" - yield data - for m in _PDF_STREAM.finditer(data): - end = data.find(b"endstream", m.end()) - if end < 0: +def _run_length(data: bytes) -> bytes: + out, i = bytearray(), 0 + while i < len(data): + n = data[i] + if n == 128: break + if n < 128: + out += data[i + 1:i + 2 + n] + i += 2 + n + else: + out += data[i + 1:i + 2] * (257 - n) + i += 2 + return bytes(out) + + +def _ascii_hex(data: bytes) -> bytes: + digits = re.sub(rb"\s", b"", data.split(b">", 1)[0]) + return bytes.fromhex((digits + b"0" * (len(digits) % 2)).decode("ascii")) + + +def _ascii85(data: bytes) -> bytes: + text = re.sub(rb"\s", b"", data) + text = text[2:] if text.startswith(b"<~") else text + return base64.a85decode(text.split(b"~>", 1)[0]) + + +DECODERS = {"FlateDecode": lambda data: zlib.decompressobj().decompress(data), "ASCIIHexDecode": _ascii_hex, + "ASCII85Decode": _ascii85, "RunLengthDecode": _run_length} +DECODERS.update({"Fl": DECODERS["FlateDecode"], "AHx": _ascii_hex, "A85": _ascii85, "RL": _run_length}) + + +def pdf_parts(data: bytes) -> tuple: + """([the PDF's bytes, then each stream decoded], [why each stream that could not be decoded could not]). + Image streams and cross-reference streams are skipped: neither can hold a font.""" + parts, undecodable = [data], [] + headers = [m.end() for m in _PDF_OBJ.finditer(data)] + pos = 0 + while True: + m = _PDF_STREAM.search(data, pos) + if m is None: + break + start = m.end() + header = bisect.bisect_right(headers, m.start()) - 1 + dictionary = data[headers[header] if header >= 0 else max(0, m.start() - 4096):m.start() + 2] + length = _PDF_LENGTH.search(dictionary) + end = start + int(length.group(1)) if length else -1 + if not (length and data[end:end + 20].lstrip().startswith(b"endstream")): + end = data.find(b"endstream", start) + if end < 0: + break + body, pos = data[start:end], end + filter_value = _PDF_FILTER.search(dictionary) + filters = ([_unescape_name(n).decode("latin-1") for n in re.findall(_PDF_NAME, filter_value.group(1))] + if filter_value else []) + if _PDF_IMAGE.search(dictionary) or _PDF_XREF.search(dictionary) or IMAGE_FILTERS & set(filters): + continue + predictor = _PDF_PREDICTOR.search(dictionary) + if predictor and int(predictor.group(1)) > 1: + undecodable.append(f"predictor {int(predictor.group(1))}") + continue try: - yield zlib.decompressobj().decompress(data[m.end():end]) - except zlib.error: + for name in filters: + if name not in DECODERS: + raise ValueError(name) + body = DECODERS[name](body) + except (ValueError, zlib.error) as exc: + undecodable.append(f"{name}" + ("" if str(exc) == name else f": {exc}")) continue + parts.append(body) + return parts, undecodable def cmap_needs(data: bytes, cmap_names: set) -> list: """Why pdf.js would fetch a CMap file for this PDF, if it would; see the docstring.""" - needs = set() - for part in pdf_parts(data): + if _PDF_ENCRYPT.search(data): + return ["is encrypted, so its fonts cannot be read"] + parts, undecodable = pdf_parts(data) + needs = {f"has a stream this step cannot decode ({why}), so its fonts cannot all be checked" + for why in undecodable} + for part in parts: for m in list(_PDF_ENCODING.finditer(part)) + list(_PDF_USECMAP.finditer(part)): name = _unescape_name(m.group(1)).decode("latin-1") if name in cmap_names: diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py index 9992c994..bddb9bef 100644 --- a/scripts/test_drop_unread_rows.py +++ b/scripts/test_drop_unread_rows.py @@ -84,6 +84,17 @@ def _flate(body: bytes) -> bytes: return b"<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream" % (len(data), data) +def _stream(data: bytes, entries: bytes = b"") -> bytes: + return b"<< /Length %d %s >>\nstream\n%s\nendstream" % (len(data), entries, data) + + +def _run_length(data: bytes) -> bytes: + """RunLengthDecode data in 3-byte literal runs, so no name survives whole in the raw file.""" + return b"".join(bytes([len(data[i:i + 3]) - 1]) + data[i:i + 3] for i in range(0, len(data), 3)) + b"\x80" + + +JAPANESE_FONT = b"<< /Type /Font /Subtype /Type0 /BaseFont /MS-Gothic /Encoding /90ms-RKSJ-H >>" + PLAIN_PDF = _pdf(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>", b"<< /Type /Font /Subtype /Type0 /BaseFont /ABCDEF+Noto /Encoding /Identity-H >>", b"<< /Registry (Adobe) /Ordering (Identity) /Supplement 0 >>", @@ -492,6 +503,43 @@ def test_a_pdf_that_needs_a_cmap_refuses(tmp_path: Path): _refused(tmp_path / str(i), rows, f"d/cjk.pdf {expected}") +def test_a_pdf_whose_fonts_cannot_all_be_read_refuses(tmp_path: Path): + import base64 + cmap = "uses the predefined CMap 90ms-RKSJ-H" + cases = [ + (_pdf(_stream(base64.a85encode(zlib.compress(JAPANESE_FONT), adobe=True), + b"/Type /ObjStm /N 1 /First 0 /Filter [/ASCII85Decode /FlateDecode]")), cmap), + (_pdf(_stream(JAPANESE_FONT.hex().encode() + b">", b"/Filter /ASCIIHexDecode")), cmap), + (_pdf(_stream(_run_length(JAPANESE_FONT), b"/Filter /RunLengthDecode")), cmap), + (_pdf(_stream(base64.a85encode(JAPANESE_FONT), b"/Filter /A85")), cmap), + (PLAIN_PDF + b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R >>\n", "is encrypted, so its fonts cannot be read"), + (_pdf(_stream(b"\x80\x0b\x60\x50", b"/Filter /LZWDecode")), "cannot decode (LZWDecode)"), + (_pdf(_stream(b"not zlib data", b"/Filter /FlateDecode")), "cannot decode (FlateDecode: "), + (_pdf(_stream(zlib.compress(b"\x02\x00\x00\x00\x01"), + b"/Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns 4 >>")), + "cannot decode (predictor 12)"), + ] + for i, (pdf, expected) in enumerate(cases): + rows = _base_rows() + rows["d/odd.pdf"] = ("application/pdf", pdf) + _refused(tmp_path / str(i), rows, f"d/odd.pdf {expected}" if expected == cmap else expected) + + +def test_images_and_cross_reference_streams_do_not_refuse(tmp_path: Path): + content = zlib.compress(b"BT /F1 12 Tf (Hello) Tj ET") + pdf = _pdf(_stream(b"\xff\xd8\xff\xe0 not decodable", b"/Type /XObject /Subtype /Image /Filter /DCTDecode"), + _stream(b"\x00\x00\x00\x0cjP ", b"/Filter /JPXDecode"), + _stream(b"\x80\x0b\x60\x50", b"/Subtype /Image /Filter /LZWDecode"), + _stream(b"\x02\x01\x00\x10\x00", b"/Type /XRef /Filter /FlateDecode /DecodeParms << /Predictor 12 >>"), + b"<< /Length 7 0 R /Filter /FlateDecode >>\nstream\n" + content + b"\nendstream", + b"%d" % len(content), + _stream(content, b"/Filter /FlateDecode")) + rows = _base_rows() + rows["d/images.pdf"] = ("application/pdf", pdf) + code, _out, err = _run(_database(tmp_path, rows)) + assert code == 0, err + + def test_font_names_are_read_as_pdf_js_reads_them(): names = list(tool.font_names(b"/BaseFont /A#42C /FontName (x\\(y\\)z(n)\\101\\\nq\\\\) /FontName<4142 4>" b" /FontName <>")) From 315366d6ea174d04a038a74caa9274b8ee405d13 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 21:41:49 -0700 Subject: [PATCH 7/9] ADFA-3305: Stop ".." at the root when resolving links, as browsers do link_paths discarded any resolved path that climbed past the root, but a browser resolves ../../x/a.html from i/clamp.html to /x/a.html. When a kept row shared the file name, the file-name rule did not fire either, so such a link to a deleted row went unseen. Leading ".." segments are now dropped at the root. PR #46 review finding R46-05. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/drop_unread_rows.py | 4 +++- scripts/test_drop_unread_rows.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/drop_unread_rows.py b/scripts/drop_unread_rows.py index 6fe9fe6c..95c8a82f 100644 --- a/scripts/drop_unread_rows.py +++ b/scripts/drop_unread_rows.py @@ -308,7 +308,9 @@ def link_paths(ref: str, base_dirs, nested: bool = False) -> list: rooted = candidate.startswith("/") for base in ("",) if rooted else base_dirs: path = posixpath.normpath(candidate.lstrip("/") if rooted else posixpath.join(base, candidate)) - if path not in (".", "..") and not path.startswith("../"): + while path == ".." or path.startswith("../"): # ".." stops at the root, as in a browser + path = path[3:] + if path not in ("", "."): out.append(path) return out diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py index bddb9bef..54638907 100644 --- a/scripts/test_drop_unread_rows.py +++ b/scripts/test_drop_unread_rows.py @@ -377,6 +377,18 @@ def test_each_kind_of_link_to_a_deleted_row_refuses(tmp_path: Path): _refused(tmp_path / str(i), rows, f"{path} {expected}" if not expected.startswith(path) else expected) +def test_a_link_climbing_past_the_root_resolves_as_a_browser_resolves_it(tmp_path: Path): + # A browser stops ".." at the root, so both links reach x/a.html. A kept i/sub/a.html shares the file name, + # so only resolving the path, not the file-name rule, can catch them. + for i, link in enumerate((b"../../x/a.html", b"/../x/a.html")): + rows = _base_rows() + rows["i/sub/a.html"] = ("text/html", b"

kept

") + rows["i/clamp.html"] = ("text/html", b'a' % link) + _refused(tmp_path / str(i), rows, "i/clamp.html links to x/a.html") + assert tool.link_paths("../../../a/b.html", ("i",)) == ["a/b.html"] + assert tool.link_paths("..", ("i",)) == [] + + def test_a_reference_the_allowlist_names_only_for_another_row_still_refuses(tmp_path: Path): rows = _base_rows() rows["p/web/other.mjs"] = ("text/javascript", b'import("../build/pdf.sandbox.mjs")') From 6c33cac1b0e2162b3ac7a92fc1dc2a695916759d Mon Sep 17 00:00:00 2001 From: David Schachter Date: Sun, 18 Oct 2026 21:44:03 -0700 Subject: [PATCH 8/9] ADFA-3305: Test the guards three surviving mutants showed untested Three guards could be broken without a test failing: turning scripting off when the sandbox rows are already gone (the run said "Nothing to do" and left it on), refusing a row to delete that cannot be read, and refusing a stored PDF that cannot be read. Each now has a test. The template test also gains a link that resolves only from the directory of the row using the template, since stopping ".." at the root made its first link resolve from the root as well. The tooltip-shape test gains a schema-3 database whose Tooltips still holds a summary column, which a mutant of that guard passed. The module docstring now lists what the tests cover. All 75 mutants (the earlier 49 and 26 of the guards these fixes add) fail at least one test. PR #46 review finding R46-06. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/test_drop_unread_rows.py | 51 +++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py index 54638907..94738516 100644 --- a/scripts/test_drop_unread_rows.py +++ b/scripts/test_drop_unread_rows.py @@ -11,12 +11,15 @@ test_drop_unread_rows.py Covers drop_unread_rows.py: that each group deletes its rows, their chunk rows -and nothing else; that scripting is turned off in the viewer; and that every -check refuses the run, writing nothing, when a kept row, template, tooltip or -Bookshelf entry refers to a deleted row, when a row to delete is not what its -group expects, when a stored PDF needs a CMap, when the viewer cannot be -edited, or when a kept row cannot be read. A run is all or nothing, a dry run -writes nothing, and a second run changes nothing. +and nothing else; that scripting is turned off in the viewer, even once the +sandbox rows are gone; that tooltips are read in documentation schema 2 and 3; +and that every check refuses the run, writing nothing, when a kept row, +template, tooltip or Bookshelf entry refers to a deleted row, when a row to +delete is not what its group expects, when a stored PDF needs a CMap or its +fonts cannot all be read, when the viewer cannot be edited, when the tooltip +tables are in neither shape, or when a kept row, a row to delete or a PDF +cannot be read. A run is all or nothing, a dry run writes nothing, and a +second run changes nothing. Run it either way - as a plain script: @@ -398,8 +401,14 @@ def test_a_reference_the_allowlist_names_only_for_another_row_still_refuses(tmp_ def test_a_template_link_refuses(tmp_path: Path): rows = _base_rows() rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) - _refused(tmp_path, rows, "template page.peb links to x/a.html", + _refused(tmp_path / "up", rows, "template page.peb links to x/a.html", templates={1: ("page.peb", b'{{ title }}')}) + # A link that resolves only from the directory of the row using the template: from the root it names no + # row, and its file name, B.html, is also a kept row's. + rows = _base_rows() + rows["j/html/api/m/a/T.html"] = ("text/html", b'{"title": "t"}', 1) + _refused(tmp_path / "here", rows, "template page.peb links to j/html/api/m/a/class-use/B.html", + templates={1: ("page.peb", b'{{ title }}')}) def test_a_tooltip_button_refuses_in_either_schema(tmp_path: Path): @@ -434,6 +443,8 @@ def test_tooltip_tables_in_neither_shape_refuse_before_the_scan(tmp_path: Path): # Tooltips without their text anywhere. (2, "CREATE TABLE T2 (id INTEGER, categoryId INTEGER, tag TEXT); DROP TABLE Tooltips;" " ALTER TABLE T2 RENAME TO Tooltips;"), + # The schema-3 tables, but Tooltips still holding text. + (3, "ALTER TABLE Tooltips ADD COLUMN summary TEXT;"), ] for i, (schema, change) in enumerate(cases): db = _database(tmp_path / str(i), _base_rows(), schema=schema) @@ -572,6 +583,32 @@ def test_an_unreadable_kept_row_refuses(tmp_path: Path): _refused(tmp_path, rows, "i/broken.html could not be read, so its references were not checked") +def test_an_unreadable_row_to_delete_refuses(tmp_path: Path): + # Its group's check cannot see what it holds, so it is not deleted on trust. + rows = _base_rows() + rows["x/c.html"] = ("text/html", b"not brotli at all", 0, True) + _refused(tmp_path, rows, "x/c.html could not be read") + + +def test_an_unreadable_pdf_refuses(tmp_path: Path): + rows = _base_rows() + rows["d/broken.pdf"] = ("application/pdf", b"not brotli at all", 0, True) + _refused(tmp_path, rows, "d/broken.pdf could not be read") + + +def test_scripting_is_turned_off_when_the_sandbox_rows_are_already_gone(tmp_path: Path): + rows = {path: row for path, row in _base_rows().items() if path not in tool.PDFJS_EXTRAS} + db = _database(tmp_path, rows) + before = _stored(db) + code, out, err = _run(db, groups=["pdfjs-extras"]) + after = _stored(db) + assert code == 0, err + assert "enableScripting default set to false" in out and not out.startswith("Nothing to do") + assert set(after) == set(before) and _decompress(after[tool.VIEWER][1]) == VIEWER_MJS_OFF + code, out, _err = _run(db, groups=["pdfjs-extras"]) + assert code == 0 and out.startswith("Nothing to do") and "scripting is already off" in out + + def test_a_chunked_row_goes_with_its_base_and_others_stay(tmp_path: Path): import random rng = random.Random(7) From e31aa00c40a83a9456a49ddcab6773f11339ae8e Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 19 Oct 2026 07:22:22 -0700 Subject: [PATCH 9/9] ADFA-3305: Decode templates before reading their links from version 3 From documentation major version 3, migrate_tooltip_schema.py stores every template Brotli-compressed against the dictionary, with no marker but the version. The reference check read the stored bytes as UTF-8, so on a 3.0.0 file it saw no template link at all (1,126 real templates: 0 link-like strings as stored, 29,210 decoded) and a template linking to a deleted row was not refused, though the docstring says the step may run after the migration. Templates are now decoded with the database's codec when the newest DocumentationDatabaseVersion row (ordered as the app orders them) declares major 3 or later, and read as plain text below it or with no version. A template that does not decode refuses the run, writing nothing. Four tests: a compressed template's link refuses from version 3, a plain one's still refuses below it, a template that does not decode refuses, and a version-3 database with compressed templates deletes the same rows. The first and third fail on 6c33cac1 (exit 0). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- CLAUDE.md | 3 +- scripts/drop_unread_rows.py | 35 ++++++++++++-- scripts/test_drop_unread_rows.py | 82 +++++++++++++++++++++++++++++--- 3 files changed, 107 insertions(+), 13 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b319b090..3c515692 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -171,7 +171,8 @@ uses. Any future Android/Java tooltip work should target the normalized `Tooltip `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), `create_empty_database.py`, `list_database_documents.py`, `drop_unread_rows.py` (deletes Content rows nothing reads - javadoc Use pages, pdf.js CMaps, pdf.js fallbacks and scripting sandbox, x/ - placeholders - after checking that nothing kept refers to them), `clear_repeated_tooltip_detail.py` + placeholders - after checking that nothing kept refers to them, before or after the schema-3 + migration, whose templates it decodes), `clear_repeated_tooltip_detail.py` (empties a tooltip detail that repeats its summary). **Deprecated/dead** (target the removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, diff --git a/scripts/drop_unread_rows.py b/scripts/drop_unread_rows.py index 95c8a82f..880df58e 100644 --- a/scripts/drop_unread_rows.py +++ b/scripts/drop_unread_rows.py @@ -54,7 +54,11 @@ links. The tooltip tables are read in either documentation schema: 2 (TooltipButtons.uri, Tooltips.summary and detail) or 3, after migrate_tooltip_schema.py (TooltipButtonUris, TooltipTexts); tables in - neither shape refuse the run before anything is read. Links are: HTML and + neither shape refuse the run before anything is read. From documentation + major version 3 (the newest DocumentationDatabaseVersion row, ordered as the + app orders them) every template is stored Brotli-compressed against the + dictionary, so templates are decoded first there and read as plain text + below it. Links are: HTML and SVG attributes, srcset, CSS url() and @import, quoted file names in scripts, the string values of a templated row's JSON, viewer.html?file= parameters and http://localhost:6174/ URLs. A link that resolves (relative to its row, or @@ -83,7 +87,8 @@ the phone. - The viewer edit (pdfjs-extras). p/web/viewer.mjs must hold exactly one enableScripting default; if it holds none or more, nothing is written. -- A kept text row that cannot be decoded cannot be checked, so it refuses too. +- A kept text row or template that cannot be decoded cannot be checked, so it + refuses too. Runs in one transaction (rolled back on any error), backs the database up first, and VACUUMs afterwards. --dry-run does all the checks and reports what @@ -92,8 +97,8 @@ Pipeline position: after every step that writes page content, links or tooltips, so the reference check sees the final pages, and before rebuild_database.py. It may run before or after migrate_tooltip_schema.py, -since it reads both tooltip shapes. Deleting rows does not call for a -dictionary re-mint. +since it reads both tooltip shapes and both template encodings. Deleting +rows does not call for a dictionary re-mint. uv run scripts/drop_unread_rows.py documentation.db --dry-run uv run scripts/drop_unread_rows.py documentation.db [--only GROUP ...] [--workers N] [--verbose] @@ -130,6 +135,8 @@ "application/x-typescript") VIEWER = "p/web/viewer.mjs" +# From this documentation major version on, migrate_tooltip_schema.py has stored every template compressed. +COMPRESSED_TEMPLATES_MAJOR = 3 PDFJS_EXTRAS = ("p/web/wasm/openjpeg_nowasm_fallback.js", "p/web/wasm/jbig2_nowasm_fallback.js", "p/web/wasm/quickjs-eval.wasm", "p/web/wasm/quickjs-eval.js", "p/build/pdf.sandbox.mjs") PLACEHOLDER = b"Placeholder for XML layout attribute" @@ -506,6 +513,17 @@ def cmap_needs(data: bytes, cmap_names: set) -> list: return sorted(needs) +def declared_major(conn) -> int | None: + """The documentation major version the database declares - its newest DocumentationDatabaseVersion row, ordered + as Code On The Go orders them - or None if it declares none.""" + if not conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name = 'DocumentationDatabaseVersion'").fetchone(): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion " + "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone() + return None if row is None else int(row[0]) + + def tooltip_queries(conn): """(buttons query, texts query) for the database's tooltip tables, schema 2 or 3, or None for any other shape. The buttons query yields (tooltipId, uri), the texts query (id, summary, detail).""" @@ -624,9 +642,16 @@ def scan(item): for path, row in rows.items(): if row[4]: dirs_by_template[row[4]].add(posixpath.dirname(path)) + major = declared_major(conn) + compressed = major is not None and major >= COMPRESSED_TEMPLATES_MAJOR for template_id, name, content in conn.execute("SELECT id, name, content FROM Templates"): dirs = ("",) + tuple(sorted(dirs_by_template[template_id])) - text = bytes(content).decode("utf-8", errors="replace") + try: + raw = codec.decompress(bytes(content), "brotli") if compressed else bytes(content) + except Exception as exc: # noqa: BLE001 - reported, and the run refused + problems.append(f"template {name} could not be decoded, so its links were not checked: {exc}") + continue + text = raw.decode("utf-8", errors="replace") found.update((f"template {name}", target, how) for target, how in refs.in_text(text, dirs)) for tooltip_id, uri in conn.execute(buttons_query): found.update((f"tooltip {tooltip_id} button", target, how) for target, how in refs.targets(uri, ("",))) diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py index 94738516..e90c28ad 100644 --- a/scripts/test_drop_unread_rows.py +++ b/scripts/test_drop_unread_rows.py @@ -12,13 +12,14 @@ Covers drop_unread_rows.py: that each group deletes its rows, their chunk rows and nothing else; that scripting is turned off in the viewer, even once the -sandbox rows are gone; that tooltips are read in documentation schema 2 and 3; +sandbox rows are gone; that tooltips are read in documentation schema 2 and 3, +and templates as plain text below documentation version 3 and decoded from it; and that every check refuses the run, writing nothing, when a kept row, template, tooltip or Bookshelf entry refers to a deleted row, when a row to delete is not what its group expects, when a stored PDF needs a CMap or its fonts cannot all be read, when the viewer cannot be edited, when the tooltip -tables are in neither shape, or when a kept row, a row to delete or a PDF -cannot be read. A run is all or nothing, a dry run writes nothing, and a +tables are in neither shape, or when a kept row, a template, a row to delete or +a PDF cannot be read. A run is all or nothing, a dry run writes nothing, and a second run changes nothing. Run it either way - as a plain script: @@ -29,7 +30,8 @@ uv run --with brotli --with Pillow --with pytest pytest scripts/test_drop_unread_rows.py -The dictionary test needs the `brotli` CLI on PATH, as the tool itself does. +The dictionary and template-version tests need the `brotli` CLI on PATH, as the +tool itself does. """ from __future__ import annotations @@ -183,10 +185,11 @@ def _decompress(blob: bytes, dictionary: bytes = None) -> bytes: def _database(tmp_path: Path, rows: dict, dictionary: bytes = None, templates: dict = None, - buttons: list = None, tooltips: list = None, schema: int = 2) -> Path: + buttons: list = None, tooltips: list = None, schema: int = 2, versions: list = None) -> Path: """A miniature documentation database with the production schema's triggers and the tooltip tables of - documentation schema 2 or 3. `templates` maps id -> (name, body), `buttons` lists (tooltipId, uri) and - `tooltips` (id, summary, detail).""" + documentation schema 2 or 3. `templates` maps id -> (name, stored body), `buttons` lists (tooltipId, uri), + `tooltips` (id, summary, detail) and `versions` the DocumentationDatabaseVersion rows as (major, minor, patch, + changeTime); without it there is no such table.""" tmp_path.mkdir(parents=True, exist_ok=True) db_path = tmp_path / "documentation.db" conn = sqlite3.connect(db_path) @@ -219,6 +222,11 @@ def _database(tmp_path: Path, rows: dict, dictionary: bytes = None, templates: d (path, stored, type_id, template_id)) for template_id, (name, body) in (templates or {}).items(): conn.execute("INSERT INTO Templates (id, name, content) VALUES (?, ?, ?)", (template_id, name, body)) + if versions is not None: + conn.execute("CREATE TABLE DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL," + " patch INT NOT NULL, who TEXT NOT NULL, comment TEXT NOT NULL," + " changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") + conn.executemany("INSERT INTO DocumentationDatabaseVersion VALUES (?, ?, ?, 'test', '', ?)", versions) for tooltip_id, summary, detail in tooltips or []: if schema == 2: conn.execute("INSERT INTO Tooltips (id, categoryId, tag, summary, detail) VALUES (?, 1, ?, ?, ?)", @@ -411,6 +419,66 @@ def test_a_template_link_refuses(tmp_path: Path): templates={1: ("page.peb", b'{{ title }}')}) +# A template that links to a deleted row, and a dictionary that holds it, as the real ones hold theirs. +LINKING_TEMPLATE = b'' * 3 + \ + b'{{ title }}' +TEMPLATE_DICTIONARY = VIEWER_MJS * 4 + LINKING_TEMPLATE + + +def test_a_compressed_template_link_refuses_from_version_3(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + stored = _compress(LINKING_TEMPLATE, TEMPLATE_DICTIONARY) + assert b"x/a.html" not in stored, "the link survives compression, so this tests nothing" + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + # The newest row decides, by changeTime and then rowid, whatever order the rows were written in. + for i, versions in enumerate([[(3, 0, 0, "2026-10-19 07:09:52")], + [(3, 0, 0, "2026-10-19 07:09:52"), (2, 1, 0, "2026-10-18 10:00:00")], + [(2, 1, 0, "2026-10-19 07:09:52"), (3, 0, 0, "2026-10-19 07:09:52")]]): + _refused(tmp_path / str(i), rows, "template page.peb links to x/a.html", dictionary=TEMPLATE_DICTIONARY, + schema=3, versions=versions, templates={1: ("page.peb", stored)}) + + +def test_a_version_3_database_with_compressed_templates_deletes_the_same_rows(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + kept_link = LINKING_TEMPLATE.replace(b"../x/a.html", b"../x.html") + db = _database(tmp_path, rows, dictionary=TEMPLATE_DICTIONARY, schema=3, + versions=[(3, 0, 0, "2026-10-19 07:09:52")], + templates={1: ("page.peb", _compress(kept_link, TEMPLATE_DICTIONARY))}) + before = _stored(db) + code, _out, err = _run(db) + assert code == 0, err + assert set(before) - set(_stored(db)) == DELETED + + +def test_a_plain_template_link_refuses_below_version_3(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + # An empty version table declares no version, as a missing one does. + for i, versions in enumerate([[(2, 1, 0, "2026-10-18 10:00:00")], [], + [(3, 0, 0, "2026-10-17 10:00:00"), (2, 1, 0, "2026-10-18 10:00:00")], + [(3, 0, 0, "2026-10-18 10:00:00"), (2, 1, 0, "2026-10-18 10:00:00")]]): + _refused(tmp_path / str(i), rows, "template page.peb links to x/a.html", dictionary=TEMPLATE_DICTIONARY, + versions=versions, templates={1: ("page.peb", LINKING_TEMPLATE)}) + + +def test_a_template_that_does_not_decode_from_version_3_refuses(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + # A step run after the migration left this one plain; its link to a kept row would not refuse. + _refused(tmp_path, rows, "template page.peb could not be decoded, so its links were not checked", + dictionary=TEMPLATE_DICTIONARY, schema=3, versions=[(3, 0, 0, "2026-10-19 07:09:52")], + templates={1: ("page.peb", b'{{ title }}')}) + + def test_a_tooltip_button_refuses_in_either_schema(tmp_path: Path): for schema in (2, 3): _refused(tmp_path / str(schema), _base_rows(), "tooltip 3 button links to j/html/api/m/a/package-use.html",