diff --git a/CLAUDE.md b/CLAUDE.md index 060c04d2..d50ec3e7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -226,7 +226,11 @@ shape: every tool here reads and writes that, and only the shipped copy holds th - **`scripts/`** — the original CLI toolbox. Live/current: `DocumentationDatabase.py` (Content ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), - `create_empty_database.py`, `list_database_documents.py`, `migrate_tooltip_schema.py` (ADFA-3305: + `create_empty_database.py`, `list_database_documents.py`, `drop_unread_rows.py` (deletes Content rows + nothing reads - javadoc Use pages, pdf.js CMaps, pdf.js fallbacks and scripting sandbox, x/ + placeholders - after checking that nothing kept refers to them, before or after the schema-3 + migration, whose templates it decodes), `clear_repeated_tooltip_detail.py` (empties a tooltip detail + that repeats its summary), `migrate_tooltip_schema.py` (ADFA-3305: converts to documentation schema 3.0.0, which Code On The Go reads from 3.0.0 on: the WITHOUT ROWID, lookup-table tooltip shape, and every template Brotli-compressed against the dictionary; run it on the copy that ships, after every step that writes tooltips or templates, since they all write the old diff --git a/scripts/clear_repeated_tooltip_detail.py b/scripts/clear_repeated_tooltip_detail.py new file mode 100644 index 00000000..5e91119d --- /dev/null +++ b/scripts/clear_repeated_tooltip_detail.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# ] +# /// +""" +clear_repeated_tooltip_detail.py + +Empties Tooltips.detail where it is the same text as Tooltips.summary. + +In the 2026-10-18 database 1,605 tooltips (1,592 Kotlin, 13 Java) repeat their +summary as their detail, 151,274 bytes in all; 25,285 others already have an +empty detail. + +What the user sees differently (ToolTipManager.kt, the only reader of detail): +- The first popup shows the summary, as before. +- "See More" shows when the detail is not blank or the tooltip has buttons. + For a tooltip with buttons it still shows, and the second popup now lists + just the buttons' links instead of the summary again, a blank line, then the + links. All 1,605 tooltips in that database have buttons. +- A tooltip with no buttons loses "See More", whose popup held only the + summary again (and the sponsor button every second popup shows). This step + counts such tooltips and says so. +- The info dialog and the feedback e-mail show an empty Detail. + +The comparison is exact (Python string equality), whatever collation the +schema gives the columns. Tooltips are not Brotli-compressed, so the shared +dictionary does not need re-minting. + +Measured on the 2026-10-18 database after ADFA-6627, packed with +rebuild_database.py --pack Content,Tooltips --page-size 4096: the file is +151,552 bytes smaller; its brotli -q 11 -w 24 is 10,049 bytes smaller, which is +within the 0.2 MB that repacking alone moves it. + +Runs in one transaction (rolled back on any error), backs the database up +first, and VACUUMs afterwards. --dry-run reports what would change without +writing. A second run finds nothing to do and exits 0. + +Pipeline position: after every step that writes tooltip text, and before +migrate_tooltip_schema.py, which moves summary and detail out of Tooltips into +TooltipTexts (documentation schema 3). This step reads only the schema-2 shape +and refuses a migrated database, writing nothing. rebuild_database.py comes +after both. + + uv run scripts/clear_repeated_tooltip_detail.py documentation.db [--dry-run] [--verbose] +""" +from __future__ import annotations + +import argparse +import sqlite3 +import sys +from collections import Counter +from pathlib import Path + +from optimize_db_media import backup_database, human + + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + + conn = sqlite3.connect(db_path) + try: + columns = {row[1] for row in conn.execute("PRAGMA table_info(Tooltips)")} + if conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'TooltipTexts'").fetchone(): + print("error: this database's tooltip text is in TooltipTexts (documentation schema 3); run this step " + "before migrate_tooltip_schema.py", file=sys.stderr) + return 1 + if not {"id", "categoryId", "tag", "summary", "detail"} <= columns: + print("error: this database has no Tooltips table with id, categoryId, tag, summary and detail", + file=sys.stderr) + return 1 + rows = conn.execute( + "SELECT T.id, T.tag, T.summary, T.detail, COALESCE(TC.category, '?'), " + "EXISTS (SELECT 1 FROM TooltipButtons B WHERE B.tooltipId = T.id) " + "FROM Tooltips T LEFT JOIN TooltipCategories TC ON TC.id = T.categoryId ORDER BY T.id").fetchall() + repeated = [row for row in rows if isinstance(row[3], str) and row[3] != "" and row[3] == row[2]] + saved = sum(len(row[3].encode("utf-8")) for row in repeated) + by_category = Counter(row[4] for row in repeated) + no_buttons = [row for row in repeated if not row[5]] + + if not repeated: + print(f"Nothing to do: none of {len(rows)} tooltip(s) repeats its summary as its detail.") + return 0 + print(f"{len(repeated)} of {len(rows)} tooltip(s) repeat their summary as their detail " + f"({', '.join(f'{category} {count}' for category, count in sorted(by_category.items()))}); " + f"emptying it saves {human(saved)} byte(s).") + if no_buttons: + print(f" {len(no_buttons)} of them have no buttons, so their tooltip loses its See More link, " + "which led only to the same text again.") + + if cfg["dry_run"]: + print("Dry run: nothing written.") + return 0 + print(f"Backing up {db_path} ...") + print(f"Backup written to {backup_database(db_path)}") + conn.execute("BEGIN") + try: + for tooltip_id, tag, _summary, _detail, category, _buttons in repeated: + conn.execute("UPDATE Tooltips SET detail = '' WHERE id = ?", (tooltip_id,)) + if cfg["verbose"]: + print(f" [CLEAR] {category} {tag}") + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + print(f"\nDone. emptied the detail of {len(repeated)} tooltip(s), saving {human(saved)} byte(s).") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to edit in place, e.g. documentation.db") + p.add_argument("--dry-run", action="store_true", + help="Report what would change, without writing or backing up") + p.add_argument("--verbose", action="store_true", help="Log every tooltip changed") + return p + + +def main() -> None: + args = build_parser().parse_args() + sys.exit(run({"db_path": args.db_path, "dry_run": args.dry_run, "verbose": args.verbose})) + + +if __name__ == "__main__": + main() diff --git a/scripts/drop_unread_rows.py b/scripts/drop_unread_rows.py new file mode 100644 index 00000000..880df58e --- /dev/null +++ b/scripts/drop_unread_rows.py @@ -0,0 +1,756 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "scour", +# "cairosvg", +# ] +# /// +""" +drop_unread_rows.py + +Deletes Content rows that nothing in the app or the documentation reads, after +checking that no row it keeps refers to one of them. + +The rows, in four groups (pick some with --only; the default is all four): + +- javadoc-use: the javadoc Use pages (j/html/api/**/class-use/*.html and + *package-use.html) and the module graphs (j/html/api/**/*module-graph.svg). + Only other Use pages link to a Use page, and no page names a module graph. + 4,896 + 60 rows. +- pdfjs-cmaps: pdf.js's predefined CMaps (p/web/cmaps/, 168 .bcmap files and + their LICENSE). pdf.js fetches one by name only for a font that needs it; see + "PDF check" below. +- pdfjs-extras: pdf.js's no-WebAssembly fallbacks (openjpeg_nowasm_fallback.js, + jbig2_nowasm_fallback.js), which its worker imports only when WebAssembly + cannot start, and its scripting sandbox (pdf.sandbox.mjs, quickjs-eval.js, + quickjs-eval.wasm), which the viewer loads only with enableScripting on. The + step also sets enableScripting's default to false in p/web/viewer.mjs, so the + viewer never asks for the sandbox: with it off, the viewer passes no + scripting manager to its PDFViewer, which is the only caller of the one that + imports pdf.sandbox.mjs. The generic viewer never writes the pdfjs.preferences + entry that could turn it back on. +- x-placeholders: the x/ pages, each "Placeholder for XML layout attribute + ...". XML tooltips link to a/ pages instead. 1,238 rows. x.html and x.pdf + are not in this group. + +A row's chunk continuations ("-1", "-2", ...) go with it. + +Measured on the 2026-10-18 database after ADFA-6627, each group alone, packed +with rebuild_database.py --pack Content,Tooltips --page-size 4096, as bytes +saved in the file and in its brotli -q 11 -w 24: javadoc-use 5,042,176 and +4,330,908; pdfjs-cmaps 860,160 and 851,736; pdfjs-extras 339,968 and 346,809; +x-placeholders 122,880 and 103,920. Repacking alone moves the .br by up to +about 0.2 MB, so the last figure is within that. + +Before deleting anything the step checks, and refuses with the list and exit 1 +when any check fails, writing nothing: + +- References. Every kept text row (HTML, CSS, JS, SVG, JSON, XML, Markdown and + plain text, decoded, chunks joined), every Templates row, every tooltip + button's uri, every tooltip's text and the Bookshelf entries are read for + links. The tooltip tables are read in either documentation schema: 2 + (TooltipButtons.uri, Tooltips.summary and detail) or 3, after + migrate_tooltip_schema.py (TooltipButtonUris, TooltipTexts); tables in + neither shape refuse the run before anything is read. From documentation + major version 3 (the newest DocumentationDatabaseVersion row, ordered as the + app orders them) every template is stored Brotli-compressed against the + dictionary, so templates are decoded first there and read as plain text + below it. Links are: HTML and + SVG attributes, srcset, CSS url() and @import, quoted file names in scripts, + the string values of a templated row's JSON, viewer.html?file= parameters and + http://localhost:6174/ URLs. A link that resolves (relative to its row, or + from the root) to a deleted row is refused. So is a quoted file name that + resolves to no row at all but is the name of a deleted row and of no kept + one, which is how a script that builds its URLs names a file. Three such + references are expected and allowed; ALLOWED below says why each is dead. +- Content. Each x/ row must be a placeholder page, each javadoc-use HTML row a + "Uses of" page and each module graph an SVG, and p/web/cmaps/ may hold only + .bcmap files and LICENSE, so a real page that lands on one of these paths is + never deleted. +- PDF check (pdfjs-cmaps). pdf.js 6.1.200 fetches a CMap file for: a font whose + /Encoding names a predefined CMap other than Identity-H or Identity-V; an + embedded CMap that uses one (usecmap); a CID font of the Adobe GB1, CNS1, + Japan1 or Korea1 collection (Adobe--UCS2, unless the font has its + own ToUnicode map); and a font with one of ten Chinese names (GBK-EUC-H), + read from both /BaseFont and the FontDescriptor's /FontName, which pdf.js + prefers when the two differ, as a name or a string. + Every stored PDF is read and each stream decoded through its filters + (FlateDecode, ASCII85Decode, ASCIIHexDecode and RunLengthDecode, in any + chain), skipping image and cross-reference streams, which cannot hold a font; + any sign of these refuses the run. So does an encrypted PDF, or a stream it + cannot decode (another filter, a predictor, corrupt data), since its fonts + cannot be read. The collection check ignores ToUnicode, so it refuses more + than pdf.js needs, never less. It cannot see PDFs that a plugin installs on + the phone. +- The viewer edit (pdfjs-extras). p/web/viewer.mjs must hold exactly one + enableScripting default; if it holds none or more, nothing is written. +- A kept text row or template that cannot be decoded cannot be checked, so it + refuses too. + +Runs in one transaction (rolled back on any error), backs the database up +first, and VACUUMs afterwards. --dry-run does all the checks and reports what +would change without writing. A second run finds nothing to do and exits 0. + +Pipeline position: after every step that writes page content, links or +tooltips, so the reference check sees the final pages, and before +rebuild_database.py. It may run before or after migrate_tooltip_schema.py, +since it reads both tooltip shapes and both template encodings. Deleting +rows does not call for a dictionary re-mint. + + uv run scripts/drop_unread_rows.py documentation.db --dry-run + uv run scripts/drop_unread_rows.py documentation.db [--only GROUP ...] [--workers N] [--verbose] + +Recovering: the timestamped backup taken before the first write is the whole +undo. +""" +from __future__ import annotations + +import argparse +import base64 +import bisect +import html +import json +import posixpath +import re +import sqlite3 +import sys +import zlib +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import unquote + +from optimize_db_media import ( + BrotliCodec, backup_database, delete_media_row, human, is_continuation_path, load_dictionary, + owned_fragment_paths, reassemble, +) +from update_media_references import replace_row + +# Content types read for references. Everything else is media. +SCANNED_TYPES = ("text/html", "text/css", "text/javascript", "image/svg+xml", "application/json", + "application/xml", "text/markdown", "text/plain", "text", "text/text", + "application/x-typescript") + +VIEWER = "p/web/viewer.mjs" +# From this documentation major version on, migrate_tooltip_schema.py has stored every template compressed. +COMPRESSED_TEMPLATES_MAJOR = 3 +PDFJS_EXTRAS = ("p/web/wasm/openjpeg_nowasm_fallback.js", "p/web/wasm/jbig2_nowasm_fallback.js", + "p/web/wasm/quickjs-eval.wasm", "p/web/wasm/quickjs-eval.js", "p/build/pdf.sandbox.mjs") +PLACEHOLDER = b"Placeholder for XML layout attribute" + +# References to rows this step deletes that nothing follows, as (referring row, +# deleted row) -> why. Each is printed when found, so it stays visible. +ALLOWED = { + ("p/build/pdf.worker.mjs", "p/web/wasm/openjpeg_nowasm_fallback.js"): + "the worker imports it only when WebAssembly cannot start", + ("p/build/pdf.worker.mjs", "p/web/wasm/jbig2_nowasm_fallback.js"): + "the worker imports it only when WebAssembly cannot start", + ("p/web/viewer.mjs", "p/build/pdf.sandbox.mjs"): + "the viewer imports it only with enableScripting on, and this step turns that off", +} + +# The enableScripting entry of the viewer's default options, minified or not. +_SCRIPTING_DEFAULT = re.compile(rb"(\benableScripting\s*:\s*\{\s*value\s*:\s*)(true|false|!0|!1)\b") + +_ATTR = re.compile( + r"""(?`]+))""", re.I) +_SRCSET = re.compile(r"""(?]{1,400}?\.(?:html?|m?js|css|png|gif|webp|svg|webm|mp4|json|pdf|wasm|bcmap|pfb|ttf|""" + r"""otf|woff2?|icc|ico|md|xml|txt|jpe?g|ogv|mov|peb))(?:[?#][^"'`\s]*)?["'`]""", re.I) +_META_REFRESH = re.compile(r"\s*\d+\s*;\s*url\s*=\s*", re.I) +_FILE_PARAM = re.compile(r"[?&]file=([^&#]+)") +_SCHEME = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*:") +LOCAL_ORIGINS = ("http://localhost:6174/", "http://127.0.0.1:6174/") + +# What pdf.js 6.1.200 reads to decide it needs a CMap file; see the docstring. +_PDF_NAME = rb"/((?:[^\s/<>\[\]()%{}#]|#[0-9A-Fa-f]{2})+)" +_PDF_ENCODING = re.compile(rb"/(?:Encoding|UseCMap)\s*" + _PDF_NAME) +_PDF_USECMAP = re.compile(_PDF_NAME + rb"\s+usecmap\b") +_PDF_ORDERING = re.compile(rb"/Ordering\s*(?:\(([^)]*)\)|<([0-9A-Fa-f\s]*)>)") +_PDF_FONT_NAME_KEY = re.compile(rb"/(?:BaseFont|FontName)\s*") +_PDF_NAME_AT = re.compile(_PDF_NAME) +_PDF_HEX_AT = re.compile(rb"<([0-9A-Fa-f\s]*)>") +_PDF_ESCAPES = {ord("n"): b"\n", ord("r"): b"\r", ord("t"): b"\t", ord("b"): b"\b", ord("f"): b"\f"} +# A stream starts after its dictionary's ">>"; matching that keeps "endstream" from reading as a start. +_PDF_STREAM = re.compile(rb">>\s*stream(?:\r\n|\n|\r)") +_PDF_OBJ = re.compile(rb"\d+\s+\d+\s+obj\b") +_PDF_LENGTH = re.compile(rb"/Length\s+(\d+)\b(?!\s+\d+\s+R)") +_PDF_FILTER = re.compile(rb"/Filter\s*(\[[^\]]*\]|" + _PDF_NAME + rb")") +_PDF_PREDICTOR = re.compile(rb"/Predictor\s+(\d+)") +_PDF_IMAGE = re.compile(rb"/Subtype\s*/Image\b") +_PDF_XREF = re.compile(rb"/Type\s*/XRef\b") +_PDF_ENCRYPT = re.compile(rb"/Encrypt\s*(?:\d+\s+\d+\s+R|<<)") +# Filters that only ever hold image data, so their streams cannot hold a font. +IMAGE_FILTERS = {"DCTDecode", "DCT", "JPXDecode", "CCITTFaxDecode", "CCF", "JBIG2Decode"} +CJK_ORDERINGS = (b"GB1", b"CNS1", b"Japan1", b"Korea1") +# pdf.js's chineseFontNames, as the bytes it compares a font name with. +CHINESE_FONT_NAMES = {name.encode("latin-1") for name in ( + "\u00cb\u00ce\u00cc\u00e5", "\u00ba\u00da\u00cc\u00e5", "\u00bf\u00ac\u00cc\u00e5", "\u00b7\u00c2\u00cb\u00ce", + "\u00bf\u00ac\u00cc\u00e5_GB2312", "\u00b7\u00c2\u00cb\u00ce_GB2312", "\u00c1\u00a5\u00ca\u00e9", + "\u00d0\u00c2\u00cb\u00ce", "\u00b7\u00c2\u00cb\u00ce\u00cc\u00e5", "\u00d0\u00a1\u00b1\u00ea\u00cb\u00ce")} +IDENTITY_CMAPS = {"Identity-H", "Identity-V"} + + +def _javadoc_use(path: str) -> bool: + if not path.startswith("j/html/api/"): + return False + directory, name = posixpath.split(path) + return ((directory.endswith("/class-use") and name.endswith(".html")) or name.endswith("package-use.html") + or name.endswith("module-graph.svg")) + + +def _check_javadoc_use(path: str, raw: bytes): + if path.endswith(".svg"): + return None if b"Uses of " in raw else "is not a javadoc Use page" + + +def _check_cmap(path: str, raw: bytes): + name = posixpath.basename(path) + return None if name.endswith(".bcmap") or name == "LICENSE" else "is not a CMap or its LICENSE" + + +def _check_placeholder(path: str, raw: bytes): + return None if PLACEHOLDER in raw else "is not a placeholder page" + + +# name -> (what it holds, which rows it selects, a check on each row's decoded bytes) +GROUPS = { + "javadoc-use": ("javadoc Use pages and module graphs", _javadoc_use, _check_javadoc_use), + "pdfjs-cmaps": ("pdf.js CMaps", lambda path: path.startswith("p/web/cmaps/"), _check_cmap), + "pdfjs-extras": ("pdf.js no-WebAssembly fallbacks and scripting sandbox", lambda path: path in PDFJS_EXTRAS, + None), + "x-placeholders": ("x/ placeholder pages", lambda path: path.startswith("x/"), _check_placeholder), +} + + +def turn_scripting_off(viewer: bytes) -> tuple: + """(viewer with enableScripting's default false, whether that changed it), or (None, why not).""" + found = list(_SCRIPTING_DEFAULT.finditer(viewer)) + if len(found) != 1: + return None, f"holds {len(found)} enableScripting defaults, not 1" + m = found[0] + if m.group(2) in (b"false", b"!1"): + return viewer, False + return viewer[:m.start(2)] + b"false" + viewer[m.end(2):], True + + +def link_candidates(text: str, is_json: bool) -> set: + """Every string in text that could be a link to a stored file.""" + found = set() + blobs = [text] + if is_json: + try: + strings = [] + _json_strings(json.loads(text), strings) + except ValueError: + pass # not JSON after all; read it as text + else: + blobs = strings + found.update(s for s in strings if len(s) < 400 and "<" not in s) + for blob in blobs: + for m in _ATTR.finditer(blob): + value = next(g for g in m.groups() if g is not None) + if "&" in value: + value = html.unescape(value) + refresh = _META_REFRESH.match(value) + found.add(value[refresh.end():] if refresh else value) + for m in _SRCSET.finditer(blob): + for part in (m.group(1) or m.group(2) or "").split(","): + if part.strip(): + found.add(html.unescape(part.split()[0])) + for rx in (_CSS_URL, _CSS_IMPORT): + for m in rx.finditer(blob): + found.add(next((g for g in m.groups() if g is not None), "")) + for m in _QUOTED.finditer(blob): + found.add(m.group(1)) + found.discard("") + return found + + +def _json_strings(value, out: list) -> None: + if isinstance(value, str): + out.append(value) + elif isinstance(value, dict): + for item in value.values(): + _json_strings(item, out) + elif isinstance(value, list): + for item in value: + _json_strings(item, out) + + +def is_external(ref: str) -> bool: + """Whether ref points off this server: another scheme, or a host of its own.""" + ref = ref.strip() + if ref.lower().startswith(LOCAL_ORIGINS): + return False + return ref.startswith("//") or bool(_SCHEME.match(ref)) + + +def link_paths(ref: str, base_dirs, nested: bool = False) -> list: + """The stored paths ref could name, read from each of base_dirs (from the root if it starts with "/").""" + ref = ref.strip() + out = [] + param = _FILE_PARAM.search(ref) + if param and not nested: + out.extend(link_paths(unquote(param.group(1)), base_dirs, nested=True)) + lowered = ref.lower() + for origin in LOCAL_ORIGINS: + if lowered.startswith(origin): + ref = "/" + ref[len(origin):] + break + if not ref or ref.startswith(("#", "//")) or _SCHEME.match(ref): + return out + ref = ref.split("#", 1)[0].split("?", 1)[0] + for candidate in {ref, unquote(ref)}: + if not candidate: + continue + rooted = candidate.startswith("/") + for base in ("",) if rooted else base_dirs: + path = posixpath.normpath(candidate.lstrip("/") if rooted else posixpath.join(base, candidate)) + while path == ".." or path.startswith("../"): # ".." stops at the root, as in a browser + path = path[3:] + if path not in ("", "."): + out.append(path) + return out + + +class References: + """Finds the references a text makes to rows being deleted.""" + + def __init__(self, stored: set, deleted: set): + self.stored = stored + self.deleted = deleted + kept_names = {posixpath.basename(p) for p in stored - deleted} + self.only_deleted = defaultdict(list) + for path in deleted: + name = posixpath.basename(path) + if name not in kept_names: + self.only_deleted[name].append(path) + + def targets(self, ref: str, base_dirs) -> list: + """[(deleted path, how)] for one reference.""" + out, resolved = [], False + for path in link_paths(ref, base_dirs): + for variant in (path, path + ".html", path + "/index.html"): + if variant in self.stored: + resolved = True + if variant in self.deleted: + out.append((variant, "links to")) + break + if not resolved and not is_external(ref): + name = unquote(posixpath.basename(ref.strip().split("#", 1)[0].split("?", 1)[0])) + out.extend((target, "names") for target in self.only_deleted.get(name, ())) + return out + + def in_text(self, text: str, base_dirs, is_json: bool = False) -> set: + found = set() + for ref in link_candidates(text, is_json): + found.update(self.targets(ref, base_dirs)) + return found + + +def _unescape_name(name: bytes) -> bytes: + return re.sub(rb"#([0-9A-Fa-f]{2})", lambda m: bytes([int(m.group(1), 16)]), name) + + +def _literal_string(data: bytes, start: int): + """The bytes of the PDF literal string whose "(" is at start (escapes decoded, nested parentheses kept), + or None if it does not end.""" + out, depth, i = bytearray(), 0, start + 1 + while i < len(data): + c = data[i] + if c == 0x5C: # backslash + i += 1 + if i >= len(data): + return None + e = data[i] + if e in _PDF_ESCAPES: + out += _PDF_ESCAPES[e] + elif 0x30 <= e <= 0x37: + digits = re.match(rb"[0-7]{1,3}", data[i:i + 3]).group(0) + out.append(int(digits, 8) & 0xFF) + i += len(digits) - 1 + elif e == 0x0D: # a line continuation, \r or \r\n + if data[i + 1:i + 2] == b"\n": + i += 1 + elif e != 0x0A: + out.append(e) + elif c == 0x28: + depth += 1 + out.append(c) + elif c == 0x29: + if depth == 0: + return bytes(out) + depth -= 1 + out.append(c) + else: + out.append(c) + i += 1 + return None + + +def font_names(part: bytes): + """Each /BaseFont and /FontName value in part, as the bytes pdf.js compares: a name (#xx decoded), a + literal string or a hex string. pdf.js takes the FontDescriptor's FontName when it differs from the + BaseFont, so both are read.""" + for m in _PDF_FONT_NAME_KEY.finditer(part): + i = m.end() + lead = part[i:i + 1] + if lead == b"/": + name = _PDF_NAME_AT.match(part, i) + if name: + yield _unescape_name(name.group(1)) + elif lead == b"(": + text = _literal_string(part, i) + if text is not None: + yield text + elif lead == b"<" and part[i + 1:i + 2] != b"<": + hexed = _PDF_HEX_AT.match(part, i) + if hexed: + digits = re.sub(rb"\s", b"", hexed.group(1)) + yield bytes.fromhex((digits + b"0" * (len(digits) % 2)).decode()) + + +def _run_length(data: bytes) -> bytes: + out, i = bytearray(), 0 + while i < len(data): + n = data[i] + if n == 128: + break + if n < 128: + out += data[i + 1:i + 2 + n] + i += 2 + n + else: + out += data[i + 1:i + 2] * (257 - n) + i += 2 + return bytes(out) + + +def _ascii_hex(data: bytes) -> bytes: + digits = re.sub(rb"\s", b"", data.split(b">", 1)[0]) + return bytes.fromhex((digits + b"0" * (len(digits) % 2)).decode("ascii")) + + +def _ascii85(data: bytes) -> bytes: + text = re.sub(rb"\s", b"", data) + text = text[2:] if text.startswith(b"<~") else text + return base64.a85decode(text.split(b"~>", 1)[0]) + + +DECODERS = {"FlateDecode": lambda data: zlib.decompressobj().decompress(data), "ASCIIHexDecode": _ascii_hex, + "ASCII85Decode": _ascii85, "RunLengthDecode": _run_length} +DECODERS.update({"Fl": DECODERS["FlateDecode"], "AHx": _ascii_hex, "A85": _ascii85, "RL": _run_length}) + + +def pdf_parts(data: bytes) -> tuple: + """([the PDF's bytes, then each stream decoded], [why each stream that could not be decoded could not]). + Image streams and cross-reference streams are skipped: neither can hold a font.""" + parts, undecodable = [data], [] + headers = [m.end() for m in _PDF_OBJ.finditer(data)] + pos = 0 + while True: + m = _PDF_STREAM.search(data, pos) + if m is None: + break + start = m.end() + header = bisect.bisect_right(headers, m.start()) - 1 + dictionary = data[headers[header] if header >= 0 else max(0, m.start() - 4096):m.start() + 2] + length = _PDF_LENGTH.search(dictionary) + end = start + int(length.group(1)) if length else -1 + if not (length and data[end:end + 20].lstrip().startswith(b"endstream")): + end = data.find(b"endstream", start) + if end < 0: + break + body, pos = data[start:end], end + filter_value = _PDF_FILTER.search(dictionary) + filters = ([_unescape_name(n).decode("latin-1") for n in re.findall(_PDF_NAME, filter_value.group(1))] + if filter_value else []) + if _PDF_IMAGE.search(dictionary) or _PDF_XREF.search(dictionary) or IMAGE_FILTERS & set(filters): + continue + predictor = _PDF_PREDICTOR.search(dictionary) + if predictor and int(predictor.group(1)) > 1: + undecodable.append(f"predictor {int(predictor.group(1))}") + continue + try: + for name in filters: + if name not in DECODERS: + raise ValueError(name) + body = DECODERS[name](body) + except (ValueError, zlib.error) as exc: + undecodable.append(f"{name}" + ("" if str(exc) == name else f": {exc}")) + continue + parts.append(body) + return parts, undecodable + + +def cmap_needs(data: bytes, cmap_names: set) -> list: + """Why pdf.js would fetch a CMap file for this PDF, if it would; see the docstring.""" + if _PDF_ENCRYPT.search(data): + return ["is encrypted, so its fonts cannot be read"] + parts, undecodable = pdf_parts(data) + needs = {f"has a stream this step cannot decode ({why}), so its fonts cannot all be checked" + for why in undecodable} + for part in parts: + for m in list(_PDF_ENCODING.finditer(part)) + list(_PDF_USECMAP.finditer(part)): + name = _unescape_name(m.group(1)).decode("latin-1") + if name in cmap_names: + needs.add(f"uses the predefined CMap {name}") + for m in _PDF_ORDERING.finditer(part): + ordering = m.group(1) if m.group(1) is not None else bytes.fromhex(m.group(2).decode().replace(" ", "")) + if ordering.strip() in CJK_ORDERINGS: + needs.add(f"has a CID font of the Adobe {ordering.strip().decode()} collection") + for name in font_names(part): + if name in CHINESE_FONT_NAMES or name.split(b"+", 1)[-1] in CHINESE_FONT_NAMES: + needs.add("has a font pdf.js reads as Chinese (GBK-EUC-H)") + return sorted(needs) + + +def declared_major(conn) -> int | None: + """The documentation major version the database declares - its newest DocumentationDatabaseVersion row, ordered + as Code On The Go orders them - or None if it declares none.""" + if not conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name = 'DocumentationDatabaseVersion'").fetchone(): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion " + "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone() + return None if row is None else int(row[0]) + + +def tooltip_queries(conn): + """(buttons query, texts query) for the database's tooltip tables, schema 2 or 3, or None for any other + shape. The buttons query yields (tooltipId, uri), the texts query (id, summary, detail).""" + def columns(table): + return {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} + + tooltips, buttons, texts, uris = (columns(t) for t in ("Tooltips", "TooltipButtons", "TooltipTexts", + "TooltipButtonUris")) + if {"id", "summary", "detail"} <= tooltips and {"tooltipId", "uri"} <= buttons and not texts and not uris: + return ("SELECT tooltipId, uri FROM TooltipButtons WHERE uri IS NOT NULL", + "SELECT id, summary, detail FROM Tooltips") + if ({"id", "categoryId", "tag"} <= tooltips and not {"summary", "detail"} & tooltips + and {"tooltipId", "uriId"} <= buttons and {"id", "summary", "detail"} <= texts and {"id", "uri"} <= uris): + return ("SELECT B.tooltipId, U.uri FROM TooltipButtons B JOIN TooltipButtonUris U ON U.id = B.uriId " + "WHERE U.uri IS NOT NULL", + "SELECT id, summary, detail FROM TooltipTexts") + return None + + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + groups = cfg["groups"] or list(GROUPS) + + conn = sqlite3.connect(db_path) + codec = None + try: + queries = tooltip_queries(conn) + if queries is None: + print("error: the tooltip tables are in neither the schema-2 shape (Tooltips.summary and detail, " + "TooltipButtons.uri) nor the schema-3 shape (TooltipTexts, TooltipButtonUris), so their links " + "cannot be read; nothing was changed.", file=sys.stderr) + return 1 + buttons_query, texts_query = queries + codec = BrotliCodec(load_dictionary(conn)) + rows = {path: (row_id, length, mime, compression, template_id) + for row_id, path, length, mime, compression, template_id in conn.execute( + "SELECT c.id, c.path, LENGTH(c.content), ct.value, ct.compression, c.templateId " + "FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id ORDER BY c.path")} + lengths = {path: row[1] for path, row in rows.items()} + bases = [path for path in rows if not is_continuation_path(lengths, path)] + + def stored_bytes(path: str) -> bytes: + """A row's stored bytes, chunks joined. Main thread only: it queries.""" + first = conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0] + return reassemble(conn, path, first) + + doomed = {} # base row -> its group + for group in groups: + doomed.update((path, group) for path in bases if GROUPS[group][1](path)) + deleted = {} # every row deleted, chunk rows included -> its group + for path, group in doomed.items(): + deleted[path] = group + deleted.update((fragment, group) for fragment in owned_fragment_paths(conn, path, lengths[path])) + + problems = [] + viewer_new, viewer_changed = None, False + if "pdfjs-extras" in groups: + if VIEWER in rows: + viewer_new, viewer_changed = turn_scripting_off(codec.decompress(stored_bytes(VIEWER), + rows[VIEWER][3])) + if viewer_new is None: + problems.append(f"{VIEWER} {viewer_changed}, so scripting cannot be turned off") + viewer_changed = False + elif any(group == "pdfjs-extras" for group in doomed.values()): + problems.append(f"{VIEWER} is not stored, so this step cannot tell that scripting is off") + + if not doomed and not viewer_changed and not problems: + print("Nothing to do: no row in " + ", ".join(groups) + + (", and scripting is already off." if "pdfjs-extras" in groups else ".")) + return 0 + + def decode(item): + path, full = item + try: + if path == VIEWER and viewer_new is not None: + return path, viewer_new, None + return path, codec.decompress(full, rows[path][3]), None + except Exception as exc: # noqa: BLE001 - reported, and the run refused + return path, None, str(exc) + + print(f"Checking the {len(doomed)} row(s) to delete...") + checked = [(path, stored_bytes(path)) for path in sorted(doomed) if GROUPS[doomed[path]][2] is not None] + with ThreadPoolExecutor(max_workers=cfg["workers"]) as pool: + for path, raw, error in pool.map(decode, checked): + problem = f"could not be read: {error}" if raw is None else GROUPS[doomed[path]][2](path, raw) + if problem: + problems.append(f"{path} {problem}") + checked = None + + refs = References(set(rows), set(deleted)) + found = set() # (referring row, deleted row, how) + kept = [path for path in bases if path not in deleted and rows[path][2] in SCANNED_TYPES] + print(f"Reading {len(kept)} kept text row(s) for references...") + texts = [(path, stored_bytes(path)) for path in kept] + + def scan(item): + path, raw, error = decode(item) + if raw is None: + return path, None, error + text = raw.decode("utf-8", errors="replace") + return path, refs.in_text(text, (posixpath.dirname(path),), rows[path][4] != 0), None + + with ThreadPoolExecutor(max_workers=cfg["workers"]) as pool: + for path, targets, error in pool.map(scan, texts): + if targets is None: + problems.append(f"{path} could not be read, so its references were not checked: {error}") + else: + found.update((path, target, how) for target, how in targets) + texts = None + + # A template renders at the paths of the rows that use it. + dirs_by_template = defaultdict(set) + for path, row in rows.items(): + if row[4]: + dirs_by_template[row[4]].add(posixpath.dirname(path)) + major = declared_major(conn) + compressed = major is not None and major >= COMPRESSED_TEMPLATES_MAJOR + for template_id, name, content in conn.execute("SELECT id, name, content FROM Templates"): + dirs = ("",) + tuple(sorted(dirs_by_template[template_id])) + try: + raw = codec.decompress(bytes(content), "brotli") if compressed else bytes(content) + except Exception as exc: # noqa: BLE001 - reported, and the run refused + problems.append(f"template {name} could not be decoded, so its links were not checked: {exc}") + continue + text = raw.decode("utf-8", errors="replace") + found.update((f"template {name}", target, how) for target, how in refs.in_text(text, dirs)) + for tooltip_id, uri in conn.execute(buttons_query): + found.update((f"tooltip {tooltip_id} button", target, how) for target, how in refs.targets(uri, ("",))) + for tooltip_id, summary, detail in conn.execute(texts_query): + found.update((f"tooltip {tooltip_id} text", target, how) + for target, how in refs.in_text(f"{summary}\n{detail}", ("",))) + by_id = {rows[path][0]: path for path in deleted} + for (content_id,) in conn.execute("SELECT contentID FROM Bookshelf"): + if content_id in by_id: + found.add(("Bookshelf", by_id[content_id], "lists")) + + allowed = sorted({(src, dst) for src, dst, _how in found if (src, dst) in ALLOWED}) + problems.extend(f"{src} {how} {dst}" for src, dst, how in sorted(found) if (src, dst) not in ALLOWED) + + cmap_names = {posixpath.basename(path)[:-len(".bcmap")] for path, group in doomed.items() + if group == "pdfjs-cmaps" and path.endswith(".bcmap")} - IDENTITY_CMAPS + if cmap_names: + pdfs = [path for path in bases if path not in deleted and rows[path][2] == "application/pdf"] + for path, raw, error in map(decode, [(path, stored_bytes(path)) for path in pdfs]): + needs = [f"could not be read: {error}"] if raw is None else cmap_needs(raw, cmap_names) + problems.extend(f"{path} {need}" for need in needs) + print(f"Read {len(pdfs)} PDF(s) for fonts that need a CMap.") + + for src, dst in allowed: + print(f" allowed: {src} refers to {dst}; {ALLOWED[(src, dst)]}") + if problems: + for problem in problems: + print(f" error: {problem}", file=sys.stderr) + print(f"error: {len(problems)} problem(s) found; nothing was changed.", file=sys.stderr) + return 1 + + freed_total = sum(lengths[path] for path in deleted) + for group in groups: + count = sum(1 for g in doomed.values() if g == group) + chunks = sum(1 for path, g in deleted.items() if g == group and path not in doomed) + freed = sum(lengths[path] for path, g in deleted.items() if g == group) + print(f" {group}: {count} row(s) of {GROUPS[group][0]}" + + (f" and {chunks} chunk row(s)" if chunks else "") + f", {human(freed)} stored byte(s)") + if viewer_changed: + print(f" {VIEWER}: enableScripting default set to false") + + if cfg["dry_run"]: + print("Dry run: nothing written.") + else: + print(f"Backing up {db_path} ...") + print(f"Backup written to {backup_database(db_path)}") + conn.execute("BEGIN") + try: + for path in sorted(doomed): + delete_media_row(conn, path, lengths[path]) + if cfg["verbose"]: + print(f" [DEL] {path}") + if viewer_changed: + language_id, content_type_id, template_id = conn.execute( + "SELECT languageID, contentTypeID, templateId FROM Content WHERE path = ?", + (VIEWER,)).fetchone() + replace_row(conn, VIEWER, lengths[VIEWER], codec.compress(viewer_new, rows[VIEWER][3]), + language_id, content_type_id, template_id, lambda m: print(m, file=sys.stderr)) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + if codec is not None: + codec.close() + conn.close() + + if not cfg["dry_run"]: + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + + verb = "would delete" if cfg["dry_run"] else "deleted" + print(f"\nDone. {verb} {len(deleted)} row(s), {human(freed_total)} stored byte(s)" + + ("; scripting turned off in the viewer." if viewer_changed else ".")) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to edit in place, e.g. documentation.db") + p.add_argument("--dry-run", action="store_true", + help="Do all the checks and report what would change, without writing or backing up") + p.add_argument("--only", action="append", choices=list(GROUPS), dest="groups", + help="Delete only this group; repeat for more (default: all of them)") + p.add_argument("--workers", type=int, default=8, + help="Threads used to decompress and read rows (default: 8)") + p.add_argument("--verbose", action="store_true", help="Log every row deleted") + return p + + +def main() -> None: + args = build_parser().parse_args() + sys.exit(run({"db_path": args.db_path, "dry_run": args.dry_run, "groups": args.groups, + "workers": args.workers, "verbose": args.verbose})) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_clear_repeated_tooltip_detail.py b/scripts/test_clear_repeated_tooltip_detail.py new file mode 100644 index 00000000..f51adfd4 --- /dev/null +++ b/scripts/test_clear_repeated_tooltip_detail.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_clear_repeated_tooltip_detail.py + +Covers clear_repeated_tooltip_detail.py: that it empties a detail only when it +is exactly its summary, whatever collation the schema gives the columns; that +it says how many such tooltips have no buttons; and that a run is all or +nothing, a dry run writes nothing, and a second run changes nothing. + +Run it either way - as a plain script: + + uv run scripts/test_clear_repeated_tooltip_detail.py + +or under pytest: + + uv run --with brotli --with Pillow --with pytest pytest scripts/test_clear_repeated_tooltip_detail.py +""" +from __future__ import annotations + +import contextlib +import io +import re +import sqlite3 +import sys +import tempfile +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import clear_repeated_tooltip_detail as tool # noqa: E402 + +# (id, category, summary, detail, buttons) +TOOLTIPS = [ + (1, 4, "Returns the size.", "Returns the size.", 1), + (2, 4, "Adds an element.", "Adds an element, if there is room.", 1), + (3, 4, "Removes it.", "", 1), + (4, 3, "Splits a string.", "Splits a string.", 2), + (5, 4, "Trims it.", "Trims it. ", 1), + (6, 4, "Counts.", "counts.", 1), + (7, 1, "Opens the file.", "Opens the file.", 0), + (8, 4, "", "", 1), +] + + +def _database(tmp_path: Path, tooltips=TOOLTIPS, collate: str = "") -> Path: + db_path = tmp_path / "documentation.db" + conn = sqlite3.connect(db_path) + conn.executescript( + "CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL);" + "INSERT INTO TooltipCategories VALUES (1, 'ide'), (3, 'java'), (4, 'kotlin');" + f"CREATE TABLE Tooltips (id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," + f" summary TEXT NOT NULL {collate}, detail TEXT NOT NULL {collate}, UNIQUE (categoryId, tag));" + "CREATE TABLE TooltipButtons (tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT);" + ) + for tooltip_id, category, summary, detail, buttons in tooltips: + conn.execute("INSERT INTO Tooltips VALUES (?, ?, ?, ?, ?)", (tooltip_id, category, f"tag{tooltip_id}", + summary, detail)) + for number in range(buttons): + conn.execute("INSERT INTO TooltipButtons VALUES (?, ?, 'View full documentation', 'a/x.html')", + (tooltip_id, number + 1)) + conn.commit() + conn.close() + return db_path + + +def _details(db_path: Path) -> dict: + conn = sqlite3.connect(db_path) + try: + return {row[0]: row[1:] for row in conn.execute("SELECT id, summary, detail FROM Tooltips")} + finally: + conn.close() + + +def _run(db_path: Path, **overrides) -> tuple: + cfg = {"db_path": db_path, "dry_run": False, "verbose": False} + cfg.update(overrides) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = tool.run(cfg) + return code, out.getvalue(), err.getvalue() + + +def test_only_a_detail_that_is_exactly_its_summary_is_emptied(tmp_path: Path): + db = _database(tmp_path) + before = _details(db) + code, out, err = _run(db) + after = _details(db) + assert code == 0, err + assert {i for i in before if after[i] != before[i]} == {1, 4, 7} + for i in (1, 4, 7): + assert after[i] == (before[i][0], "") + assert "3 of 8 tooltip(s)" in out and "ide 1, java 1, kotlin 1" in out + assert "saves 48 byte(s)" in out + + +def test_a_case_insensitive_schema_does_not_widen_the_match(tmp_path: Path): + db = _database(tmp_path, collate="COLLATE NOCASE") + code, _out, err = _run(db) + assert code == 0, err + assert _details(db)[6] == ("Counts.", "counts.") + + +def test_a_detail_that_is_not_text_is_left_alone(tmp_path: Path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + conn.execute("UPDATE Tooltips SET summary = CAST(summary AS BLOB), detail = CAST(detail AS BLOB) WHERE id = 1") + conn.commit() + conn.close() + code, out, err = _run(db) + assert code == 0, err + assert _details(db)[1] == (b"Returns the size.", b"Returns the size.") and "2 of 8 tooltip(s)" in out + + +def test_tooltips_without_buttons_are_counted(tmp_path: Path): + code, out, _err = _run(_database(tmp_path)) + assert code == 0 and "1 of them have no buttons, so their tooltip loses its See More link" in out + + +def test_dry_run_writes_nothing(tmp_path: Path): + db = _database(tmp_path) + before = _details(db) + code, out, _err = _run(db, dry_run=True) + assert code == 0 and "3 of 8 tooltip(s)" in out and "Dry run" in out + assert _details(db) == before and len(list(tmp_path.iterdir())) == 1 + + +def test_a_second_run_changes_nothing(tmp_path: Path): + db = _database(tmp_path) + assert _run(db)[0] == 0 + once, files = _details(db), sorted(tmp_path.iterdir()) + code, out, _err = _run(db) + assert code == 0 and out.startswith("Nothing to do") + assert _details(db) == once and sorted(tmp_path.iterdir()) == files + + +def test_the_backup_holds_the_database_as_it_was(tmp_path: Path): + db = _database(tmp_path) + before = _details(db) + code, out, _err = _run(db) + backup = Path(re.search(r"Backup written to (.+)", out).group(1).strip()) + assert code == 0 and _details(backup) == before + + +def test_an_error_while_writing_rolls_everything_back(tmp_path: Path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + conn.execute("CREATE TRIGGER fail BEFORE UPDATE ON Tooltips WHEN OLD.id = 4 BEGIN SELECT RAISE(ABORT, 'disk full');" + " END") + conn.commit() + conn.close() + before = _details(db) + try: + _run(db) + except sqlite3.DatabaseError: + pass + else: + raise AssertionError("the failure did not propagate") + assert _details(db) == before + + +def test_a_migrated_database_is_refused_by_name(tmp_path: Path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + conn.executescript("CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY, summary TEXT, detail TEXT);" + "INSERT INTO TooltipTexts SELECT id, summary, detail FROM Tooltips;" + "CREATE TABLE T2 (id INTEGER, categoryId INTEGER, tag TEXT);" + "INSERT INTO T2 SELECT id, categoryId, tag FROM Tooltips;" + "DROP TABLE Tooltips; ALTER TABLE T2 RENAME TO Tooltips;") + conn.close() + code, out, err = _run(db) + assert code == 1 and "run this step before migrate_tooltip_schema.py" in err, err + assert sorted(p.name for p in tmp_path.iterdir()) == ["documentation.db"] + # Even with the old columns still on Tooltips, a TooltipTexts means the text lives there. + (tmp_path / "half").mkdir() + db2 = _database(tmp_path / "half") + conn = sqlite3.connect(db2) + conn.execute("CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY, summary TEXT, detail TEXT)") + conn.close() + before = _details(db2) + code, _out, err = _run(db2) + assert code == 1 and "migrate_tooltip_schema.py" in err and _details(db2) == before + + +def test_the_script_declares_what_its_imports_need(): + # `uv run scripts/clear_repeated_tooltip_detail.py` installs only what its PEP 723 block lists, and + # optimize_db_media, which it imports, imports brotli and PIL at module level. This test file declares both + # itself, so without this check a missing block shows up only when the step is run. + script = Path(tool.__file__).read_text() + block = re.search(r"^# /// script\n(.*?)^# ///$", script, re.S | re.M) + assert block, "no PEP 723 script block" + declared = set(re.findall(r'"([A-Za-z0-9_.-]+)', block.group(1).split("dependencies", 1)[-1])) + imported = Path(tool.__file__).with_name("optimize_db_media.py").read_text() + needed = {"brotli": r"^import brotli\b", "Pillow": r"^from PIL import"} + for package, pattern in needed.items(): + assert re.search(pattern, imported, re.M), f"optimize_db_media no longer imports {package}; update this test" + assert package in declared, f"{package} is not declared" + + +def test_a_database_without_tooltips_is_an_error(tmp_path: Path): + db = tmp_path / "documentation.db" + sqlite3.connect(db).close() + code, _out, err = _run(db) + assert code == 1 and "no Tooltips table" in err + code, _out, err = _run(tmp_path / "absent.db") + assert code == 1 and "does not exist" in err + + +def _main() -> int: + """Runs the tests without pytest (`uv run scripts/test_clear_repeated_tooltip_detail.py`).""" + failures = [] + for name, function in sorted(globals().items()): + if not name.startswith("test_") or not callable(function): + continue + with tempfile.TemporaryDirectory() as directory: + needs_tmp = "tmp_path" in function.__code__.co_varnames[:function.__code__.co_argcount] + try: + function(Path(directory)) if needs_tmp else function() + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: # noqa: BLE001 - a failure is the result here + frame = traceback.extract_tb(exc.__traceback__)[-1] + failures.append(f"{name} (line {frame.lineno}: {frame.line}): {type(exc).__name__}: {exc}") + print(f"FAIL {name}", file=sys.stderr) + else: + print(f"ok {name}") + print() + print(f"{len(failures)} failure(s)" if failures else "all tests passed") + for failure in failures: + print(f" {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/scripts/test_drop_unread_rows.py b/scripts/test_drop_unread_rows.py new file mode 100644 index 00000000..e90c28ad --- /dev/null +++ b/scripts/test_drop_unread_rows.py @@ -0,0 +1,801 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_drop_unread_rows.py + +Covers drop_unread_rows.py: that each group deletes its rows, their chunk rows +and nothing else; that scripting is turned off in the viewer, even once the +sandbox rows are gone; that tooltips are read in documentation schema 2 and 3, +and templates as plain text below documentation version 3 and decoded from it; +and that every check refuses the run, writing nothing, when a kept row, +template, tooltip or Bookshelf entry refers to a deleted row, when a row to +delete is not what its group expects, when a stored PDF needs a CMap or its +fonts cannot all be read, when the viewer cannot be edited, when the tooltip +tables are in neither shape, or when a kept row, a template, a row to delete or +a PDF cannot be read. A run is all or nothing, a dry run writes nothing, and a +second run changes nothing. + +Run it either way - as a plain script: + + uv run scripts/test_drop_unread_rows.py + +or under pytest: + + uv run --with brotli --with Pillow --with pytest pytest scripts/test_drop_unread_rows.py + +The dictionary and template-version tests need the `brotli` CLI on PATH, as the +tool itself does. +""" +from __future__ import annotations + +import contextlib +import io +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import traceback +import zlib +from pathlib import Path + +import brotli + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import drop_unread_rows as tool # noqa: E402 + +CHUNK_SIZE = 1024 * 1024 + +TYPES = {"text/css": (1, "brotli"), "image/svg+xml": (2, "brotli"), "text/javascript": (6, "brotli"), + "application/json": (8, "brotli"), "text/html": (12, "brotli"), "text": (13, "none"), + "application/pdf": (14, "brotli"), "application/wasm": (19, "brotli"), + "application/octet-stream": (27, "brotli")} + +VIEWER_MJS = (b'const defaultOptions={enableScripting:{value:true,kind:OptionKind.VIEWER+OptionKind.PREFERENCE},' + b'wasmUrl:{value:"../web/wasm/"}};defaultOptions.sandboxBundleSrc={value:"../build/pdf.sandbox.mjs"};' + b'scriptingManager:AppOptions.get("enableScripting")&&pdfScriptingManager') +VIEWER_MJS_OFF = VIEWER_MJS.replace(b"enableScripting:{value:true", b"enableScripting:{value:false") +WORKER_MJS = (b'class JpxImage{_filename="openjpeg.wasm";_noWasmFilename="openjpeg_nowasm_fallback.js"}' + b'class JBig2{_filename="jbig2.wasm";_noWasmFilename="jbig2_nowasm_fallback.js"}') +SANDBOX_MJS = b'import("./quickjs-eval.js");fetch("../web/wasm/quickjs-eval.wasm")' +CLASS_PAGE = b'BB' +USE_PAGE = (b'Uses of Class a.BB' + b'package') +PACKAGE_USE = b'Uses of Package aB' +MODULE_GRAPH = b'' + + +def _placeholder(name: str) -> bytes: + return (b"Placeholder for %s" + b"Placeholder for XML layout attribute %s" % (name.encode(), name.encode())) + + +def _pdf(*objects: bytes) -> bytes: + return (b"%PDF-1.7\n" + b"".join(b"%d 0 obj\n%s\nendobj\n" % (i + 1, o) for i, o in enumerate(objects)) + + b"%%EOF\n") + + +def _flate(body: bytes) -> bytes: + data = zlib.compress(body) + return b"<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream" % (len(data), data) + + +def _stream(data: bytes, entries: bytes = b"") -> bytes: + return b"<< /Length %d %s >>\nstream\n%s\nendstream" % (len(data), entries, data) + + +def _run_length(data: bytes) -> bytes: + """RunLengthDecode data in 3-byte literal runs, so no name survives whole in the raw file.""" + return b"".join(bytes([len(data[i:i + 3]) - 1]) + data[i:i + 3] for i in range(0, len(data), 3)) + b"\x80" + + +JAPANESE_FONT = b"<< /Type /Font /Subtype /Type0 /BaseFont /MS-Gothic /Encoding /90ms-RKSJ-H >>" + +PLAIN_PDF = _pdf(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>", + b"<< /Type /Font /Subtype /Type0 /BaseFont /ABCDEF+Noto /Encoding /Identity-H >>", + b"<< /Registry (Adobe) /Ordering (Identity) /Supplement 0 >>", + _flate(b"BT /F1 12 Tf (Hello) Tj ET")) + +# What run() deletes from _base_rows() with every group on. +DELETED = { + "j/html/api/m/a/class-use/B.html", "j/html/api/m/a/package-use.html", "j/html/api/m/module-graph.svg", + "p/web/cmaps/90ms-RKSJ-H.bcmap", "p/web/cmaps/Adobe-Japan1-UCS2.bcmap", "p/web/cmaps/LICENSE", + "p/web/wasm/openjpeg_nowasm_fallback.js", "p/web/wasm/jbig2_nowasm_fallback.js", + "p/web/wasm/quickjs-eval.wasm", "p/web/wasm/quickjs-eval.js", "p/build/pdf.sandbox.mjs", + "x/a.html", "x/b.html", +} + + +def _base_rows() -> dict: + """path -> (content type, decoded bytes [, templateId [, stored as they are]]) for a database holding a row + of every group, the pdf.js files that refer to them, and kept pages.""" + return { + "i/index.html": ("text/html", b'xguide'), + "j/html/api/m/a/B.html": ("text/html", CLASS_PAGE), + "j/html/api/m/a/class-use/B.html": ("text/html", USE_PAGE), + "j/html/api/m/a/package-use.html": ("text/html", PACKAGE_USE), + "j/html/api/m/module-graph.svg": ("image/svg+xml", MODULE_GRAPH), + "j/html/api/script.js": ("text/javascript", b'var page = "B.html"; var tree = "package-tree.html";'), + "j/html/api/m/a/package-tree.html": ("text/html", b"

tree

"), + "p/web/cmaps/90ms-RKSJ-H.bcmap": ("application/octet-stream", b"\xe0\x01cmap"), + "p/web/cmaps/Adobe-Japan1-UCS2.bcmap": ("application/octet-stream", b"\xe0\x02cmap"), + "p/web/cmaps/LICENSE": ("text", b"license text"), + "p/web/wasm/openjpeg.wasm": ("application/wasm", b"\x00asm\x01"), + "p/web/wasm/openjpeg_nowasm_fallback.js": ("text/javascript", b"export default function(){}"), + "p/web/wasm/jbig2_nowasm_fallback.js": ("text/javascript", b"export default function(){ }"), + "p/web/wasm/quickjs-eval.wasm": ("application/wasm", b"\x00asm\x02"), + "p/web/wasm/quickjs-eval.js": ("text/javascript", b'fetch("quickjs-eval.wasm")'), + "p/build/pdf.sandbox.mjs": ("text/javascript", SANDBOX_MJS), + "p/build/pdf.worker.mjs": ("text/javascript", WORKER_MJS), + "p/web/viewer.mjs": ("text/javascript", VIEWER_MJS), + "p/web/viewer.html": ("text/html", b''), + "x/a.html": ("text/html", _placeholder("a")), + "x/b.html": ("text/html", _placeholder("b")), + "x.html": ("text", b"

Hello, App Dev for All

"), + "x.pdf": ("application/pdf", PLAIN_PDF), + "d/guide.pdf": ("application/pdf", PLAIN_PDF + b" "), + } + + +def _compress(raw: bytes, dictionary: bytes = None) -> bytes: + """Stored bytes for a brotli row, at quality 5, so a row the tool re-compressed (at 11) looks different.""" + if dictionary is None: + return brotli.compress(raw, quality=5) + with tempfile.TemporaryDirectory() as directory: + dict_path = Path(directory) / "dictionary.bin" + dict_path.write_bytes(dictionary) + return subprocess.run(["brotli", "-D", str(dict_path), "-q", "5", "-c"], input=raw, + stdout=subprocess.PIPE, check=True).stdout + + +def _decompress(blob: bytes, dictionary: bytes = None) -> bytes: + if dictionary is None: + return brotli.decompress(blob) + with tempfile.TemporaryDirectory() as directory: + dict_path = Path(directory) / "dictionary.bin" + dict_path.write_bytes(dictionary) + return subprocess.run(["brotli", "-D", str(dict_path), "-d", "-c"], input=blob, + stdout=subprocess.PIPE, check=True).stdout + + +TOOLTIP_TABLES = { + 2: "CREATE TABLE Tooltips (id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," + " summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag));" + "CREATE TABLE TooltipButtons (tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT);", + # What migrate_tooltip_schema.py leaves (documentation schema 3). + 3: "CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY AUTOINCREMENT, summary TEXT NOT NULL, detail TEXT NOT NULL);" + "CREATE TABLE Tooltips (id INTEGER NOT NULL, categoryId INTEGER NOT NULL, tag TEXT NOT NULL," + " PRIMARY KEY (categoryId, tag)) WITHOUT ROWID;" + "CREATE TABLE TooltipButtonDescriptions (id INTEGER PRIMARY KEY, description TEXT NOT NULL);" + "CREATE TABLE TooltipButtonUris (id INTEGER PRIMARY KEY, uri TEXT NOT NULL);" + "CREATE TABLE TooltipButtons (tooltipId INTEGER NOT NULL, buttonNumberId INTEGER NOT NULL," + " descriptionId INTEGER NOT NULL, uriId INTEGER NOT NULL, PRIMARY KEY (tooltipId, buttonNumberId)) WITHOUT ROWID;" + "INSERT INTO TooltipButtonDescriptions VALUES (1, 'View full documentation');", +} + + +def _database(tmp_path: Path, rows: dict, dictionary: bytes = None, templates: dict = None, + buttons: list = None, tooltips: list = None, schema: int = 2, versions: list = None) -> Path: + """A miniature documentation database with the production schema's triggers and the tooltip tables of + documentation schema 2 or 3. `templates` maps id -> (name, stored body), `buttons` lists (tooltipId, uri), + `tooltips` (id, summary, detail) and `versions` the DocumentationDatabaseVersion rows as (major, minor, patch, + changeTime); without it there is no such table.""" + tmp_path.mkdir(parents=True, exist_ok=True) + db_path = tmp_path / "documentation.db" + conn = sqlite3.connect(db_path) + conn.executescript( + "CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL);" + "CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL," + " content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE(path));" + "CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL," + " UNIQUE(name));" + "CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL);" + + TOOLTIP_TABLES[schema] + + "CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT ''," + " bookCategoryID INTEGER, UNIQUE(title, bookCategoryId));" + "CREATE TRIGGER AddBook AFTER INSERT ON Content WHEN NEW.path LIKE '%.pdf' BEGIN" + " INSERT INTO Bookshelf (contentID, title) VALUES (NEW.id, CURRENT_TIMESTAMP || NEW.id); END;" + "CREATE TRIGGER DeleteBook AFTER DELETE ON Content WHEN OLD.path LIKE '%.pdf' BEGIN" + " DELETE FROM Bookshelf WHERE contentID = OLD.id; END;" + "INSERT INTO TooltipCategories VALUES (1, 'ide');" + ) + conn.executemany("INSERT INTO ContentTypes (id, value, compression) VALUES (?, ?, ?)", + [(type_id, value, compression) for value, (type_id, compression) in TYPES.items()]) + if dictionary is not None: + conn.execute("CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY, data BLOB NOT NULL)") + conn.execute("INSERT INTO CompressionDictionary (id, data) VALUES (1, ?)", (dictionary,)) + for path, (content_type, raw, *extra) in rows.items(): + template_id, as_is = (list(extra) + [0, False][len(extra):])[:2] + type_id, compression = TYPES[content_type] + stored = raw if as_is or compression == "none" else _compress(raw, dictionary) + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, ?, ?)", + (path, stored, type_id, template_id)) + for template_id, (name, body) in (templates or {}).items(): + conn.execute("INSERT INTO Templates (id, name, content) VALUES (?, ?, ?)", (template_id, name, body)) + if versions is not None: + conn.execute("CREATE TABLE DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL," + " patch INT NOT NULL, who TEXT NOT NULL, comment TEXT NOT NULL," + " changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") + conn.executemany("INSERT INTO DocumentationDatabaseVersion VALUES (?, ?, ?, 'test', '', ?)", versions) + for tooltip_id, summary, detail in tooltips or []: + if schema == 2: + conn.execute("INSERT INTO Tooltips (id, categoryId, tag, summary, detail) VALUES (?, 1, ?, ?, ?)", + (tooltip_id, f"tag{tooltip_id}", summary, detail)) + else: + conn.execute("INSERT INTO TooltipTexts (id, summary, detail) VALUES (?, ?, ?)", (tooltip_id, summary, detail)) + conn.execute("INSERT INTO Tooltips (id, categoryId, tag) VALUES (?, 1, ?)", (tooltip_id, f"tag{tooltip_id}")) + for number, (tooltip_id, uri) in enumerate(buttons or [], start=1): + if schema == 2: + conn.execute("INSERT INTO TooltipButtons VALUES (?, ?, 'View full documentation', ?)", + (tooltip_id, number, uri)) + else: + conn.execute("INSERT INTO TooltipButtonUris (id, uri) VALUES (?, ?)", (number, uri)) + conn.execute("INSERT INTO TooltipButtons VALUES (?, ?, 1, ?)", (tooltip_id, number, number)) + conn.commit() + conn.close() + return db_path + + +def _stored(db_path: Path) -> dict: + conn = sqlite3.connect(db_path) + try: + return {row[0]: row[1:] for row in conn.execute( + "SELECT path, languageID, content, contentTypeID, templateId FROM Content")} + finally: + conn.close() + + +def _table(db_path: Path, sql: str) -> list: + conn = sqlite3.connect(db_path) + try: + return sorted(conn.execute(sql).fetchall()) + finally: + conn.close() + + +def _run(db_path: Path, **overrides) -> tuple: + """(exit code, stdout, stderr) of one run of the tool.""" + cfg = {"db_path": db_path, "dry_run": False, "groups": None, "workers": 2, "verbose": False} + cfg.update(overrides) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = tool.run(cfg) + return code, out.getvalue(), err.getvalue() + + +def _refused(tmp_path: Path, rows: dict, expected: str, **database) -> None: + """Asserts that a run over rows exits 1 naming `expected` and leaves the database and directory as they were.""" + db = _database(tmp_path, rows, **database) + before = _stored(db) + code, _out, err = _run(db) + assert code == 1, f"exit {code}, expected a refusal naming {expected!r}" + assert expected in err, err + assert _stored(db) == before + assert len(list(tmp_path.iterdir())) == 1, "a refused run wrote a backup" + + +def _skip(reason: str) -> None: + try: + import pytest + except ImportError: + raise _Skipped(reason) from None + pytest.skip(reason) + + +class _Skipped(Exception): + pass + + +def test_every_group_deletes_its_rows_and_nothing_else(tmp_path: Path): + rows = _base_rows() + db = _database(tmp_path, rows) + before = _stored(db) + code, out, err = _run(db) + after = _stored(db) + assert code == 0, err + assert set(before) - set(after) == DELETED + for path in after: + if path != tool.VIEWER: + assert after[path] == before[path], path + assert _decompress(after[tool.VIEWER][1]) == VIEWER_MJS_OFF + assert after[tool.VIEWER][0] == before[tool.VIEWER][0] and after[tool.VIEWER][2:] == before[tool.VIEWER][2:] + assert "javadoc-use: 3 row(s)" in out and "pdfjs-cmaps: 3 row(s)" in out + assert "pdfjs-extras: 5 row(s)" in out and "x-placeholders: 2 row(s)" in out + for src, dst in tool.ALLOWED: + assert f"allowed: {src} refers to {dst}" in out + + +def test_only_the_named_groups_run(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + code, out, err = _run(db, groups=["x-placeholders", "javadoc-use"]) + after = _stored(db) + assert code == 0, err + assert set(before) - set(after) == {p for p in DELETED if p.startswith(("x/", "j/"))} + assert after[tool.VIEWER] == before[tool.VIEWER] + assert "pdfjs" not in out + + +def test_the_viewer_is_edited_in_either_form(): + unminified = b"const defaultOptions = {\n enableScripting: {\n value: true,\n kind: OptionKind.VIEWER\n }};" + new, changed = tool.turn_scripting_off(unminified) + assert changed and new == unminified.replace(b"value: true", b"value: false") + new, changed = tool.turn_scripting_off(b"enableScripting:{value:!0,kind:1}") + assert changed and new == b"enableScripting:{value:false,kind:1}" + assert tool.turn_scripting_off(VIEWER_MJS_OFF) == (VIEWER_MJS_OFF, False) + + +def test_a_viewer_without_exactly_one_scripting_default_refuses(tmp_path: Path): + for i, viewer in enumerate((b"const defaultOptions={};", VIEWER_MJS + b";" + VIEWER_MJS)): + rows = _base_rows() + rows[tool.VIEWER] = ("text/javascript", viewer) + _refused(tmp_path / str(i), rows, "so scripting cannot be turned off") + + +def test_a_missing_viewer_refuses_while_the_sandbox_is_stored(tmp_path: Path): + rows = _base_rows() + del rows[tool.VIEWER] + del rows["p/web/viewer.html"] + _refused(tmp_path, rows, "cannot tell that scripting is off") + + +def test_a_database_without_pdfjs_needs_no_viewer(tmp_path: Path): + rows = {path: row for path, row in _base_rows().items() if not path.startswith("p/")} + db = _database(tmp_path, rows) + code, _out, err = _run(db) + assert code == 0, err + + +def test_each_kind_of_link_to_a_deleted_row_refuses(tmp_path: Path): + cases = [ + ("i/a.html", "text/html", b'use', + "links to j/html/api/m/a/class-use/B.html"), + ("i/b.html", "text/html", b"", "links to x/a.html"), + ("i/c.html", "text/html", b'', "links to x/b.html"), + ("i/d.html", "text/html", b'', "links to x/a.html"), + ("i/e.css", "text/css", b".a { background: url( ../p/web/cmaps/LICENSE ) }", "links to p/web/cmaps/LICENSE"), + ("i/f.css", "text/css", b"@import '../p/web/cmaps/LICENSE';", "links to p/web/cmaps/LICENSE"), + ("i/g.svg", "image/svg+xml", b'', + "links to j/html/api/m/module-graph.svg"), + ("i/h.js", "text/javascript", b'load("../p/web/wasm/quickjs-eval.js")', "links to p/web/wasm/quickjs-eval.js"), + ("k/i.js", "text/javascript", b'const name = `${base}/package-use.html`;', + "names j/html/api/m/a/package-use.html"), + ("i/j.html", "text/html", b'a', "links to x/a.html"), + ("i/k.html", "text/html", b'b', "links to x/b.html"), + ("i/l.html", "text/html", b'pdf', "links to x/a.html"), + ("i/l2.html", "text/html", b'pdf', + "links to x/b.html"), + ("i/m.html", "text/html", b'b', "links to x/b.html"), + ("i/n.html", "text/html", b'a', "links to x/a.html"), + ("k/o.html", "text/html", b'{"link": "../x/b.html", "body": "a"}', + "k/o.html links to x/a.html"), + ("k/p.html", "text/html", b'{"next": "../x/b.html"}', "k/p.html links to x/b.html"), + ] + for i, (path, content_type, body, expected) in enumerate(cases): + rows = _base_rows() + rows[path] = (content_type, body, 7) if path.startswith("k/") and path.endswith(".html") else \ + (content_type, body) + _refused(tmp_path / str(i), rows, f"{path} {expected}" if not expected.startswith(path) else expected) + + +def test_a_link_climbing_past_the_root_resolves_as_a_browser_resolves_it(tmp_path: Path): + # A browser stops ".." at the root, so both links reach x/a.html. A kept i/sub/a.html shares the file name, + # so only resolving the path, not the file-name rule, can catch them. + for i, link in enumerate((b"../../x/a.html", b"/../x/a.html")): + rows = _base_rows() + rows["i/sub/a.html"] = ("text/html", b"

kept

") + rows["i/clamp.html"] = ("text/html", b'a' % link) + _refused(tmp_path / str(i), rows, "i/clamp.html links to x/a.html") + assert tool.link_paths("../../../a/b.html", ("i",)) == ["a/b.html"] + assert tool.link_paths("..", ("i",)) == [] + + +def test_a_reference_the_allowlist_names_only_for_another_row_still_refuses(tmp_path: Path): + rows = _base_rows() + rows["p/web/other.mjs"] = ("text/javascript", b'import("../build/pdf.sandbox.mjs")') + _refused(tmp_path, rows, "p/web/other.mjs links to p/build/pdf.sandbox.mjs") + + +def test_a_template_link_refuses(tmp_path: Path): + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + _refused(tmp_path / "up", rows, "template page.peb links to x/a.html", + templates={1: ("page.peb", b'{{ title }}')}) + # A link that resolves only from the directory of the row using the template: from the root it names no + # row, and its file name, B.html, is also a kept row's. + rows = _base_rows() + rows["j/html/api/m/a/T.html"] = ("text/html", b'{"title": "t"}', 1) + _refused(tmp_path / "here", rows, "template page.peb links to j/html/api/m/a/class-use/B.html", + templates={1: ("page.peb", b'{{ title }}')}) + + +# A template that links to a deleted row, and a dictionary that holds it, as the real ones hold theirs. +LINKING_TEMPLATE = b'' * 3 + \ + b'{{ title }}' +TEMPLATE_DICTIONARY = VIEWER_MJS * 4 + LINKING_TEMPLATE + + +def test_a_compressed_template_link_refuses_from_version_3(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + stored = _compress(LINKING_TEMPLATE, TEMPLATE_DICTIONARY) + assert b"x/a.html" not in stored, "the link survives compression, so this tests nothing" + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + # The newest row decides, by changeTime and then rowid, whatever order the rows were written in. + for i, versions in enumerate([[(3, 0, 0, "2026-10-19 07:09:52")], + [(3, 0, 0, "2026-10-19 07:09:52"), (2, 1, 0, "2026-10-18 10:00:00")], + [(2, 1, 0, "2026-10-19 07:09:52"), (3, 0, 0, "2026-10-19 07:09:52")]]): + _refused(tmp_path / str(i), rows, "template page.peb links to x/a.html", dictionary=TEMPLATE_DICTIONARY, + schema=3, versions=versions, templates={1: ("page.peb", stored)}) + + +def test_a_version_3_database_with_compressed_templates_deletes_the_same_rows(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + kept_link = LINKING_TEMPLATE.replace(b"../x/a.html", b"../x.html") + db = _database(tmp_path, rows, dictionary=TEMPLATE_DICTIONARY, schema=3, + versions=[(3, 0, 0, "2026-10-19 07:09:52")], + templates={1: ("page.peb", _compress(kept_link, TEMPLATE_DICTIONARY))}) + before = _stored(db) + code, _out, err = _run(db) + assert code == 0, err + assert set(before) - set(_stored(db)) == DELETED + + +def test_a_plain_template_link_refuses_below_version_3(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + # An empty version table declares no version, as a missing one does. + for i, versions in enumerate([[(2, 1, 0, "2026-10-18 10:00:00")], [], + [(3, 0, 0, "2026-10-17 10:00:00"), (2, 1, 0, "2026-10-18 10:00:00")], + [(3, 0, 0, "2026-10-18 10:00:00"), (2, 1, 0, "2026-10-18 10:00:00")]]): + _refused(tmp_path / str(i), rows, "template page.peb links to x/a.html", dictionary=TEMPLATE_DICTIONARY, + versions=versions, templates={1: ("page.peb", LINKING_TEMPLATE)}) + + +def test_a_template_that_does_not_decode_from_version_3_refuses(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + rows = _base_rows() + rows["k/topic.html"] = ("text/html", b'{"title": "t"}', 1) + # A step run after the migration left this one plain; its link to a kept row would not refuse. + _refused(tmp_path, rows, "template page.peb could not be decoded, so its links were not checked", + dictionary=TEMPLATE_DICTIONARY, schema=3, versions=[(3, 0, 0, "2026-10-19 07:09:52")], + templates={1: ("page.peb", b'{{ title }}')}) + + +def test_a_tooltip_button_refuses_in_either_schema(tmp_path: Path): + for schema in (2, 3): + _refused(tmp_path / str(schema), _base_rows(), "tooltip 3 button links to j/html/api/m/a/package-use.html", + tooltips=[(3, "s", "")], buttons=[(3, "i/index.html"), (3, "j/html/api/m/a/package-use.html#x")], + schema=schema) + + +def test_a_link_in_tooltip_text_refuses_in_either_schema(tmp_path: Path): + for schema in (2, 3): + _refused(tmp_path / f"s{schema}", _base_rows(), "tooltip 4 text links to x/b.html", + tooltips=[(4, 'See b.', "")], schema=schema) + _refused(tmp_path / f"d{schema}", _base_rows(), "tooltip 5 text links to x/a.html", + tooltips=[(5, "s", 'a')], schema=schema) + + +def test_a_schema_3_database_deletes_the_same_rows(tmp_path: Path): + db = _database(tmp_path, _base_rows(), tooltips=[(1, "s", "d")], buttons=[(1, "j/html/api/m/a/B.html")], schema=3) + before = _stored(db) + code, _out, err = _run(db) + assert code == 0, err + assert set(before) - set(_stored(db)) == DELETED + + +def test_tooltip_tables_in_neither_shape_refuse_before_the_scan(tmp_path: Path): + cases = [ + # Half migrated: the schema-2 tables beside a TooltipTexts. + (2, "CREATE TABLE TooltipTexts (id INTEGER PRIMARY KEY, summary TEXT, detail TEXT);"), + # Schema 3 without its uri table. + (3, "DROP TABLE TooltipButtonUris;"), + # Tooltips without their text anywhere. + (2, "CREATE TABLE T2 (id INTEGER, categoryId INTEGER, tag TEXT); DROP TABLE Tooltips;" + " ALTER TABLE T2 RENAME TO Tooltips;"), + # The schema-3 tables, but Tooltips still holding text. + (3, "ALTER TABLE Tooltips ADD COLUMN summary TEXT;"), + ] + for i, (schema, change) in enumerate(cases): + db = _database(tmp_path / str(i), _base_rows(), schema=schema) + conn = sqlite3.connect(db) + conn.executescript(change) + conn.close() + before = _stored(db) + code, out, err = _run(db) + assert code == 1 and "in neither the schema-2 shape" in err, (i, err) + assert "Reading" not in out and "Checking" not in out, (i, out) + assert _stored(db) == before and len(list((tmp_path / str(i)).iterdir())) == 1 + + +def test_a_bookshelf_entry_refuses(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + conn = sqlite3.connect(db) + conn.execute("INSERT INTO Bookshelf (contentID, title) SELECT id, 'placeholder book' FROM Content " + "WHERE path = 'x/a.html'") + conn.commit() + conn.close() + before, shelf = _stored(db), _table(db, "SELECT * FROM Bookshelf") + code, _out, err = _run(db) + assert code == 1 and "Bookshelf lists x/a.html" in err, err + assert _stored(db) == before and _table(db, "SELECT * FROM Bookshelf") == shelf + + +def test_links_to_kept_rows_are_not_refused(tmp_path: Path): + # B.html is kept and class-use/B.html shares only its file name; package-use.html on another host is not + # this database's. + rows = _base_rows() + rows["i/q.html"] = ("text/html", b'B' + b'elsewhere') + db = _database(tmp_path, rows, buttons=[(1, "j/html/api/m/a/B.html")]) + code, _out, err = _run(db) + assert code == 0, err + + +def test_a_row_to_delete_that_is_not_what_its_group_holds_refuses(tmp_path: Path): + cases = [ + ("x/c.html", ("text/html", b"

A real page

"), "x/c.html is not a placeholder page"), + ("j/html/api/m/b/class-use/C.html", ("text/html", b"C"), + "j/html/api/m/b/class-use/C.html is not a javadoc Use page"), + ("j/html/api/n/module-graph.svg", ("image/svg+xml", b"

not svg

"), + "j/html/api/n/module-graph.svg is not an SVG"), + ("p/web/cmaps/notes.html", ("text/html", b"

notes

"), "p/web/cmaps/notes.html is not a CMap"), + ] + for i, (path, row, expected) in enumerate(cases): + rows = _base_rows() + rows[path] = row + _refused(tmp_path / str(i), rows, expected) + + +def test_a_pdf_that_needs_a_cmap_refuses(tmp_path: Path): + cases = [ + (_pdf(b"<< /Type /Font /Subtype /Type0 /BaseFont /MS-Gothic /Encoding /90ms-RKSJ-H >>"), + "uses the predefined CMap 90ms-RKSJ-H"), + (_pdf(_flate(b"/CIDInit /ProcSet findresource begin /90ms-RKSJ-H usecmap begincmap endcmap")), + "uses the predefined CMap 90ms-RKSJ-H"), + (_pdf(_flate(b"<< /Type /Font /Subtype /Type0 /Encoding /Adobe-Japan1-UCS2 >>")), + "uses the predefined CMap Adobe-Japan1-UCS2"), + (_pdf(b"<< /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >>"), + "has a CID font of the Adobe Japan1 collection"), + (_pdf(b"<< /Registry (Adobe) /Ordering <4B 6F72656131> /Supplement 2 >>"), + "has a CID font of the Adobe Korea1 collection"), + (_pdf(b"<< /Type /Font /Subtype /TrueType /BaseFont /#CB#CE#CC#E5 /Encoding /WinAnsiEncoding >>"), + "has a font pdf.js reads as Chinese"), + # pdf.js takes the FontDescriptor's FontName when it differs from the BaseFont. + (_pdf(b"<< /Type /Font /Subtype /TrueType /BaseFont /SimSunX /FontDescriptor 2 0 R" + b" /Encoding /WinAnsiEncoding >>", b"<< /Type /FontDescriptor /FontName /#CB#CE#CC#E5 >>"), + "has a font pdf.js reads as Chinese"), + (_pdf(b"<< /Type /FontDescriptor /FontName (\\313\\316\\314\\345) >>"), "has a font pdf.js reads as Chinese"), + (_pdf(b"<< /Type /FontDescriptor /FontName >>"), "has a font pdf.js reads as Chinese"), + (_pdf(_flate(b"<< /Type /FontDescriptor /FontName (ABCDEF+\xcb\xce\xcc\xe5) >>")), + "has a font pdf.js reads as Chinese"), + ] + for i, (pdf, expected) in enumerate(cases): + rows = _base_rows() + rows["d/cjk.pdf"] = ("application/pdf", pdf) + _refused(tmp_path / str(i), rows, f"d/cjk.pdf {expected}") + + +def test_a_pdf_whose_fonts_cannot_all_be_read_refuses(tmp_path: Path): + import base64 + cmap = "uses the predefined CMap 90ms-RKSJ-H" + cases = [ + (_pdf(_stream(base64.a85encode(zlib.compress(JAPANESE_FONT), adobe=True), + b"/Type /ObjStm /N 1 /First 0 /Filter [/ASCII85Decode /FlateDecode]")), cmap), + (_pdf(_stream(JAPANESE_FONT.hex().encode() + b">", b"/Filter /ASCIIHexDecode")), cmap), + (_pdf(_stream(_run_length(JAPANESE_FONT), b"/Filter /RunLengthDecode")), cmap), + (_pdf(_stream(base64.a85encode(JAPANESE_FONT), b"/Filter /A85")), cmap), + (PLAIN_PDF + b"trailer\n<< /Root 1 0 R /Encrypt 5 0 R >>\n", "is encrypted, so its fonts cannot be read"), + (_pdf(_stream(b"\x80\x0b\x60\x50", b"/Filter /LZWDecode")), "cannot decode (LZWDecode)"), + (_pdf(_stream(b"not zlib data", b"/Filter /FlateDecode")), "cannot decode (FlateDecode: "), + (_pdf(_stream(zlib.compress(b"\x02\x00\x00\x00\x01"), + b"/Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns 4 >>")), + "cannot decode (predictor 12)"), + ] + for i, (pdf, expected) in enumerate(cases): + rows = _base_rows() + rows["d/odd.pdf"] = ("application/pdf", pdf) + _refused(tmp_path / str(i), rows, f"d/odd.pdf {expected}" if expected == cmap else expected) + + +def test_images_and_cross_reference_streams_do_not_refuse(tmp_path: Path): + content = zlib.compress(b"BT /F1 12 Tf (Hello) Tj ET") + pdf = _pdf(_stream(b"\xff\xd8\xff\xe0 not decodable", b"/Type /XObject /Subtype /Image /Filter /DCTDecode"), + _stream(b"\x00\x00\x00\x0cjP ", b"/Filter /JPXDecode"), + _stream(b"\x80\x0b\x60\x50", b"/Subtype /Image /Filter /LZWDecode"), + _stream(b"\x02\x01\x00\x10\x00", b"/Type /XRef /Filter /FlateDecode /DecodeParms << /Predictor 12 >>"), + b"<< /Length 7 0 R /Filter /FlateDecode >>\nstream\n" + content + b"\nendstream", + b"%d" % len(content), + _stream(content, b"/Filter /FlateDecode")) + rows = _base_rows() + rows["d/images.pdf"] = ("application/pdf", pdf) + code, _out, err = _run(_database(tmp_path, rows)) + assert code == 0, err + + +def test_font_names_are_read_as_pdf_js_reads_them(): + names = list(tool.font_names(b"/BaseFont /A#42C /FontName (x\\(y\\)z(n)\\101\\\nq\\\\) /FontName<4142 4>" + b" /FontName <>")) + assert names == [b"ABC", b"x(y)z(n)Aq\\", b"AB@"] + + +def test_the_pdf_check_runs_only_with_the_cmaps(tmp_path: Path): + rows = _base_rows() + rows["d/cjk.pdf"] = ("application/pdf", _pdf(b"<< /Registry (Adobe) /Ordering (Japan1) >>")) + db = _database(tmp_path, rows) + code, _out, err = _run(db, groups=["x-placeholders"]) + assert code == 0, err + + +def test_an_unreadable_kept_row_refuses(tmp_path: Path): + rows = _base_rows() + rows["i/broken.html"] = ("text/html", b"not brotli at all", 0, True) + _refused(tmp_path, rows, "i/broken.html could not be read, so its references were not checked") + + +def test_an_unreadable_row_to_delete_refuses(tmp_path: Path): + # Its group's check cannot see what it holds, so it is not deleted on trust. + rows = _base_rows() + rows["x/c.html"] = ("text/html", b"not brotli at all", 0, True) + _refused(tmp_path, rows, "x/c.html could not be read") + + +def test_an_unreadable_pdf_refuses(tmp_path: Path): + rows = _base_rows() + rows["d/broken.pdf"] = ("application/pdf", b"not brotli at all", 0, True) + _refused(tmp_path, rows, "d/broken.pdf could not be read") + + +def test_scripting_is_turned_off_when_the_sandbox_rows_are_already_gone(tmp_path: Path): + rows = {path: row for path, row in _base_rows().items() if path not in tool.PDFJS_EXTRAS} + db = _database(tmp_path, rows) + before = _stored(db) + code, out, err = _run(db, groups=["pdfjs-extras"]) + after = _stored(db) + assert code == 0, err + assert "enableScripting default set to false" in out and not out.startswith("Nothing to do") + assert set(after) == set(before) and _decompress(after[tool.VIEWER][1]) == VIEWER_MJS_OFF + code, out, _err = _run(db, groups=["pdfjs-extras"]) + assert code == 0 and out.startswith("Nothing to do") and "scripting is already off" in out + + +def test_a_chunked_row_goes_with_its_base_and_others_stay(tmp_path: Path): + import random + rng = random.Random(7) + noise = bytes(rng.getrandbits(8) for _ in range(CHUNK_SIZE + 5000)) + rows = _base_rows() + # Stored as they are: a deleted placeholder spanning two rows, a kept page spanning two, and a page + # that only looks like a chunk row because its base is short. + page = brotli.compress(_placeholder("big") + b"", quality=0) + assert len(page) > CHUNK_SIZE + rows["x/big.html"] = ("text/html", page[:CHUNK_SIZE], 0, True) + rows["x/big.html-1"] = ("text/html", page[CHUNK_SIZE:], 0, True) + kept = brotli.compress(b"

" + noise.hex().encode() + b"

", quality=0) + rows["i/big.html"] = ("text/html", kept[:CHUNK_SIZE], 0, True) + rows["i/big.html-1"] = ("text/html", kept[CHUNK_SIZE:], 0, True) + rows["x/a.html-1"] = ("text/html", _placeholder("a-1")) + db = _database(tmp_path, rows) + code, out, err = _run(db) + after = _stored(db) + assert code == 0, err + assert not {"x/big.html", "x/big.html-1", "x/a.html-1"} & set(after) + assert {"i/big.html", "i/big.html-1"} <= set(after) + assert "x-placeholders: 4 row(s) of x/ placeholder pages and 1 chunk row(s)" in out + + +def test_dry_run_writes_nothing(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + code, out, _err = _run(db, dry_run=True) + assert code == 0 and "would delete 13 row(s)" in out + assert _stored(db) == before + assert len(list(tmp_path.iterdir())) == 1 + + +def test_a_second_run_changes_nothing(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + assert _run(db)[0] == 0 + once = _stored(db) + files = sorted(tmp_path.iterdir()) + code, out, _err = _run(db) + assert code == 0 and out.startswith("Nothing to do") + assert _stored(db) == once and sorted(tmp_path.iterdir()) == files + + +def test_the_backup_holds_the_database_as_it_was(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + code, out, _err = _run(db) + backup = Path(re.search(r"Backup written to (.+)", out).group(1).strip()) + assert code == 0 and backup.is_file() + assert _stored(backup) == before + + +def test_an_error_while_writing_rolls_everything_back(tmp_path: Path): + db = _database(tmp_path, _base_rows()) + before = _stored(db) + real = tool.delete_media_row + calls = [] + + def failing(conn, path, length): + calls.append(path) + if len(calls) == 3: + raise RuntimeError("disk full") + return real(conn, path, length) + + tool.delete_media_row = failing + try: + try: + _run(db) + except RuntimeError: + pass + else: + raise AssertionError("the failure did not propagate") + finally: + tool.delete_media_row = real + assert len(calls) == 3 and _stored(db) == before + + +def test_a_dictionary_database_is_read_and_written_through_the_dictionary(tmp_path: Path): + if shutil.which("brotli") is None: + _skip("brotli is not on PATH") + dictionary = VIEWER_MJS * 4 + USE_PAGE * 4 + db = _database(tmp_path, _base_rows(), dictionary=dictionary) + code, _out, err = _run(db) + assert code == 0, err + assert _decompress(_stored(db)[tool.VIEWER][1], dictionary) == VIEWER_MJS_OFF + + +def test_a_missing_database_is_an_error(tmp_path: Path): + code, _out, err = _run(tmp_path / "absent.db") + assert code == 1 and "does not exist" in err + + +def _main() -> int: + """Runs the tests without pytest installed as a runner (`uv run scripts/test_drop_unread_rows.py`).""" + failures = [] + for name, function in sorted(globals().items()): + if not name.startswith("test_") or not callable(function): + continue + with tempfile.TemporaryDirectory() as directory: + needs_tmp = "tmp_path" in function.__code__.co_varnames[:function.__code__.co_argcount] + try: + function(Path(directory)) if needs_tmp else function() + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: # noqa: BLE001 - a failure is the result here + if type(exc).__name__ in ("Skipped", "_Skipped"): + print(f"skip {name}: {exc}") + continue + frame = traceback.extract_tb(exc.__traceback__)[-1] + failures.append(f"{name} (line {frame.lineno}: {frame.line}): {type(exc).__name__}: {exc}") + print(f"FAIL {name}", file=sys.stderr) + else: + print(f"ok {name}") + print() + print(f"{len(failures)} failure(s)" if failures else "all tests passed") + for failure in failures: + print(f" {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(_main())