diff --git a/.github/workflows/build-android-docs-local.yaml b/.github/workflows/build-android-docs-local.yaml deleted file mode 100644 index ad888039..00000000 --- a/.github/workflows/build-android-docs-local.yaml +++ /dev/null @@ -1,333 +0,0 @@ -name: Build Android Docs (Local) - -# Local-filesystem counterpart of build-android-docs.yaml: the same five steps -# (android_docs_to_json -> load_android_json_db -> verify_android_json_db -> -# remint_dictionary -> verify_remint_dictionary), but reading its -# documentation.db input from a path on the runner's own disk (db_path) instead -# of Google Drive, and writing its outputs (the built database, a run-numbered -# copy) back to disk (output_dir / db_path) instead of uploading them to Drive. -# No GCP Workload Identity Federation, Drive API, or associated secrets are -# used anywhere in this file. The Java equivalent is build-java-docs-local.yaml; -# this file deliberately mirrors its shape. -# -# Since a GitHub-hosted runner is a fresh, disposable VM with no access to -# anyone's actual local disk, db_path/output_dir only make sense here against a -# self-hosted runner, or when this workflow is run locally (e.g. via -# https://github.com/nektos/act) with those host paths bind-mounted into the -# job's container at the paths you pass as inputs. run-build-android-docs-with-act.sh -# does exactly that. -# -# CAUTION when invoking act by hand: act does NOT apply workflow_dispatch input -# defaults. Every "default:" below applies on real GitHub and is simply absent -# under act, so an input you don't pass arrives empty. That matters most for -# dry_run, whose default is true: with it unset, "${{ !inputs.dry_run }}" -# evaluates to true and the final step writes the rebuilt database back over -# db_path. Always pass --input dry_run=true/false explicitly. -# -# NOTE ON DISK AND TIME: the conversion writes ~350 MB of JSON and the load -# copies the whole database, so a run needs roughly 1 GB of free space beside -# the checkout. The re-mint recompresses every row in the database at Brotli -# q11 - not just the Android ones - and is most of the wall clock: ~5.5 minutes -# against the real database, where converting takes ~1 and loading ~1.5. Use -# --only for a subset run that exercises the whole pipeline in seconds, or -# --skip-remint to leave the dictionary alone. -# -# NOTE ON MEMORY: nothing here is memory-hungry the way the Java pipeline's -# Dokka pass is. The conversion runs one BeautifulSoup parse per page across -# worker processes, and the largest page in the corpus (android.app.Activity, -# 1.2 MB of HTML) parses in well under a second. A default container is fine. -# -# Optional secret (Slack notifications are skipped with a warning if unset) - -# same as build-android-docs.yaml: -# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps -# below ("Grabbing baton" on start, "...Dropping -# baton" on finish - org shorthand for lock -# acquire/release, since this workflow mutates a -# single shared local file, db_path). - -permissions: - contents: read - -# This workflow overwrites a single shared local file (db_path) - never let -# two runs race to write it at the same time. -concurrency: - group: build-android-docs-local - cancel-in-progress: false - -on: - workflow_dispatch: - inputs: - db_path: - description: >- - Path on this runner's disk to the input documentation.db. Read - directly (no download/unzip) and, unless dry_run is true, written - back to this same path when the run finishes. - required: true - output_dir: - description: >- - Directory on this runner's disk to write outputs into: a - run-numbered copy of the built database - (documentation-db-.db). Created if it doesn't already - exist. - required: false - default: 'build-android-docs-output' - only: - description: >- - Convert only the pages whose path contains this substring, e.g. - "android/app/". Turns a multi-minute run into seconds, which is the - way to smoke-test a change. Links into the rest of the tree still - resolve (the whole corpus is indexed either way), but the database - will hold a mix of JSON and untouched HTML rows, so pair it with - verify_complete=false. - required: false - default: '' - sample_size: - description: >- - How many stored pages to pull back out of the built database and - render through their stored template. The check is not free - each - one starts a JVM - so this samples rather than rendering all 12,906. - required: false - default: '40' - verify_complete: - description: >- - Fail if any Android row is still serving raw HTML, i.e. if the load - did not cover the whole corpus. Turn it off only for an `only` run, - which by definition leaves the rest behind. - required: false - default: true - type: boolean - skip_remint: - description: >- - Skip re-training the shared Brotli dictionary and recompressing every - row. Right in two cases: a structural check whose output nobody will - ship, and a re-run against a database whose dictionary was already - minted for this corpus and whose content has not materially changed - since - re-minting is not cumulative, and a second pass over an - already-minted database measured -0.7%, i.e. five and a half minutes - to get marginally worse. Otherwise leave it off: on a database coming - from HTML it is worth about 16% (see WHY THE RE-MINT IS PART OF THE - JOB above). - required: false - default: false - type: boolean - dry_run: - description: >- - If true, build and verify everything but do NOT write the result - back to db_path - the input file on disk is left untouched. Set to - false only once you trust a given commit. - required: false - default: true - type: boolean - -jobs: - build-android-docs-local: - runs-on: ubuntu-latest - timeout-minutes: 90 - env: - DB_PATH: ${{ inputs.db_path }} - OUTPUT_DIR: ${{ inputs.output_dir }} - ONLY: ${{ inputs.only }} - SAMPLE_SIZE: ${{ inputs.sample_size }} - # Where each stage writes. JSON_OUT is ~350 MB and STAGED_DB is the - # database being built; both stay in the workspace. - JSON_OUT: ${{ github.workspace }}/android-json-build - STAGED_DB: ${{ github.workspace }}/documentation-android.db - PREREMINT_DB: ${{ github.workspace }}/preremint.db - MODULE: ProcessDocs/AndroidDocs/AndroidDocsToJson - KOTLIN_PIPELINE: ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON - steps: - - name: Checkout OfflineDocumentationTools - uses: actions/checkout@v4 - - - name: Resolve local file paths - run: | - if [ ! -f "$DB_PATH" ]; then - echo "Error: db_path '$DB_PATH' does not exist on this runner - for a self-hosted runner this must be a path on that machine; for act, bind-mount it into the container so it's visible at this exact path" >&2 - exit 1 - fi - mkdir -p "$OUTPUT_DIR" - echo "Resolved DB_PATH: $DB_PATH" - echo "Resolved OUTPUT_DIR: $OUTPUT_DIR" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Set up JDK (for the Pebble renderer the verification step runs) - uses: actions/setup-java@v4 - with: - distribution: temurin - # The renderer's Gradle build targets 17 and its wrapper is happy on - # anything from 17 up. Nothing here is pinned to an old Kotlin the - # way kdoc-to-json is, so this has no upper bound. - java-version: '17' - - - name: Install system dependencies - run: | - sudo apt-get update -y - # brotli: the CLI, not the Python package. DictionaryCompressor shells - # out to it because no Python binding exposes a custom dictionary - # (ADFA-5153). - # zstd: its --train-fastcover trainer is what mints the Brotli - # dictionary; Brotli has no trainer of its own (see train_dictionary). - sudo apt-get install -y unzip zip sqlite3 brotli zstd - - - name: Install Python dependencies - run: | - pip install -r requirements.txt - # pytest: the "Run the unit tests" step below needs it, and nothing else installs it. - # setup-python gives a clean interpreter, so the runner image's own packages are not - # on this path - without this line that step dies on `No module named pytest`. - pip install pytest - - - name: Build the Pebble renderer - run: | - # verify_android_json_db.py renders stored pages through their stored - # templates to prove the database works, and shells out to this jar to - # do it. Built once here rather than per page. - (cd "$MODULE/renderer" && ./gradlew --console=plain -q installDist) - - - name: Run the unit tests - run: | - # Cheap (under a second for both suites) and worth having in front of - # a run that rewrites a shared database. - (cd "$MODULE" && python3 -m pytest -q) - (cd "$MODULE/renderer" && ./gradlew --console=plain test) - - - name: Copy documentation.db from local disk - run: | - cp "$DB_PATH" documentation.db - sqlite3 documentation.db "SELECT 1;" > /dev/null - echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" - - # dry_run-gated, and paired with the release below. The baton stands for the lock on the - # shared file this workflow rewrites, so a dry run - which rewrites nothing - should never - # take it. Before this was gated, the default dispatch (dry_run is true) grabbed a baton - # that the release, gated on dry_run being false, then never dropped. - - name: 'Notify Slack: build started' - id: slack_baton_taken - if: ${{ !inputs.dry_run }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - run: | - if [ -z "$SLACK_WEBHOOK_URL" ]; then - echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 - else - curl -sS -X POST -H 'Content-type: application/json' \ - --data '{"text": "Grabbing baton"}' \ - "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 - fi - - - name: 'Step 1/5: android_docs_to_json.py (scraped HTML -> JSON documentation tree)' - run: | - ARGS=(ProcessDocs/AndroidDocs/android ProcessDocs/AndroidDocs/androidx --out "$JSON_OUT") - [ -n "$ONLY" ] && ARGS+=(--only "$ONLY") - python3 "$MODULE/android_docs_to_json.py" "${ARGS[@]}" - - - name: 'Step 2/5: load_android_json_db.py (JSON -> Content rows, templates, stylesheet)' - run: | - python3 "$MODULE/load_android_json_db.py" "$JSON_OUT" \ - --source documentation.db --out "$STAGED_DB" --force - - - name: 'Step 3/5: verify_android_json_db.py (render stored pages from the database)' - run: | - # The question the loader cannot answer: do the rows it wrote render? - # This pulls the stored JSON and the stored template back out and runs - # them through Pebble configured the way the server configures it. - ARGS=("$STAGED_DB" --sample "$SAMPLE_SIZE") - python3 "$MODULE/verify_android_json_db.py" "${ARGS[@]}" - - - name: 'Step 4/5: remint_dictionary.py (re-train the shared dictionary, recompress)' - if: ${{ !inputs.skip_remint }} - run: | - cp "$STAGED_DB" "$PREREMINT_DB" - python3 "$KOTLIN_PIPELINE/remint_dictionary.py" "$STAGED_DB" --no-backup - - - name: 'Step 5/5: verify_remint_dictionary.py (every row still decodes to the same bytes)' - if: ${{ !inputs.skip_remint }} - run: | - # Not paranoia: a row recompressed against a mismatched dictionary - # decodes without error into *different* bytes (measured at 38% in - # that script's own header), so nothing at runtime would catch it. - python3 "$KOTLIN_PIPELINE/verify_remint_dictionary.py" "$PREREMINT_DB" "$STAGED_DB" - - - name: Summary - run: | - python3 - "$STAGED_DB" "$JSON_OUT" "${{ inputs.verify_complete }}" <<'PYEOF' - import pathlib - import sqlite3 - import sys - - db_path, json_out, verify_complete = sys.argv[1], sys.argv[2], sys.argv[3] == "true" - conn = sqlite3.connect(db_path) - - def count(where, params=()): - return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] - - android = "path LIKE 'a/android/%' OR path LIKE 'a/androidx/%' OR path = 'a/index.html'" - templated = count(f"({android}) AND templateId != 0") - raw_html = count(f"({android}) AND templateId = 0") - stored = conn.execute( - f"SELECT SUM(length(content)) FROM Content WHERE ({android}) AND templateId != 0" - ).fetchone()[0] or 0 - pages_on_disk = sum(1 for _ in pathlib.Path(json_out).rglob("*.json")) - - print(f"Database: {db_path}") - print(f" JSON pages generated : {pages_on_disk:,}") - print(f" a/android* rows, JSON+template : {templated:,}") - print(f" a/android* rows, still raw HTML : {raw_html:,}") - summaries = count("path LIKE 'a/%package-summary.html'") - templates = [n for (n,) in conn.execute("SELECT name FROM Templates ORDER BY id")] - print(f" package summaries : {summaries:,}") - print(f" stored bytes of the JSON rows : {stored:,}") - print(f" templates : {templates}") - conn.close() - - if templated == 0: - print("FAIL: no Android rows are pointing at a template - the load did nothing.") - sys.exit(1) - # Fewer templated rows than pages generated means the load dropped some. More is - # normal and not an error: an `only` run against a database a previous full run - # already covered leaves every other page templated from before. - if templated < pages_on_disk: - print(f"FAIL: {pages_on_disk:,} pages were generated but only {templated:,} rows " - f"are templated - the load did not cover what the conversion produced.") - sys.exit(1) - if verify_complete and raw_html: - print(f"FAIL: {raw_html:,} Android row(s) are still serving raw HTML. Every page in " - f"the corpus should have been replaced; pass verify_complete=false only for " - f"an `only` run, which deliberately leaves the rest behind.") - sys.exit(1) - print("PASS") - PYEOF - - - name: Write built database to output_dir - run: | - cp "$STAGED_DB" "$OUTPUT_DIR/documentation-db-${{ github.run_number }}.db" - echo "Wrote $OUTPUT_DIR/documentation-db-${{ github.run_number }}.db" - - - name: Write updated database back to db_path - if: ${{ !inputs.dry_run }} - run: | - cp "$STAGED_DB" "$DB_PATH" - echo "Wrote updated documentation.db back to $DB_PATH" - - # always(), keyed off whether the acquire actually ran rather than off dry_run again. - # Everything from the checkout to the Drive download can fail before "build started" is - # reached, and a release gated only on dry_run announces a drop for a baton nobody took; - # without always(), a failure after it leaves the baton held. The acquire's own conclusion - # answers both - 'skipped' on a dry run and on an early failure, 'success' exactly when - # Slack was told the baton had been taken. - - name: 'Notify Slack: build complete' - if: ${{ always() && steps.slack_baton_taken.conclusion == 'success' }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - run: | - if [ -z "$SLACK_WEBHOOK_URL" ]; then - echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 - else - curl -sS -X POST -H 'Content-type: application/json' \ - --data '{"text": "Updated Android documentation. Dropping baton"}' \ - "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 - fi diff --git a/.github/workflows/build-android-docs.yaml b/.github/workflows/build-android-docs.yaml deleted file mode 100644 index 2beb2dc1..00000000 --- a/.github/workflows/build-android-docs.yaml +++ /dev/null @@ -1,382 +0,0 @@ -name: Build Android Docs - -# CI counterpart of ProcessDocs/AndroidDocs/AndroidDocsToJson's pipeline - the -# same five steps (android_docs_to_json -> load_android_json_db -> -# verify_android_json_db -> remint_dictionary -> verify_remint_dictionary), but -# reading and writing the real database on Google Drive (GOOGLE_DRIVE_FILE_ID) -# instead of a local copy. build-android-docs-local.yaml is the same workflow -# against a path on disk; this file deliberately mirrors build-kotlin-docs.yaml -# and build-java-docs.yaml in shape. -# -# What it does: converts the scraped developer.android.com reference under -# ProcessDocs/AndroidDocs into a JSON documentation tree, replaces the 12,106 -# rows of scraped HTML in the database with it, adds the 800 package summaries -# and root index the scrape has no pages for, and re-mints the shared Brotli -# dictionary over the changed corpus. -# -# NO EXTERNAL SOURCE CHECKOUT: unlike the Kotlin workflow (which clones -# kotlin-web-site) and the Java one (which unpacks a JDK's lib/src.zip), the -# input is already in this repository - the 12,106 derived HTML pages under -# ProcessDocs/AndroidDocs/{android,androidx}, committed here. The 9 GB raw -# scrape they were derived from lives on Drive and is not needed: see -# ProcessDocs/AndroidDocs/README.md. So a run is reproducible from a commit -# alone, and there is no ref input to pin. -# -# WHY THE RE-MINT IS PART OF THE JOB, not a follow-up: content is compressed -# against one shared dictionary, and the database's was trained when the corpus -# was mostly HTML with the site's stylesheet inlined thousands of times over. It -# compresses those old rows extremely well and has never seen the shape of the -# new ones, so swapping HTML for JSON *alone* makes the Android pages about 7% -# larger. Re-training over the changed corpus takes that back and more - 29.7 MB -# of HTML becomes 27.4 MB of JSON across 800 more pages. skip_remint exists for -# a quick structural check, not for a run whose output anyone will ship. -# -# TIMING, measured on the real corpus (a GitHub-hosted runner will differ, but -# the proportions hold): ~1 min to convert, ~1.5 min to load, ~5.5 min to -# re-mint and ~2 min to verify the re-mint. The re-mint recompresses every row -# in the database at Brotli q11, not just the Android ones, which is why it -# dominates. Budget ~15 minutes for a full run. -# -# Required secrets (already configured - see docdb-regression-test.yaml for -# their other use in this repo): -# GCP_WIF_PROVIDER - Workload Identity Federation provider name -# GCP_WIF_SERVICE_ACCOUNT - Service account email for WIF (needs write -# access - not just view - to the database file, -# since this workflow overwrites it) -# GOOGLE_DRIVE_FILE_ID - File ID of the production documentation.db -# (stored on Drive as a zip) -# -# Optional secret (Slack notifications are skipped with a warning if unset): -# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps -# below ("Grabbing baton" on start, "...Dropping -# baton" on finish - org shorthand for lock -# acquire/release, since this workflow mutates a -# single shared Drive file). - -permissions: - contents: read - id-token: write - -# Every workflow that rewrites the shared documentation.db on Drive shares this group, not just -# the two runs of this one. Each does download -> modify -> upload against the same file, so an -# Android run overlapping a Kotlin run means both start from the same revision and whichever -# uploads second silently discards the other's work. A per-workflow group never saw that. -concurrency: - group: documentation-db - cancel-in-progress: false - -on: - workflow_dispatch: - inputs: - only: - description: >- - Convert only the pages whose path contains this substring, e.g. - "android/app/". Turns a multi-minute run into seconds, which is the - way to smoke-test a change. Links into the rest of the tree still - resolve (the whole corpus is indexed either way), but the database - will hold a mix of JSON and untouched HTML rows, so pair it with - verify_complete=false. - required: false - default: '' - sample_size: - description: >- - How many stored pages to pull back out of the built database and - render through their stored template. The check is not free - each - one starts a JVM - so this samples rather than rendering all 12,906. - required: false - default: '40' - verify_complete: - description: >- - Fail if any Android row is still serving raw HTML, i.e. if the load - did not cover the whole corpus. Turn it off only for an `only` run, - which by definition leaves the rest behind. - required: false - default: true - type: boolean - skip_remint: - description: >- - Skip re-training the shared Brotli dictionary and recompressing every - row. Right in two cases: a structural check whose output nobody will - ship, and a re-run against a database whose dictionary was already - minted for this corpus and whose content has not materially changed - since - re-minting is not cumulative, and a second pass over an - already-minted database measured -0.7%, i.e. five and a half minutes - to get marginally worse. Otherwise leave it off: on a database coming - from HTML it is worth about 16% (see WHY THE RE-MINT IS PART OF THE - JOB above). - required: false - default: false - type: boolean - dry_run: - description: >- - If true, build and verify everything but do NOT upload the result - back to Google Drive - the production database is left untouched. - Set to false only once you trust a given commit. - required: false - default: true - type: boolean - -jobs: - build-android-docs: - runs-on: ubuntu-latest - timeout-minutes: 90 - env: - DB_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_FILE_ID }} - ONLY: ${{ inputs.only }} - SAMPLE_SIZE: ${{ inputs.sample_size }} - # Where each stage writes. JSON_OUT is ~350 MB and STAGED_DB is the - # database being built; both stay in the workspace. - JSON_OUT: ${{ github.workspace }}/android-json-build - STAGED_DB: ${{ github.workspace }}/documentation-android.db - PREREMINT_DB: ${{ github.workspace }}/preremint.db - MODULE: ProcessDocs/AndroidDocs/AndroidDocsToJson - KOTLIN_PIPELINE: ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON - # --- Hard-coded override for one-off manual testing ------------------ - # Fill in with a literal Google Drive file ID to bypass the secret above - # for a quick, repeatable test run (e.g. against a scratch copy of the - # database on Drive). Leave empty ('') for normal operation. - TEST_DB_FILE_ID: '' - steps: - - name: Checkout OfflineDocumentationTools - uses: actions/checkout@v4 - - - name: Resolve Google Drive file ID - run: | - DB_FILE_ID="${TEST_DB_FILE_ID:-$DB_FILE_ID_SECRET}" - if [ -z "$DB_FILE_ID" ]; then - echo "Error: no database file ID resolved - set the GOOGLE_DRIVE_FILE_ID secret, or TEST_DB_FILE_ID above for a test run" >&2 - exit 1 - fi - echo "Resolved DB_FILE_ID: ${DB_FILE_ID:+(set)}" - echo "DB_FILE_ID=$DB_FILE_ID" >> "$GITHUB_ENV" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Set up JDK (for the Pebble renderer the verification step runs) - uses: actions/setup-java@v4 - with: - distribution: temurin - # The renderer's Gradle build targets 17 and its wrapper is happy on - # anything from 17 up. Nothing here is pinned to an old Kotlin the - # way kdoc-to-json is, so this has no upper bound. - java-version: '17' - - - name: Install system dependencies - run: | - sudo apt-get update -y - # brotli: the CLI, not the Python package. DictionaryCompressor shells - # out to it because no Python binding exposes a custom dictionary - # (ADFA-5153). - # zstd: its --train-fastcover trainer is what mints the Brotli - # dictionary; Brotli has no trainer of its own (see train_dictionary). - sudo apt-get install -y unzip zip sqlite3 brotli zstd - - - name: Install Python dependencies - run: | - pip install -r requirements.txt - # pytest: the "Run the unit tests" step below needs it, and nothing else installs it. - # setup-python gives a clean interpreter, so the runner image's own packages are not - # on this path - without this line that step dies on `No module named pytest`. - pip install pytest - # google-api-python-client & friends: Drive download/upload, same - # libraries check-tools/download_database.py already depends on. - pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib - - - name: Build the Pebble renderer - run: | - # verify_android_json_db.py renders stored pages through their stored - # templates to prove the database works, and shells out to this jar to - # do it. Built once here rather than per page. - (cd "$MODULE/renderer" && ./gradlew --console=plain -q installDist) - - - name: Run the unit tests - run: | - # Cheap (under a second for both suites) and worth having in front of - # a run that rewrites a shared database. - (cd "$MODULE" && python3 -m pytest -q) - (cd "$MODULE/renderer" && ./gradlew --console=plain test) - - - name: Authenticate to Google Cloud using Workload Identity Federation - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} - service_account: ${{ secrets.GCP_WIF_SERVICE_ACCOUNT }} - # drive.readonly as well as drive.file: drive.file covers only files this app - # created or opened itself, and the step below downloads a database it did not. - # The same pair docdb-regression-test.yaml uses against the same file. - access_token_scopes: | - https://www.googleapis.com/auth/drive.readonly - https://www.googleapis.com/auth/drive.file - - - name: Download current documentation.db from Google Drive - run: | - python3 check-tools/download_database.py "$DB_FILE_ID" documentation.zip - unzip -o documentation.zip - if [ ! -f documentation.db ]; then - found="$(find . -maxdepth 2 -name documentation.db | head -n1)" - [ -n "$found" ] && mv "$found" documentation.db - fi - test -f documentation.db - sqlite3 documentation.db "SELECT 1;" > /dev/null - rm -f documentation.zip - echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" - - # dry_run-gated, and paired with the release below. The baton stands for the lock on the - # shared file this workflow rewrites, so a dry run - which rewrites nothing - should never - # take it. Before this was gated, the default dispatch (dry_run is true) grabbed a baton - # that the release, gated on dry_run being false, then never dropped. - - name: 'Notify Slack: build started' - id: slack_baton_taken - if: ${{ !inputs.dry_run }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - run: | - if [ -z "$SLACK_WEBHOOK_URL" ]; then - echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 - else - curl -sS -X POST -H 'Content-type: application/json' \ - --data '{"text": "Grabbing baton"}' \ - "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 - fi - - - name: 'Step 1/5: android_docs_to_json.py (scraped HTML -> JSON documentation tree)' - run: | - ARGS=(ProcessDocs/AndroidDocs/android ProcessDocs/AndroidDocs/androidx --out "$JSON_OUT") - [ -n "$ONLY" ] && ARGS+=(--only "$ONLY") - python3 "$MODULE/android_docs_to_json.py" "${ARGS[@]}" - - - name: 'Step 2/5: load_android_json_db.py (JSON -> Content rows, templates, stylesheet)' - run: | - python3 "$MODULE/load_android_json_db.py" "$JSON_OUT" \ - --source documentation.db --out "$STAGED_DB" --force - - - name: 'Step 3/5: verify_android_json_db.py (render stored pages from the database)' - run: | - # The question the loader cannot answer: do the rows it wrote render? - # This pulls the stored JSON and the stored template back out and runs - # them through Pebble configured the way the server configures it. - ARGS=("$STAGED_DB" --sample "$SAMPLE_SIZE") - python3 "$MODULE/verify_android_json_db.py" "${ARGS[@]}" - - - name: 'Step 4/5: remint_dictionary.py (re-train the shared dictionary, recompress)' - if: ${{ !inputs.skip_remint }} - run: | - cp "$STAGED_DB" "$PREREMINT_DB" - python3 "$KOTLIN_PIPELINE/remint_dictionary.py" "$STAGED_DB" --no-backup - - - name: 'Step 5/5: verify_remint_dictionary.py (every row still decodes to the same bytes)' - if: ${{ !inputs.skip_remint }} - run: | - # Not paranoia: a row recompressed against a mismatched dictionary - # decodes without error into *different* bytes (measured at 38% in - # that script's own header), so nothing at runtime would catch it. - python3 "$KOTLIN_PIPELINE/verify_remint_dictionary.py" "$PREREMINT_DB" "$STAGED_DB" - - - name: Summary - run: | - python3 - "$STAGED_DB" "$JSON_OUT" "${{ inputs.verify_complete }}" <<'PYEOF' - import pathlib - import sqlite3 - import sys - - db_path, json_out, verify_complete = sys.argv[1], sys.argv[2], sys.argv[3] == "true" - conn = sqlite3.connect(db_path) - - def count(where, params=()): - return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] - - android = "path LIKE 'a/android/%' OR path LIKE 'a/androidx/%' OR path = 'a/index.html'" - templated = count(f"({android}) AND templateId != 0") - raw_html = count(f"({android}) AND templateId = 0") - stored = conn.execute( - f"SELECT SUM(length(content)) FROM Content WHERE ({android}) AND templateId != 0" - ).fetchone()[0] or 0 - pages_on_disk = sum(1 for _ in pathlib.Path(json_out).rglob("*.json")) - - print(f"Database: {db_path}") - print(f" JSON pages generated : {pages_on_disk:,}") - print(f" a/android* rows, JSON+template : {templated:,}") - print(f" a/android* rows, still raw HTML : {raw_html:,}") - summaries = count("path LIKE 'a/%package-summary.html'") - templates = [n for (n,) in conn.execute("SELECT name FROM Templates ORDER BY id")] - print(f" package summaries : {summaries:,}") - print(f" stored bytes of the JSON rows : {stored:,}") - print(f" templates : {templates}") - conn.close() - - if templated == 0: - print("FAIL: no Android rows are pointing at a template - the load did nothing.") - sys.exit(1) - # Fewer templated rows than pages generated means the load dropped some. More is - # normal and not an error: an `only` run against a database a previous full run - # already covered leaves every other page templated from before. - if templated < pages_on_disk: - print(f"FAIL: {pages_on_disk:,} pages were generated but only {templated:,} rows " - f"are templated - the load did not cover what the conversion produced.") - sys.exit(1) - if verify_complete and raw_html: - print(f"FAIL: {raw_html:,} Android row(s) are still serving raw HTML. Every page in " - f"the corpus should have been replaced; pass verify_complete=false only for " - f"an `only` run, which deliberately leaves the rest behind.") - sys.exit(1) - print("PASS") - PYEOF - - - name: Upload built database as workflow artifact - uses: actions/upload-artifact@v4 - with: - name: documentation-db-${{ github.run_number }} - path: ${{ env.STAGED_DB }} - retention-days: 14 - - - name: Zip updated database for upload - if: ${{ !inputs.dry_run }} - run: | - # The member inside the zip has to be called documentation.db, not whatever this - # workflow happened to name its staging file: every reader of the Drive artifact looks - # for that exact name and fails outright without it - this workflow's own download step - # above, docdb-regression-test.yaml's daily run, and build-kotlin-docs.yaml. `zip -j` - # drops the directory but keeps the basename, so the rename has to happen first. - cp "$STAGED_DB" documentation.db - zip -j documentation.zip documentation.db - - - name: Upload updated database to Google Drive - if: ${{ !inputs.dry_run }} - run: | - python3 - <<'PYEOF' - import os - from google.auth import default - from googleapiclient.discovery import build - from googleapiclient.http import MediaFileUpload - - file_id = os.environ["DB_FILE_ID"] - credentials, _ = default() - service = build("drive", "v3", credentials=credentials) - media = MediaFileUpload("documentation.zip", mimetype="application/zip", resumable=True) - updated = service.files().update( - fileId=file_id, media_body=media, fields="id, modifiedTime, md5Checksum" - ).execute() - print(f"Uploaded new revision of {file_id}: {updated}") - PYEOF - - # always(), keyed off whether the acquire actually ran rather than off dry_run again. - # Everything from the checkout to the Drive download can fail before "build started" is - # reached, and a release gated only on dry_run announces a drop for a baton nobody took; - # without always(), a failure after it leaves the baton held. The acquire's own conclusion - # answers both - 'skipped' on a dry run and on an early failure, 'success' exactly when - # Slack was told the baton had been taken. - - name: 'Notify Slack: build complete' - if: ${{ always() && steps.slack_baton_taken.conclusion == 'success' }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - run: | - if [ -z "$SLACK_WEBHOOK_URL" ]; then - echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 - else - curl -sS -X POST -H 'Content-type: application/json' \ - --data '{"text": "Updated Android documentation. Dropping baton"}' \ - "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 - fi diff --git a/.github/workflows/build-kotlin-docs.yaml b/.github/workflows/build-kotlin-docs.yaml index fa65ba96..549eddba 100644 --- a/.github/workflows/build-kotlin-docs.yaml +++ b/.github/workflows/build-kotlin-docs.yaml @@ -47,8 +47,6 @@ permissions: # This workflow overwrites a single shared Drive file - never let two runs # race to upload against each other. -# Shared with build-android-docs.yaml: both rewrite the same documentation.db on Drive, so they -# have to serialise against each other, not merely against a second run of themselves. concurrency: group: documentation-db cancel-in-progress: false diff --git a/CLAUDE.md b/CLAUDE.md index 5149c163..051142b0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -186,6 +186,27 @@ uses. Any future Android/Java tooltip work should target the normalized `Tooltip (ADFA-6589); this step ships after both. A media rollback (`--restore-originals`) cannot undo its pages: start the rollback from this step's pre-run backup, keep that backup with the rollback bundle, and run this step again afterwards. +- **`scripts/android_pages_to_templates.py`** - stores the Android reference pages under `a/` + (except `a/devsite/`) as JSON plus `android-reference.peb` and one shared include per + inherited-member block header (`android-inherited-.peb`), with every text of 32 or more + characters that 5 or more pages hold (link holes excepted) stored once in a shared-text bundle + (`android-text-.peb`, at most 8,000 bytes of source each, texts grouped by the pages that use + them), after rendering every page back + byte for byte with the app's engine (`scripts/hm_render_check/`, as above); a page that does not + render back, or is over the app's render limit, stays HTML (ADFA-3305). Run it on the ship-time + copy only, never on the master: after the steps that write `a/` pages, `drop_unread_rows.py` and + the Kotlin trim, before the 3.0.0 migration (which compresses every template; the app reads + compressed templates from documentation schema 3.0.0, CodeOnTheGo stacked on #2163) and the final + dictionary re-mint. + It refuses a database that already holds its templates, or that declares documentation version 3 + or later (whose templates are compressed, so its plain ones would not render). + Following `drop_unread_rows.py` is an exception to that step's "after every page-writing step" + rule, and safe: every converted page renders back byte for byte, so it links to nothing the row + drops did not already check. + ADFA-5548's `ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py` converts the same + `a/android/` and `a/androidx/` pages its own way. This step is the one that ships: the workflows + that ran that pipeline on the master (`build-android-docs.yaml`, `build-android-docs-local.yaml`) + were retired in ADFA-3305, never having run, since after it this step would convert almost nothing. - **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/README.md b/ProcessDocs/AndroidDocs/AndroidDocsToJson/README.md index 8c5467e1..1675d325 100644 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/README.md +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/README.md @@ -331,36 +331,14 @@ a row whose type and template are not changing) but not here, where a scraped pa 0 and has to come out templated. So the columns that say how to serve a row are set afterwards, on the base row and on any continuation row. -## Running it in CI - -Two workflows, mirroring the Kotlin and Java pairs: - -| | | -|---|---| -| `.github/workflows/build-android-docs.yaml` | reads and writes the real database on Google Drive | -| `.github/workflows/build-android-docs-local.yaml` | the same five steps against a path on disk | -| `run-build-android-docs-with-act.sh` | drives the local one under [act](https://github.com/nektos/act) | - -The steps between the two are identical -- convert, load, verify, re-mint, -verify the re-mint -- and only the ends differ: Drive download/upload behind -Workload Identity Federation, or `cp` from and to `db_path`. The local workflow -exists because WIF validates the OIDC token's issuer against GitHub's own token -endpoint, so act can never get past the Drive workflow's auth step no matter -what secrets it is given. - -Unlike the Kotlin workflow (which clones kotlin-web-site) and the Java one -(which unpacks a JDK's `lib/src.zip`), there is no external source to fetch: the -12,106 derived HTML pages are committed in this repository, so a run is -reproducible from a commit alone and there is no ref to pin. The 9 GB raw scrape -they came from stays on Drive and is not needed. - -Two inputs are worth knowing about. `only` converts a substring of the corpus -and turns a twelve-minute run into seconds, which is how to smoke-test a change; -it leaves the rest of the rows as they were, so it needs `verify_complete=false` -beside it. `skip_remint` drops the two slowest steps, which is right for a -structural check and for a re-run against a database already minted for this -corpus -- re-minting is not cumulative, and a second pass over an already-minted -database measured -0.7%. +## Not run in CI + +The two workflows that ran these five steps against the master (`build-android-docs.yaml` on +Google Drive, `build-android-docs-local.yaml` on disk) and `run-build-android-docs-with-act.sh` +were retired in ADFA-3305, never having run. The `a/` pages ship converted on the ship-time copy +by `scripts/android_pages_to_templates.py` instead, which leaves a row already in another template +alone, so running this pipeline on the master first would leave it almost nothing to convert. The +scripts and their tests stay; the workflows are in git history (61e62469, 0ae218f8). ## The renderer diff --git a/run-build-android-docs-with-act.sh b/run-build-android-docs-with-act.sh deleted file mode 100755 index 0fcb6a05..00000000 --- a/run-build-android-docs-with-act.sh +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env bash -# Runs the "Build Android Docs (Local)" GitHub Actions workflow -# (.github/workflows/build-android-docs-local.yaml) locally via act -# (https://github.com/nektos/act). -# -# This drives the *local* workflow, not build-android-docs.yaml. The Drive -# workflow authenticates to Google Cloud with Workload Identity Federation, and -# WIF validates the OIDC token's issuer against GitHub's own token endpoint for -# a specific repo and run - act cannot mint a token GCP will accept, so that -# workflow can never get past its auth step locally no matter what secrets you -# supply. build-android-docs-local.yaml exists precisely to be runnable here: it -# reads its documentation.db from disk and writes its outputs back to disk, and -# is otherwise step-for-step identical to the Drive workflow (same -# android_docs_to_json -> load_android_json_db -> verify_android_json_db -> -# remint_dictionary -> verify_remint_dictionary, same verification). -# -# The Java counterpart is run-build-java-docs-with-act.sh; this mirrors it. -# -# Requires: -# - act (https://github.com/nektos/act#installation) on PATH -# - a running Docker daemon (act executes each step inside a container) -# -# CONTAINER SIZE: unlike the Java pipeline, nothing here is memory-hungry - the -# conversion is one HTML parse per page across worker processes, and the largest -# page in the corpus parses in well under a second. A default container VM is -# fine. What it does want is disk and time: -# -# ~730 MB the scraped HTML, already in the checkout act copies in -# ~350 MB the JSON tree the conversion writes -# ~2x the database, since the load builds a new one beside the input -# -# so allow roughly 2 GB of free space in the container beyond the checkout. -# -# TIMING, measured against the real corpus and database on a 10-core machine: -# -# convert ~1 min 12,106 pages -> 12,906 JSON documents -# load ~1.5 min rows, templates, stylesheet -# verify (40 pages) ~1 min one JVM per sampled page -# re-mint ~5.5 min recompresses EVERY row in the database, not -# just the Android ones, at Brotli q11 -# verify re-mint ~2 min decompresses both databases and compares -# -# About 12 minutes all told. --only cuts the first three to seconds and is the -# way to smoke-test a change; --skip-remint drops the last two. -# -# Secrets: none are required. SLACK_WEBHOOK_URL is the only secret this workflow -# reads, and it is optional - the two "Notify Slack" steps print a skip notice -# and continue when it is unset. Export it if you want to see them actually fire -# (both need --live: the baton stands for the lock on db_path, so neither half -# of the pair runs on a dry run, which rewrites nothing). GitHub never exposes a -# stored secret's value through any API or CLI, so if you do want the real -# webhook you have to supply your own copy. -# -# Inputs are host paths, bind-mounted into the job container at fixed locations -# and passed to the workflow as those in-container paths (a GitHub-hosted runner -# has no access to your disk, so the workflow only ever sees the mounted paths). -# Note this means the host paths must live somewhere your container runtime is -# allowed to share - under $HOME is safe for both colima and Docker Desktop; -# /tmp on macOS often is not. -# -# Usage: -# ./run-build-android-docs-with-act.sh --db-path PATH [options] [-- ] -# -# Options: -# --db-path PATH Host path to the input documentation.db (required). -# With --live this file is overwritten in place. -# --output-dir PATH Host directory for outputs - a run-numbered copy of the -# built database. Created if absent. -# (default: ./build-android-docs-output) -# --live dry_run=false: write the rebuilt database back over -# --db-path when the run finishes. Also required for -# either Slack notification to fire. Default is -# dry_run=true. -# --only SUBSTRING Convert only pages whose path contains SUBSTRING, e.g. -# "android/app/". Turns a multi-minute run into seconds. -# Requires --no-verify-complete, since the rest of the -# corpus is deliberately left as it was. -# --sample-size N How many stored pages to render back out of the built -# database (default: 40). Each one starts a JVM. -# --no-verify-complete verify_complete=false: do not fail when Android rows are -# still serving raw HTML. Only for an --only run. -# --skip-remint skip_remint=true: leave the shared Brotli dictionary -# alone. Saves most of the run time. Right for a -# structural check, and for a re-run against a database -# already minted for this corpus - re-minting is not -# cumulative, and a second pass over an already-minted -# database measured -0.7%. Otherwise it is worth about -# 16% on a database coming from HTML. -# -# Every workflow input is passed explicitly on every run, including the ones -# whose YAML "default:" would cover them. act does not apply workflow_dispatch -# input defaults - an input you don't pass arrives empty - and for dry_run that -# inverts the intended behaviour: "${{ !inputs.dry_run }}" on an empty value is -# true, so the step that writes the database back over --db-path would run. -# Passing all of them keeps a local run's semantics identical to a real dispatch. -# -# On Apple Silicon act warns about container architecture; append -# `-- --container-architecture linux/arm64` if you want to silence it (the -# default works). -set -euo pipefail - -if ! command -v act >/dev/null 2>&1; then - echo "error: act is required - see https://github.com/nektos/act#installation" >&2 - exit 1 -fi - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKFLOW="$REPO_ROOT/.github/workflows/build-android-docs-local.yaml" - -# Where the host paths below get bind-mounted inside the job container, and -# therefore what the workflow itself is told its inputs are. -CONTAINER_DB_PATH="/mnt/act-inputs/documentation.db" -CONTAINER_OUTPUT_DIR="/mnt/act-output" - -DB_PATH="" -OUTPUT_DIR="$REPO_ROOT/build-android-docs-output" -# Mirrors build-android-docs-local.yaml's own defaults. Restated here because -# act does not apply workflow_dispatch defaults (see the note above); passing -# every input unconditionally is what keeps a local run equivalent to a real one. -ONLY="" -SAMPLE_SIZE="40" -VERIFY_COMPLETE="true" -SKIP_REMINT="false" -DRY_RUN="true" - -EXTRA_ACT_ARGS=() - -while [ $# -gt 0 ]; do - case "$1" in - --db-path) DB_PATH="$2"; shift 2 ;; - --output-dir) OUTPUT_DIR="$2"; shift 2 ;; - --live) DRY_RUN="false"; shift ;; - --only) ONLY="$2"; shift 2 ;; - --sample-size) SAMPLE_SIZE="$2"; shift 2 ;; - --no-verify-complete) VERIFY_COMPLETE="false"; shift ;; - --skip-remint) SKIP_REMINT="true"; shift ;; - --) shift; EXTRA_ACT_ARGS+=("$@"); break ;; - *) echo "error: unrecognized argument '$1'" >&2; exit 1 ;; - esac -done - -if [ -z "$DB_PATH" ]; then - echo "error: --db-path is required (host path to the documentation.db to build against)" >&2 - exit 1 -fi -if [ ! -f "$DB_PATH" ]; then - echo "error: --db-path '$DB_PATH' does not exist or is not a file" >&2 - exit 1 -fi -DB_PATH="$(cd "$(dirname "$DB_PATH")" && pwd)/$(basename "$DB_PATH")" - -if [ -n "$ONLY" ] && [ "$VERIFY_COMPLETE" = "true" ]; then - echo "error: --only converts part of the corpus, so the rest of the Android rows stay as" >&2 - echo "error: they were and the completeness check is guaranteed to fail. Pass" >&2 - echo "error: --no-verify-complete alongside it." >&2 - exit 1 -fi - -mkdir -p "$OUTPUT_DIR" -OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)" - -# One -v per input. Mounting the database individually (rather than its parent -# directory) keeps the container's view to exactly what the run needs, and lets -# --db-path and --output-dir live in unrelated places on the host. -# -# act takes --container-options as one string and splits it with shell-style -# quoting rules, so each mount spec is emitted double-quoted: an unquoted join -# would break the moment a host path contained a space. -CONTAINER_OPTIONS="" -add_mount() { CONTAINER_OPTIONS+=" -v \"$1:$2\""; } -add_mount "$DB_PATH" "$CONTAINER_DB_PATH" -add_mount "$OUTPUT_DIR" "$CONTAINER_OUTPUT_DIR" - -if [ "$DRY_RUN" = "true" ]; then - echo "note: dry_run=true - '$DB_PATH' will NOT be modified; the built database is" >&2 - echo "note: written to '$OUTPUT_DIR' only. Neither Slack notification fires: the" >&2 - echo "note: baton stands for the lock on db_path, and a dry run never takes it." >&2 - echo "note: Pass --live to write back and see both." >&2 -else - echo "WARNING: --live - '$DB_PATH' will be OVERWRITTEN in place when the run finishes." >&2 -fi - -if [ "$SKIP_REMINT" = "true" ]; then - echo "note: --skip-remint - the shared Brotli dictionary is left as it is. On a database" >&2 - echo "note: coming from HTML that costs about 16% on the Android pages; on one already" >&2 - echo "note: minted for this corpus it costs nothing, and skipping saves ~5.5 minutes." >&2 -fi - -# The workflow reads SLACK_WEBHOOK_URL and tolerates it being unset, so pass it -# through when it's in the environment and stay silent when it isn't. -# -# SECRET_ARGS and EXTRA_ACT_ARGS are expanded below as ${arr[@]+"${arr[@]}"} -# rather than plain "${arr[@]}": macOS still ships bash 3.2, where `set -u` -# treats an empty array's "${arr[@]}" as an unbound variable and aborts. Both -# arrays are empty on a normal run. -SECRET_ARGS=() -if [ -n "${SLACK_WEBHOOK_URL:-}" ]; then - SECRETS_FILE="$(mktemp)" - trap 'rm -f "$SECRETS_FILE"' EXIT - printf 'SLACK_WEBHOOK_URL=%s\n' "$SLACK_WEBHOOK_URL" > "$SECRETS_FILE" - SECRET_ARGS=(--secret-file "$SECRETS_FILE") -fi - -echo "== Running $WORKFLOW via act ==" -echo " db_path $DB_PATH -> $CONTAINER_DB_PATH" -echo " output_dir $OUTPUT_DIR -> $CONTAINER_OUTPUT_DIR" -echo " only=${ONLY:-(whole corpus)} sample_size=$SAMPLE_SIZE dry_run=$DRY_RUN" -echo " verify_complete=$VERIFY_COMPLETE skip_remint=$SKIP_REMINT" - -# --container-daemon-socket - : act otherwise bind-mounts the host's Docker -# socket into the job container so steps can run Docker themselves. Nothing in -# this workflow does, and the mount outright fails on runtimes whose socket -# isn't a plain bind-mountable file - under colima it aborts the run with -# "error while creating mount source path ...: operation not supported". -act workflow_dispatch \ - -W "$WORKFLOW" \ - -P ubuntu-latest=catthehacker/ubuntu:act-latest \ - --container-daemon-socket - \ - --container-options "$CONTAINER_OPTIONS" \ - --input db_path="$CONTAINER_DB_PATH" \ - --input output_dir="$CONTAINER_OUTPUT_DIR" \ - --input only="$ONLY" \ - --input sample_size="$SAMPLE_SIZE" \ - --input verify_complete="$VERIFY_COMPLETE" \ - --input skip_remint="$SKIP_REMINT" \ - --input dry_run="$DRY_RUN" \ - ${SECRET_ARGS[@]+"${SECRET_ARGS[@]}"} \ - ${EXTRA_ACT_ARGS[@]+"${EXTRA_ACT_ARGS[@]}"} diff --git a/scripts/android_pages_to_templates.py b/scripts/android_pages_to_templates.py new file mode 100644 index 00000000..5f538efc --- /dev/null +++ b/scripts/android_pages_to_templates.py @@ -0,0 +1,880 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# ] +# /// +""" +android_pages_to_templates.py + +Stores the Android reference pages under a/ as JSON plus shared Pebble +templates, instead of as finished HTML pages (ADFA-3305). + +An a/ page repeats the same structures many times: summary rows, detail +headers, parameter rows, and above all the "Inherited methods" blocks, which +copy a superclass's member rows onto every subclass page (ListView, CheckBox, +Switch and the other widgets each carry about 0.5 MB of View's). A shared +256 KiB dictionary cannot hold those, so every page pays for them. This step +moves them into templates: + + - Records. A page is cut into segments. A segment is either raw HTML or a + record of one "shape": HTML with named holes, written once in SHAPES. The + same shape text gives the regex that reads a record out of a page and the + template branch that writes it back, so a record renders to exactly the + text it was read from. A hole used twice in one shape is stored once, and + a link into the page itself is stored as its fragment only. The fixed head + and footer variants are found in the pages themselves (each used by at + least MIN_SHARED_PAGES pages) and stored with the title as a hole. + - Shared includes. Each inherited-member block (Doclava's "expandable + jd-inherited-apis" or Dackka's ) is keyed by its exact + header. The rows every page's block for one key shows are merged into one + order; the include template android-inherited-.peb holds the header, + each row behind {% if x contains %}, and the footer. A page stores + {"k": ..., "i": , "x": [rows it leaves out]} in place of the block: the + members its class overrides. A key whose pages disagree on the order of + their rows, or repeat a row, keeps its blocks in the pages. + - Shared texts. A text of INTERN_MIN_CHARS or more that INTERN_MIN_PAGES or + more pages hold - a raw segment, or a hole other than a link hole - is stored + once, in a bundle template android-text-.peb of at most + INTERN_MAX_TEMPLATE bytes of source, as one branch of + {% if j == 0 %}...{% elseif j == 1 %}...{% endif %}. A page stores + [, ] under "@" ("r@" for a raw segment) in place of the text. + Only the holes that hold one get that alternative in the main template. + Link holes stay in the pages: a same-page link's fragment often equals a + member's id, but it is written relative to the page. (A shared text may + still hold a link in its markup; every such link in the a/ pages is + rooted.) A text too long for a + bundle stays in its pages. A page compiles every bundle it uses, so texts go + in bundles by the pages that use them (INTERN_BUNDLE_COST): the same pages' + texts together, other groups merged only when the pages that use both gain + more than the rest lose. In the 2026-10-19 ship-time build this interned + 6,441 texts (repeated paragraphs, inherited descriptions, annotations, + parameters, types) into 524 bundles, the largest 7,998 bytes, and saved + 905,216 file and 928,748 .br bytes (84,803,584 and 66,204,977) against the + same build without it. The median page compiles 2,653 bytes of bundle + source, the 95th percentile 14,411, the largest 144,127 (TextView 121,032, + in 32 bundles). + - The main template android-reference.peb loops over a page's segments. + +Pages: every text/html row under a/ with templateId 0, except a/devsite/ +(guide pages of another shape). A row in another template, a page that is not +UTF-8 or holds a NUL is left as it is and named. + +ADFA-5548's ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py +does the same job another way: it replaces the scraped a/android/ and +a/androidx/ rows with JSON from the committed source tree, rendered through its +own android-class.peb, android-package.peb and android-index.peb. This step is +the one that ships, on the ship-time copy. The workflows that ran that pipeline +on the master were retired (ADFA-3305) before they ever ran: rows already +rendered through another template are left as they are, so after it this step +would convert almost nothing and save almost nothing. + +Before anything is written, every converted page is rendered with the real +Pebble engine (scripts/hm_render_check/Render.java, built as the app builds +it, with the app's 4 Mi-character render limit) and compared with the stored +page byte for byte. A page that does not render back exactly, or is over the +limit (a/android/R.styleable.html, 4,455,003 characters), stays HTML and is +named. Rendering needs `java` (17 or later) and the Pebble 4.1.1, unbescape, +slf4j-api and Gson jars, from the Gradle or Maven cache or --classpath. + +Templates are written as plain UTF-8. On the ship-time copy the 3.0.0 +migration then compresses every template against the shared dictionary, +which the app reads in documentation schema 3.0.0 (CodeOnTheGo, stacked on +#2163); today's app reads templates as plain text, so a database from this +step needs that app. For the same reason the step refuses a database that +already declares documentation version 3 or later (its newest +DocumentationDatabaseVersion row, ordered as the app orders them: changeTime, +then rowid): its plain templates would sit among compressed ones, and the app +would fail every converted page. + +Runs in one transaction, backs the database up first and VACUUMs afterwards. +--dry-run does everything, the render check included, and writes nothing. +The step does not run twice: on a database that already holds its templates +it refuses. Exit 1 always means nothing was written: a database declaring +version 3 or later, a missing renderer, an a/ row that cannot be read, or +templates already present. + +Pipeline position: on the ship-time copy, never on the master database the +other tools edit. After minify_text.py (which writes a/ pages), +drop_unread_rows.py and trim_kotlin_json.py; before the 3.0.0 migration +(migrate_tooltip_schema.py, which compresses the templates) and the final +dictionary re-mint (remint_dictionary.py, which re-mints templates too), then +packing (PR #38's rebuild_database.py). Coming after drop_unread_rows.py is an +exception to that step's rule (PR #46) that it runs after every step writing +page content. It is safe because this step adds no link: every page it +converts renders back byte for byte, so it links to exactly what +drop_unread_rows.py already checked in it as HTML. (drop_unread_rows.py reads +templates and templated rows' JSON too, so it could also run after this step.) + +Measured for the shapes and inherited-member includes, before the shared +texts, on the 2026-10-18 schema-3 test database (98,201,600 bytes), against +the same database with only its templates compressed; both re-minted with 4 +seeds (templates sampled and recompressed with the rows), packed with +rebuild_database.py --pack Content,TooltipTexts --page-size 4096, .br with +brotli -q 11 -w 24: + + pages converted 12,827 of 12,828 (R.styleable stays HTML) + templates 1 main + 1,119 includes, 12.1 MB as text + file 98,271,232 -> 90,857,472 (-7,413,760, -7.5%) + .br 80,139,063 -> 72,644,272 (-7,494,791, -9.4%) + +The seed-to-seed spread within each re-mint was at most 0.4 MB. +Render time on a desktop JVM, JSON parse included: median 0.15 ms per page, +largest page (R.attr, 2.1 M characters) 13 ms. + +Usage: + uv run scripts/android_pages_to_templates.py documentation.db [--dry-run] [--workers N] [--verbose] +""" +from __future__ import annotations + +import argparse +import graphlib +import heapq +import json +import os +import re +import sqlite3 +import sys +from collections import Counter, defaultdict +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +from pathlib import Path + +from hm_pages_to_templates import PebbleHarness, find_classpath, first_difference +from optimize_db_media import BrotliCodec, backup_database, human, is_continuation_path, load_dictionary, reassemble +from update_media_references import replace_row + +PREFIX = "a/" +EXCLUDED = ("a/devsite/",) +CONTENT_TYPE = "text/html" +MAIN_TEMPLATE = "android-reference.peb" +INCLUDE_PREFIX = "android-inherited-" +INCLUDE_SUFFIX = ".peb" + +# A head or footer variant becomes a template shape only when this many pages share it exactly. +MIN_SHARED_PAGES = 5 + +TEXT_PREFIX = "android-text-" +# A text is interned when this many pages hold it. Measured with the first prototype, which filled bundles in +# first-use order, on the 2026-10-19 ship-time build (4-seed re-mint, packed, .br): 5 pages saved 839,680 file +# and 850,755 .br bytes; 2 pages saved less (614,400 and 799,100, in 1,047 bundles where a page pulls in more of +# them), and so did 20 pages (643,072 and 588,189). +INTERN_MIN_PAGES = 5 +# A shorter text costs about as much as the reference that would replace it. +INTERN_MIN_CHARS = 32 +# Most source a bundle template holds. A page compiles every bundle it uses. With the packing below, offline on the +# 2026-10-19 ship-time build: bundles of 16,000 put TextView's texts in 28 bundles instead of 32 for about the +# same source (122,038 bytes against 121,032), and bundles of 32,000 made TextView compile 174,317. +INTERN_MAX_TEMPLATE = 8000 +# Texts go in bundles by the pages that use them: two groups share a bundle when the pages using both, each saved +# one bundle, outweigh the bytes the other pages then compile for nothing, a bundle counting as this many bytes. +# Against first-use order, offline on the 2026-10-19 ship-time build, this cut the bundle source the median page +# compiles from 15,419 to 2,653 bytes, the 95th percentile from 61,211 to 14,411 and TextView from 290,523 to +# 121,032 (32 bundles, not 38). A bundle cost of 2,000 compiled less per page (median 958) in 938 bundles that +# stored about 22 KB more; 32,000 compiled more (median 7,973). +INTERN_BUNDLE_COST = 8000 + +# From this documentation major on, migrate_tooltip_schema.py (ADFA-3305) has stored every template compressed +# against the dictionary, and the app decodes every one: the plain templates this step writes would not render. +COMPRESSED_TEMPLATES_MAJOR = 3 + +# Hole marker inside SHAPES text: H + name + H. Pages holding a NUL are not converted, so it never collides. +H = "\x00" + +# Hole name -> regex. Multi-line holes are bounded so that a match that fails cannot scan a whole page. +# A name with trailing digits (id2) uses its base name's regex. +HOLES = { + "t": r"[^<]*", + "v": r'[^"\n]*', "lv": r'[^<\n]*', "id": r'[^"\n]*', + "n": r'[^<"\n]*', + "h": r'[^"\n]*', + "ps": r"[^\n]*?", "c": r"[^\n]*?", "a": r"[^\n]*", "w": r"[^<\n]*", "cl": r'[^"\n]*', "x": r"[^\n]*", + "ty": r"(?s:.{0,4000}?)", "d": r"(?s:.{0,20000}?)", "sig": r"(?s:.{0,8000}?)", +} +# Holes holding a link: a link into the page itself is stored as its fragment, under "#". +LINK_HOLES = {"h"} + +API = 'Added in API level ' +PRE = '
'
+SHAPES = [
+    # Doclava summary rows: constant or field, method, constructor.
+    ('\n\0ty\0\n\n'
+     '\0n\0\n\0d\0\n\n'),
+    ('\n\n\0ty\0\n\n\n'
+     '\n\0n\0\0ps\0\n\n\0d\0\n\n'),
+    ('\n\n'
+     '\n\0n\0\0ps\0\n\n\0d\0\n\n'),
+    # Doclava detail headers: method or constructor (api-level in an inner div), field or constant.
+    ('
\n
\0n\0
\n'
+     '
\n
\n' + API + '\0v\0\n
\n
\n' + PRE + '\n\0sig\0
\n'), + ('
\n

\0n\0

\n' + '
\n' + API + '\0v\0\n
\n' + PRE + '\n\0sig\0\n'), + # Parameter, return and exception rows, and their table heads. + ('\n\0n\0\n\n' + '\0ty\0\0d\0\n\n'), + '\n\0ty\0\n\0d\0\n\n', + '\n\n', + '

Constant Value:\n\0c\0\n(\0x\0)\n

', + # Doclava section and table headings, and an inherited block kept in the page. + '

\0n\0

\n', + ('
\0n\0
\n\n'), + '
' + '

\0n\0

\n\n', + ('\n\n', + # Dackka summary rows, table heads and ends, detail headers with and without version metadata, sections. + ('\n\n\n\n'), + '\n\n\n', + ('
\n

\0n\0

\n\n
\0ty\0\n' + '
\0n\0\0ps\0
\n\0d\0
\n
\0c\0
\n\0d\0
\n\n\n\n' + '\n\n\n\n' + '\n\n\n'), + '\n

\0n\0

\n
\n', + ('
\0a\0\n
\n
\n' + '

\0n\0

\n
\n
\n' + '
\n
Added in \0lv\0
\n' + '
\n
\n
\n' + PRE + '\0sig\0\n'), + ('
\0a\0\n
\n
\n' + '

\0n\0

\n
\n
\n' + PRE + '\0sig\0\n'), + '
\n

\0n\0

\n', +] + +# Inherited-member blocks: (header regex, footer). Between them, only table rows. +BLOCK_STYLES = ( + (re.compile(r'\n\n\n\n'), + (re.compile(r'\nFrom [^\n]*\n' + r'
\n\n\n\n' + r'\n\n\n'), + '\n
\n
\n \n\n'), +) +KEY_JOIN = "\0" # between a block's header and footer in its key +_TR_OPEN = re.compile(r"]") + +# Pebble drops the line break right after any tag: one match of these, in this order +# (io.pebbletemplates.pebble.lexer.Syntax), at the tag's end. +_PEBBLE_NEWLINE = re.compile("\r\n|\n\r|\r|\n|\u0085|\u2028|\u2029") +_PEBBLE_SYNTAX = ("{{", "{%", "{#") +_END_VERBATIM = re.compile(r"\{%-?\s*endverbatim") +# What Pebble 4.1.1 strips from the start of a verbatim block, all of it: Java's Character.isWhitespace plus +# U+0085 (measured, one character at a time). +_VERBATIM_STRIPS = frozenset("\t\n\u000b\f\r\u001c\u001d\u001e\u001f \u0085\u1680\u2000\u2001\u2002\u2003" + "\u2004\u2005\u2006\u2008\u2009\u200a\u2028\u2029\u205f\u3000") + + +class Unrepresentable(ValueError): + """Literal text a template cannot hold as written (it would end a verbatim block).""" + + +def after_tag(text: str) -> str: + """Text that follows a tag, written so Pebble's newline trimming leaves it whole: when it starts with a + line break, that character is doubled, and the match Pebble drops is then exactly the extra character + (a doubled character never forms one of the two-character breaks).""" + if text and _PEBBLE_NEWLINE.match(text): + return text[0] + text + return text + + +def static_text(text: str, after_a_tag: bool = True) -> str: + """Pebble source for literal text, by default text that follows a tag. Text holding Pebble syntax goes in + a verbatim block, its leading whitespace before the block (Pebble strips all of it inside); a final "{" + is printed from a string literal, since a tag right after it would read as "{{".""" + if not text: + return "" + if text.endswith("{"): + return static_text(text[:-1], after_a_tag) + '{{ "{" }}' + if any(s in text for s in _PEBBLE_SYNTAX): + if _END_VERBATIM.search(text): + raise Unrepresentable(text[:80]) + i = 0 + while text[i] in _VERBATIM_STRIPS: + i += 1 + lead = text[:i] + return (after_tag(lead) if after_a_tag else lead) + "{% verbatim %}" + text[i:] + "{% endverbatim %}" + return after_tag(text) if after_a_tag else text + + +def hole_regex(name: str) -> str: + return HOLES[name] if name in HOLES else HOLES[name.rstrip("0123456789")] + + +def compile_shape(text: str): + parts = text.split(H) + pattern, seen = [], set() + for i, part in enumerate(parts): + if i % 2 == 0: + pattern.append(re.escape(part)) + elif part in seen: + pattern.append(f"(?P={part})") + else: + seen.add(part) + pattern.append(f"(?P<{part}>{hole_regex(part)})") + return re.compile("".join(pattern)) + + +def text_include(ref: str) -> str: + """Pebble source writing the shared text `ref` ([bundle, index]) points at.""" + return f'{{% include ("{TEXT_PREFIX}" ~ {ref}[0] ~ "{INCLUDE_SUFFIX}") with {{"j": {ref}[1]}} %}}' + + +def hole_source(name: str, interned: frozenset = frozenset()) -> str: + if name in interned: + ref = f's["{name}@"]' + return f"{{% if {ref} is not null %}}{text_include(ref)}{{% else %}}{{{{ s.{name} | raw }}}}{{% endif %}}" + if name in LINK_HOLES: + return (f'{{% if s["{name}#"] is not null %}}{{{{ u | raw }}}}#{{{{ s["{name}#"] | raw }}}}' + f'{{% else %}}{{{{ s.{name} | raw }}}}{{% endif %}}') + return f"{{{{ s.{name} | raw }}}}" + + +def branch_source(shape: str, interned: frozenset = frozenset()) -> str: + """The template source writing one record of `shape`. Every literal part follows a tag: the elseif, or + a hole's print or endif. A hole in `interned` may hold a shared text instead of its own.""" + out, parts = [], shape.split(H) + for i, part in enumerate(parts): + out.append(hole_source(part, interned) if i % 2 else static_text(part)) + return "".join(out) + + +def head_and_tail_shapes(pages: list) -> list: + """The fixed head (with the title as a hole) and footer variants that at least MIN_SHARED_PAGES pages + share exactly, most shared first.""" + mark = '
\n' + heads, tails = Counter(), Counter() + for page in pages: + i = page.find(mark) + if i > 0: + heads[re.sub(r"[^<]*", f"{H}t{H}", page[:i + len(mark)], count=1)] += 1 + j = page.rfind(" 0: + tails[page[j:]] += 1 + out = [] + for shape, count in heads.most_common() + tails.most_common(): + if count < MIN_SHARED_PAGES: + continue + try: + branch_source(shape) + except Unrepresentable: + continue + out.append(shape) + return out + + +class Converter: + def __init__(self, shapes: list, include_kind: int = None): + self.shapes = shapes + self.regexes = [compile_shape(s) for s in shapes] + self.firsts = [s.split(H, 1)[0][:12] for s in shapes] + self.include_kind = len(shapes) if include_kind is None else include_kind + + def template(self, interned: frozenset = frozenset()) -> str: + """The main template. Only the holes in `interned` ("r" for raw segments) get the shared-text branch.""" + raw = "{{ s.r | raw }}" + if "r" in interned: + ref = 's["r@"]' + raw = f"{{% if {ref} is not null %}}{text_include(ref)}{{% else %}}{raw}{{% endif %}}" + out = ["{% for s in b %}{% if s.k is null %}" + raw] + for k, shape in enumerate(self.shapes): + out.append(f"{{% elseif s.k == {k} %}}" + branch_source(shape, interned)) + out.append(f'{{% elseif s.k == {self.include_kind} %}}' + f'{{% include ("{INCLUDE_PREFIX}" ~ s.i ~ "{INCLUDE_SUFFIX}") with {{"x": s.x}} %}}') + out.append("{% endif %}{% endfor %}") + return "".join(out) + + def segments(self, page: str, url: str) -> list: + segs, raw, pos, n = [], [], 0, len(page) + while pos < n: + for k, rx in enumerate(self.regexes): + if not page.startswith(self.firsts[k], pos): + continue + m = rx.match(page, pos) + if m and m.end() > pos: + if raw: + segs.append({"r": "".join(raw)}) + raw = [] + rec = {"k": k} + for name, value in m.groupdict().items(): + if name in LINK_HOLES and value.startswith(url + "#"): + rec[name + "#"] = value[len(url) + 1:] + else: + rec[name] = value + segs.append(rec) + pos = m.end() + break + else: + nl = page.find("\n", pos) + end = n if nl < 0 else nl + 1 + raw.append(page[pos:end]) + pos = end + if raw: + segs.append({"r": "".join(raw)}) + return segs + + +def split_rows(page: str, pos: int): + """The table rows starting at pos, each a whole ... with any nested rows, and where they end.""" + rows = [] + while page.startswith("") + p = nl + 1 + if depth <= 0: + break + rows.append(page[pos:p]) + pos = p + return rows, pos + + +def find_blocks(page: str) -> list: + """(start, end, key, rows) for each inherited-member block, in page order.""" + blocks = [] + for rx, foot in BLOCK_STYLES: + for m in rx.finditer(page): + rows, pos = split_rows(page, m.end()) + if rows and page.startswith(foot, pos): + blocks.append((m.start(), pos + len(foot), m.group(0) + KEY_JOIN + foot, rows)) + blocks.sort() + return blocks + + +def plan_includes(found: list) -> dict: + """key -> {row: its index in the merged order}, for each key whose pages agree on their rows' order.""" + seqs = defaultdict(list) + for blocks in found: + for _start, _end, key, rows in blocks: + seqs[key].append(rows) + keys = {} + for key, sequences in seqs.items(): + # A row repeated within one block is a cycle too (x -> ... -> x), so the sort refuses it. + sorter = graphlib.TopologicalSorter() + for s in sequences: + for i, row in enumerate(s): + sorter.add(row, *([s[i - 1]] if i else [])) + try: + order = list(sorter.static_order()) + except graphlib.CycleError: + continue + try: + include_source(key, {row: i for i, row in enumerate(order)}) + except Unrepresentable: + continue + keys[key] = {row: i for i, row in enumerate(order)} + return keys + + +def include_source(key: str, order: dict) -> str: + head, foot = key.split(KEY_JOIN) + out = [static_text(head, after_a_tag=False)] # the template's first text follows no tag + for row, i in sorted(order.items(), key=lambda t: t[1]): + out.append(f"{{% if x contains {i} %}}{{% else %}}{static_text(row)}{{% endif %}}") + out.append(static_text(foot)) + return "".join(out) + + +# Worker state, set by _init in each process. +_CONVERTER = None +_INCLUDES = None + + +def _init(shapes: list, includes: dict) -> None: + global _CONVERTER, _INCLUDES + _CONVERTER = Converter(shapes) + _INCLUDES = includes + + +def _find(page: str) -> list: + return find_blocks(page) + + +def _convert(item: tuple) -> dict: + """A page's JSON context: its own URL and its segments, with each inherited block whose key has an + include replaced by an include record.""" + path, page, blocks = item + url = "/" + path + segs, pos = [], 0 + for start, end, key, rows in blocks: + if key not in _INCLUDES: + continue + order, number = _INCLUDES[key] + segs += _CONVERTER.segments(page[pos:start], url) + rec = {"k": _CONVERTER.include_kind, "i": number} + present = {order[row] for row in rows} + left_out = [i for i in range(len(order)) if i not in present] + if left_out: + rec["x"] = left_out + segs.append(rec) + pos = end + segs += _CONVERTER.segments(page[pos:], url) + return {"u": url, "b": segs} + + +def internable(name: str, value) -> bool: + """Whether a segment's field may hold a shared text instead: text long enough, and never a link.""" + return (isinstance(value, str) and len(value) >= INTERN_MIN_CHARS and name not in LINK_HOLES + and not name.endswith("#")) + + +def intern_texts(contexts: list) -> tuple: + """Moves every text that INTERN_MIN_PAGES or more pages hold into bundle templates of at most + INTERN_MAX_TEMPLATE bytes of source, grouped by the pages that use them (see pack_texts), and points each page + at it as [bundle, index] under "@" ("r@" for a raw segment). A text too long for a bundle stays in its + pages. Returns the bundles' sources by name and the holes that now hold a reference.""" + pages_of, order, sources_of = defaultdict(set), [], {} + for number, context in enumerate(contexts): + for seg in context["b"]: + for name, value in seg.items(): + if internable(name, value): + pages_of[value].add(number) + end = "{% endif %}" + for context in contexts: + for seg in context["b"]: + for name, value in seg.items(): + if not internable(name, value) or len(pages_of[value]) < INTERN_MIN_PAGES or value in sources_of: + continue + try: + source = static_text(value) + except Unrepresentable: + continue + if len("{% if j == 0 %}".encode()) + len(source.encode()) + len(end) <= INTERN_MAX_TEMPLATE: + sources_of[value] = source + order.append(value) + bundles = pack_texts(order, {v: frozenset(pages_of[v]) for v in order}, sources_of) + refs = {v: [b, j] for b, texts in enumerate(bundles) for j, v in enumerate(texts)} + interned = set() + for context in contexts: + for seg in context["b"]: + for name in [n for n, v in seg.items() if internable(n, v) and v in refs]: + seg[name + "@"] = list(refs[seg.pop(name)]) + interned.add(name) + sources = {text_name(b): bundle_source([sources_of[v] for v in texts]) for b, texts in enumerate(bundles)} + return sources, frozenset(interned) + + +def bundle_source(texts: list) -> str: + """A bundle's source: one branch per text, chosen by j.""" + return "".join(("{% if j == 0 %}" if j == 0 else f"{{% elseif j == {j} %}}") + t for j, t in enumerate(texts)) + \ + "{% endif %}" + + +def pack_texts(order: list, pages: dict, sources: dict) -> list: + """Bundles of texts, each a list in first-use order (`order`), the bundles numbered by their first text's use. + Texts that the same pages use start in one group, split to fit; then the two groups whose merge most lowers + the sum over pages of compiled bundle source plus INTERN_BUNDLE_COST per bundle merge, while the result fits a + bundle, until no merge lowers it.""" + rank = {v: i for i, v in enumerate(order)} + by_pages = defaultdict(list) + for v in order: + by_pages[pages[v]].append(v) + groups = {} # id -> [texts, pages, source bytes] + for page_set, texts in by_pages.items(): + chunk = [] + for v in texts: + if chunk and len(bundle_source([sources[t] for t in chunk + [v]]).encode()) > INTERN_MAX_TEMPLATE: + groups[len(groups)] = [chunk, page_set, len(bundle_source([sources[t] for t in chunk]).encode())] + chunk = [] + chunk.append(v) + groups[len(groups)] = [chunk, page_set, len(bundle_source([sources[t] for t in chunk]).encode())] + using = defaultdict(set) + for gid, (_texts, page_set, _size) in groups.items(): + for p in page_set: + using[p].add(gid) + heap, version = [], defaultdict(int) + + def gain(a: int, b: int) -> int: + _ta, pa, sa = groups[a] + _tb, pb, sb = groups[b] + both = len(pa & pb) + return INTERN_BUNDLE_COST * both - sb * (len(pa) - both) - sa * (len(pb) - both) + + # A merge drops one "{% endif %}" and one "{% if j == 0 %}" and adds at least one "{% elseif j == n %}", so a + # merged bundle is at least this much smaller than its two parts: never refuse here one that would fit. + least_saved = len("{% endif %}") + len("{% if j == 0 %}") - len("{% elseif j == 1 %}") + + def offer(a: int) -> None: + for b in {b for p in groups[a][1] for b in using[p]} - {a}: + if groups[a][2] + groups[b][2] - least_saved <= INTERN_MAX_TEMPLATE and (g := gain(a, b)) > 0: + lo, hi = min(a, b), max(a, b) + heapq.heappush(heap, (-g, lo, hi, version[lo], version[hi])) + + for a in list(groups): + offer(a) + while heap: + _g, a, b, va, vb = heapq.heappop(heap) + if a not in groups or b not in groups or (version[a], version[b]) != (va, vb): + continue + texts = sorted(groups[a][0] + groups[b][0], key=rank.get) + size = len(bundle_source([sources[t] for t in texts]).encode()) + if size > INTERN_MAX_TEMPLATE: + continue + for p in groups[b][1]: + using[p].discard(b) + using[p].add(a) + groups[a] = [texts, groups[a][1] | groups[b][1], size] + del groups[b] + version[a] += 1 + offer(a) + return sorted((g[0] for g in groups.values()), key=lambda texts: rank[texts[0]]) + + +def text_name(number: int) -> str: + return f"{TEXT_PREFIX}{number}{INCLUDE_SUFFIX}" + + +def context_json(context: dict) -> bytes: + return json.dumps(context, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def include_name(number: int) -> str: + return f"{INCLUDE_PREFIX}{number}{INCLUDE_SUFFIX}" + + +def _pool_map(fn, items: list, workers: int, initargs=None) -> list: + if workers <= 1: + if initargs is not None: + _init(*initargs) + return [fn(item) for item in items] + kwargs = {"initializer": _init, "initargs": initargs} if initargs is not None else {} + with ProcessPoolExecutor(max_workers=workers, **kwargs) as pool: + return list(pool.map(fn, items, chunksize=max(1, len(items) // (workers * 8) or 1))) + + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + log = (lambda m: print(m)) if cfg["verbose"] else (lambda m: None) + workers = max(1, cfg["workers"]) + + conn = sqlite3.connect(db_path) + codec = None + harness = cfg.get("renderer") + own_harness = False + try: + # First, so a refusal needs no renderer and writes nothing, in a dry run too. + refusal = compressed_templates_refusal(conn) + if refusal: + print(f"error: {refusal}. Nothing written.", file=sys.stderr) + return 1 + taken = [name for (name,) in conn.execute("SELECT name FROM Templates") + if name == MAIN_TEMPLATE or name.startswith((INCLUDE_PREFIX, TEXT_PREFIX))] + if taken: + print(f"error: this database already holds this step's templates ({', '.join(sorted(taken)[:3])}" + f"{', ...' if len(taken) > 3 else ''}); run it on a database it has not converted. " + "Nothing written.", file=sys.stderr) + return 1 + if harness is None: + try: + classpath = cfg["classpath"] or find_classpath() + except RuntimeError as exc: + print(f"error: {exc}; nothing written.", file=sys.stderr) + return 1 + harness = PebbleHarness(classpath, cfg["java"]) + own_harness = True + codec = BrotliCodec(load_dictionary(conn)) + + # The prefix is matched exactly: LIKE ignores case and reads "_" as a wildcard. + rows = conn.execute( + "SELECT c.path, c.content, c.languageID, c.contentTypeID, c.templateId, ct.compression " + "FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id " + "WHERE ct.value = ? AND substr(c.path, 1, ?) = ? ORDER BY c.path", + (CONTENT_TYPE, len(PREFIX), PREFIX)).fetchall() + lengths = {r[0]: len(r[1]) for r in rows} + left, candidates = [], [] + for path, blob, language_id, type_id, template_id, compression in rows: + if not path.startswith(PREFIX) or is_continuation_path(lengths, path) or path.startswith(EXCLUDED): + continue + if template_id: + left.append((path, f"rendered through another template (templateId {template_id})")) + continue + candidates.append((path, reassemble(conn, path, blob), len(blob), language_id, type_id, compression)) + + def decode(c): + try: + return codec.decompress(c[1], c[5]), None + except Exception as exc: # noqa: BLE001 - reported below, and nothing is written + return None, f"could not be read: {exc}" + + with ThreadPoolExecutor(max_workers=workers) as pool: + decoded = list(pool.map(decode, candidates)) + unreadable = [(c[0], why) for c, (_d, why) in zip(candidates, decoded) if why] + if unreadable: + for path, why in unreadable: + print(f" {path}: {why}", file=sys.stderr) + print(f"error: {len(unreadable)} a/ row(s) could not be read, so nothing was written.", file=sys.stderr) + return 1 + + pages, meta = [], {} + for c, (data, _why) in zip(candidates, decoded): + path = c[0] + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + left.append((path, "is not UTF-8")) + continue + if H in text: + left.append((path, "holds a NUL character")) + continue + pages.append((path, text)) + meta[path] = c + print(f"Found {human(len(pages))} a/ page(s) to convert" + f"{f'; {len(left)} left as they are' if left else ''}.") + if not pages: + for path, why in left: + print(f" left as it is: {path} - {why}", file=sys.stderr) + print("Nothing to convert.") + return 0 + + found = _pool_map(_find, [p for _, p in pages], workers) + keys = plan_includes(found) + includes = {key: (order, n) for n, (key, order) in enumerate(keys.items())} + include_sources = {include_name(n): include_source(key, order) for key, (order, n) in includes.items()} + shapes = head_and_tail_shapes([p for _, p in pages]) + SHAPES + contexts = _pool_map(_convert, [(path, page, blocks) for (path, page), blocks in zip(pages, found)], + workers, (shapes, includes)) + text_sources, interned = intern_texts(contexts) + template = Converter(shapes).template(interned) + blocks_total = sum(len(b) for b in found) + print(f"{human(blocks_total)} inherited-member block(s) under {human(len({b[2] for bl in found for b in bl}))} " + f"header(s); {human(len(includes))} include(s) cover " + f"{human(sum(1 for bl in found for b in bl if b[2] in includes))} of them.") + + harness.define(MAIN_TEMPLATE, template) + for name, source in {**include_sources, **text_sources}.items(): + harness.define(name, source) + converted, stay = [], [] + for (path, page), context in zip(pages, contexts): + data = context_json(context) + ok, body, _nanos = harness.render(MAIN_TEMPLATE, data) + expected = page.encode("utf-8") + if ok and body == expected: + converted.append((path, data, context)) + elif not ok and "Tried to write more than" in body: + stay.append((path, "is over the app's render limit")) + elif not ok: + stay.append((path, f"does not render: {body.strip().splitlines()[0][:200] if body.strip() else body}")) + else: + stay.append((path, f"renders differently from byte {first_difference(body, expected)}")) + used = {include_name(seg["i"]) for _p, _d, context in converted for seg in context["b"] + if seg.get("k") == len(shapes)} + include_sources = {name: source for name, source in include_sources.items() if name in used} + used_texts = {text_name(seg[f][0]) for _p, _d, context in converted for seg in context["b"] + for f in seg if f.endswith("@")} + text_sources = {name: source for name, source in text_sources.items() if name in used_texts} + + with ThreadPoolExecutor(max_workers=workers) as pool: + stored = list(pool.map(lambda t: codec.compress(t[1], meta[t[0]][5]), converted)) + page_of = dict(pages) + before = sum(len(meta[p][1]) for p, _d, _c in converted) # chunks joined + after = sum(map(len, stored)) + decoded_before = sum(len(page_of[p].encode("utf-8")) for p, _d, _c in converted) + decoded_after = sum(len(d) for _p, d, _c in converted) + template_bytes = len(template.encode("utf-8")) + sum( + len(s.encode("utf-8")) for s in {**include_sources, **text_sources}.values()) + + for path, why in left: + print(f" left as it is: {path} - {why}", file=sys.stderr) + for path, why in stay: + print(f" stays HTML: {path} - {why}", file=sys.stderr) + print(f"{'Would convert' if cfg['dry_run'] else 'Converting'} {human(len(converted))} of {human(len(pages))} " + f"page(s) to JSON through {MAIN_TEMPLATE}, {human(len(include_sources))} include(s) and " + f"{human(len(text_sources))} shared-text bundle(s) (largest " + f"{human(max((len(v.encode('utf-8')) for v in text_sources.values()), default=0))} bytes): " + f"{human(before)} -> {human(after)} stored byte(s) against the current dictionary, " + f"{human(decoded_before)} -> {human(decoded_after)} decoded, plus {human(template_bytes)} bytes of " + f"templates as text.") + if not converted: + print("Nothing to convert.") + return 0 + if cfg["dry_run"]: + print("Dry run: nothing written.") + return 0 + + print(f"Backing up {db_path} ...") + print(f"Backup written to {backup_database(db_path)}") + conn.execute("BEGIN") + try: + conn.execute("INSERT INTO Templates (name, content) VALUES (?, ?)", + (MAIN_TEMPLATE, template.encode("utf-8"))) + main_id = conn.execute("SELECT id FROM Templates WHERE name = ?", (MAIN_TEMPLATE,)).fetchone()[0] + for name, source in sorted({**include_sources, **text_sources}.items()): + conn.execute("INSERT INTO Templates (name, content) VALUES (?, ?)", (name, source.encode("utf-8"))) + for (path, _data, _context), blob in zip(converted, stored): + _p, _full, base_len, language_id, type_id, _compression = meta[path] + replace_row(conn, path, base_len, blob, language_id, type_id, main_id, + lambda m: print(m, file=sys.stderr)) + conn.execute("UPDATE Content SET templateId = ? WHERE path = ?", (main_id, path)) + log(f" [JSON] {path}") + conn.commit() + except Exception: + conn.rollback() + raise + finally: + if codec is not None: + codec.close() + conn.close() + if own_harness: + harness.close() + + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + print("Next: the 3.0.0 migration (compresses the templates), the dictionary re-mint, then packing.") + return 0 + + +def declared_major(conn) -> int | None: + """The documentation major the database declares - its newest DocumentationDatabaseVersion row, ordered as + Code On The Go orders them (changeTime, then rowid) - or None if it declares none.""" + if not conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name = 'DocumentationDatabaseVersion'").fetchone(): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion " + "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone() + return None if row is None else int(row[0]) + + +def compressed_templates_refusal(conn) -> str | None: + """Why this step may not write its plain templates into the database, or None.""" + major = declared_major(conn) + if major is None or major < COMPRESSED_TEMPLATES_MAJOR: + return None + return (f"the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 every " + "template is stored compressed, and this step writes plain ones. Run it before migrate_tooltip_schema.py") + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to edit in place: a ship-time copy") + p.add_argument("--dry-run", action="store_true", + help="Do all the work, the render check included, and report what would change without writing") + p.add_argument("--workers", type=int, default=os.cpu_count() or 4, + help="Processes used to convert pages and threads used to (de)compress rows (default: CPUs)") + p.add_argument("--verbose", action="store_true", help="Log every page written") + p.add_argument("--classpath", default=os.environ.get("HM_RENDER_CLASSPATH"), + help="Classpath for Render.java: Pebble 4.1.1, unbescape, slf4j-api and Gson jars " + "(default: $HM_RENDER_CLASSPATH, else found in the Gradle or Maven cache)") + p.add_argument("--java", default="java", help="The java launcher, 17 or later (default: java)") + return p + + +def main() -> None: + args = build_parser().parse_args() + sys.exit(run({"db_path": args.db_path, "dry_run": args.dry_run, "workers": args.workers, + "verbose": args.verbose, "classpath": args.classpath, "java": args.java})) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_android_pages_to_templates.py b/scripts/test_android_pages_to_templates.py new file mode 100644 index 00000000..f404b30f --- /dev/null +++ b/scripts/test_android_pages_to_templates.py @@ -0,0 +1,1287 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_android_pages_to_templates.py + +Covers android_pages_to_templates.py: the Pebble text rules (newline trimming, +verbatim, a final "{"), every record shape read out of a page and written back, +the inherited-member includes (one per header, rows a page leaves out, headers +whose pages disagree kept in the page), and whole runs on small synthetic +databases of Doclava- and Dackka-style pages: every converted page read back +and rendered through the stored templates equals the original; a page that +does not render back, or is over the render limit, stays HTML and is named; +a page whose render fails stays HTML and says why; rows the step must leave +alone are left alone; a dry run, a second run, an unreadable row and a +missing renderer write nothing; a chunked page is read +whole and its JSON written in chunks; a failure while writing rolls back. + +Run it either way - as a plain script: + + uv run scripts/test_android_pages_to_templates.py + +or under pytest: + + uv run --with brotli --with Pillow --with pytest pytest scripts/test_android_pages_to_templates.py + +Tests that render need `java` (17 or later) and the Pebble 4.1.1, unbescape, +slf4j-api and Gson jars in the Gradle or Maven cache (or $HM_RENDER_CLASSPATH); +without them they are skipped. Every test that builds a database needs the +`brotli` CLI on PATH. +""" +from __future__ import annotations + +import atexit +import contextlib +import hashlib +import io +import json +import os +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import android_pages_to_templates as apt # noqa: E402 +import hm_pages_to_templates as hm # noqa: E402 +import optimize_db_media # noqa: E402 +import update_media_references # noqa: E402 + + +class Skip(Exception): + pass + + +def _skip(reason: str): + try: + import pytest + except ImportError: + raise Skip(reason) + pytest.skip(reason) + + +_HARNESS = [] + + +def _harness() -> hm.PebbleHarness: + """One renderer for the whole run: templates are redefined by name, so tests do not see each other's.""" + if not _HARNESS: + try: + if shutil.which("java") is None: + raise RuntimeError("no java on PATH") + harness = hm.PebbleHarness(os.environ.get("HM_RENDER_CLASSPATH") or hm.find_classpath()) + atexit.register(harness.close) + _HARNESS.append(harness) + except RuntimeError as exc: + _HARNESS.append(exc) + if isinstance(_HARNESS[0], Exception): + _skip(f"needs the Pebble renderer: {_HARNESS[0]}") + return _HARNESS[0] + + +def _render(template: str, context: dict) -> bytes: + h = _harness() + h.define("t", template) + ok, body, _ = h.render("t", json.dumps(context).encode()) + assert ok, body + return body + + +# --- synthetic pages ------------------------------------------------------------- + +MARK = '
\n' +# The style holds Pebble syntax, so the head shape needs a verbatim block. +HEAD = ('{title}
\n' + '
\n' + MARK) +TAIL = ('\n
\n' + '\n\n
\n') +API_LINK = '' +PRE = '
'
+
+
+def doclava_row(owner: str, name: str, params: str = "()", added: str = "1") -> str:
+    return (f'\n\nboolean\n\n'
+            f'\n\n{name}'
+            f'{params}\n\n
About {name}, café.\n
\n\n\n')
+
+
+def doclava_block(owner: str, rows: list, kind: str = "class") -> str:
+    dotted = owner.replace("/", ".")
+    return (f'\n\n\n\n')
+
+
+OBJECT_ROWS = ["clone", "equals", "hashCode", "toString"]
+
+
+def doclava_page(cls: str, blocks: str = None, extra: str = "") -> str:
+    """A Doclava-style class page under a/pkg/ using every Doclava shape."""
+    url = f"/a/pkg/{cls}.html"
+    blocks = doclava_block("java/lang/Object", OBJECT_ROWS) if blocks is None else blocks
+    return (HEAD.format(title=cls) + f'\n
\n' + f'

{cls}

\n

About {cls} {{{{ braces }}}} and {{% more %}}.

\n' + extra + + '

Summary

\n' + '\n\n' + f'\n\n\n\n' + '

Constants

int\n' + f'MAX\n

The maximum.\n

\n
\n\n\n' + f'\n\n\n

Public constructors

\n\n' + f'{cls}()\n\n

Makes one.\n

\n
\n' + '\n\n' + blocks + '

Inherited methods

\n' + '

Constants

\n' + '
\n

MAX

\n' + f'
\nAdded in {API_LINK}API level 1\n
\n{PRE}\n' + 'public static final int MAX\n

The maximum value.

\n' + '

Constant Value:\n7\n(0x00000007)\n

\n' + '

Public methods

\n' + '
\n

run

\n' + f'
\n
\nAdded in {API_LINK}API level 21\n
\n
\n{PRE}\n' + 'public void run (int times)\n

Runs it.

\n' + '\n\n\n' + '\n\n\n
Parameters
times\n' + 'int: how often.
\n' + '\n\n\n' + '\n\n\n
Returns
voidnothing.
\n' + '
\n' + TAIL) + + +def dackka_block(owner: str, rows: list) -> str: + dotted = owner.replace("/", ".") + body = "".join(f'\nint\n\n' + f'
{n}()
\n' + f'\n\n' for n in rows) + return (f'\nFrom {dotted}' + '\n
\n\n\n\n' + '\n\n\n' + body + + '\n
\n
\n \n\n') + + +def dackka_table(title: str, ident: str, rows: str) -> str: + return ('
\n\n\n\n\n' + f'\n\n\n\n\n\n\n' + rows + '\n

{title}' + '

\n
\n') + + +def dackka_page(cls: str, inherited: list = None) -> str: + """A Dackka-style class page under a/x/ using every Dackka shape.""" + url = f"/a/x/{cls}.html" + inherited = ["getA", "getB", "getC"] if inherited is None else inherited + methods = (f'\nboolean\n\n' + f'
equals(Object other)' + f'
\n\n\n\n\n
public final class ' + f'{cls}.Inner
\n

An inner class.

\n\n\n') + return (HEAD.format(title=cls) + f'\n
\n' + f'

{cls}

\n

About {cls}.

\n' + + dackka_table("Public methods", "public-methods", methods) + + dackka_table("Inherited methods", "inherited-methods", dackka_block("x/Base", inherited)) + + '
\n

Public methods

\n' + '
\n
\n' + '
\n

equals

\n
\n
\n' + f'{PRE}public boolean equals(Object other)\n' + '
\n
\n
\n
\n' + '

size

\n
\n
\n' + '
\n
Added in 1.0.0
\n
\n
\n
\n' + f'{PRE}public int size()\n
\n
\n' + TAIL) + + +def standard_pages() -> dict: + """Six Doclava and six Dackka pages (enough for their head and footer to become shapes). C3 overrides + hashCode, so it leaves that inherited row out; D2 overrides getB.""" + pages = {} + for i in range(6): + rows = [r for r in OBJECT_ROWS if not (i == 3 and r == "hashCode")] + pages[f"a/pkg/C{i}.html"] = doclava_page(f"C{i}", doclava_block("java/lang/Object", rows)) + pages[f"a/x/D{i}.html"] = dackka_page(f"D{i}", ["getA", "getC"] if i == 2 else None) + return pages + + +# --- a synthetic documentation.db -------------------------------------------------- + +DICTIONARY = (b'\n' * 40 + HEAD.encode() * 4) +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE('path')); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, + UNIQUE('name')); +CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL); +""" + + +def _brotli(data: bytes, decompress: bool = False) -> bytes: + """Dictionary Brotli at quality 5, independent of the tool's codec.""" + with tempfile.TemporaryDirectory() as directory: + dict_path = Path(directory) / "dictionary.bin" + dict_path.write_bytes(DICTIONARY) + args = ["brotli", "-D", str(dict_path)] + (["-d"] if decompress else ["-q", "5"]) + ["-c"] + return subprocess.run(args, input=data, stdout=subprocess.PIPE, check=True).stdout + + +def make_db(path: Path, pages: dict = None, extra: dict = None, template_ids: dict = None) -> Path: + """A database holding `pages` (default: standard_pages()) as text/html rows, plus a guide page under + a/devsite/, a page outside a/, an image under a/ and an unrelated template. `extra` adds raw rows: + path -> (bytes, compress?).""" + conn = sqlite3.connect(path) + conn.executescript(SCHEMA) + conn.execute("INSERT INTO Languages (value) VALUES ('en-US')") + conn.execute("INSERT INTO ContentTypes (id, value, compression) VALUES (12, 'text/html', 'brotli')") + conn.execute("INSERT INTO ContentTypes (id, value, compression) VALUES (3, 'image/png', 'none')") + conn.execute("INSERT INTO CompressionDictionary (id, data) VALUES (1, ?)", (DICTIONARY,)) + conn.execute("INSERT INTO Templates (name, content) VALUES ('page.peb', ?)", (b"

{{ title }}

",)) + rows = {p: (t.encode("utf-8"), True) for p, t in (standard_pages() if pages is None else pages).items()} + rows.setdefault("a/devsite/guide.html", (doclava_page("Guide").encode("utf-8"), True)) + rows.setdefault("k/other.html", (doclava_page("Other").encode("utf-8"), True)) + rows.update(extra or {}) + for p, (data, compress) in sorted(rows.items()): + type_id = 3 if p.endswith(".png") else 12 + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, 1, ?, ?, ?)", + (p, _brotli(data) if compress else data, type_id, (template_ids or {}).get(p, 0))) + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID) VALUES ('a/img.png', 1, ?, 3)", + (b"\x89PNG not really",)) + conn.commit() + conn.close() + return path + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def run_tool(db: Path, **extra) -> tuple: + cfg = {"db_path": db, "dry_run": False, "workers": 1, "verbose": True, "classpath": None, "java": "java"} + if "renderer" not in extra: + cfg["renderer"] = _harness() + cfg.update(extra) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = apt.run(cfg) + return code, out.getvalue(), err.getvalue() + + +def served(db: Path) -> dict: + """What the app would serve for every Content item: rows with a templateId rendered through the stored + templates, chunks joined.""" + conn = sqlite3.connect(db) + try: + lengths = dict(conn.execute("SELECT path, length(content) FROM Content")) + names = dict(conn.execute("SELECT id, name FROM Templates")) + h = _harness() + for name, content in conn.execute("SELECT name, content FROM Templates"): + h.define(name, bytes(content).decode("utf-8")) + out = {} + for path, blob, tid, compression in conn.execute( + "SELECT c.path, c.content, c.templateId, ct.compression FROM Content c " + "JOIN ContentTypes ct ON ct.id = c.contentTypeID"): + if optimize_db_media.is_continuation_path(lengths, path): + continue + data = optimize_db_media.reassemble(conn, path, blob) + data = _brotli(data, decompress=True) if compression == "brotli" else data + if tid and names[tid] == apt.MAIN_TEMPLATE: + ok, body, _ = h.render(names[tid], data) + assert ok, body + data = body + out[path] = (data, tid) + return out + finally: + conn.close() + + +def templates(db: Path) -> dict: + conn = sqlite3.connect(db) + try: + return {n: bytes(c).decode("utf-8") for n, c in conn.execute("SELECT name, content FROM Templates")} + finally: + conn.close() + + +def stored_context(db: Path, path: str) -> dict: + conn = sqlite3.connect(db) + try: + blob = optimize_db_media.reassemble(conn, path, conn.execute( + "SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0]) + finally: + conn.close() + return json.loads(_brotli(blob, decompress=True)) + + +# --- Pebble text rules --------------------------------------------------------------- + +BREAKS = ["\n", "\r", "\r\n", "\n\r", "\u0085", "\u2028", "\u2029", "\n\n", "x\n"] + + +def test_after_tag_doubles_only_a_leading_line_break(): + for text in BREAKS: + expected = text[0] + text if text[0] in "\r\n\u0085\u2028\u2029" else text + assert apt.after_tag(text) == expected, repr(text) + assert apt.after_tag("") == "" + + +def test_text_after_a_tag_renders_whole(): + for text in BREAKS + ["\r\nplain", "\n{{ x }}", "\r{% y %}", "a {# z #} b"]: + source = "{% if true %}{% endif %}" + apt.static_text(text) + assert _render(source, {}) == text.encode("utf-8"), repr(text) + + +# Pebble strips all leading whitespace inside a verbatim block (measured), so it is written before the block. +SYNTAX_AFTER_SPACE = ["\n{{ x }}", "\r{% y %}", " {{ x }}", "\t\n {# c #}", "\u3000{{ z }}", "\u2028\u2029{%", + "\u00a0{{ kept inside }}", "\u0085{{ x }}", " \r\n\t{{ x }} \n"] + + +def test_whitespace_before_pebble_syntax_is_kept(): + for text in SYNTAX_AFTER_SPACE: + assert _render("{% if true %}{% endif %}" + apt.static_text(text), {}) == text.encode("utf-8"), repr(text) + + +def test_text_at_the_start_of_a_template_is_not_doubled(): + for text in BREAKS + SYNTAX_AFTER_SPACE + ["\nplain", "\n"]: + assert _render(apt.static_text(text, after_a_tag=False) + "{{ '' }}", {}) == text.encode("utf-8"), repr(text) + + +def test_pebble_syntax_and_a_final_brace_are_written_literally(): + for text in ["a {{ b }} c", "{% if x %}", "{# note #}", "ends {", "{", "{\n{", "x{{"]: + source = "{{ '' }}" + apt.static_text(text) + "{% if true %}{% endif %}" + assert _render(source, {}) == text.encode("utf-8"), repr(text) + + +def test_text_that_would_end_a_verbatim_block_is_refused(): + for text in ["{{ a }}{% endverbatim %}", "{% x %}{%endverbatim%}", "{{ y }}{%- endverbatim -%}"]: + try: + apt.static_text(text) + except apt.Unrepresentable: + continue + raise AssertionError(f"accepted {text!r}") + + +# --- record shapes --------------------------------------------------------------------- + +URL = "/a/pkg/Sample.html" +FILL = {"t": "Title", "v": "21", "lv": "1.0.0", "id": "run(int)", "id2": "pubmethods", "n": "run", + "h": URL + "#run(int)", "ps": "(int times)", "c": "public final class X", "a": '', + "w": "class", "cl": "responsive methods", "x": "0x00000007", "ty": "int", "d": "

Runs.\n

\n", + "sig": "public void run (int times)"} + + +def sample(shape: str) -> str: + parts = shape.split(apt.H) + return "".join(FILL[p] if i % 2 else p for i, p in enumerate(parts)) + + +def test_every_shape_reads_one_record_and_writes_it_back(): + converter = apt.Converter(apt.SHAPES) + template = converter.template() + for k, shape in enumerate(apt.SHAPES): + text = sample(shape) + segs = converter.segments(text, URL) + assert [s.get("k") for s in segs] == [k], (k, segs) + if "\0h\0" in shape: + assert segs[0]["h#"] == "run(int)" and "h" not in segs[0], segs[0] + assert _render(template, {"u": URL, "b": segs}) == text.encode("utf-8"), k + + +def test_a_link_elsewhere_is_stored_whole(): + converter = apt.Converter(apt.SHAPES) + text = sample(apt.SHAPES[0]).replace(URL + "#run(int)", "/a/other/Page.html#run(int)") + segs = converter.segments(text, URL) + assert segs[0]["h"] == "/a/other/Page.html#run(int)" and "h#" not in segs[0] + assert _render(converter.template(), {"u": URL, "b": segs}) == text.encode("utf-8") + + +def test_text_no_shape_matches_stays_raw_line_by_line(): + segs = apt.Converter(apt.SHAPES).segments("

one

\n

two

", URL) + assert segs == [{"r": "

one

\n

two

"}] + + +def test_head_and_footer_become_shapes_only_when_enough_pages_share_them(): + pages = [doclava_page(f"C{i}") for i in range(apt.MIN_SHARED_PAGES)] + shapes = apt.head_and_tail_shapes(pages) + assert len(shapes) == 2 and "\0t\0" in shapes[0] and shapes[1] == TAIL + assert apt.head_and_tail_shapes(pages[:-1]) == [] + + +# --- includes --------------------------------------------------------------------------- + +def test_rows_of_one_header_merge_into_one_order(): + found = [apt.find_blocks(doclava_page("A", doclava_block("java/lang/Object", ["clone", "equals", "toString"]))), + apt.find_blocks(doclava_page("B", doclava_block("java/lang/Object", ["clone", "hashCode", "toString"])))] + (key, order), = apt.plan_includes(found).items() + names = [row.split("#", 1)[1].split("(", 1)[0] for row, _ in sorted(order.items(), key=lambda t: t[1])] + assert names[0] == "clone" and names[-1] == "toString" and set(names) == {"clone", "equals", "hashCode", + "toString"} + + +def test_a_header_whose_pages_disagree_on_order_gets_no_include(): + found = [apt.find_blocks(doclava_page("A", doclava_block("java/lang/Object", ["clone", "equals"]))), + apt.find_blocks(doclava_page("B", doclava_block("java/lang/Object", ["equals", "clone"])))] + assert apt.plan_includes(found) == {} + + +def test_a_header_whose_block_repeats_a_row_gets_no_include(): + found = [apt.find_blocks(doclava_page("A", doclava_block("java/lang/Object", ["clone", "clone"])))] + assert apt.plan_includes(found) == {} + + +def test_a_block_without_its_footer_is_not_a_block(): + """A header and rows followed by anything but the style's own footer stay in the page, even when a whole + block of the same style follows.""" + doclava, dackka = doclava_block("java/lang/Object", ["clone"]), dackka_block("pkg/Base", ["getA"]) + for block, foot in ((doclava, "\n
\n\n"), + (dackka, "\n\n
\n \n\n")): + assert len(apt.find_blocks(block)) == 1 + broken = block.replace(foot, "\n
\n\n\n") + assert apt.find_blocks(broken) == [] + (start, end, _key, _rows), = apt.find_blocks(broken + block) + assert (start, end) == (len(broken), len(broken) + len(block)) + + +def test_an_include_starting_with_a_line_break_renders_it_once(): + """An include's first text follows no tag, so a leading line break must not be doubled.""" + source = apt.include_source("\n\n" + apt.KEY_JOIN + "\n", {"row\n": 0}) + assert _render(source, {"x": []}) == b"\n\nrow\n\n" + assert _render(source, {"x": [0]}) == b"\n\n\n" + + +def test_an_include_holds_every_row_behind_its_guard(): + found = [apt.find_blocks(doclava_page("A"))] + (key, order), = apt.plan_includes(found).items() + source = apt.include_source(key, order) + for i in range(len(OBJECT_ROWS)): + assert f"{{% if x contains {i} %}}{{% else %}}" in source + assert source.startswith(key.split(apt.KEY_JOIN)[0]) and source.endswith(key.split(apt.KEY_JOIN)[1]) + + +# --- whole runs --------------------------------------------------------------------------- + +def test_every_page_converts_and_serves_the_same_bytes(tmp_path): + db = make_db(tmp_path / "doc.db") + before = served(db) + code, out, err = run_tool(db) + assert code == 0, err + after = served(db) + assert after.keys() == before.keys() + for path, (data, _tid) in before.items(): + assert after[path][0] == data, path + converted = {p for p, (_d, tid) in after.items() if tid and p.startswith("a/")} + assert converted == set(standard_pages()), converted + assert after["a/devsite/guide.html"][1] == 0 and after["k/other.html"][1] == 0 + t = templates(db) + assert apt.MAIN_TEMPLATE in t and "page.peb" in t + assert sorted(n for n in t if n.startswith(apt.INCLUDE_PREFIX)) == [apt.include_name(0), apt.include_name(1)] + assert f'"{apt.INCLUDE_PREFIX}" ~ s.i ~ "{apt.INCLUDE_SUFFIX}"' in t[apt.MAIN_TEMPLATE] + assert any(p.name.startswith("doc.db.backup-") for p in tmp_path.iterdir()) + conn = sqlite3.connect(db) + assert conn.execute("PRAGMA freelist_count").fetchone()[0] == 0 + conn.close() + + +def guarded_rows(source: str) -> list: + """The member name of each row of an include, in guard order.""" + out = [] + for part in source.split("{% if x contains ")[1:]: + number, rest = part.split(" %}{% else %}", 1) + assert int(number) == len(out) + out.append(rest.split("{% endif %}", 1)[0].split("#", 1)[1].split("(", 1)[0]) + return out + + +def test_the_space_the_pages_gave_up_is_vacuumed(tmp_path): + """Pages that lose a large inherited block leave free pages behind until the VACUUM.""" + import random + rng = random.Random(3305) + names = [f"m{i}x{rng.getrandbits(64):016x}" for i in range(400)] + pages = {f"a/pkg/C{i}.html": doclava_page(f"C{i}", doclava_block("java/lang/Object", names)) for i in range(6)} + db = make_db(tmp_path / "doc.db", pages) + assert run_tool(db)[0] == 0 + conn = sqlite3.connect(db) + assert conn.execute("PRAGMA freelist_count").fetchone()[0] == 0 + conn.close() + + +def test_a_page_stores_the_rows_its_class_overrides(tmp_path): + db = make_db(tmp_path / "doc.db") + assert run_tool(db)[0] == 0 + includes = lambda path: [s for s in stored_context(db, path)["b"] if "i" in s] # noqa: E731 + (c0,), (c3,) = includes("a/pkg/C0.html"), includes("a/pkg/C3.html") + assert c0["i"] == c3["i"] and "x" not in c0 + assert c3["x"] == [guarded_rows(templates(db)[apt.include_name(c3["i"])]).index("hashCode")] + (d2,) = includes("a/x/D2.html") + assert d2["x"] == [1] + + +def test_blocks_whose_pages_disagree_stay_in_the_pages(tmp_path): + pages = standard_pages() + pages["a/pkg/C1.html"] = doclava_page("C1", doclava_block("java/lang/Object", OBJECT_ROWS) + + doclava_block("pkg/I", ["m1", "m2"], "interface")) + pages["a/pkg/C2.html"] = doclava_page("C2", doclava_block("java/lang/Object", OBJECT_ROWS) + + doclava_block("pkg/I", ["m2", "m1"], "interface")) + db = make_db(tmp_path / "doc.db", pages) + before = served(db) + code, out, err = run_tool(db) + assert code == 0, err + after = served(db) + assert after["a/pkg/C1.html"][0] == before["a/pkg/C1.html"][0] and after["a/pkg/C1.html"][1] + assert after["a/pkg/C2.html"][0] == before["a/pkg/C2.html"][0] and after["a/pkg/C2.html"][1] + assert not any("pkg.I" in s for n, s in templates(db).items() if n.startswith(apt.INCLUDE_PREFIX)) + kept = [s for s in stored_context(db, "a/pkg/C1.html")["b"] if s.get("w") == "interface"] + assert len(kept) == 1 and kept[0]["n"] == "pkg.I" # the kept block's header, a record of its own + + +def test_text_that_would_end_a_verbatim_block_stays_in_the_pages(tmp_path): + """A head shared by enough pages, and an inherited row, each holding text that would end a verbatim + block: neither becomes template text, and the pages still convert and serve the same bytes.""" + odd_head = "\n" + pages = standard_pages() + for i in range(6): + rows = OBJECT_ROWS + (["odd"] if i == 0 else []) + page = doclava_page(f"C{i}", doclava_block("java/lang/Object", rows)) + pages[f"a/pkg/C{i}.html"] = odd_head + page + pages["a/pkg/C0.html"] = pages["a/pkg/C0.html"].replace("About odd, caf", "About odd {% endverbatim %}, caf") + db = make_db(tmp_path / "doc.db", pages) + before = served(db) + code, out, err = run_tool(db) + assert code == 0, err + after = served(db) + for path in pages: + assert after[path][0] == before[path][0] and after[path][1], path + sources = templates(db) + assert not any("endverbatim %} -->" in v or "About odd {%" in v for v in sources.values()) + assert not any("java.lang.Object" in v for n, v in sources.items() if n.startswith(apt.INCLUDE_PREFIX)) + + +def test_a_database_without_a_pages_is_left_alone(tmp_path): + db = make_db(tmp_path / "doc.db", pages={}) + digest = sha(db) + code, out, err = run_tool(db) + assert code == 0 and "Nothing to convert." in out + assert sha(db) == digest + + +class Altering: + """The real renderer, except that pages whose JSON holds `marker` come back with one byte more.""" + + def __init__(self, real, marker: bytes): + self.real, self.marker = real, marker + + def define(self, name, text): + self.real.define(name, text) + + def render(self, name, context): + ok, body, nanos = self.real.render(name, context) + return (ok, body + b"!", nanos) if ok and self.marker in context else (ok, body, nanos) + + def close(self): + pass + + +def test_a_page_that_does_not_render_back_stays_html_and_its_include_is_not_written(tmp_path): + pages = standard_pages() + pages["a/pkg/C5.html"] = doclava_page("C5", doclava_block("java/lang/Object", OBJECT_ROWS) + + doclava_block("pkg/Only", ["solo"])) + db = make_db(tmp_path / "doc.db", pages) + conn = sqlite3.connect(db) + c5 = conn.execute("SELECT content FROM Content WHERE path = 'a/pkg/C5.html'").fetchone()[0] + conn.close() + code, out, err = run_tool(db, renderer=Altering(_harness(), b'"/a/pkg/C5.html"')) + assert code == 0, err + assert "stays HTML: a/pkg/C5.html - renders differently from byte" in err + conn = sqlite3.connect(db) + row = conn.execute("SELECT content, templateId FROM Content WHERE path = 'a/pkg/C5.html'").fetchone() + assert row == (c5, 0) + assert conn.execute("SELECT templateId FROM Content WHERE path = 'a/pkg/C4.html'").fetchone()[0] > 0 + conn.close() + assert not any("pkg.Only" in s for s in templates(db).values()) + + +class Failing: + """The real renderer, except that pages whose JSON holds `marker` fail with an error other than the limit.""" + + def __init__(self, real, marker: bytes): + self.real, self.marker = real, marker + + def define(self, name, text): + self.real.define(name, text) + + def render(self, name, context): + if self.marker in context: + return False, "PebbleException: boom (?:?)\n\tat Render.main", 0 + return self.real.render(name, context) + + def close(self): + pass + + +def test_a_page_that_fails_to_render_stays_html_and_says_why(tmp_path): + db = make_db(tmp_path / "doc.db") + conn = sqlite3.connect(db) + c5 = conn.execute("SELECT content FROM Content WHERE path = 'a/pkg/C5.html'").fetchone()[0] + conn.close() + code, out, err = run_tool(db, renderer=Failing(_harness(), b'"/a/pkg/C5.html"')) + assert code == 0, err + assert "stays HTML: a/pkg/C5.html - does not render: PebbleException: boom (?:?)\n" in err + conn = sqlite3.connect(db) + assert conn.execute("SELECT content, templateId FROM Content WHERE path = 'a/pkg/C5.html'").fetchone() == (c5, 0) + assert conn.execute("SELECT templateId FROM Content WHERE path = 'a/pkg/C4.html'").fetchone()[0] > 0 + conn.close() + + +def test_a_page_over_the_render_limit_stays_html(tmp_path): + pages = standard_pages() + pages["a/pkg/Huge.html"] = doclava_page("Huge", extra="

" + "x" * (4 * 1024 * 1024) + "

\n") + db = make_db(tmp_path / "doc.db", pages) + code, out, err = run_tool(db) + assert code == 0, err + assert "stays HTML: a/pkg/Huge.html - is over the app's render limit" in err + conn = sqlite3.connect(db) + assert conn.execute("SELECT templateId FROM Content WHERE path = 'a/pkg/Huge.html'").fetchone()[0] == 0 + conn.close() + + +def test_rows_it_must_not_convert_are_left_and_named(tmp_path): + db = make_db(tmp_path / "doc.db", extra={ + "a/pkg/Templated.html": (doclava_page("Templated").encode(), True), + "a/pkg/Latin1.html": ("

café

".encode("latin-1"), True), + "a/pkg/Nul.html": (b"

\x00

\n", True)}, template_ids={"a/pkg/Templated.html": 1}) + before = served(db) + code, out, err = run_tool(db) + assert code == 0, err + assert "left as it is: a/pkg/Templated.html - rendered through another template (templateId 1)" in err + assert "left as it is: a/pkg/Latin1.html - is not UTF-8" in err + assert "left as it is: a/pkg/Nul.html - holds a NUL character" in err + conn = sqlite3.connect(db) + left = dict(conn.execute("SELECT path, templateId FROM Content WHERE path IN " + "('a/pkg/Latin1.html', 'a/pkg/Nul.html', 'a/pkg/Templated.html', 'a/img.png')")) + conn.close() + assert left == {"a/pkg/Latin1.html": 0, "a/pkg/Nul.html": 0, "a/pkg/Templated.html": 1, "a/img.png": 0} + after = served(db) + assert after["a/img.png"] == before["a/img.png"] + + +def test_a_dry_run_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + digest = sha(db) + code, out, err = run_tool(db, dry_run=True) + assert code == 0, err + assert "Would convert 12 of 12" in out and "Dry run: nothing written." in out + assert sha(db) == digest and [p.name for p in tmp_path.iterdir()] == ["doc.db"] + + +def test_a_second_run_refuses_and_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + assert run_tool(db)[0] == 0 + digest = sha(db) + backups = sorted(p.name for p in tmp_path.iterdir()) + code, out, err = run_tool(db) + assert code == 1 + assert "already holds this step's templates" in err and "Nothing written." in err + assert sha(db) == digest and sorted(p.name for p in tmp_path.iterdir()) == backups + + +def test_an_unreadable_row_refuses_and_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db", extra={"a/pkg/Broken.html": (b"not brotli at all", False)}) + digest = sha(db) + code, out, err = run_tool(db) + assert code == 1 + assert "a/pkg/Broken.html: could not be read" in err and "nothing was written" in err + assert sha(db) == digest and [p.name for p in tmp_path.iterdir()] == ["doc.db"] + + +def test_a_missing_renderer_refuses_and_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + digest = sha(db) + saved = apt.find_classpath + + def missing(): + raise RuntimeError("cannot find org.pebbletemplates:pebble:4.1.1") + apt.find_classpath = missing + try: + code, out, err = run_tool(db, renderer=None) + finally: + apt.find_classpath = saved + assert code == 1 and "cannot find org.pebbletemplates" in err and "nothing written" in err + assert sha(db) == digest + + +def test_a_missing_database_refuses(tmp_path): + code, out, err = run_tool(tmp_path / "absent.db") + assert code == 1 and "does not exist" in err + assert not (tmp_path / "absent.db").exists() + + +def add_versions(db: Path, versions: list) -> None: + """DocumentationDatabaseVersion rows, inserted in the order given: (major, changeTime) each.""" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE IF NOT EXISTS DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL, " + "patch INT NOT NULL, who TEXT NOT NULL, comment TEXT NOT NULL, " + "changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") + conn.executemany("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " + "VALUES (?, 0, 0, 'test', 'test', ?)", versions) + conn.commit() + conn.close() + + +def test_a_database_declaring_version_3_refuses_and_writes_nothing(tmp_path): + """Its templates are compressed, and this step writes plain ones the app could not decode.""" + db = make_db(tmp_path / "doc.db") + add_versions(db, [(2, "2026-08-20 22:52:54"), (3, "2026-10-19 07:09:52")]) + digest = sha(db) + for dry_run in (False, True): + code, out, err = run_tool(db, dry_run=dry_run, renderer=None) + assert code == 1, (dry_run, out, err) + assert "error: the database declares documentation version 3" in err and "Nothing written." in err + assert "Found" not in out + assert sha(db) == digest and [p.name for p in tmp_path.iterdir()] == ["doc.db"] + + +def test_the_newest_version_row_is_read_by_change_time_then_rowid(tmp_path): + """Ordered as the app orders them: a later changeTime wins over a later rowid, and a tie goes to the + later rowid. Rowid alone, changeTime alone, or a tie broken by the earlier rowid each reads another row.""" + cases = [ + ([(3, "2026-10-19 07:00:00"), (2, "2026-08-20 00:00:00")], 1), # the older row is the newer change + ([(2, "2026-10-19 07:00:00"), (3, "2026-08-20 00:00:00")], 0), + ([(2, "2026-10-19 07:00:00"), (3, "2026-10-19 07:00:00")], 1), # a tie: the later rowid + ([(3, "2026-10-19 07:00:00"), (2, "2026-10-19 07:00:00")], 0), + ] + for i, (versions, expected) in enumerate(cases): + db = make_db(tmp_path / f"doc{i}.db") + add_versions(db, versions) + code, out, err = run_tool(db, dry_run=True) + assert code == expected, (versions, out, err) + assert ("Would convert 12 of 12" in out) == (expected == 0), (versions, out, err) + + +def test_a_database_declaring_version_2_or_none_still_converts(tmp_path): + """No version table, an empty one, and version 2 all hold plain templates.""" + for name, versions in (("none", None), ("empty", []), ("two", [(2, "2026-08-20 22:52:54")])): + db = make_db(tmp_path / f"{name}.db") + if versions is not None: + add_versions(db, versions) + code, out, err = run_tool(db) + assert code == 0, (name, err) + assert "Converting 12 of 12" in out, (name, out) + assert apt.MAIN_TEMPLATE in templates(db), name + + +def test_a_chunked_page_is_read_whole_and_its_json_written_in_chunks(tmp_path): + saved = optimize_db_media.CHUNK_SIZE, update_media_references.CHUNK_SIZE + optimize_db_media.CHUNK_SIZE = update_media_references.CHUNK_SIZE = 64 + try: + db = make_db(tmp_path / "doc.db") + conn = sqlite3.connect(db) + blob = conn.execute("SELECT content FROM Content WHERE path = 'a/pkg/C0.html'").fetchone()[0] + assert len(blob) > 128 and len(blob) % 64 + conn.execute("UPDATE Content SET content = ? WHERE path = 'a/pkg/C0.html'", (blob[:64],)) + for n, start in enumerate(range(64, len(blob), 64), start=1): + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID) VALUES (?, 1, ?, 12)", + (f"a/pkg/C0.html-{n}", blob[start:start + 64])) + conn.commit() + conn.close() + before = served(db) + code, out, err = run_tool(db) + assert code == 0, err + assert "stays HTML" not in err + after = served(db) + assert after["a/pkg/C0.html"][0] == before["a/pkg/C0.html"][0] and after["a/pkg/C0.html"][1] + conn = sqlite3.connect(db) + chunks = conn.execute("SELECT count(*), min(templateId) FROM Content WHERE path LIKE 'a/pkg/C0.html-%'" + ).fetchone() + conn.close() + assert chunks[0] >= 1 and chunks[1] > 0 + finally: + optimize_db_media.CHUNK_SIZE, update_media_references.CHUNK_SIZE = saved + + +def test_a_failure_while_writing_rolls_everything_back(tmp_path): + db = make_db(tmp_path / "doc.db") + conn = sqlite3.connect(db) + content = sorted(conn.execute("SELECT * FROM Content")) + names = sorted(conn.execute("SELECT name FROM Templates")) + conn.close() + saved, calls = apt.replace_row, [] + + def failing(*args, **kwargs): + calls.append(args[1]) + if len(calls) == 2: + raise RuntimeError("disk full") + return saved(*args, **kwargs) + apt.replace_row = failing + try: + try: + run_tool(db) + except RuntimeError as exc: + assert "disk full" in str(exc) + else: + raise AssertionError("the failure was swallowed") + finally: + apt.replace_row = saved + conn = sqlite3.connect(db) + assert sorted(conn.execute("SELECT * FROM Content")) == content + assert sorted(conn.execute("SELECT name FROM Templates")) == names + conn.close() + + +def test_worker_processes_give_the_same_result(tmp_path): + one, two = make_db(tmp_path / "one.db"), make_db(tmp_path / "two.db") + assert run_tool(one)[0] == 0 + assert run_tool(two, workers=2)[0] == 0 + contexts = lambda db: {p: stored_context(db, p) for p in standard_pages()} # noqa: E731 + assert contexts(one) == contexts(two) + assert templates(one) == templates(two) + + +# --- shared texts -------------------------------------------------------------------- + +SHARED = "

Every one of these pages repeats this paragraph, word for word.

\n" +DESCRIPTION = "Returns the same long description on every one of these pages." + + +def notes(title: str = "Notes") -> str: + """A section heading, which is a record: the text around it becomes raw segments of their own.""" + return f'

{title}

\n' + + +def shared_extra() -> str: + """SHARED as a raw segment of its own, and DESCRIPTION in a returns row's description hole.""" + return (notes() + SHARED + notes() + '\n\n\n' + f'\n\n\n' + '
Returns
int{DESCRIPTION}
\n') + + +def shared_pages(holders: int, others: bool = True) -> dict: + """The first `holders` Doclava pages carry shared_extra(); `others` adds the rest of standard_pages().""" + pages = standard_pages() if others else {} + for i in range(holders): + pages[f"a/pkg/C{i}.html"] = doclava_page(f"C{i}", extra=shared_extra()) + return pages + + +def bundle_texts(db: Path) -> dict: + """(bundle, index) -> text, read back out of the stored bundles (their texts hold no Pebble syntax).""" + out = {} + for name, source in templates(db).items(): + if not name.startswith(apt.TEXT_PREFIX): + continue + number = int(name[len(apt.TEXT_PREFIX):-len(apt.INCLUDE_SUFFIX)]) + assert source.endswith("{% endif %}") + for part in source[:-len("{% endif %}")].split("{% ")[1:]: + head, text = part.split(" %}", 1) + out[(number, int(head.rsplit("==", 1)[1]))] = text + return out + + +def refs(db: Path, path: str) -> dict: + """The shared-text references a stored page holds: (bundle, index) -> field.""" + return {tuple(v): f[:-1] for seg in stored_context(db, path)["b"] for f, v in seg.items() if f.endswith("@")} + + +def test_texts_on_five_pages_are_interned_and_serve_the_same_bytes(tmp_path): + db = make_db(tmp_path / "doc.db", shared_pages(5)) + before = served(db) + code, out, err = run_tool(db) + assert code == 0, err + after = served(db) + for path, (data, _tid) in before.items(): + assert after[path][0] == data, path + assert all(after[path][1] for path in standard_pages()), "a page stayed HTML" + texts = bundle_texts(db) + (shared,) = [k for k, v in texts.items() if v == SHARED] + (description,) = [k for k, v in texts.items() if v == DESCRIPTION] + for i in range(5): + held = refs(db, f"a/pkg/C{i}.html") + assert held[shared] == "r" and held[description] == "d", held + assert shared not in refs(db, "a/pkg/C5.html") and description not in refs(db, "a/pkg/C5.html") + # Only the holes that hold a shared text somewhere read one. + fields = {f for path in standard_pages() for f in refs(db, path).values()} + assert set(re.findall(r's\["(\w+)@"\]', templates(db)[apt.MAIN_TEMPLATE])) == fields + assert {"r", "d"} <= fields and "ty" not in fields + + +def test_a_text_on_four_pages_stays_in_the_pages(tmp_path): + db = make_db(tmp_path / "doc.db", shared_pages(4)) + assert run_tool(db)[0] == 0 + assert SHARED not in bundle_texts(db).values() and DESCRIPTION not in bundle_texts(db).values() + segs = stored_context(db, "a/pkg/C0.html")["b"] + assert {"r": SHARED} in segs and any(seg.get("d") == DESCRIPTION for seg in segs) + + +def contexts_sharing(texts: list, pages: int = 5) -> list: + """`pages` contexts, each holding every one of `texts` as a raw segment.""" + return [{"u": f"/a/p{i}.html", "b": [{"r": text} for text in texts]} for i in range(pages)] + + +def with_setting(name: str, value: int, function): + saved = getattr(apt, name) + setattr(apt, name, value) + try: + return function() + finally: + setattr(apt, name, saved) + + +def with_bundle_size(size: int, function): + return with_setting("INTERN_MAX_TEMPLATE", size, function) + + +def test_bundles_fill_up_to_the_bundle_size_and_no_further(): + # Lengths vary, so some text lands right at a bundle's limit. + texts = [f"

Shared text number {i:03d}{'x' * (i % 13)}, long enough to intern.

\n" for i in range(40)] + contexts = contexts_sharing(texts) + # No merging here, only the split: one page set's texts, which gain nothing from a merge when a bundle costs 0. + sources, interned = with_setting("INTERN_BUNDLE_COST", 0, + lambda: with_bundle_size(400, lambda: apt.intern_texts(contexts))) + assert interned == {"r"} + names = [apt.text_name(b) for b in range(len(sources))] + assert sorted(sources) == sorted(names) and len(names) > 3 + for b, name in enumerate(names): + size = len(sources[name].encode()) + assert size <= 400, (name, size) + if b + 1 < len(names): # the next bundle's first text would not have fitted in this one + first = sources[names[b + 1]].removeprefix("{% if j == 0 %}").split("{% ", 1)[0] + grown = size + len(f"{{% elseif j == {sources[name].count('{% elseif') + 1} %}}{first}".encode()) + assert grown > 400, (name, grown) + # Each text has its own place, numbered in order of first use. + places = [tuple(seg["r@"]) for seg in contexts[0]["b"]] + assert places == sorted(places) and len(set(places)) == len(texts) + assert all(c["b"] == contexts[0]["b"] for c in contexts) + + +def test_a_bundle_may_reach_the_bundle_size_exactly(): + # 15 bytes of "{% if j == 0 %}", 19 of "{% elseif j == 1 %}" and 11 of "{% endif %}": 355 bytes of text make 400. + texts = ["

" + "a" * 193 + "

", "

" + "b" * 148 + "

"] + # The split alone: with no merging (a bundle costing 0), a split that left a second bundle would show. + sources, _interned = with_setting("INTERN_BUNDLE_COST", 0, lambda: with_bundle_size( + 400, lambda: apt.intern_texts(contexts_sharing(texts)))) + assert list(sources) == [apt.text_name(0)] and len(sources[apt.text_name(0)].encode()) == 400 + + +def test_only_the_holes_that_hold_a_shared_text_read_one(): + converter = apt.Converter(apt.SHAPES) + assert "@" not in converter.template() + some = converter.template(frozenset({"d"})) + assert 's["d@"]' in some and 's["r@"]' not in some and 's["ty@"]' not in some + assert 's["r@"]' in converter.template(frozenset({"r"})) + + +def contexts_using(texts_by_pages: list, pages: int) -> list: + """`pages` contexts; each (texts, page numbers) pair puts those texts, as raw segments, on those pages.""" + out = [{"u": f"/a/p{i}.html", "b": []} for i in range(pages)] + for texts, numbers in texts_by_pages: + for i in numbers: + out[i]["b"] += [{"r": t} for t in texts] + return out + + +def bundle_of(contexts: list, page: int) -> set: + return {seg["r@"][0] for seg in contexts[page]["b"] if "r@" in seg} + + +def test_texts_the_same_pages_use_share_a_bundle_and_others_get_their_own(): + a = [f"

Text {i} that the first five pages share.

\n" for i in range(3)] + b = [f"

Text {i} that the next five pages share.

\n" for i in range(3)] + contexts = contexts_using([(a, range(5)), (b, range(5, 10))], 10) + sources, _ = apt.intern_texts(contexts) + assert len(sources) == 2 + assert all(bundle_of(contexts, i) == {0} for i in range(5)) and all(bundle_of(contexts, i) == {1} for i in range(5, 10)) + assert [seg["r@"] for seg in contexts[0]["b"]] == [[0, 0], [0, 1], [0, 2]] # in order of first use + + +def big(name: str, size: int) -> str: + return f"

{name} " + "x" * (size - len(name) - 9) + "

\n" + + +def test_groups_merge_when_the_pages_both_serve_outweigh_the_bytes_wasted(): + """A (5,000 bytes, pages 0-4) and B (pages 0-5): merged, five pages save a bundle (5 x 8,000) and page 5 compiles + A for nothing (5,000). With a bundle worth a tenth as much, they would stay apart.""" + a, b = big("A", 5000), "

B, on one page more than A.

\n" + contexts = contexts_using([([a], range(5)), ([b], range(6))], 6) + sources, _ = apt.intern_texts(contexts) + assert len(sources) == 1 and bundle_of(contexts, 0) == {0} + + +def test_groups_stay_apart_when_the_bytes_wasted_outweigh_the_pages_both_serve(): + """A (5,000 bytes, pages 0-4) and B (pages 0-14): merged, five pages would save a bundle (40,000) and ten would + compile A for nothing (50,000).""" + a, b = big("A", 5000), "

B, on ten pages more than A.

\n" + contexts = contexts_using([([a], range(5)), ([b], range(15))], 15) + sources, _ = apt.intern_texts(contexts) + assert len(sources) == 2 and bundle_of(contexts, 0) == {0, 1} and bundle_of(contexts, 10) == {1} + + +def tiny(name: str) -> str: + return f"

{name}, a short text long enough to share.

\n" + + +def test_a_merge_that_would_overflow_a_bundle_is_refused(): + """Merged, B's ten texts move to indexes 10-19, one digit longer each: the bundle outgrows the two parts it was + made of, so a cap of exactly their sum must refuse it.""" + a = [tiny(f"A{i}") for i in range(10)] + b = [tiny(f"B{i}") for i in range(10)] + size = lambda texts: len(apt.bundle_source(texts).encode()) # noqa: E731 + cap = size(a) + size(b) + assert size(a + b) > cap + contexts = contexts_using([(a, range(5)), (b, range(6))], 6) + sources, _ = with_bundle_size(cap, lambda: apt.intern_texts(contexts)) + assert len(sources) == 2 and all(len(s.encode()) <= cap for s in sources.values()) + + +def test_a_merged_bundle_keeps_its_texts_in_order_of_first_use(): + a1, b1, a2 = tiny("A1"), tiny("B1"), tiny("A2") + contexts = [{"u": f"/a/p{i}.html", "b": [{"r": a1}, {"r": b1}, {"r": a2}] if i < 5 else [{"r": b1}]} + for i in range(6)] + sources, _ = apt.intern_texts(contexts) + assert len(sources) == 1 + assert [seg["r@"] for seg in contexts[0]["b"]] == [[0, 0], [0, 1], [0, 2]] + + +def test_bundles_are_numbered_by_their_first_texts_use(): + """P's texts are first used at 0, 1, 3 and 4, Q's at 2; with room for two texts a bundle, P splits in two and Q's + bundle comes between them.""" + p = [tiny(f"P{i}") for i in range(4)] + q = [tiny("Q0")] + contexts = [{"u": f"/a/p{i}.html", "b": [{"r": t} for t in (p[:2] + q + p[2:] if i < 5 else q)]} for i in range(6)] + cap = len(apt.bundle_source(p[:2]).encode()) + sources, _ = with_bundle_size(cap, lambda: apt.intern_texts(contexts)) + assert [seg["r@"][0] for seg in contexts[0]["b"]] == [0, 0, 1, 2, 2] + + +def test_a_merge_offered_before_its_group_grew_is_not_taken(): + """A (pages 0-9) merges with B (pages 0-17) first. A and C (6,000 bytes, pages 0-4 and 18) were worth merging + while A was alone; merged with A and B, C would cost thirteen pages 6,000 bytes each, so it stays apart.""" + a, b, c = tiny("A"), tiny("B"), big("C", 6000) + contexts = contexts_using([([a], range(10)), ([b], range(18)), ([c], list(range(5)) + [18])], 19) + sources, _ = apt.intern_texts(contexts) + holding = lambda text: {name for name, source in sources.items() if text in source} # noqa: E731 + assert len(sources) == 2 and holding(a) == holding(b) != holding(c) + + +def test_a_merged_group_can_still_merge_with_a_lower_numbered_one(): + """X (pages 0-11) comes first, so it has the lowest number. A (pages 0-9 and 12) and B (pages 0-9 and 13) merge + first; the merged AB must still be able to merge with X, whose number is lower than its own (R56-01).""" + x, a, b = tiny("X"), tiny("A"), tiny("B") + contexts = contexts_using([([x], range(12)), ([a], list(range(10)) + [12]), ([b], list(range(10)) + [13])], 14) + sources, _ = apt.intern_texts(contexts) + assert len(sources) == 1, sorted(sources) + + +def test_a_text_that_fills_a_bundle_exactly_is_shared_and_one_byte_more_is_not(): + # A bundle of one text is "{% if j == 0 %}" (15 bytes), the text and "{% endif %}" (11). + fits, over = "

" + "f" * (300 - 26 - 8) + "

\n", "

" + "o" * (300 - 26 - 7) + "

\n" + assert len(fits) == 274 and len(over) == 275 + contexts = contexts_sharing([fits, over]) + sources, _ = with_bundle_size(300, lambda: apt.intern_texts(contexts)) + assert contexts[0]["b"] == [{"r@": [0, 0]}, {"r": over}] + assert [len(s.encode()) for s in sources.values()] == [300] + + +def test_a_bundle_holds_at_most_8000_bytes_of_source(): + fits, over = "

" + "f" * (8000 - 26 - 8) + "

\n", "

" + "o" * (8000 - 26 - 7) + "

\n" + contexts = contexts_sharing([fits, over]) + sources, _ = apt.intern_texts(contexts) + assert contexts[0]["b"] == [{"r@": [0, 0]}, {"r": over}] + assert [len(s.encode()) for s in sources.values()] == [8000] + + +def test_a_text_of_32_characters_is_shared_and_one_of_31_is_not(): + t32, t31 = "

" + "a" * 25 + "

", "

" + "b" * 24 + "

" + assert (len(t32), len(t31)) == (32, 31) + contexts = contexts_sharing([t32, t31]) + apt.intern_texts(contexts) + assert contexts[0]["b"] == [{"r@": [0, 0]}, {"r": t31}] + + +def test_a_merge_that_fills_a_bundle_exactly_is_taken(): + # Merged: "{% if j == 0 %}" + A + "{% elseif j == 1 %}" + B + "{% endif %}", 45 bytes of tags. + a, b = "

" + "a" * (150 - 8) + "

\n", "

" + "b" * (155 - 8) + "

\n" + cap = 45 + len(a) + len(b) + contexts = contexts_using([([a], range(5)), ([b], range(6))], 6) + sources, _ = with_bundle_size(cap, lambda: apt.intern_texts(contexts)) + assert [len(s.encode()) for s in sources.values()] == [cap] + + +def test_a_merge_that_gains_nothing_is_not_taken(): + """A (pages 0-5) and B (500 bytes as a bundle, pages 0-4): merged, five pages would save a bundle each, worth 5 x + 100 here, and page 5 would compile B's 500 bytes for nothing.""" + a, b = tiny("A"), "

" + "b" * (500 - 26 - 8) + "

\n" + assert len(apt.bundle_source([b]).encode()) == 500 + contexts = contexts_using([([a], range(6)), ([b], range(5))], 6) + sources, _ = with_setting("INTERN_BUNDLE_COST", 100, lambda: apt.intern_texts(contexts)) + assert len(sources) == 2 + more, _ = with_setting("INTERN_BUNDLE_COST", 101, lambda: apt.intern_texts(contexts_using([([a], range(6)), ([b], range(5))], 6))) + assert len(more) == 1 + + +PEBBLE_LIKE = [ + "

Prints {{ like a variable }} and is shared by five pages.

\n", + "

Opens {% like a tag %} and is shared by five pages too.

\n", + "

Opens {# like a comment and is shared by five pages.

\n", + "

An unclosed {{ and an unclosed {% on five pages, word for word.

\n", + "

Whitespace control {%- like this -%} on five pages, word for word.

\n", + "

Ends with an opening brace on five pages, word for word {

\n", + "

A Windows line end on five pages,\r\nword for word.

\n", + "

Line separator\u2028and paragraph separator\u2029on five pages.

\n", + "

Next line\u0085on five pages, word for word, shared.

\n", + "

Non-ASCII: café, naïve, Ελληνικά, 中文, on five pages.

\n", +] + + +def test_texts_holding_pebble_syntax_and_line_breaks_are_shared_and_serve_the_same_bytes(tmp_path): + """A text that Pebble would read as syntax, or whose line breaks it would trim, goes in a bundle as literal text. + If it did not, the render check would keep its pages as HTML, so this asserts that they convert.""" + extra = "".join(notes() + text for text in PEBBLE_LIKE) + notes() + pages = standard_pages() + for i in range(5): + pages[f"a/pkg/C{i}.html"] = doclava_page(f"C{i}", extra=extra) + db = make_db(tmp_path / "doc.db", pages) + before = served(db) + code, out, err = run_tool(db) + assert code == 0, err + after = served(db) + for path, (data, _tid) in before.items(): + assert after[path][0] == data, path + assert all(after[f"a/pkg/C{i}.html"][1] for i in range(5)), err + shared = {seg["r@"][0] for seg in stored_context(db, "a/pkg/C0.html")["b"] if "r@" in seg} + held = "".join(t for n, t in templates(db).items() if n.startswith(apt.TEXT_PREFIX)) + assert shared and "{% verbatim %}" in held + raws = [seg.get("r", "") for seg in stored_context(db, "a/pkg/C0.html")["b"]] + assert not any(text in raw for text in PEBBLE_LIKE for raw in raws), "a text stayed in the page" + + +def test_several_bundles_serve_the_same_bytes(tmp_path): + extra = "".join(notes() + f"

Shared paragraph number {i:03d}, repeated on five pages.

\n" for i in range(30)) + pages = standard_pages() + for i in range(5): + pages[f"a/pkg/C{i}.html"] = doclava_page(f"C{i}", extra=extra + notes()) + db = make_db(tmp_path / "doc.db", pages) + before = served(db) + code, out, err = with_bundle_size(400, lambda: run_tool(db)) + assert code == 0, err + assert sum(n.startswith(apt.TEXT_PREFIX) for n in templates(db)) > 3 + after = served(db) + for path, (data, _tid) in before.items(): + assert after[path][0] == data, path + assert all(after[path][1] for path in pages), "a page stayed HTML" + assert all(refs(db, f"a/pkg/C{i}.html") for i in range(5)) + + +def test_a_text_longer_than_a_bundle_stays_in_its_pages(): + long_text = "

" + "a long shared paragraph " * (apt.INTERN_MAX_TEMPLATE // 20) + "

\n" + short_text = "

A short shared paragraph, long enough to intern.

\n" + contexts = contexts_sharing([long_text, short_text]) + sources, _interned = apt.intern_texts(contexts) + assert contexts[0]["b"] == [{"r": long_text}, {"r@": [0, 0]}] + assert list(sources) == [apt.text_name(0)] and long_text not in sources[apt.text_name(0)] + + +def test_links_are_never_interned(): + """A same-page link is stored as its fragment, which often equals a member's id: the id may be shared, the + link may not, or the page would lose its own URL.""" + fragment = "aVeryLongMemberName(android.content.Context,%20int)" + link = "/a/android/view/SomeVeryLongClassName.html#member(int)" + contexts = [{"u": f"/a/p{i}.html", "b": [{"k": 3, "id": fragment, "h#": fragment, "h": link}]} for i in range(5)] + sources, interned = apt.intern_texts(contexts) + assert interned == {"id"} + assert contexts[0]["b"] == [{"k": 3, "h#": fragment, "h": link, "id@": [0, 0]}] + + +def test_a_database_holding_shared_text_bundles_refuses_and_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + conn = sqlite3.connect(db) + conn.execute("INSERT INTO Templates (name, content) VALUES (?, ?)", (apt.text_name(0), b"{% if j == 0 %}x{% endif %}")) + conn.commit() + conn.close() + digest = sha(db) + code, out, err = run_tool(db) + assert code == 1 and "already holds this step's templates" in err and apt.text_name(0) in err + assert sha(db) == digest and [p.name for p in tmp_path.iterdir()] == ["doc.db"] + + +def test_a_dry_run_with_shared_texts_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db", shared_pages(5)) + digest = sha(db) + code, out, err = run_tool(db, dry_run=True) + assert code == 0, err + assert re.search(r"and [1-9][0-9]* shared-text bundle\(s\)", out) and "Dry run: nothing written." in out + assert sha(db) == digest and [p.name for p in tmp_path.iterdir()] == ["doc.db"] + + +class AlteringAll(Altering): + """Every page whose JSON holds one of `markers` comes back with one byte more.""" + + def __init__(self, real, markers: list): + super().__init__(real, b"") + self.markers = markers + + def render(self, name, context): + ok, body, nanos = self.real.render(name, context) + return (ok, body + b"!", nanos) if ok and any(m in context for m in self.markers) else (ok, body, nanos) + + +def test_a_bundle_only_pages_that_stay_html_use_is_not_written(tmp_path): + """Only the five holders share texts, and one more page, which shares none, converts. When every holder stays + HTML, no bundle has a reader.""" + pages = {**shared_pages(5, others=False), "a/x/D0.html": dackka_page("D0")} + db = make_db(tmp_path / "doc.db", pages) + dry = run_tool(db, dry_run=True) + assert "and 0 shared-text bundle(s)" not in dry[1], dry[1] + markers = [f'"/a/pkg/C{i}.html"'.encode() for i in range(5)] + code, out, err = run_tool(db, renderer=AlteringAll(_harness(), markers)) + assert code == 0, err + assert err.count("stays HTML: a/pkg/C") == 5 + assert served(db)["a/x/D0.html"][1] and refs(db, "a/x/D0.html") == {} + assert not any(n.startswith(apt.TEXT_PREFIX) for n in templates(db)) + + +def _main() -> int: + """Runs the tests without pytest, so this file works the way the tools + beside it do (`uv run scripts/test_android_pages_to_templates.py`).""" + failures, skipped = [], [] + for name, function in sorted(globals().items()): + if not name.startswith("test_") or not callable(function): + continue + with tempfile.TemporaryDirectory() as directory: + needs_tmp = "tmp_path" in function.__code__.co_varnames[:function.__code__.co_argcount] + try: + function(Path(directory)) if needs_tmp else function() + except Skip as exc: + skipped.append(name) + print(f"skip {name}: {exc}") + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: # noqa: BLE001 - a failure is the result here + if type(exc).__name__ == "Skipped": # pytest.skip, when pytest is installed + skipped.append(name) + print(f"skip {name}: {exc}") + continue + frame = traceback.extract_tb(exc.__traceback__)[-1] + failures.append(f"{name} (line {frame.lineno}: {frame.line}): {type(exc).__name__}: {exc}") + print(f"FAIL {name}", file=sys.stderr) + else: + print(f"ok {name}") + print() + print(f"{len(failures)} failure(s), {len(skipped)} skipped" if failures or skipped else "all tests passed") + for failure in failures: + print(f" {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(_main())