docs-sidebar toc-element"""
+
+ // A Y with diaeresis (2 bytes in UTF-8) and an ellipsis (3 bytes).
+ const val NON_ASCII = "
Ÿ…
"
+ }
+}
diff --git a/common/src/main/java/com/itsaky/androidide/documentation/ContentBlocks.kt b/common/src/main/java/com/itsaky/androidide/documentation/ContentBlocks.kt
new file mode 100644
index 0000000000..c212137f2a
--- /dev/null
+++ b/common/src/main/java/com/itsaky/androidide/documentation/ContentBlocks.kt
@@ -0,0 +1,154 @@
+/*
+ * This file is part of Code on the Go.
+ *
+ * AndroidIDE is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * AndroidIDE is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with AndroidIDE. If not, see
.
+ */
+
+package com.itsaky.androidide.documentation
+
+import android.database.sqlite.SQLiteDatabase
+
+/**
+ * One item of a decoded content block (ADFA-6683, documentation version 4): bytes
+ * `[offset, offset + length)`.
+ *
+ * The range is checked as 64-bit before it is narrowed to an index: `ContentBlockItems` columns are
+ * SQLite INTEGERs, and an offset past [Int.MAX_VALUE] would otherwise wrap and slice the wrong bytes
+ * without an error.
+ *
+ * @throws IllegalStateException When the range is not inside [block], which means the database is
+ * corrupt.
+ */
+internal fun sliceBlockItem(
+ block: ByteArray,
+ blockId: Long,
+ offset: Long,
+ length: Long,
+): ByteArray {
+ check(offset >= 0 && length >= 0 && offset <= block.size - length) {
+ "Item [$offset, +$length) is outside content block $blockId, which decodes to ${block.size} bytes"
+ }
+ return block.copyOfRange(offset.toInt(), (offset + length).toInt())
+}
+
+/**
+ * Block [blockId]'s compressed bytes, in pieces of at most [pieceBytes], to be decoded back to back
+ * (see [chunksAsStream]).
+ *
+ * A cursor holds its row in a CursorWindow, 2 MiB on Android, and a row that does not fit fails the
+ * read. Only a block's decoded size is capped, not its compressed size, so a block over [pieceBytes]
+ * is read a piece per query with `substr()`, which counts a BLOB's bytes from 1. SQLite loads the
+ * whole blob for each of those queries, so a block of n pieces costs n reads of it; the largest
+ * block in the 4.0.0 build, 559,581 bytes, is one piece and one query.
+ *
+ * @param path The page being served, for the failure.
+ * @throws IllegalStateException When the database has no such block, or it has no data.
+ */
+internal fun readContentBlock(
+ database: SQLiteDatabase,
+ blockId: Long,
+ path: String,
+ pieceBytes: Int = DocumentationContentSource.CONTENT_CHUNK_SIZE,
+): List
{
+ val id = blockId.toString()
+ val pieces = mutableListOf()
+ val size =
+ database.rawQuery(BLOCK_QUERY, arrayOf(pieceBytes.toString(), id)).use { cursor ->
+ check(cursor.moveToFirst()) { "Content block $blockId not found, for path '$path'" }
+ check(!cursor.isNull(0)) { "Content block $blockId has no data, for path '$path'" }
+ // substr() of an empty blob is NULL; that block then fails to decode, naming itself.
+ pieces += cursor.getBlob(1) ?: ByteArray(0)
+ cursor.getLong(0)
+ }
+ var read = pieces[0].size.toLong()
+ while (read < size) {
+ val piece =
+ database.rawQuery(BLOCK_PIECE_QUERY, arrayOf((read + 1).toString(), pieceBytes.toString(), id)).use { cursor ->
+ check(cursor.moveToFirst()) { "Content block $blockId not found, for path '$path'" }
+ cursor.getBlob(0) ?: ByteArray(0)
+ }
+ // Empty only when the blob shrank under the read, which would otherwise loop for good.
+ check(piece.isNotEmpty()) { "Content block $blockId ended at byte $read of $size, for path '$path'" }
+ pieces += piece
+ read += piece.size
+ }
+ return pieces
+}
+
+// The block's size and its first piece in one query, so a block of one piece costs one.
+private const val BLOCK_QUERY = "SELECT length(data), substr(data, 1, ?) FROM ContentBlocks WHERE id = ?"
+
+private const val BLOCK_PIECE_QUERY = "SELECT substr(data, ?, ?) FROM ContentBlocks WHERE id = ?"
+
+/**
+ * Decoded content blocks by key, least recently used out once their plain bytes would pass
+ * [maxBytes]. A block larger than the whole budget is not cached.
+ *
+ * Thread-safe: every access holds this object's monitor. A caller decodes outside it, so a decode
+ * never blocks another request's hit; two requests that miss the same block both decode it, and the
+ * second [put] replaces the first.
+ *
+ * @param K What names a block: its `ContentBlocks.id`, and whatever else tells two files' blocks of
+ * that id apart.
+ */
+internal class ContentBlockCache(
+ private val maxBytes: Long,
+) {
+ // Access order, so iteration starts at the least recently used block.
+ private val blocks = LinkedHashMap(16, 0.75f, true)
+
+ private var plainBytes = 0L
+
+ /** The cached block, now the most recently used; null on a miss. */
+ @Synchronized
+ fun get(key: K): ByteArray? = blocks[key]
+
+ /**
+ * Caches [plain] as [key]'s decoded block and evicts least recently used blocks to fit.
+ *
+ * @return The plain bytes cached after the insert.
+ */
+ @Synchronized
+ fun put(
+ key: K,
+ plain: ByteArray,
+ ): Long {
+ blocks.remove(key)?.let { plainBytes -= it.size }
+ if (plain.size > maxBytes) return plainBytes
+
+ blocks[key] = plain
+ plainBytes += plain.size
+ // Never reaches the block just added: it is the newest, and it fits alone.
+ val eldest = blocks.values.iterator()
+ while (plainBytes > maxBytes) {
+ plainBytes -= eldest.next().size
+ eldest.remove()
+ }
+ return plainBytes
+ }
+
+ @Synchronized
+ fun clear() {
+ blocks.clear()
+ plainBytes = 0
+ }
+
+ /** The plain bytes cached now. */
+ @Synchronized
+ fun plainBytes(): Long = plainBytes
+
+ companion object {
+ const val MAX_PLAIN_BYTES = 8L * 1024 * 1024
+ }
+}
diff --git a/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt b/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt
index 847f0c2ba1..66fcde940a 100644
--- a/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt
+++ b/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt
@@ -17,6 +17,7 @@
package com.itsaky.androidide.documentation
+import android.database.Cursor
import android.database.sqlite.SQLiteDatabase
import androidx.annotation.VisibleForTesting
import com.google.gson.Gson
@@ -25,6 +26,7 @@ import com.google.gson.ToNumberPolicy
import com.google.gson.reflect.TypeToken
import com.itsaky.androidide.utils.BrotliDictionaryCodec
import com.itsaky.androidide.utils.expectsCompressedTemplates
+import com.itsaky.androidide.utils.expectsContentBlocks
import com.itsaky.androidide.utils.loadCompressionDictionary
import io.pebbletemplates.pebble.PebbleEngine
import io.pebbletemplates.pebble.error.PebbleException
@@ -32,6 +34,7 @@ import org.slf4j.LoggerFactory
import java.io.ByteArrayInputStream
import java.io.Closeable
import java.io.File
+import java.io.IOException
import java.io.InputStream
import java.io.SequenceInputStream
import java.io.StringWriter
@@ -152,9 +155,20 @@ data class RequestLookup(
* are Brotli-compressed against the same dictionary as the rows; an earlier database's are plain
* UTF-8 and are read as they are, since an app update never replaces the installed database.
*
+ * From documentation version 4 (ADFA-6683) some pages are stored up to 64 to a Brotli block: such a
+ * row's own content is empty, and its bytes are a slice of the decoded block, which is then served
+ * exactly as a decoded row would be. A block that is missing or does not decode, or an item outside
+ * its block, fails the page. Below 4 every row holds its own content, whatever tables the file has.
+ *
* Thread-safe: [lookup] and [withDatabase] hold a read lock for the whole read, and the swap takes
* the write lock, since swapping closes the handle a reader could be using. Readers never block
- * each other.
+ * each other. A block is decoded and cached under the read lock, and a swap clears the cache under
+ * the write lock, so a block decoded from one database is never served from another. A file
+ * rewritten in place before the swap check notices is caught by keying on the file's state: each
+ * cached block (see [decodedBlock]), and what the file's version and dictionary said (see
+ * [FileBound]). Compiled templates and template names are not keyed so, as that would mean
+ * clearing the engine's caches under the read lock or keying them in the loader; until the swap
+ * check reopens the file, they are the old file's.
*
* @param decodeTemplate Turns a `Templates` row's bytes into the template's text, given the codec
* for the database it came from. A seam for `:common`'s tests only, which have no brotli4j desktop
@@ -220,24 +234,38 @@ class DocumentationContentSource internal constructor(
// Decodes Content's brotli rows against the shared dictionary they were compressed with (see
// ADFA-5153). Built on the first read that needs it after a swap rather than eagerly, then
- // cached for that database. Holds no dictionary -- and so decodes plain brotli -- when the
+ // cached for that file. Holds no dictionary -- and so decodes plain brotli -- when the
// active database declares a version below the dictionary migration. Access only through
// [codec], which rebuilds it when stale.
- private var codec: BrotliDictionaryCodec? = null
- private var codecStale = true
+ private var codec: FileBound? = null
- // Whether the active database's templates are Brotli-compressed (documentation version 3 on), or
+ // Whether the active file's templates are Brotli-compressed (documentation version 3 on), or
// null until its first render asks. Cleared on every swap: the next database may be either kind.
@Volatile
- private var templatesCompressed: Boolean? = null
+ private var templatesCompressed: FileBound? = null
+
+ // Whether the active file stores pages in content blocks (documentation version 4 on), or null
+ // until its first read asks. Cleared with the blocks cached from the previous database: a block
+ // id names a different block in another database.
+ @Volatile
+ private var contentBlocks: FileBound? = null
+
+ // Keyed by the file's state as well as the block id: see FileState.
+ private val blockCache = ContentBlockCache(ContentBlockCache.MAX_PLAIN_BYTES)
+
+ // Its own tag, so logcat can be filtered to the block decodes alone. A cache hit is not logged:
+ // IdeLogger has debug on in every build, and a hit happens for most requests a page makes.
+ private val blockLog = LoggerFactory.getLogger("DocBlocks")
// The loader reads Templates rows, so a template can reference another one (ADFA-5405). It also
// makes the engine's own cache the compiled-template cache, keyed by name: a partial pulled in
// by several pages is compiled once, and dropping a database means invalidating that cache too.
// A compressed row is decoded with the active database's codec, the one its Content rows use.
+ // Both are the file's as it is now, which a rewrite in place may have changed since the page's row.
private val templateLoader =
DatabaseTemplateLoader({ database }) { database, body ->
- if (templatesCompressed(database)) decodeTemplate(codec(database), body) else body
+ val file = activeFileState()
+ if (templatesCompressed(database, file)) decodeTemplate(codec(database, file), body) else body
}
private val pebbleEngine =
@@ -412,7 +440,11 @@ class DocumentationContentSource internal constructor(
name: String,
path: String,
contextJson: (SQLiteDatabase) -> ByteArray,
- ): ByteArray = withDatabase { database -> renderNamed(database, name, contextJson(database), path) }
+ ): ByteArray =
+ withDatabase { database ->
+ val file = activeFileState()
+ renderNamed(database, name, contextJson(database), path, file)
+ }
/**
* Clears all cached templates, compiled and by name.
@@ -456,6 +488,7 @@ class DocumentationContentSource internal constructor(
log.error("Cannot close the documentation database", e)
}
database = null
+ forgetDatabaseState()
}
}
@@ -489,45 +522,194 @@ class DocumentationContentSource internal constructor(
): DocumentationLookup {
// Primed before the row is read, not inside decompressBrotli, so a database's dictionary is
// loaded on its first content fetch (ADFA-5153's contract) rather than on the first fetch
- // that happens to be Brotli-compressed. Still at most once per database.
+ // that happens to be Brotli-compressed. Still at most once per database and file state.
//
// Best-effort, unlike the call inside the decode: this is an optimisation, and letting a
// transient failure here propagate would fail *every* lookup -- including rows with
- // compression = 'none', which need no dictionary at all. The staleness flag is left set, so
- // the next read retries, and a brotli row that genuinely cannot resolve its dictionary still
- // fails loudly from decompressBrotli.
+ // compression = 'none', which need no dictionary at all. Nothing is cached, so the next read
+ // retries, and a brotli row that genuinely cannot resolve its dictionary still fails loudly
+ // from decompressBrotli.
+ //
+ // The file's state is taken before any of its reads, so everything this read decides from
+ // the file is keyed by a state no later than the one it read (see FileBound).
+ val fileBefore = activeFileState()
try {
- codec(database)
+ codec(database, fileBefore)
} catch (e: Exception) {
log.warn("Could not prime the compression dictionary; will retry on the next read: {}", e.message)
}
- database.rawQuery(CONTENT_QUERY, arrayOf(path)).use { cursor ->
+ val blocked = contentBlocks(database, fileBefore)
+ queryContent(database, path, blocked).use { cursor ->
if (cursor.count == 0) return DocumentationLookup.NotFound
if (cursor.count != 1) return DocumentationLookup.Ambiguous(cursor.count)
cursor.moveToFirst()
- val firstChunk = cursor.getBlob(0)
val mimeType = cursor.getString(1)
- val compression = cursor.getString(2)
val templateId = cursor.getInt(3)
- val decoded = decodeRow(database, path, firstChunk, compression)
- val bytes = if (templateId > 0) render(database, templateId, decoded, path) else decoded
+ val decoded = rowBytes(database, path, cursor, fileBefore, blocked)
+ val bytes = if (templateId > 0) render(database, templateId, decoded, path, fileBefore) else decoded
return DocumentationLookup.Found(DocumentationContent(bytes, mimeType))
}
}
- /** A row's content: its chunks joined, and Brotli-decoded when its content type says so. */
+ /**
+ * [path]'s Content row: content, MIME type, compression and template id, then -- when [blocked],
+ * the file declaring content blocks -- the row's block id, offset and length, NULL when it has no
+ * block.
+ *
+ * Such a file always gets the blocked query: one missing the block tables fails the read with
+ * SQLite's error rather than serving its blocked rows' empty content.
+ */
+ private fun queryContent(
+ database: SQLiteDatabase,
+ path: String,
+ blocked: Boolean,
+ ): Cursor = database.rawQuery(if (blocked) BLOCKED_CONTENT_QUERY else CONTENT_QUERY, arrayOf(path))
+
+ /**
+ * Whether the active file, in state [file], stores pages in content blocks, from the version it
+ * declares; asked once per file state (see [FileBound]). A read asks once, before its query, and
+ * both picks the query and decodes the row by that one answer. A failure to read the version is
+ * not cached, so it fails this read and the next read asks again.
+ */
+ private fun contentBlocks(
+ database: SQLiteDatabase,
+ file: FileState,
+ ): Boolean =
+ contentBlocks.valueFor(file)
+ ?: expectsContentBlocks(database).also { contentBlocks = FileBound(file, it) }
+
+ /**
+ * The plain bytes of the row [cursor] is on, a [queryContent] result: the row's slice of its
+ * decoded block when it has one, else its own content decoded (see [decodeRow]).
+ *
+ * @param fileBefore The active file's state, taken before [cursor]'s query ran (see [decodedBlock]).
+ * @param blocked What [contentBlocks] said for [fileBefore], which picked [cursor]'s query.
+ */
+ private fun rowBytes(
+ database: SQLiteDatabase,
+ path: String,
+ cursor: Cursor,
+ fileBefore: FileState,
+ blocked: Boolean,
+ ): ByteArray {
+ if (!blocked || cursor.isNull(BLOCK_ID_COLUMN)) {
+ val content = cursor.getBlob(0)
+ val compression = cursor.getString(2)
+ // A blocked row's own content is empty, and every blocked row is Brotli. Such a row with
+ // no item has lost its way to its bytes; an empty Brotli row is never valid anyway, so the
+ // read fails here, naming the likely cause. An empty uncompressed row is an empty file.
+ check(!(blocked && compression == "brotli" && content.isEmpty())) {
+ "Content row '$path' is empty and has no ContentBlockItems row"
+ }
+ return decodeRow(database, path, content, compression, fileBefore)
+ }
+ val blockId = cursor.getLong(BLOCK_ID_COLUMN)
+ // Version 4 empties a blocked row's content (x''). Bytes there as well mean the file is not
+ // what it declares, and nothing says which copy is the page.
+ check(cursor.getBlob(0)?.isEmpty() != false) {
+ "Content row '$path' has a ContentBlockItems row in block $blockId and content of its own"
+ }
+ // Both NOT NULL in the schema; read as 0 they would slice an empty page out of the block.
+ check(!cursor.isNull(BLOCK_ID_COLUMN + 1) && !cursor.isNull(BLOCK_ID_COLUMN + 2)) {
+ "ContentBlockItems row for '$path' in block $blockId has no offset or length"
+ }
+ val offset = cursor.getLong(BLOCK_ID_COLUMN + 1)
+ val length = cursor.getLong(BLOCK_ID_COLUMN + 2)
+ return sliceBlockItem(decodedBlock(database, blockId, path, fileBefore), blockId, offset, length)
+ }
+
+ /**
+ * Block [blockId] decoded, from the cache or else read and Brotli-decoded with the codec the
+ * database's Content rows use, then cached.
+ *
+ * Served only if the active file is still in [fileBefore], the state it was in before the row
+ * naming the block was read, and cached under that state. An installer rewrites the installed
+ * file in place, under the open handle, which then reads the new file's rows until the rewrite
+ * check reopens it, up to a check interval later. A block cached from the old file is under the
+ * old state, so it is never sliced for a row of the new one; and a rewrite that lands between the
+ * row and its block fails the read, since the two may come from different files.
+ *
+ * @param path The page being served, for the failure and the log line.
+ * @throws IllegalStateException When the database has no such block, or it has no data, or the
+ * file changed during the read.
+ * @throws IOException When the block does not decode.
+ */
+ private fun decodedBlock(
+ database: SQLiteDatabase,
+ blockId: Long,
+ path: String,
+ fileBefore: FileState,
+ ): ByteArray {
+ val key = BlockKey(blockId, fileBefore)
+ blockCache.get(key)?.let { block ->
+ checkFileUnchanged(fileBefore, path)
+ return block
+ }
+
+ val compressed = readContentBlock(database, blockId, path)
+
+ val codec = codec(database, fileBefore)
+ val started = System.nanoTime()
+ val block =
+ try {
+ codec.decompress(chunksAsStream(compressed))
+ } catch (e: Exception) {
+ // Logged with the database, as a template that does not decode is: the message leaves
+ // it out, and a debug database on the sdcard is the usual suspect.
+ log.error("Content block {} in '{}' does not decode; page '{}' fails", blockId, database.path, path, e)
+ throw IOException("Content block $blockId does not decode, for path '$path'", e)
+ }
+ val decodeNanos = System.nanoTime() - started
+ checkFileUnchanged(fileBefore, path)
+ val cached = blockCache.put(key, block)
+
+ if (blockLog.isDebugEnabled) {
+ blockLog.debug(
+ "decode block={} path={} compressed={} plain={} decodeMs={} cachePlain={}",
+ blockId,
+ path,
+ compressed.sumOf { it.size },
+ block.size,
+ String.format(Locale.ROOT, "%.3f", decodeNanos / 1_000_000.0),
+ cached,
+ )
+ }
+ return block
+ }
+
+ /** The active file's state now: see [FileState]. Two stat calls, cheap beside the query. */
+ private fun activeFileState(): FileState {
+ val file = File(checkNotNull(activeDatabasePath) { "no documentation database is active" })
+ return FileState(file.length(), file.lastModified())
+ }
+
+ /** Fails the read when the active file is no longer in [before]: see [decodedBlock]. */
+ private fun checkFileUnchanged(
+ before: FileState,
+ path: String,
+ ) = check(activeFileState() == before) { "The documentation database changed while '$path' was read" }
+
+ /** The decoded bytes the block cache holds now. */
+ @VisibleForTesting
+ fun cachedBlockBytes(): Long = blockCache.plainBytes()
+
+ /**
+ * A row's content: its chunks joined, and Brotli-decoded when its content type says so, with the
+ * codec for [file], the active file's state before the row was read.
+ */
private fun decodeRow(
database: SQLiteDatabase,
path: String,
firstChunk: ByteArray,
compression: String,
+ file: FileState,
): ByteArray {
val chunks = readChunks(database, path, firstChunk)
- return if (compression == "brotli") decompressBrotli(database, chunks) else joinChunks(chunks)
+ return if (compression == "brotli") decompressBrotli(database, chunks, file) else joinChunks(chunks)
}
/**
@@ -546,7 +728,9 @@ class DocumentationContentSource internal constructor(
path: String,
pagePath: String,
): DocumentationLookup? {
- database.rawQuery(CONTENT_QUERY, arrayOf(pagePath)).use { cursor ->
+ val fileBefore = activeFileState()
+ val blocked = contentBlocks(database, fileBefore)
+ queryContent(database, pagePath, blocked).use { cursor ->
if (cursor.count > 1) return DocumentationLookup.Ambiguous(cursor.count)
if (!cursor.moveToFirst()) return null
val templateId = cursor.getInt(3)
@@ -554,8 +738,9 @@ class DocumentationContentSource internal constructor(
val pageTemplate = templateNames.getOrPut(templateId) { templateName(database, templateId, pagePath) }
if (pageTemplate != HM_TOPIC_TEMPLATE || !templateLoader.resourceExists(JSTOPIC_TEMPLATE)) return null
- val json = decodeRow(database, pagePath, cursor.getBlob(0), cursor.getString(2))
- return DocumentationLookup.Found(DocumentationContent(renderNamed(database, JSTOPIC_TEMPLATE, json, path), JSTOPIC_MIME_TYPE))
+ val json = rowBytes(database, pagePath, cursor, fileBefore, blocked)
+ val rendered = renderNamed(database, JSTOPIC_TEMPLATE, json, path, fileBefore)
+ return DocumentationLookup.Found(DocumentationContent(rendered, JSTOPIC_MIME_TYPE))
}
}
@@ -564,6 +749,7 @@ class DocumentationContentSource internal constructor(
*
* @param templateId The identifier of the template to render.
* @param contextJson The JSON-encoded context supplied to the template.
+ * @param file The active file's state, taken before the row holding [contextJson] was read.
* @return The rendered template content encoded as UTF-8 bytes.
*/
private fun render(
@@ -571,6 +757,7 @@ class DocumentationContentSource internal constructor(
templateId: Int,
contextJson: ByteArray,
path: String,
+ file: FileState,
): ByteArray {
val name =
templateNames.getOrPut(templateId) {
@@ -578,7 +765,7 @@ class DocumentationContentSource internal constructor(
templateName(database, templateId, path)
}
- return renderNamed(database, name, contextJson, path)
+ return renderNamed(database, name, contextJson, path, file)
}
/**
@@ -591,6 +778,7 @@ class DocumentationContentSource internal constructor(
* @param name The template's `Templates.name`.
* @param contextJson The JSON-encoded context supplied to the template.
* @param path The content path associated with the rendering request.
+ * @param file The active file's state, taken before this render's database reads.
* @return The rendered template content encoded as UTF-8 bytes.
*/
private fun renderNamed(
@@ -598,10 +786,11 @@ class DocumentationContentSource internal constructor(
name: String,
contextJson: ByteArray,
path: String,
+ file: FileState,
): ByteArray {
// Resolved here, before the engine runs, so a failure to read the version arrives as itself
// rather than inside the loader, which would report it as a template that does not decode.
- templatesCompressed(database)
+ templatesCompressed(database, file)
val contextString = contextJson.toString(Charsets.UTF_8)
if (contextString.isBlank() || contextString.trim() == "null") {
@@ -631,11 +820,16 @@ class DocumentationContentSource internal constructor(
}
/**
- * Whether [database]'s templates are Brotli-compressed, from the version it declares; asked once
- * per database. A failure to read the version is not cached, so the next render asks again.
+ * Whether the active file's templates, in state [file], are Brotli-compressed, from the version it
+ * declares; asked once per file state (see [FileBound]). A failure to read the version is not
+ * cached, so the next render asks again.
*/
- private fun templatesCompressed(database: SQLiteDatabase): Boolean =
- templatesCompressed ?: expectsCompressedTemplates(database).also { templatesCompressed = it }
+ private fun templatesCompressed(
+ database: SQLiteDatabase,
+ file: FileState,
+ ): Boolean =
+ templatesCompressed.valueFor(file)
+ ?: expectsCompressedTemplates(database).also { templatesCompressed = FileBound(file, it) }
/**
* Resolves a template id to the name the engine loads it by.
@@ -717,33 +911,34 @@ class DocumentationContentSource internal constructor(
*
* @param database The database used to obtain the Brotli dictionary.
* @param chunks The compressed content chunks.
+ * @param file The active file's state, taken before the row was read.
* @return The decompressed content.
*/
private fun decompressBrotli(
database: SQLiteDatabase,
chunks: List,
- ): ByteArray = codec(database).decompress(chunksAsStream(chunks))
+ file: FileState,
+ ): ByteArray = codec(database, file).decompress(chunksAsStream(chunks))
/**
- * The codec for the active database, (re)built on the first use after a swap.
+ * The codec for the active file in state [file], (re)built on the first use after a swap or a
+ * change of state (see [FileBound]): a file rewritten in place may carry another dictionary.
*
- * Only clears the staleness flag on a clean build -- a definitive dictionary or a definitive
- * absence, per [loadCompressionDictionary]'s contract -- so an unexpected exception leaves it
- * set and the next read retries, rather than caching a transient failure as "no dictionary"
- * for the rest of this database's lifetime.
+ * Only cached on a clean build -- a definitive dictionary or a definitive absence, per
+ * [loadCompressionDictionary]'s contract -- so an unexpected exception caches nothing and the
+ * next read retries, rather than caching a transient failure as "no dictionary" for the rest of
+ * this file's lifetime.
*
* @param database The active documentation database.
- * @return The codec holding that database's dictionary, dictionary-free when it declares none.
+ * @return The codec holding that file's dictionary, dictionary-free when it declares none.
*/
- private fun codec(database: SQLiteDatabase): BrotliDictionaryCodec =
+ private fun codec(
+ database: SQLiteDatabase,
+ file: FileState,
+ ): BrotliDictionaryCodec =
synchronized(this) {
- var current = codec
- if (codecStale || current == null) {
- current = BrotliDictionaryCodec(loadCompressionDictionary(database))
- codec = current
- codecStale = false
- }
- current
+ codec.valueFor(file)
+ ?: BrotliDictionaryCodec(loadCompressionDictionary(database)).also { codec = FileBound(file, it) }
}
/**
@@ -852,16 +1047,7 @@ class DocumentationContentSource internal constructor(
database = opened
activeDatabasePath = path
databaseTimestamp = timestamp
- // Nulled as well as marked stale: a different database can carry a different dictionary
- // (or none), and decoding its rows against the previous one can succeed with wrong bytes
- // rather than fail (see BrotliDictionaryCodec). A null codec can only be rebuilt, never
- // reused.
- codec = null
- codecStale = true
- // The new database may store its templates the other way, and against its own dictionary:
- // the compiled templates go with the cache clear below.
- templatesCompressed = null
- clearTemplateCache()
+ forgetDatabaseState()
generation++
try {
@@ -871,6 +1057,55 @@ class DocumentationContentSource internal constructor(
}
}
+ /**
+ * Drops everything read from the active database, on every swap -- to the debug database, or a
+ * reopen of the rewritten installed file -- and on [close]. Callers hold the write lock.
+ */
+ private fun forgetDatabaseState() {
+ // A different database can carry a different dictionary (or none), and decoding its rows
+ // against the previous one can succeed with wrong bytes rather than fail (see
+ // BrotliDictionaryCodec). A null codec can only be rebuilt, never reused.
+ codec = null
+ // The new database may store its templates the other way, and against its own dictionary:
+ // the compiled templates go with the cache clear below.
+ templatesCompressed = null
+ clearTemplateCache()
+ // It may store its pages the other way too, and its block ids name different blocks.
+ contentBlocks = null
+ blockCache.clear()
+ }
+
+ /**
+ * The active file's length and modification time, as the file system reports them: what an
+ * installer's rewrite in place changes. The rewrite check relies on the modification time moving
+ * too; neither installer sets it.
+ */
+ private data class FileState(
+ val length: Long,
+ val lastModified: Long,
+ )
+
+ /**
+ * [value], read from the active file while it was in state [file]. An installer rewrites the
+ * installed file in place, under the open handle, which reads the new file at once; the swap
+ * check reopens it, dropping everything read from the old one, up to a check interval later. In
+ * between, a value read from the old file -- its version, its dictionary -- is not the new one's,
+ * and acting on it fails the new file's pages or decodes them wrong. Kept with the state it was
+ * read in, it is read again when the state moves.
+ */
+ private class FileBound(
+ val file: FileState,
+ val value: T,
+ )
+
+ /** This value, if it was read while the active file was in state [file]: see [FileBound]. */
+ private fun FileBound?.valueFor(file: FileState): T? = this?.takeIf { it.file == file }?.value
+
+ private data class BlockKey(
+ val blockId: Long,
+ val file: FileState,
+ )
+
companion object {
const val CONTENT_CHUNK_SIZE = 1024 * 1024
@@ -897,6 +1132,19 @@ class DocumentationContentSource internal constructor(
AND C.path = ?
"""
+ // CONTENT_QUERY's four columns, then the row's item; a row with no item joins to NULLs. The
+ // item's key is ContentBlockItems' primary key, so the join cannot duplicate a row. "offset"
+ // is quoted because OFFSET is an SQL keyword.
+ private const val BLOCKED_CONTENT_QUERY = """
+ SELECT C.content, CT.value, CT.compression, C.templateId, CBI.blockId, CBI."offset", CBI.length
+ FROM Content C
+ JOIN ContentTypes CT ON C.contentTypeID = CT.id
+ LEFT JOIN ContentBlockItems CBI ON CBI.contentId = C.id
+ WHERE C.path = ?
+ """
+
+ private const val BLOCK_ID_COLUMN = 4
+
private const val CHUNK_QUERY = "SELECT content FROM Content WHERE path = ? AND languageId = 1"
// A Help+Manual topic's JavaScript copy, beside its page: i/jstopics/X.js for i/X.html.
diff --git a/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt b/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt
index 7ad62718e8..b57e6bc1fd 100644
--- a/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt
+++ b/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt
@@ -34,6 +34,12 @@ object DatabaseVersionResolver {
// the installed database, so a 2.x file has to keep rendering its templated pages.
const val MAJOR_VERSION_WITH_COMPRESSED_TEMPLATES = 3
+ // From this MAJOR version on (ADFA-6683), some Content rows are stored in compressed blocks:
+ // such a row's content is empty, and its ContentBlockItems row names its block and the item's
+ // [offset, offset + length) in the decoded block. The version says so, not the tables' presence:
+ // below it a database carrying the tables is read as it always was.
+ const val MAJOR_VERSION_WITH_CONTENT_BLOCKS = 4
+
private const val QUERY_VERSION_TABLE_EXISTS = """
SELECT 1
FROM sqlite_master
diff --git a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt
index 486c510716..1edf2f18f6 100644
--- a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt
+++ b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt
@@ -136,6 +136,16 @@ fun expectsCompressedTemplates(db: SQLiteDatabase): Boolean {
return majorVersion != null && majorVersion >= DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSED_TEMPLATES
}
+/**
+ * Whether [db] declares a version that stores some `Content` rows in `ContentBlocks` (see
+ * [DatabaseVersionResolver.MAJOR_VERSION_WITH_CONTENT_BLOCKS]); below it, and for a database
+ * declaring none, every row holds its own content, whatever tables the file has.
+ */
+fun expectsContentBlocks(db: SQLiteDatabase): Boolean {
+ val majorVersion = DatabaseVersionResolver.resolveMajorVersion(db)
+ return majorVersion != null && majorVersion >= DatabaseVersionResolver.MAJOR_VERSION_WITH_CONTENT_BLOCKS
+}
+
/**
* Compresses and decompresses documentation `Content` blobs against [dictionary], the shared Brotli
* dictionary from the database those blobs live in (see [loadCompressionDictionary]). A null
diff --git a/common/src/test/java/com/itsaky/androidide/documentation/ContentBlockReadTest.kt b/common/src/test/java/com/itsaky/androidide/documentation/ContentBlockReadTest.kt
new file mode 100644
index 0000000000..e3c4ae9f03
--- /dev/null
+++ b/common/src/test/java/com/itsaky/androidide/documentation/ContentBlockReadTest.kt
@@ -0,0 +1,107 @@
+package com.itsaky.androidide.documentation
+
+import android.database.sqlite.SQLiteDatabase
+import com.google.common.truth.Truth.assertThat
+import org.junit.After
+import org.junit.Assert.assertThrows
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.annotation.Config
+import org.robolectric.annotation.ConscryptMode
+import java.io.File
+import kotlin.random.Random
+
+/**
+ * [readContentBlock] against a real SQLite file. A block over the piece size is read a piece per
+ * query, so no row has to fit Android's 2 MiB CursorWindow whole; the pieces here are a few bytes,
+ * to test the piece boundaries without megabytes of data. A block over 1 MiB read through the
+ * content source is in :app's ContentBlockSchemaTest, which has Brotli.
+ *
+ * Conscrypt off, as in DocumentationRequestInterceptorResponseTest, for SelfSignedCertTest's sake.
+ */
+@RunWith(RobolectricTestRunner::class)
+@Config(manifest = Config.NONE, sdk = [28])
+@ConscryptMode(ConscryptMode.Mode.OFF)
+class ContentBlockReadTest {
+ @get:Rule
+ val folder = TemporaryFolder()
+
+ private lateinit var database: SQLiteDatabase
+
+ @Before
+ fun setUp() {
+ database = SQLiteDatabase.openOrCreateDatabase(File(folder.root, "documentation.db"), null)
+ database.execSQL("CREATE TABLE ContentBlocks (id INTEGER PRIMARY KEY AUTOINCREMENT, data BLOB NOT NULL)")
+ }
+
+ @After
+ fun tearDown() {
+ database.close()
+ }
+
+ // Every byte value, including 0, which a TEXT read would stop at.
+ @Test
+ fun `a block over the piece size is read in pieces that join to its bytes`() {
+ val data = ByteArray(1_000) { it.toByte() }
+ block(1, data)
+
+ val pieces = readContentBlock(database, 1, "j/x.html", pieceBytes = 300)
+
+ assertThat(pieces.map { it.size }).containsExactly(300, 300, 300, 100).inOrder()
+ assertThat(joinChunks(pieces)).isEqualTo(data)
+ }
+
+ @Test
+ fun `a block of whole pieces has no empty piece after them`() {
+ val data = Random(6683).nextBytes(900)
+ block(1, data)
+
+ val pieces = readContentBlock(database, 1, "j/x.html", pieceBytes = 300)
+
+ assertThat(pieces.map { it.size }).containsExactly(300, 300, 300).inOrder()
+ assertThat(joinChunks(pieces)).isEqualTo(data)
+ }
+
+ @Test
+ fun `a block within the piece size is one piece`() {
+ val data = Random(6683).nextBytes(300)
+ block(1, data)
+
+ assertThat(readContentBlock(database, 1, "j/x.html", pieceBytes = 300).single()).isEqualTo(data)
+ }
+
+ @Test
+ fun `the default piece is the 1 MiB a continuation row holds`() {
+ val data = Random(6683).nextBytes(DocumentationContentSource.CONTENT_CHUNK_SIZE + 1)
+ block(1, data)
+
+ val pieces = readContentBlock(database, 1, "j/x.html")
+
+ assertThat(pieces.map { it.size }).containsExactly(DocumentationContentSource.CONTENT_CHUNK_SIZE, 1).inOrder()
+ assertThat(joinChunks(pieces)).isEqualTo(data)
+ }
+
+ @Test
+ fun `only the named block is read`() {
+ block(1, ByteArray(10) { 1 })
+ block(2, ByteArray(10) { 2 })
+
+ assertThat(joinChunks(readContentBlock(database, 2, "j/x.html", pieceBytes = 4))).isEqualTo(ByteArray(10) { 2 })
+ }
+
+ @Test
+ fun `a missing block fails, naming the block and the path`() {
+ val e = assertThrows(IllegalStateException::class.java) { readContentBlock(database, 3, "j/x.html", pieceBytes = 4) }
+
+ assertThat(e).hasMessageThat().isEqualTo("Content block 3 not found, for path 'j/x.html'")
+ }
+
+ private fun block(
+ id: Long,
+ data: ByteArray,
+ ) = database.execSQL("INSERT INTO ContentBlocks (id, data) VALUES (?, ?)", arrayOf(id, data))
+}
diff --git a/common/src/test/java/com/itsaky/androidide/documentation/ContentBlocksTest.kt b/common/src/test/java/com/itsaky/androidide/documentation/ContentBlocksTest.kt
new file mode 100644
index 0000000000..651ad4f1e8
--- /dev/null
+++ b/common/src/test/java/com/itsaky/androidide/documentation/ContentBlocksTest.kt
@@ -0,0 +1,175 @@
+package com.itsaky.androidide.documentation
+
+import com.google.common.truth.Truth.assertThat
+import org.junit.Assert.assertThrows
+import org.junit.Test
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+
+/**
+ * The pure logic of the ADFA-6683 content blocks: slicing an item out of a decoded block, and the
+ * cache of decoded blocks. Serving a page from a real Brotli block is covered in :app, by
+ * ContentBlockSourceTest, since this module's tests have no brotli4j native.
+ */
+class ContentBlocksTest {
+ private val block = "firstsecondthird".toByteArray()
+
+ @Test
+ fun `an item is its offset and length in the decoded block`() {
+ assertThat(slice(offset = 5, length = 6)).isEqualTo("second")
+ assertThat(slice(offset = 0, length = 5)).isEqualTo("first")
+ assertThat(slice(offset = 11, length = 5)).isEqualTo("third")
+ }
+
+ @Test
+ fun `an empty item at the end of the block is empty`() {
+ assertThat(slice(offset = block.size.toLong(), length = 0)).isEmpty()
+ }
+
+ @Test
+ fun `an item running past the end of the block is rejected`() {
+ val e = assertThrows(IllegalStateException::class.java) { slice(offset = 11, length = 6) }
+ assertThat(e).hasMessageThat().contains("content block 9")
+ }
+
+ @Test
+ fun `a negative offset or length is rejected`() {
+ assertThrows(IllegalStateException::class.java) { slice(offset = -1, length = 2) }
+ assertThrows(IllegalStateException::class.java) { slice(offset = 2, length = -1) }
+ }
+
+ // Narrowed to Int without the 64-bit check, 2^32 + 5 would wrap to 5 and serve "second".
+ @Test
+ fun `an offset past Int range is rejected, not wrapped`() {
+ assertThrows(IllegalStateException::class.java) { slice(offset = (1L shl 32) + 5, length = 6) }
+ }
+
+ @Test
+ fun `blocks are evicted once their plain bytes pass the budget`() {
+ val cache = ContentBlockCache(maxBytes = 10)
+ assertThat(cache.put(1, ByteArray(4))).isEqualTo(4L)
+ assertThat(cache.put(2, ByteArray(4))).isEqualTo(8L)
+
+ assertThat(cache.put(3, ByteArray(4))).isEqualTo(8L)
+
+ assertThat(cache.get(1)).isNull()
+ assertThat(cache.get(2)).isNotNull()
+ assertThat(cache.get(3)).isNotNull()
+ }
+
+ @Test
+ fun `a block filling the budget exactly is kept`() {
+ val cache = ContentBlockCache(maxBytes = 10)
+ cache.put(1, ByteArray(4))
+
+ assertThat(cache.put(2, ByteArray(10))).isEqualTo(10L)
+
+ assertThat(cache.get(1)).isNull()
+ assertThat(cache.get(2)).isNotNull()
+ }
+
+ @Test
+ fun `the least recently used block goes first, not the oldest`() {
+ val cache = ContentBlockCache(maxBytes = 9)
+ cache.put(1, ByteArray(3))
+ cache.put(2, ByteArray(3))
+ cache.put(3, ByteArray(3))
+ cache.get(1)
+
+ cache.put(4, ByteArray(3))
+
+ assertThat(cache.get(2)).isNull()
+ assertThat(cache.get(1)).isNotNull()
+ assertThat(cache.get(3)).isNotNull()
+ assertThat(cache.get(4)).isNotNull()
+ }
+
+ // OfflineDocumentationTools closes a block before it passes 4 MiB decoded, so that two fit here.
+ @Test
+ fun `the app's budget holds two blocks at the 4 MiB cap`() {
+ val cache = ContentBlockCache(ContentBlockCache.MAX_PLAIN_BYTES)
+ cache.put(1, ByteArray(4 * 1024 * 1024))
+ cache.put(2, ByteArray(4 * 1024 * 1024))
+
+ assertThat(cache.get(1)).isNotNull()
+ assertThat(cache.get(2)).isNotNull()
+ }
+
+ @Test
+ fun `a block over the whole budget is not cached and evicts nothing`() {
+ val cache = ContentBlockCache(maxBytes = 10)
+ cache.put(1, ByteArray(4))
+
+ assertThat(cache.put(2, ByteArray(11))).isEqualTo(4L)
+
+ assertThat(cache.get(2)).isNull()
+ assertThat(cache.get(1)).isNotNull()
+ }
+
+ @Test
+ fun `caching a block again replaces it and counts it once`() {
+ val cache = ContentBlockCache(maxBytes = 10)
+ cache.put(1, ByteArray(4))
+
+ assertThat(cache.put(1, ByteArray(6))).isEqualTo(6L)
+
+ assertThat(cache.get(1)).hasLength(6)
+ }
+
+ @Test
+ fun `clear drops every block and its bytes`() {
+ val cache = ContentBlockCache(maxBytes = 10)
+ cache.put(1, ByteArray(4))
+ cache.put(2, ByteArray(4))
+
+ cache.clear()
+
+ assertThat(cache.plainBytes()).isEqualTo(0L)
+ assertThat(cache.get(1)).isNull()
+ assertThat(cache.get(2)).isNull()
+ // Counting from zero again, not from the bytes cleared.
+ assertThat(cache.put(3, ByteArray(10))).isEqualTo(10L)
+ }
+
+ // An access-ordered LinkedHashMap restructures itself on get(), so unsynchronised readers corrupt
+ // it or throw ConcurrentModificationException; the byte count must also stay within the budget.
+ @Test
+ fun `concurrent requests keep the cache within its budget`() {
+ val cache = ContentBlockCache(maxBytes = 64)
+ val threads = 8
+ val pool = Executors.newFixedThreadPool(threads)
+ val start = CountDownLatch(1)
+ val failures = mutableListOf()
+ try {
+ val tasks =
+ (0 until threads).map { thread ->
+ pool.submit {
+ start.await()
+ try {
+ repeat(20_000) { i ->
+ val id = ((thread * 31 + i) % 40).toLong()
+ if (cache.get(id) == null) cache.put(id, ByteArray(1 + (id % 9).toInt()))
+ check(cache.plainBytes() <= 64)
+ }
+ } catch (e: Throwable) {
+ synchronized(failures) { failures.add(e) }
+ }
+ }
+ }
+ start.countDown()
+ tasks.forEach { it.get(60, TimeUnit.SECONDS) }
+ } finally {
+ pool.shutdownNow()
+ }
+
+ assertThat(failures).isEmpty()
+ val cached = (0L until 40L).mapNotNull { cache.get(it) }.sumOf { it.size.toLong() }
+ assertThat(cache.plainBytes()).isEqualTo(cached)
+ }
+
+ private fun slice(
+ offset: Long,
+ length: Long,
+ ) = sliceBlockItem(block, blockId = 9, offset = offset, length = length).toString(Charsets.UTF_8)
+}
diff --git a/common/src/test/java/com/itsaky/androidide/documentation/DocumentationContentSourceTest.kt b/common/src/test/java/com/itsaky/androidide/documentation/DocumentationContentSourceTest.kt
index 623a5ac97d..3a34e4b4c0 100644
--- a/common/src/test/java/com/itsaky/androidide/documentation/DocumentationContentSourceTest.kt
+++ b/common/src/test/java/com/itsaky/androidide/documentation/DocumentationContentSourceTest.kt
@@ -5,6 +5,7 @@ import android.database.sqlite.SQLiteDatabase
import android.database.sqlite.SQLiteException
import com.google.common.truth.Truth.assertThat
import com.itsaky.androidide.utils.BrotliDictionaryCodec
+import io.mockk.clearMocks
import io.mockk.every
import io.mockk.mockk
import io.mockk.mockkStatic
@@ -77,6 +78,8 @@ class DocumentationContentSourceTest {
every { getString(1) } returns mimeType
every { getString(2) } returns compression
every { getInt(3) } returns templateId
+ // No ContentBlockItems row, for the query that joins it; the other query has no column 4.
+ every { isNull(4) } returns true
}
private fun database(contentCursor: Cursor): SQLiteDatabase =
@@ -159,6 +162,97 @@ class DocumentationContentSourceTest {
}
}
+ // ADFA-6683: from documentation version 4 some rows are stored in content blocks, which only the
+ // query joining ContentBlockItems finds. The declared version picks the query, never the tables:
+ // these mocks would answer either. Blocked rows themselves are served in :app, by
+ // ContentBlockSourceTest, since a block is Brotli.
+ @Test
+ fun `a database declaring version 4 reads its rows with their block items`() {
+ assertContentQuery(declaredMajorVersion = 4, blocked = true)
+ }
+
+ // The gate is a floor, not a match: a later major is read as 4.
+ @Test
+ fun `a database declaring a version above 4 reads its rows with their block items`() {
+ assertContentQuery(declaredMajorVersion = 5, blocked = true)
+ }
+
+ @Test
+ fun `a database declaring version 3 reads its rows as they are`() {
+ assertContentQuery(declaredMajorVersion = 3, blocked = false)
+ }
+
+ @Test
+ fun `a database declaring no version reads its rows as they are`() {
+ assertContentQuery(declaredMajorVersion = null, blocked = false)
+ }
+
+ @Test
+ fun `the version that picks the query is read once per database`() {
+ val database = database(contentCursor(bytes = "page".toByteArray()), declaredMajorVersion = 4)
+ every { SQLiteDatabase.openDatabase(any(), isNull(), any()) } returns database
+ val source = source()
+ source.lookup("a/page.html")
+ clearMocks(database, answers = false, recordedCalls = true)
+
+ repeat(3) { assertThat(text(source.lookup("a/page.html"))).isEqualTo("page") }
+
+ verify(exactly = 0) { database.rawQuery(match { it.contains("FROM DocumentationDatabaseVersion") }, any()) }
+ verify(exactly = 3) { database.rawQuery(match { it.contains("LEFT JOIN ContentBlockItems") }, any()) }
+ }
+
+ // Never read as "no blocks", which would serve every blocked row's empty content, and not cached:
+ // the next read asks again.
+ @Test
+ fun `a failure to read the version fails the read, and the next read asks again`() {
+ val database = database(contentCursor(bytes = "page".toByteArray()), declaredMajorVersion = 4)
+ var versionReadable = false
+ every { database.rawQuery(match { it.contains("FROM DocumentationDatabaseVersion") }, any()) } answers {
+ if (!versionReadable) throw SQLiteException("disk I/O error (code 10)")
+ mockk(relaxed = true) {
+ every { moveToFirst() } returns true
+ every { isNull(0) } returns false
+ every { getInt(0) } returns 4
+ }
+ }
+ every { SQLiteDatabase.openDatabase(any(), isNull(), any()) } returns database
+ val source = source()
+
+ val failed = source.lookup("a/page.html")
+
+ assertThat(failed).isInstanceOf(DocumentationLookup.Failed::class.java)
+ assertThat((failed as DocumentationLookup.Failed).cause).isInstanceOf(SQLiteException::class.java)
+ verify(exactly = 0) { database.rawQuery(match { it.contains("FROM Content") }, any()) }
+
+ versionReadable = true
+
+ assertThat(text(source.lookup("a/page.html"))).isEqualTo("page")
+ verify(exactly = 1) { database.rawQuery(match { it.contains("LEFT JOIN ContentBlockItems") }, any()) }
+ }
+
+ private fun assertContentQuery(
+ declaredMajorVersion: Int?,
+ blocked: Boolean,
+ ) {
+ val database = database(contentCursor(bytes = "page".toByteArray()), declaredMajorVersion)
+ every { SQLiteDatabase.openDatabase(any(), isNull(), any()) } returns database
+
+ val lookup = source().use { it.lookup("a/page.html") }
+
+ assertThat(text(lookup)).isEqualTo("page")
+ verify(exactly = if (blocked) 1 else 0) {
+ database.rawQuery(match { it.contains("LEFT JOIN ContentBlockItems") }, any())
+ }
+ verify(exactly = if (blocked) 0 else 1) {
+ database.rawQuery(match { it.contains("FROM Content") && !it.contains("ContentBlockItems") }, any())
+ }
+ }
+
+ private fun text(lookup: DocumentationLookup): String =
+ ((lookup as? DocumentationLookup.Found) ?: throw AssertionError("expected Found, got $lookup"))
+ .content.bytes
+ .toString(Charsets.UTF_8)
+
@Test
fun `lookup returns the row for a path`() {
val database = database(contentCursor(bytes = "page".toByteArray(), mimeType = "text/html"))
diff --git a/docs/documentation-database.md b/docs/documentation-database.md
index 491410ce6a..d05ff68266 100644
--- a/docs/documentation-database.md
+++ b/docs/documentation-database.md
@@ -34,7 +34,7 @@ CREATE TABLE Content (
One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points:
- **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java).
-- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. In a database declaring `MAJOR >= 2`, **every** row with `ContentTypes.compression = 'brotli'` is compressed against the single shared dictionary in `CompressionDictionary` (see below): the bulk of them converted in one pass by ADFA-5153, and plugin-contributed Tier 3 rows written that way at install time by `PluginDocumentationManager` (ADFA-5240). So the compression of a row follows from the database's declared version alone — there is no per-row flag, and none is needed. That matters because the two are not distinguishable by inspection, and only half-distinguishable by decode: attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are out of bounds for any spec-compliant decoder. Attaching the *wrong* dictionary is not reliable in either direction -- one of a different length, or with nothing valid at the offsets the stream references, usually throws, but one of the same length holding plausible bytes there decodes cleanly to content that is simply wrong (both cases verified in `BrotliDictionaryDecodeTest`). Never use decode success, or failure, as evidence about *which* dictionary was used. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `DocumentationContentSource` before returning.
+- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. In a database declaring `MAJOR >= 2`, **every** row with `ContentTypes.compression = 'brotli'` is compressed against the single shared dictionary in `CompressionDictionary` (see below): the bulk of them converted in one pass by ADFA-5153, and plugin-contributed Tier 3 rows written that way at install time by `PluginDocumentationManager` (ADFA-5240). So the compression of a row follows from the database's declared version alone — there is no per-row flag, and none is needed. That matters because the two are not distinguishable by inspection, and only half-distinguishable by decode: attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are out of bounds for any spec-compliant decoder. Attaching the *wrong* dictionary is not reliable in either direction -- one of a different length, or with nothing valid at the offsets the stream references, usually throws, but one of the same length holding plausible bytes there decodes cleanly to content that is simply wrong (both cases verified in `BrotliDictionaryDecodeTest`). Never use decode success, or failure, as evidence about *which* dictionary was used. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `DocumentationContentSource` before returning. From documentation version 4 a row can instead be **blocked**: its `content` is empty (`x''`) and its bytes are an item in a compressed block (see `ContentBlocks` below).
- **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs).
- The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below).
@@ -81,9 +81,31 @@ CREATE TABLE TooltipButtons (
### Supporting tables
-- **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. **Exactly one row**: the version the file *is*, which `populate_db.py` replaces rather than appends. Nothing in the DDL enforces that, so both readers (`DatabaseVersionResolver.resolveMajorVersion` here, `database_major_version` in docdb-studio) take the newest row rather than the one with the highest `major` (the app orders by `changeTime DESC, rowid DESC`, so the greatest `changeTime` wins and `rowid` only breaks ties) — a rebuild from an older content set is a downgrade and has to read as one, which `MAX(major)` would get wrong. The app's reader additionally logs a warning when it finds more than one row; docdb-studio's does not. `populate_db.py` collapses a file that somehow accumulated several back to one, and `resolveMajorVersion` returns null for a database predating the table. `MAJOR >= 3` says the tooltip tables have the shape above and every template is compressed (see `Templates` below). `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `DocumentationContentSource` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every brotli row would then be decoded with a dictionary attached and fail outright, since ADFA-5240 removed the plain-decode retry that used to mask this), and a migrated database that lost the table fails loudly instead of quietly.
-- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. First trained from a representative sample across the whole `Content` table by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py`, and retrained since by its `remint_dictionary.py`, which recompresses every brotli `Content` row against the new dictionary in the same transaction (from documentation version 3, every template as well: OfflineDocumentationTools #48). A dictionary-compressed row or template is decodable only against the exact dictionary it was compressed with, so a re-mint is safe only when it carries every one of them across; replacing the dictionary alone would silently orphan them all. Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. Both the reader and the writer load it through `loadCompressionDictionary` in `:common`, so they cannot disagree about whether a given database's brotli rows carry a dictionary — a row compressed against one the reader won't attach is undecodable, and vice versa. `DocumentationContentSource` loads it lazily -- not at database open or swap time, but on the first content lookup after the active database changes, whatever that row's own compression, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Each row is then decoded exactly once, with the dictionary attached or not according to that version; there is no retry, because there is no longer a second way a row might have been written.
+- **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. **Exactly one row**: the version the file *is*, which `populate_db.py` replaces rather than appends. Nothing in the DDL enforces that, so both readers (`DatabaseVersionResolver.resolveMajorVersion` here, `database_major_version` in docdb-studio) take the newest row rather than the one with the highest `major` (the app orders by `changeTime DESC, rowid DESC`, so the greatest `changeTime` wins and `rowid` only breaks ties) — a rebuild from an older content set is a downgrade and has to read as one, which `MAX(major)` would get wrong. The app's reader additionally logs a warning when it finds more than one row; docdb-studio's does not. `populate_db.py` collapses a file that somehow accumulated several back to one, and `resolveMajorVersion` returns null for a database predating the table. `MAJOR >= 4` says some `Content` rows are stored in `ContentBlocks` (see below), and promises everything 3 does. `MAJOR >= 3` says the tooltip tables have the shape above and every template is compressed (see `Templates` below). Every gate is a floor, so this build reads a database declaring major 5 or above as version 4, and has no way to refuse one. A format that breaks any promise of version 4 or below must therefore never reach an install running this build. `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `DocumentationContentSource` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every brotli row would then be decoded with a dictionary attached and fail outright, since ADFA-5240 removed the plain-decode retry that used to mask this), and a migrated database that lost the table fails loudly instead of quietly.
+- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. First trained from a representative sample across the whole `Content` table by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py`, and retrained since by its `remint_dictionary.py`, which recompresses every brotli `Content` row against the new dictionary in the same transaction (from documentation version 3, every template as well: OfflineDocumentationTools #48). A dictionary-compressed row or template is decodable only against the exact dictionary it was compressed with, so a re-mint is safe only when it carries every one of them across; replacing the dictionary alone would silently orphan them all. Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. Both the reader and the writer load it through `loadCompressionDictionary` in `:common`, so they cannot disagree about whether a given database's brotli rows carry a dictionary — a row compressed against one the reader won't attach is undecodable, and vice versa. `DocumentationContentSource` loads it lazily -- not at database open or swap time, but on the first content lookup after the active database changes, whatever that row's own compression, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one) or once the open file's length or modification time moves (see the rewrite in place under `ContentBlocks` below). Each row is then decoded exactly once, with the dictionary attached or not according to that version; there is no retry, because there is no longer a second way a row might have been written.
- **`Templates(id, name, content)`** — Pebble template source. The DDL is `id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE('name')` — worth stating, because the single-quoted `UNIQUE('name')` reads like a constant rather than a column and has twice been taken for an absent constraint. SQLite resolves the quoted string to the column, so a duplicate name, a null name and a null body are all rejected by the database, and readers here do not re-check them. `Content.templateId` names a page's outermost template by id; every reference *between* templates is by `name` (ADFA-5405), which is also how `WebServer` reaches well-known templates like `bookshelf`, and how the content source reaches `helpmanual-jstopic.peb` for Help+Manual's `jstopics` files. So a template can `extends`, `include`, `import` or `embed` another one by writing its `Templates.name` verbatim (`{% include "nav.peb" %}`) — the table is a flat namespace, with no directories, prefixes or suffixes applied. A reference to a name with no row fails the request rather than rendering the name; before ADFA-5405 the latter is exactly what happened, silently, because the engine's loader treated the name it was handed as the template body. From documentation version 3 (ADFA-3305), every `content` is the template's UTF-8 text Brotli-compressed against `CompressionDictionary`, like a brotli `Content` row. No column says so: `OfflineDocumentationTools` compresses every template when it builds the database for release, and nothing on the device writes `Templates`. Below 3, and in a database declaring no version, `content` is plain UTF-8 and is read as it is (`expectsCompressedTemplates`): an app update never replaces the installed database, so a 2.x file has to keep rendering its templated pages. In a 3.x file a template that does not decode fails the page that uses it, logged with the template's name and the database's path.
+- **`ContentBlocks(id, data)`** / **`ContentBlockItems(contentId, blockId, offset, length)`** — documentation version 4 (ADFA-6683): pages stored up to 64 to a block, so Brotli compresses what neighbouring pages share, which it cannot do one row at a time. Only the shipped copy has them; `OfflineDocumentationTools`' `store_pages_in_blocks.py` adds them at ship time.
+
+ ```sql
+ CREATE TABLE ContentBlocks (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ data BLOB NOT NULL -- Brotli (q11, w24) of the block's items, concatenated, against CompressionDictionary
+ );
+ CREATE TABLE ContentBlockItems (
+ contentId INTEGER NOT NULL PRIMARY KEY REFERENCES Content(id),
+ blockId INTEGER NOT NULL REFERENCES ContentBlocks(id),
+ offset INTEGER NOT NULL, -- into the decoded block, in bytes
+ length INTEGER NOT NULL -- in bytes
+ ) WITHOUT ROWID;
+ ```
+
+ A blocked `Content` row keeps its `path`, `languageID`, `contentTypeID` and `templateId`, and its `content` is empty. Its item is exactly the row's decoded bytes, before any template is applied, so a templated page renders from the item as it would from a decoded row. A block holds rows of one set only: `a/` templated pages (not `a/devsite/`), `j/` HTML, `k/` templated pages, and the Brotli rows under `a/devsite/` and under `i/`; every blocked row is Brotli. A row over 1 MiB decoded, a chunked row and its continuation rows, and every `plugin/` row stay rows. `DocumentationContentSource` decodes a block with the database's dictionary, slices the item out at `[offset, offset + length)`, and caches decoded blocks: 8 MiB of decoded bytes, least recently used out (a block closes before 4 MiB decoded, so two fit). That budget is per `DocumentationContentSource`, and the app has two, the interceptor's and `WebServer`'s, so up to 16 MiB of decoded blocks in all. Each drops its blocks with the database on every swap, reopen and close.
+
+ A cached block is keyed by the file's length and modification time as well as its id. The installers rewrite the installed file in place, under the open handle, and the check that reopens it runs at most once a second, so until then the handle reads the new file's rows while the old file's blocks are still cached; the key keeps a block of the old file from being sliced for a row of the new one. A rewrite that lands between a row and its block fails that page. What the file declares -- whether it stores blocks, whether its templates are compressed -- and its dictionary are cached under the same length and modification time, so the new file's rows are queried and decoded as the new file says; a read asks once, before its query, and runs the query and decodes the row by that one answer. Compiled templates are not keyed so: one compiled from the old file is served until the reopen.
+
+ Android reads a row through a 2 MiB `CursorWindow`, and nothing caps a block's compressed size, only its decoded size. So a block over 1 MiB compressed is read in 1 MiB pieces with `substr()` (`readContentBlock`). SQLite loads the whole blob for each piece, so a block of n pieces costs n reads of it. The largest block in the 4.0.0 build is 559,581 bytes, one piece.
+
+ A missing block, one that does not decode, an item outside its block, a Brotli row with empty content and no item, and a row with both an item and content of its own each fail the page; none is served short or empty. Below 4 the tables are not read, even when present, and a version 4 file missing them fails every page with SQLite's error rather than serving blocked rows empty.
- **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`).
- **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing.
- Misc `ide_tooltip_table` and `PUCC` tables are historical/example artifacts — not part of the live lookup paths above.
@@ -92,7 +114,7 @@ CREATE TABLE TooltipButtons (
Of the five sites below, only `DocumentationContentSource` and `ToolTipManager` open the file, with `SQLiteDatabase.openDatabase(..., OPEN_READONLY)` — `WebServer` and the interceptor read through the content source, and `PluginDocumentationManager` is the one write path, opening `OPEN_READWRITE` to install plugin content (see ADR 0001 for why raw SQLite is justified here instead of Room).
-- **`common/.../documentation/DocumentationContentSource.kt`** — the one Tier 3 pipeline that reads this database: row lookup (a request target is matched raw first, since stored `Content.path` values are percent-encoded, with a percent-decoded fallback on a miss — shared by both transports so they agree on which pages exist), chunked-row reassembly, dictionary-aware Brotli decode gated on the declared documentation version, and the database swaps — to a newer sdcard debug database, and reopening the installed file when an asset install rewrites it in place under the cached handle — under a read/write lock so several threads can read while a swap cannot close the handle under them. It also renders the rows that are Pebble template contexts (`templateId > 0`: the `k/` and `i/` pages in a 2.x database, the `i/` ones being Help+Manual's topic pages (ADFA-6618), and the `a/` pages too in a 3.x one) — resolving each template's references to other templates against `Templates.name` as it goes, and decoding each template with the same dictionary as the rows — so both transports below serve finished pages and neither needs the template engine itself. All of that logic exists once (ADFA-5176). One more rule renders a row that isn't stored: Help+Manual's in-page navigation loads `jstopics/X.js`, a JavaScript copy of topic `X`, and once the documentation pipeline stores each Help+Manual topic only as the JSON its page `X.html` renders from (ADFA-6618), a request for `jstopics/X.js` that matches no stored row, under either its raw or its percent-decoded path, is rendered from that page's JSON with the `helpmanual-jstopic.peb` template and served as `text/javascript`. It stays a 404 unless the page is one row rendered through `helpmanual-topic.peb` and the database has `helpmanual-jstopic.peb`; a duplicated page row is reported as ambiguous, like the page itself.
+- **`common/.../documentation/DocumentationContentSource.kt`** — the one Tier 3 pipeline that reads this database: row lookup (a request target is matched raw first, since stored `Content.path` values are percent-encoded, with a percent-decoded fallback on a miss — shared by both transports so they agree on which pages exist), chunked-row reassembly, dictionary-aware Brotli decode gated on the declared documentation version, a blocked row's item sliced from its decoded block (version 4), and the database swaps — to a newer sdcard debug database, and reopening the installed file when an asset install rewrites it in place under the cached handle — under a read/write lock so several threads can read while a swap cannot close the handle under them. It also renders the rows that are Pebble template contexts (`templateId > 0`: the `k/` and `i/` pages in a 2.x database, the `i/` ones being Help+Manual's topic pages (ADFA-6618), and the `a/` pages too in a 3.x one) — resolving each template's references to other templates against `Templates.name` as it goes, and decoding each template with the same dictionary as the rows — so both transports below serve finished pages and neither needs the template engine itself. All of that logic exists once (ADFA-5176). One more rule renders a row that isn't stored: Help+Manual's in-page navigation loads `jstopics/X.js`, a JavaScript copy of topic `X`, and once the documentation pipeline stores each Help+Manual topic only as the JSON its page `X.html` renders from (ADFA-6618), a request for `jstopics/X.js` that matches no stored row, under either its raw or its percent-decoded path, is rendered from that page's JSON with the `helpmanual-jstopic.peb` template and served as `text/javascript`. It stays a 404 unless the page is one row rendered through `helpmanual-topic.peb` and the database has `helpmanual-jstopic.peb`; a duplicated page row is reported as ambiguous, like the page itself.
- **`common/.../documentation/DocumentationRequestInterceptor.kt`** — serves Tier 3 *in-process* for the app's WebViews (`HelpActivity`, the tooltip fragment, `FAQActivity`), through `WebViewClient.shouldInterceptRequest`, so a page's assets cost a database read instead of a TCP connection each (ADFA-5176). It matches the same `http://localhost:6174/...` URL space, so the strings.xml entries, `ToolTipManager`'s link builder and the `DocumentationExtension` contract need no changes; anything it declines — a `/pr/` endpoint, an unknown path, a failed read — falls through to `WebServer` unchanged. Both transports get their `Content-Type` charset from the same place, `ContentTypeHeaders` (ADFA-5241), so a row does not describe itself differently depending on which one served it. They answer `Range` differently, both validly: the interceptor answers one satisfiable range with a 206 and `Content-Range` (`RangeResponse`, ADFA-3305), because the WebView cuts an in-process body to the range itself and a 200 would then start mid-file; `WebServer` ignores `Range` and sends the whole row with a 200. Set `/sdcard/Download/CodeOnTheGo.nointercept` to force documentation back onto the server.
- **`app/.../localWebServer/WebServer.kt`** — serves Tier 3 over HTTP on port 6174, for WebViews that are not wired to the interceptor above and for the `/pr/` developer endpoints. It reads through `DocumentationContentSource`, so it holds no database, template engine or decode logic of its own; what remains here is HTTP: request parsing, the `/pr/` pages, error responses, and the CSS/asset shortcuts. Also serves a Dynamic Bookshelf JSON payload (joining `Content`/`Bookshelf`/`BookCategories`, rendered through the `bookshelf` template) and debug-only HTML dumps at `/pr/db` (`LastChange`, last 20 rows) and `/pr/pr` (recent projects, from a *different* database).
- **`idetooltips/.../ToolTipManager.kt`** — serves Tier 1/2, through `readTooltip` in `TooltipQuery.kt`, which runs `queryTooltip` and `queryTooltipButtons` and builds the item shown. Looks up a tooltip by `(category, tag)`: the category's id from `TooltipCategories` first, so the row is found by `Tooltips`' key rather than a scan of the whole table (ADFA-6597), then its text by id. Then its buttons for the Tier 3 links shown at the bottom, by `TooltipButtons`' key, with their text by id. Refuses a database declaring a version below 3.