diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt index 740a0ae8a7..51ab70ec1d 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -1,5 +1,7 @@ package org.appdevforall.codeonthego.indexing +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable import org.appdevforall.codeonthego.indexing.api.ReadableIndex @@ -19,8 +21,8 @@ import java.util.concurrent.ConcurrentHashMap */ open class FilteredIndex( private val backing: ReadableIndex, -) : ReadableIndex, Closeable { - +) : ReadableIndex, + Closeable { /** * The set of source IDs whose entries are visible. * Uses a concurrent set for thread-safe reads during queries. @@ -57,14 +59,19 @@ open class FilteredIndex( /** * Returns the current set of active source IDs. */ - open fun activeSources(): Set = - activeSources.toSet() + fun activeSources(): Set = activeSources.toSet() + + /** + * The source IDs whose entries are visible, or `null` if every source is. + * + * Override this to change what the filter admits; `null` admits every source. + */ + protected open fun visibleSourceIds(): Collection? = activeSources /** * Returns true if the source is currently active (visible). */ - open fun isActive(sourceId: String): Boolean = - sourceId in activeSources + fun isActive(sourceId: String): Boolean = visibleSourceIds()?.contains(sourceId) ?: true /** * Returns true if the source exists in the backing index, @@ -72,35 +79,50 @@ open class FilteredIndex( * * Use this to check if a JAR needs indexing at all. */ - open suspend fun isCached(sourceId: String): Boolean = - backing.containsSource(sourceId) + open suspend fun isCached(sourceId: String): Boolean = backing.containsSource(sourceId) + + override fun query(query: IndexQuery): Sequence = backing.query(scopedToActive(query)) + + /** + * Narrows [query] to the active sources by rewriting its scope, rather than by filtering the + * rows it returns. + * + * Filtering afterwards is wrong whenever the query is limited: the backing index applies the + * limit first, so a page full of inactive rows yields nothing even though matches exist. Pushing + * the active set into the query makes the limit count only rows the caller can actually see. + */ + private fun scopedToActive(query: IndexQuery): IndexQuery { + val visible = visibleSourceIds()?.toSet() ?: return query - override fun query(query: IndexQuery): Sequence { - if (query.sourceId != null && !isActive(query.sourceId)) { - return emptySequence() + if (query.sourceId != null) { + // Already as narrow as a scope gets: it either survives the active set or matches nothing. + return if (query.sourceId in visible) query else query.copy(sourceIds = emptyList()) } - val original = backing.query(query) - return original.filter { isActive(it.sourceId) } - } - override suspend fun get(key: String): T? { - val entry = backing.get(key) ?: return null - return if (isActive(entry.sourceId)) entry else null + val requested = query.sourceIds + val scoped = requested?.filter { it in visible } ?: visible + return query.copy(sourceIds = scoped) } - override suspend fun containsSource(sourceId: String): Boolean { - return isActive(sourceId) && backing.containsSource(sourceId) - } + /** + * Returns the visible entry for [key], the one with the smallest source id among the visible + * sources that have it. + * + * This is a scoped key query rather than a check on the backing index's own [get]: that returns + * the smallest source id overall, and when that source is inactive a visible entry for the same + * key would be missed. + */ + override suspend fun get(key: String): T? = + withContext(Dispatchers.IO) { + backing.query(scopedToActive(IndexQuery.byKey(key))).firstOrNull() + } - override fun distinctValues(fieldName: String): Sequence { - // This is imprecise — the backing index may return values - // from inactive sources. For exact results, we'd need to - // query all entries and filter. For package enumeration - // (the main use case), this approximation is acceptable - // since packages from inactive JARs are harmless — they - // just produce empty results when queried further. - return backing.distinctValues(fieldName) - } + override suspend fun containsSource(sourceId: String): Boolean = isActive(sourceId) && backing.containsSource(sourceId) + + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence = backing.distinctValues(fieldName, scopedToActive(query)) override fun close() { activeSources.clear() diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index a5d4dce21c..01c8d19ec2 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -4,6 +4,7 @@ import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.requireDeclaredFields import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantReadWriteLock import kotlin.collections.iterator @@ -16,12 +17,17 @@ import kotlin.concurrent.write * Optimized for small-to-medium datasets (source files, typically * hundreds to low thousands of entries) that change frequently. * + * An entry is identified by its source and key together, matching [SQLiteIndex]: the same key in + * two sources is two entries, and a key lookup matching several returns the smallest source id. + * * Data layout: - * - [primaryMap]: key → entry (O(1) point lookup) - * - [sourceMap]: sourceId → set of keys (O(1) bulk removal) - * - [fieldMaps]: fieldName → (fieldValue → set of keys) (equality filter) - * - [prefixBuckets]: fieldName → (lowercased first char → list of (value, key)) + * - [rows]: (sourceId, key) -> entry + * - [sourceMap]: sourceId -> rows (O(1) bulk removal) + * - [keyMap]: key -> rows (key lookup) + * - [fieldMaps]: fieldName -> (fieldValue -> rows) (equality filter) + * - [prefixBuckets]: fieldName -> (lowercased first char -> list of (value, row)) * Provides a ~36-way partition for prefix search. + * - [fingerprints]: sourceId -> fingerprint recorded by [insertSource] * * All mutations go through [lock] in write mode for consistency * across the multiple maps. Reads use read mode. @@ -30,216 +36,304 @@ import kotlin.concurrent.write * @param descriptor Defines queryable fields and serialization. */ class InMemoryIndex( - override val descriptor: IndexDescriptor, - override val name: String = "memory:${descriptor.name}", + override val descriptor: IndexDescriptor, + override val name: String = "memory:${descriptor.name}", ) : Index { - - private val primaryMap = ConcurrentHashMap(256) - private val sourceMap = ConcurrentHashMap>(32) - private val fieldMaps = ConcurrentHashMap>>() - private val prefixBuckets = ConcurrentHashMap>>() - - private val lock = ReentrantReadWriteLock() - - private data class PrefixEntry(val lowerValue: String, val key: String) - - init { - for (field in descriptor.fields) { - fieldMaps[field.name] = ConcurrentHashMap() - if (field.prefixSearchable) { - prefixBuckets[field.name] = ConcurrentHashMap() - } - } - } - - override fun query(query: IndexQuery): Sequence { - val keys = resolveMatchingKeys(query) - val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit - return keys - .mapNotNull { primaryMap[it] } - .take(limit) - } - - override suspend fun get(key: String): T? = primaryMap[key] - - override suspend fun containsSource(sourceId: String): Boolean = - sourceMap.containsKey(sourceId) - - override fun distinctValues(fieldName: String): Sequence { - val fieldMap = fieldMaps[fieldName] ?: return emptySequence() - return lock.read { fieldMap.keys.toList() }.asSequence() - } - - override suspend fun insertAll(entries: Sequence) { - lock.write { - for (entry in entries) { - insertSingleLocked(entry) - } - } - } - - override suspend fun insert(entry: T) = lock.write { insertSingleLocked(entry) } - - override suspend fun removeBySource(sourceId: String) = lock.write { - removeBySourceLocked(sourceId) - } - - /** - * Remove every entry belonging to any of [sourceIds]. - * - * Acquires the write lock once and removes each source under it, so the whole - * batch is atomic with respect to concurrent readers and writers — there is no - * intermediate state in which only some of the sources have been removed. - */ - override suspend fun removeBySources(sourceIds: Collection) = lock.write { - for (sourceId in sourceIds) { - removeBySourceLocked(sourceId) - } - } - - /** - * Remove all entries for [sourceId] from the primary, source, and secondary - * indexes. Caller MUST already hold the write lock; this method does not lock. - */ - private fun removeBySourceLocked(sourceId: String) { - val keys = sourceMap.remove(sourceId) ?: return - for (key in keys) { - val entry = primaryMap.remove(key) ?: continue - removeFromSecondaryIndexes(entry) - } - } - - override suspend fun clear() = lock.write { - primaryMap.clear() - sourceMap.clear() - fieldMaps.values.forEach { it.clear() } - prefixBuckets.values.forEach { it.clear() } - } - - val size: Int get() = primaryMap.size - val sourceCount: Int get() = sourceMap.size - - /** - * Resolves the set of keys matching the query by intersecting - * the results of each predicate. - * - * Starts with the most selective predicate to minimize the - * intersection set. - */ - private fun resolveMatchingKeys(query: IndexQuery): Sequence = lock.read { - var candidates: Set? = null - - if (query.key != null) { - return@read if (primaryMap.containsKey(query.key)) { - sequenceOf(query.key) - } else { - emptySequence() - } - } - - if (query.sourceId != null) { - candidates = intersect(candidates, sourceMap[query.sourceId]) - } - - for ((field, value) in query.exactMatch) { - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - candidates = intersect(candidates, fieldMap[value]) - } - - for ((field, prefix) in query.prefixMatch) { - val buckets = prefixBuckets[field] - val matching: Set = if (buckets != null) { - // Prefix-searchable: case-insensitive match via the lowercased buckets, - // mirroring SQLite's `lowerCol LIKE 'prefix%'`. - val lowerPrefix = prefix.lowercase() - val firstChar = lowerPrefix.firstOrNull() - if (firstChar == null) { - // Empty prefix == "field present", matching SQLite's `LIKE '%'` - // (which excludes rows where the column IS NULL). - buckets.values.flatMapTo(mutableSetOf()) { entries -> entries.map { it.key } } - } else { - val bucket = buckets[firstChar] ?: return@read emptySequence() - bucket.asSequence() - .filter { it.lowerValue.startsWith(lowerPrefix) } - .map { it.key } - .toSet() - } - } else { - // Not prefix-searchable: fall back to a case-sensitive prefix scan of the - // regular field map, mirroring SQLite's `col LIKE 'prefix%'` fallback. - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - fieldMap.entries.asSequence() - .filter { (value, _) -> value.startsWith(prefix) } - .flatMap { (_, keys) -> keys.asSequence() } - .toSet() - } - - candidates = intersect(candidates, matching) - } - - for ((field, mustExist) in query.presence) { - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - val allKeysWithField = fieldMap.values.flatMapTo(mutableSetOf()) { it } - candidates = if (mustExist) { - intersect(candidates, allKeysWithField) - } else { - // Keys that DON'T have this field - val allKeys = primaryMap.keys.toMutableSet() - allKeys.removeAll(allKeysWithField) - intersect(candidates, allKeys) - } - } - - candidates?.asSequence() ?: primaryMap.keys.asSequence() - } - - private fun intersect(current: Set?, other: Set?): Set? { - if (other == null) return current - if (current == null) return other - return current.intersect(other) - } - - private fun insertSingleLocked(entry: T) { - val existing = primaryMap[entry.key] - if (existing != null) { - removeFromSecondaryIndexes(existing) - } - - primaryMap[entry.key] = entry - sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(entry.key) - - val fields = descriptor.fieldValues(entry) - for ((fieldName, value) in fields) { - if (value == null) continue - - fieldMaps[fieldName] - ?.getOrPut(value) { mutableSetOf() } - ?.add(entry.key) - - val buckets = prefixBuckets[fieldName] - if (buckets != null) { - val lower = value.lowercase() - val firstChar = lower.firstOrNull() ?: continue - buckets.getOrPut(firstChar) { mutableListOf() } - .add(PrefixEntry(lower, entry.key)) - } - } - } - - private fun removeFromSecondaryIndexes(entry: T) { - val fields = descriptor.fieldValues(entry) - for ((fieldName, value) in fields) { - if (value == null) continue - - fieldMaps[fieldName]?.get(value)?.remove(entry.key) - - val buckets = prefixBuckets[fieldName] - if (buckets != null) { - val lower = value.lowercase() - val firstChar = lower.firstOrNull() ?: continue - buckets[firstChar]?.removeAll { it.key == entry.key } - } - } - // Note: sourceMap is handled by the caller - } + private val rows = ConcurrentHashMap(256) + private val sourceMap = ConcurrentHashMap>(32) + private val keyMap = ConcurrentHashMap>(256) + private val fieldMaps = ConcurrentHashMap>>() + private val prefixBuckets = ConcurrentHashMap>>() + private val fingerprints = ConcurrentHashMap() + + private val lock = ReentrantReadWriteLock() + + private data class RowId( + val sourceId: String, + val key: String, + ) + + private data class PrefixEntry( + val lowerValue: String, + val row: RowId, + ) + + init { + for (field in descriptor.fields) { + fieldMaps[field.name] = ConcurrentHashMap() + if (field.prefixSearchable) { + prefixBuckets[field.name] = ConcurrentHashMap() + } + } + } + + override fun query(query: IndexQuery): Sequence { + val matching = resolveMatchingRows(query) + val ordered = if (query.key != null) matching.sortedBy { it.sourceId } else matching + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + return ordered + .asSequence() + .mapNotNull { rows[it] } + .take(limit) + } + + override suspend fun get(key: String): T? = + lock.read { + keyMap[key]?.minByOrNull { it.sourceId }?.let { rows[it] } + } + + override suspend fun containsSource(sourceId: String): Boolean = sourceMap.containsKey(sourceId) + + /** + * Projects [fieldName] out of the matching entries. + * + * Unlike the SQLite index there is no column to scan, so this reduces over the matched entries. + * That is acceptable at the sizes this index is built for (hundreds to low thousands of entries); + * it is not a general substitute for a column projection. + */ + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence { + if (!fieldMaps.containsKey(fieldName)) { + return emptySequence() + } + + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + return lock + .read { + val values = LinkedHashSet() + for (row in resolveMatchingRows(query)) { + if (values.size >= limit) { + break + } + val entry = rows[row] ?: continue + descriptor.fieldValues(entry)[fieldName]?.let { values.add(it) } + } + values.toList() + }.asSequence() + } + + override suspend fun insertAll(entries: Sequence) { + lock.write { + for (entry in entries) { + insertSingleLocked(entry) + } + } + } + + override suspend fun insert(entry: T) = lock.write { insertSingleLocked(entry) } + + /** Inserts under one write lock, so readers never see the fingerprint without every entry. */ + override suspend fun insertSource( + sourceId: String, + fingerprint: String, + entries: Sequence, + ) = lock.write { + for (entry in entries) { + insertSingleLocked(entry) + } + fingerprints[sourceId] = fingerprint + } + + override suspend fun sourceFingerprint(sourceId: String): String? = fingerprints[sourceId] + + override suspend fun removeBySource(sourceId: String) = + lock.write { + removeBySourceLocked(sourceId) + } + + /** + * Remove every entry belonging to any of [sourceIds]. + * + * Acquires the write lock once and removes each source under it, so the whole + * batch is atomic with respect to concurrent readers and writers - there is no + * intermediate state in which only some of the sources have been removed. + */ + override suspend fun removeBySources(sourceIds: Collection) = + lock.write { + for (sourceId in sourceIds) { + removeBySourceLocked(sourceId) + } + } + + /** + * Remove all entries and the fingerprint for [sourceId] from every map. + * Caller MUST already hold the write lock; this method does not lock. + */ + private fun removeBySourceLocked(sourceId: String) { + fingerprints.remove(sourceId) + val sourceRows = sourceMap.remove(sourceId) ?: return + for (row in sourceRows) { + val entry = rows.remove(row) ?: continue + keyMap[row.key]?.let { keyRows -> + keyRows.remove(row) + if (keyRows.isEmpty()) keyMap.remove(row.key) + } + removeFromSecondaryIndexes(row, entry) + } + } + + override suspend fun clear() = + lock.write { + rows.clear() + sourceMap.clear() + keyMap.clear() + fingerprints.clear() + fieldMaps.values.forEach { it.clear() } + prefixBuckets.values.forEach { it.clear() } + } + + val size: Int get() = rows.size + val sourceCount: Int get() = sourceMap.size + + /** + * Resolves the rows matching the query by intersecting the results of each predicate. + * + * Every predicate is ANDed, the key included, as SQLite does. A predicate naming a value no row + * has matches nothing rather than being ignored. The result is materialized under the read lock + * so callers never iterate a set a concurrent writer is mutating. + */ + private fun resolveMatchingRows(query: IndexQuery): List { + query.requireDeclaredFields(descriptor) + return lock.read { + var candidates: Set? = null + + if (query.key != null) { + candidates = intersect(candidates, keyMap[query.key].orEmpty()) + } + + if (query.sourceId != null) { + candidates = intersect(candidates, sourceMap[query.sourceId].orEmpty()) + } + + val sourceIds = query.sourceIds + if (sourceIds != null) { + // An empty scope matches nothing, as distinct from an absent scope matching anything. + val scoped = sourceIds.flatMapTo(mutableSetOf()) { sourceMap[it].orEmpty() } + candidates = intersect(candidates, scoped) + } + + for ((field, value) in query.exactMatch) { + val fieldMap = fieldMaps[field] ?: return@read emptyList() + candidates = intersect(candidates, fieldMap[value].orEmpty()) + } + + for ((field, values) in query.anyOf) { + val fieldMap = fieldMaps[field] ?: return@read emptyList() + val matching = values.flatMapTo(mutableSetOf()) { fieldMap[it].orEmpty() } + candidates = intersect(candidates, matching) + } + + for ((field, prefix) in query.prefixMatch) { + val matching = rowsWithPrefix(field, prefix) ?: return@read emptyList() + candidates = intersect(candidates, matching) + } + + for ((field, mustExist) in query.presence) { + val fieldMap = fieldMaps[field] ?: return@read emptyList() + val rowsWithField = fieldMap.values.flatMapTo(mutableSetOf()) { it } + candidates = + if (mustExist) { + intersect(candidates, rowsWithField) + } else { + intersect(candidates, rows.keys - rowsWithField) + } + } + + (candidates ?: rows.keys).toList() + } + } + + /** + * Rows whose [field] starts with [prefix], or `null` when [field] is not declared. + * + * Prefix-searchable fields match case-insensitively via the lowercased buckets, mirroring + * SQLite's lowercased column; any other field falls back to a case-sensitive scan of its values. + * An empty prefix means "field present", which excludes rows where the field is null. + */ + private fun rowsWithPrefix( + field: String, + prefix: String, + ): Set? { + val buckets = prefixBuckets[field] + if (buckets == null) { + val fieldMap = fieldMaps[field] ?: return null + return fieldMap.entries + .asSequence() + .filter { (value, _) -> value.startsWith(prefix) } + .flatMap { (_, valueRows) -> valueRows.asSequence() } + .toSet() + } + + val lowerPrefix = prefix.lowercase() + val firstChar = lowerPrefix.firstOrNull() + if (firstChar == null) { + return buckets.values.flatMapTo(mutableSetOf()) { entries -> entries.map { it.row } } + } + return buckets[firstChar] + .orEmpty() + .asSequence() + .filter { it.lowerValue.startsWith(lowerPrefix) } + .map { it.row } + .toSet() + } + + private fun intersect( + current: Set?, + other: Set, + ): Set { + if (current == null) return other + return current.intersect(other) + } + + private fun insertSingleLocked(entry: T) { + val row = RowId(entry.sourceId, entry.key) + val existing = rows[row] + if (existing != null) { + removeFromSecondaryIndexes(row, existing) + } + + rows[row] = entry + sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(row) + keyMap.getOrPut(entry.key) { mutableSetOf() }.add(row) + + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + if (value == null) continue + + fieldMaps[fieldName] + ?.getOrPut(value) { mutableSetOf() } + ?.add(row) + + val buckets = prefixBuckets[fieldName] + if (buckets != null) { + val lower = value.lowercase() + val firstChar = lower.firstOrNull() ?: continue + buckets + .getOrPut(firstChar) { mutableListOf() } + .add(PrefixEntry(lower, row)) + } + } + } + + /** Drops [row] from the field and prefix maps; [sourceMap] and [keyMap] are the caller's job. */ + private fun removeFromSecondaryIndexes( + row: RowId, + entry: T, + ) { + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + if (value == null) continue + + fieldMaps[fieldName]?.get(value)?.remove(row) + + val buckets = prefixBuckets[fieldName] + if (buckets != null) { + val lower = value.lowercase() + val firstChar = lower.firstOrNull() ?: continue + buckets[firstChar]?.removeAll { it.row == row } + } + } + } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt index 05a9bab1e7..433a551596 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt @@ -8,60 +8,67 @@ import java.io.Closeable /** * Merges query results from multiple [ReadableIndex] instances. * - * Indexes are queried sequentially in the order they are provided. - * Duplicate keys (same entry present in more than one backing index) - * are deduplicated — the first occurrence wins. + * Indexes are queried sequentially in the order they are provided. A key already seen is skipped + * and the first occurrence wins, whether the repeat comes from a later backing index or from the + * same index yielding it again under a different source, which a composite source-and-key primary + * key allows. * * @param T The indexed type. * @param indexes The indexes to merge, in priority order. */ class MergedIndex( - private val indexes: List>, -) : ReadableIndex, Closeable { + private val indexes: List>, +) : ReadableIndex, + Closeable { + constructor(vararg indexes: ReadableIndex) : this(indexes.toList()) - constructor(vararg indexes: ReadableIndex) : this(indexes.toList()) + override fun query(query: IndexQuery): Sequence = + sequence { + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + val seen = mutableSetOf() + var total = 0 + for (index in indexes) { + if (total >= limit) break + for (entry in index.query(query)) { + if (total >= limit) break + if (seen.add(entry.key)) { + yield(entry) + total++ + } + } + } + } - override fun query(query: IndexQuery): Sequence = sequence { - val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit - val seen = mutableSetOf() - var total = 0 - for (index in indexes) { - if (total >= limit) break - for (entry in index.query(query)) { - if (total >= limit) break - if (seen.add(entry.key)) { - yield(entry) - total++ - } - } - } - } + override suspend fun get(key: String): T? { + // First match wins (priority order) + for (index in indexes) { + val result = index.get(key) + if (result != null) return result + } + return null + } - override suspend fun get(key: String): T? { - // First match wins (priority order) - for (index in indexes) { - val result = index.get(key) - if (result != null) return result - } - return null - } + override suspend fun containsSource(sourceId: String): Boolean = indexes.any { it.containsSource(sourceId) } - override suspend fun containsSource(sourceId: String): Boolean { - return indexes.any { it.containsSource(sourceId) } - } + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence = + sequence { + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + val seen = mutableSetOf() + for (index in indexes) { + if (seen.size >= limit) break + for (value in index.distinctValues(fieldName, query)) { + if (seen.size >= limit) break + if (seen.add(value)) yield(value) + } + } + } - override fun distinctValues(fieldName: String): Sequence = sequence { - val seen = mutableSetOf() - for (index in indexes) { - for (value in index.distinctValues(fieldName)) { - if (seen.add(value)) yield(value) - } - } - } - - override fun close() { - for (index in indexes) { - if (index is Closeable) index.close() - } - } + override fun close() { + for (index in indexes) { + if (index is Closeable) index.close() + } + } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index e22a19f9ee..e44d05936e 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -4,18 +4,21 @@ import android.content.ContentValues import android.content.Context import android.database.sqlite.SQLiteDatabase import android.os.Looper +import androidx.annotation.VisibleForTesting import androidx.sqlite.db.SupportSQLiteDatabase import androidx.sqlite.db.SupportSQLiteOpenHelper import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withContext import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.requireDeclaredFields import org.slf4j.LoggerFactory import kotlin.collections.iterator @@ -25,22 +28,33 @@ import kotlin.collections.iterator * Creates a table dynamically based on the [IndexDescriptor]: * ``` * CREATE TABLE IF NOT EXISTS {name} ( - * _key TEXT PRIMARY KEY, + * _key TEXT NOT NULL, * _source_id TEXT NOT NULL, * f_{field1} TEXT, * f_{field1}_lower TEXT, -- if prefix-searchable * f_{field2} TEXT, * ... - * _payload BLOB NOT NULL + * _payload BLOB NOT NULL, + * PRIMARY KEY (_source_id, _key) * ); * ``` * + * A row is identified by its source and key together, so the same key (a class present in two + * JARs, say) has one row per source. The primary key leads with `_source_id`, which is what serves + * source-scoped reads and bulk removal. + * * SQL indexes are created on: - * - `_source_id` (for bulk removal) + * - `(_key, _source_id)` (for key lookups, already ordered by source) * - Each `f_{field}` (for equality filter) - * - Each `f_{field}_lower` (for prefix search via `LIKE 'prefix%'`) + * - Each `f_{field}_lower` (for prefix search) + * + * When a query has a selective predicate (a key, a prefix, or a match on a + * [selective][org.appdevforall.codeonthego.indexing.api.IndexField.selective] field), its source + * scope and its other terms only filter rows and never choose the index. [optimize] collects the + * planner statistics for everything else. * - * Uses WAL journal mode for concurrent read/write performance. + * File-backed databases use WAL journal mode: a commit appends to the write-ahead log instead + * of writing a rollback journal and the main file. In-memory databases have no journal to set. * Inserts are batched inside transactions for throughput. * * [query] and [distinctValues] eagerly collect results and return a @@ -54,335 +68,781 @@ import kotlin.collections.iterator * @param dbName Database file name. Pass `null` to create an in-memory database * that is discarded when closed. Different index types can share * a database (each gets its own table) or use separate files. + * @param formatVersion Version of the stored format: the schema and whatever produced the rows. + * It belongs to the database file, so opening a file stored at any other version + * discards every table in it; each index sharing the file recreates its own table + * when it opens. Indexes sharing a file must therefore pass the same version. * @param batchSize Number of rows per INSERT transaction. + * @param sourceIdChunkSize Max number of `_source_id` placeholders per source-scoped statement. + * See [SOURCE_ID_CHUNK_SIZE]. */ class SQLiteIndex( - override val descriptor: IndexDescriptor, - context: Context, - dbName: String?, - override val name: String = "sqlite:${descriptor.name}", - private val batchSize: Int = 500, + override val descriptor: IndexDescriptor, + context: Context, + dbName: String?, + formatVersion: Int, + override val name: String = "sqlite:${descriptor.name}", + private val batchSize: Int = 500, + @param:VisibleForTesting internal val sourceIdChunkSize: Int = SOURCE_ID_CHUNK_SIZE, ) : Index { - companion object { - private val log = LoggerFactory.getLogger(SQLiteIndex::class.java) - - /** - * Max number of `_source_id` placeholders per batched DELETE. - * Kept well under SQLite's default 999 bound-parameter limit. - */ - private const val DELETE_CHUNK_SIZE = 900 - } - - - private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") - - // Field column names: "f_{fieldName}" - private val fieldColumns = descriptor.fields.associate { field -> - field.name to "f_${field.name}" - } - - // Prefix-searchable fields also get a "_lower" column - private val prefixColumns = descriptor.fields - .filter { it.prefixSearchable } - .associate { it.name to "f_${it.name}_lower" } - - private val mutex = Mutex() - @Volatile private var closed = false - private val db: SupportSQLiteDatabase - - init { - val config = SupportSQLiteOpenHelper.Configuration.builder(context) - .name(dbName) - .callback(object : SupportSQLiteOpenHelper.Callback(1) { - override fun onCreate(db: SupportSQLiteDatabase) { - createTable(db) - } - - override fun onUpgrade( - db: SupportSQLiteDatabase, oldVersion: Int, newVersion: Int, - ) { - // TODO: Add migration support - db.execSQL("DROP TABLE IF EXISTS $tableName") - createTable(db) - } - - override fun onOpen(db: SupportSQLiteDatabase) { - } - }) - .build() - - db = FrameworkSQLiteOpenHelperFactory() - .create(config) - .writableDatabase - - // Ensure table exists (for shared databases) - createTable(db) - } - - override fun query(query: IndexQuery): Sequence = runBlocking { - ifOpen(emptySequence()) { - val (sql, args) = buildSelectQuery(query) - val cursor = db.query(sql, args.toTypedArray()) - cursor.use { - val payloadIdx = it.getColumnIndexOrThrow("_payload") - buildList { - while (it.moveToNext()) { - add(descriptor.deserialize(it.getBlob(payloadIdx))) - } - } - }.asSequence() - } - } - - override suspend fun get(key: String): T? = withContext(Dispatchers.IO) { - ifOpen(null) { - val cursor = db.query( - "SELECT _payload FROM $tableName WHERE _key = ? LIMIT 1", - arrayOf(key), - ) - cursor.use { - if (it.moveToFirst()) { - descriptor.deserialize(it.getBlob(0)) - } else null - } - } - } - - override suspend fun containsSource(sourceId: String): Boolean = - withContext(Dispatchers.IO) { - ifOpen(false) { - val cursor = db.query( - "SELECT 1 FROM $tableName WHERE _source_id = ? LIMIT 1", - arrayOf(sourceId), - ) - cursor.use { it.moveToFirst() } - } - } - - override fun distinctValues(fieldName: String): Sequence = runBlocking { - ifOpen(emptySequence()) { - val col = fieldColumns[fieldName] - ?: throw IllegalArgumentException("Unknown field: $fieldName") - val cursor = db.query("SELECT DISTINCT $col FROM $tableName WHERE $col IS NOT NULL") - cursor.use { - buildList { - while (it.moveToNext()) { - add(it.getString(0)) - } - } - }.asSequence() - } - } - - override suspend fun insertAll(entries: Sequence) = withContext(Dispatchers.IO) { - val batch = mutableListOf() - for (entry in entries) { - batch.add(entry) - if (batch.size >= batchSize) { - ifOpen { insertBatchLocked(batch) } - batch.clear() - } - } - if (batch.isNotEmpty()) { - ifOpen { insertBatchLocked(batch) } - } - } - - override suspend fun insert(entry: T) = withContext(Dispatchers.IO) { - ifOpen { insertBatchLocked(listOf(entry)) } - } - - override suspend fun removeBySource(sourceId: String) = withContext(Dispatchers.IO) { - ifOpen { db.execSQL("DELETE FROM $tableName WHERE _source_id = ?", arrayOf(sourceId)) } - } - - /** - * Remove every row whose `_source_id` is in [sourceIds] using a single SQLite - * transaction. The ids are split into chunks of at most [DELETE_CHUNK_SIZE] so - * each `DELETE ... IN (?, ?, ...)` stays within SQLite's bound-parameter limit; - * all chunks run inside the one transaction, so the batch commits atomically - * (an empty [sourceIds] is a no-op and opens no transaction). - * - * @param sourceIds Source ids whose rows should be deleted. - */ - override suspend fun removeBySources(sourceIds: Collection) = - withContext(Dispatchers.IO) { - if (sourceIds.isEmpty()) return@withContext - ifOpen { - db.beginTransaction() - try { - for (chunk in sourceIds.chunked(DELETE_CHUNK_SIZE)) { - val placeholders = chunk.joinToString(",") { "?" } - db.execSQL( - "DELETE FROM $tableName WHERE _source_id IN ($placeholders)", - chunk.toTypedArray(), - ) - } - db.setTransactionSuccessful() - } finally { - db.endTransaction() - } - } - } - - override suspend fun clear() = withContext(Dispatchers.IO) { - ifOpen { db.execSQL("DELETE FROM $tableName") } - } - - override fun close() { - if (Looper.getMainLooper() == Looper.myLooper()) { - log.warn( - "SQLiteIndex.close() called on the main thread; waiting on mutex and closing db may block and cause ANR" - ) - } - runBlocking { - mutex.withLock { - if (closed) return@withLock - closed = true - db.close() - } - } - } - - private suspend inline fun ifOpen(default: R, crossinline block: () -> R): R = - mutex.withLock { if (closed) default else block() } - - private suspend inline fun ifOpen(crossinline block: () -> Unit) = - mutex.withLock { if (!closed) block() } - - suspend fun size(): Int = withContext(Dispatchers.IO) { - ifOpen(0) { - val cursor = db.query("SELECT COUNT(*) FROM $tableName") - cursor.use { if (it.moveToFirst()) it.getInt(0) else 0 } - } - } - - private fun createTable(db: SupportSQLiteDatabase) { - val columns = buildString { - append("_key TEXT PRIMARY KEY, ") - append("_source_id TEXT NOT NULL, ") - - for (field in descriptor.fields) { - val col = fieldColumns[field.name]!! - append("$col TEXT, ") - - if (field.prefixSearchable) { - val lowerCol = prefixColumns[field.name]!! - append("$lowerCol TEXT, ") - } - } - - append("_payload BLOB NOT NULL") - } - - db.execSQL("CREATE TABLE IF NOT EXISTS $tableName ($columns)") - - // Indexes - db.execSQL( - "CREATE INDEX IF NOT EXISTS idx_${tableName}_source ON $tableName(_source_id)" - ) - - for (field in descriptor.fields) { - val col = fieldColumns[field.name]!! - db.execSQL( - "CREATE INDEX IF NOT EXISTS idx_${tableName}_$col ON $tableName($col)" - ) - - if (field.prefixSearchable) { - val lowerCol = prefixColumns[field.name]!! - db.execSQL( - "CREATE INDEX IF NOT EXISTS idx_${tableName}_$lowerCol ON $tableName($lowerCol)" - ) - } - } - } - - private fun insertBatchLocked(entries: List) { - db.beginTransaction() - try { - for (entry in entries) { - val cv = ContentValues().apply { - put("_key", entry.key) - put("_source_id", entry.sourceId) - - val fields = descriptor.fieldValues(entry) - for ((fieldName, value) in fields) { - val col = fieldColumns[fieldName] ?: continue - put(col, value) - - val lowerCol = prefixColumns[fieldName] - if (lowerCol != null) { - put(lowerCol, value?.lowercase()) - } - } - - put("_payload", descriptor.serialize(entry)) - } - - db.insert( - tableName, - SQLiteDatabase.CONFLICT_REPLACE, - cv, - ) - } - db.setTransactionSuccessful() - } finally { - db.endTransaction() - } - } - - private data class SqlQuery(val sql: String, val args: List) - - private fun buildSelectQuery(query: IndexQuery): SqlQuery { - val where = StringBuilder() - val args = mutableListOf() - - fun and(clause: String, vararg values: String) { - if (where.isNotEmpty()) where.append(" AND ") - where.append(clause) - args.addAll(values) - } - - query.key?.let { and("_key = ?", it) } - query.sourceId?.let { and("_source_id = ?", it) } - - for ((field, value) in query.exactMatch) { - val col = fieldColumns[field] ?: continue - and("$col = ?", value) - } - - for ((field, prefix) in query.prefixMatch) { - val lowerCol = prefixColumns[field] - if (lowerCol != null) { - // Use the pre-lowercased column for index-friendly LIKE - and("$lowerCol LIKE ?", "${prefix.lowercase()}%") - } else { - // Fallback: case-sensitive prefix on the regular column - val col = fieldColumns[field] ?: continue - and("$col LIKE ?", "$prefix%") - } - } - - for ((field, mustExist) in query.presence) { - val col = fieldColumns[field] ?: continue - if (mustExist) { - and("$col IS NOT NULL") - } else { - and("$col IS NULL") - } - } - - val sql = buildString { - append("SELECT _payload FROM $tableName") - if (where.isNotEmpty()) { - append(" WHERE ") - append(where) - } - if (query.limit > 0) { - append(" LIMIT ${query.limit}") - } - } - - return SqlQuery(sql, args) - } + companion object { + private val log = LoggerFactory.getLogger(SQLiteIndex::class.java) + + /** + * Max number of `_source_id` placeholders per batched DELETE. + * Kept well under SQLite's default 999 bound-parameter limit. + */ + private const val DELETE_CHUNK_SIZE = 900 + + /** + * Default max number of `_source_id` placeholders per source-scoped statement (see + * [sourceIdChunkSize]), under the same bound-parameter limit as [DELETE_CHUNK_SIZE]. A scoped + * query runs one statement per chunk; chunks are disjoint on `_source_id`, so no row can be + * returned by two of them. + * + * The gap between 900 and SQLite's 999 limit is headroom for the statement's other bound + * parameters -- an `anyOf` or an exact-match value adds one placeholder each, on top of the + * chunk's own. A single statement combining a full chunk with a large `anyOf` must still stay + * under 999 placeholders in total. + */ + private const val SOURCE_ID_CHUNK_SIZE = 900 + + /** Rows [optimize] samples per SQL index, which bounds its cost on a large table. */ + private const val ANALYSIS_LIMIT = 1000 + + /** The first SQLite version with `PRAGMA analysis_limit`. */ + private val ANALYSIS_LIMIT_SINCE = listOf(3, 32, 0) + + /** + * Whether SQLite [version] (as `sqlite_version()` reports it, e.g. `3.32.2`) supports + * `PRAGMA analysis_limit`. Compares the numeric components, so `3.9` is older than `3.32`; + * a version that does not parse counts as unsupported. + */ + @VisibleForTesting + internal fun supportsAnalysisLimit(version: String): Boolean { + val parts = version.split('.').map { it.toIntOrNull() ?: return false } + for (i in ANALYSIS_LIMIT_SINCE.indices) { + val part = parts.getOrElse(i) { 0 } + if (part != ANALYSIS_LIMIT_SINCE[i]) { + return part > ANALYSIS_LIMIT_SINCE[i] + } + } + return true + } + + /** Every table in a database except SQLite's and Android's own bookkeeping tables. */ + private const val USER_TABLES_QUERY = + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata'" + } + + private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") + + /** One row per source that was indexed with a fingerprint, see [insertSource]. */ + private val sourcesTableName = "${tableName}_sources" + + /** Field column names: `f_{fieldName}`. */ + private val fieldColumns = + descriptor.fields.associate { field -> + field.name to "f_${field.name}" + } + + /** Prefix-searchable fields also get a lowercased `f_{fieldName}_lower` column. */ + private val prefixColumns = + descriptor.fields + .filter { it.prefixSearchable } + .associate { it.name to "f_${it.name}_lower" } + + /** Fields whose value matches may choose the SQL index a query is served by. */ + private val selectiveFields = + descriptor.fields + .filter { it.selective } + .mapTo(HashSet()) { it.name } + + private val mutex = Mutex() + + @Volatile private var closed = false + private val db: SupportSQLiteDatabase + + init { + val config = + SupportSQLiteOpenHelper.Configuration + .builder(context) + .name(dbName) + .callback( + object : SupportSQLiteOpenHelper.Callback(formatVersion) { + override fun onCreate(db: SupportSQLiteDatabase) { + createTable(db) + } + + override fun onUpgrade( + db: SupportSQLiteDatabase, + oldVersion: Int, + newVersion: Int, + ) { + rebuild(db) + } + + override fun onDowngrade( + db: SupportSQLiteDatabase, + oldVersion: Int, + newVersion: Int, + ) { + rebuild(db) + } + }, + ).build() + + val helper = FrameworkSQLiteOpenHelperFactory().create(config) + // androidx disables WAL unless asked, and it only takes effect if set before the first open. + helper.setWriteAheadLoggingEnabled(true) + db = helper.writableDatabase + + // Ensure table exists (for shared databases) + createTable(db) + } + + override fun query(query: IndexQuery): Sequence = + runBlocking { + ifOpen(emptySequence()) { + val limit = effectiveLimit(query) + val results = mutableListOf() + for (chunk in sourceIdChunks(query)) { + if (results.size >= limit) { + break + } + + val (sql, args) = buildSelectQuery(query, chunk, limit - results.size) + val cursor = db.query(sql, args.toTypedArray()) + cursor.use { + val payloadIdx = it.getColumnIndexOrThrow("_payload") + while (it.moveToNext()) { + results.add(descriptor.deserialize(it.getBlob(payloadIdx))) + } + } + } + results.asSequence() + } + } + + override suspend fun get(key: String): T? = + withContext(Dispatchers.IO) { + ifOpen(null) { + val cursor = + db.query( + "SELECT _payload FROM $tableName WHERE _key = ? ORDER BY _source_id LIMIT 1", + arrayOf(key), + ) + cursor.use { + if (it.moveToFirst()) { + descriptor.deserialize(it.getBlob(0)) + } else { + null + } + } + } + } + + override suspend fun containsSource(sourceId: String): Boolean = + withContext(Dispatchers.IO) { + ifOpen(false) { + val cursor = + db.query( + "SELECT 1 FROM $tableName WHERE _source_id = ? LIMIT 1", + arrayOf(sourceId), + ) + cursor.use { it.moveToFirst() } + } + } + + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence = + runBlocking { + ifOpen(emptySequence()) { + val col = + fieldColumns[fieldName] + ?: throw IllegalArgumentException("Unknown field: $fieldName") + val limit = effectiveLimit(query) + + /* + * Deduplicated here as well as in SQL: DISTINCT only applies within one statement, and + * chunked source ids mean one statement per chunk. The same package name legitimately + * appears in many JARs, so without this the caller would see it once per chunk. + * + * Each chunk is queried with the full limit, not the remaining budget: a value already + * seen in an earlier chunk does not free up SQL-side room in a later one, since a + * chunk-local `LIMIT` counts rows before dedup -- a value repeating across chunks would + * otherwise starve a later chunk's still-unseen values out of a shrinking budget. The + * overall cap is enforced separately, while collecting a chunk's rows: a chunk queried + * with the full limit can itself hold more distinct new values than remain in the budget. + */ + val values = LinkedHashSet() + for (chunk in sourceIdChunks(query)) { + if (values.size >= limit) { + break + } + + val (sql, args) = buildDistinctQuery(col, query, chunk, limit) + db.query(sql, args.toTypedArray()).use { + while (it.moveToNext() && values.size < limit) { + values.add(it.getString(0)) + } + } + } + values.asSequence() + } + } + + override suspend fun insertAll(entries: Sequence) = insertBatched(entries, fingerprint = null) + + override suspend fun insertSource( + sourceId: String, + fingerprint: String, + entries: Sequence, + ) = insertBatched(entries, SourceFingerprint(sourceId, fingerprint)) + + override suspend fun sourceFingerprint(sourceId: String): String? = + withContext(Dispatchers.IO) { + ifOpen(null) { + db + .query( + "SELECT _fingerprint FROM $sourcesTableName WHERE _source_id = ?", + arrayOf(sourceId), + ).use { if (it.moveToFirst()) it.getString(0) else null } + } + } + + /** + * Inserts [entries] in transactions of [batchSize] rows, taking the lock per batch so reads can + * interleave with a long insert. A [fingerprint] goes into the last transaction, which runs + * even when there are no entries, so an empty source is still recorded as indexed. + * + * Checks for cancellation before each transaction: [Mutex.withLock] only checks it while + * actually suspended waiting for the lock, and its uncontended fast path never suspends, so a + * caller cancelled mid-scan (see [org.appdevforall.codeonthego.indexing.util.BackgroundIndexer]) + * would otherwise keep committing batches indefinitely. + */ + private suspend fun insertBatched( + entries: Sequence, + fingerprint: SourceFingerprint?, + ) = withContext(Dispatchers.IO) { + val batch = mutableListOf() + for (entry in entries) { + batch.add(entry) + if (batch.size >= batchSize) { + ensureActive() + ifOpen { insertBatchLocked(batch, fingerprint = null) } + batch.clear() + } + } + if (batch.isNotEmpty() || fingerprint != null) { + ensureActive() + ifOpen { insertBatchLocked(batch, fingerprint) } + } + } + + override suspend fun insert(entry: T) = + withContext(Dispatchers.IO) { + ifOpen { insertBatchLocked(listOf(entry), fingerprint = null) } + } + + override suspend fun removeBySource(sourceId: String) = removeBySources(listOf(sourceId)) + + /** + * Remove every row whose `_source_id` is in [sourceIds], with those sources' fingerprints, + * using a single SQLite transaction. The ids are split into chunks of at most + * [DELETE_CHUNK_SIZE] so each `DELETE ... IN (?, ?, ...)` stays within SQLite's + * bound-parameter limit; all chunks run inside the one transaction, so the batch commits + * atomically (an empty [sourceIds] is a no-op and opens no transaction). + * + * @param sourceIds Source ids whose rows should be deleted. + */ + override suspend fun removeBySources(sourceIds: Collection) = + withContext(Dispatchers.IO) { + if (sourceIds.isEmpty()) return@withContext + ifOpen { + db.beginTransaction() + try { + for (chunk in sourceIds.chunked(DELETE_CHUNK_SIZE)) { + val placeholders = chunk.joinToString(",") { "?" } + db.execSQL( + "DELETE FROM $tableName WHERE _source_id IN ($placeholders)", + chunk.toTypedArray(), + ) + db.execSQL( + "DELETE FROM $sourcesTableName WHERE _source_id IN ($placeholders)", + chunk.toTypedArray(), + ) + } + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + } + + override suspend fun clear() = + withContext(Dispatchers.IO) { + ifOpen { + db.beginTransaction() + try { + db.execSQL("DELETE FROM $tableName") + db.execSQL("DELETE FROM $sourcesTableName") + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + } + + /** + * Collects the statistics SQLite's query planner uses to choose between this table's indexes. + * + * This runs `ANALYZE` rather than `PRAGMA optimize`, which decides per table whether to analyze + * and, in older SQLite versions, only considers tables queried on the same connection: reads run + * on the pool's other connections, so right after indexing it could analyze nothing. The + * analysis limit samples each index instead of reading all of it. Both statements share one + * transaction so they run on the same connection. + * + * On SQLite older than 3.32 (API 30 and below) this does nothing. Those versions ignore the + * unknown analysis limit and read every index in full, which on a large table holds the lock + * that queries wait on for too long. Queries there rely on their selective predicates alone. + */ + override suspend fun optimize() = + withContext(Dispatchers.IO) { + ifOpen { + val version = db.query("SELECT sqlite_version()").use { if (it.moveToFirst()) it.getString(0) else "" } + if (!supportsAnalysisLimit(version)) { + log.debug("Not analyzing {}: SQLite {} has no analysis limit", tableName, version) + return@ifOpen + } + db.beginTransaction() + try { + db.query("PRAGMA analysis_limit = $ANALYSIS_LIMIT").close() + db.execSQL("ANALYZE $tableName") + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + } + + override fun close() { + if (Looper.getMainLooper() == Looper.myLooper()) { + log.warn( + "SQLiteIndex.close() called on the main thread; waiting on mutex and closing db may block and cause ANR", + ) + } + runBlocking { + mutex.withLock { + if (closed) return@withLock + closed = true + db.close() + } + } + } + + private suspend inline fun ifOpen( + default: R, + crossinline block: () -> R, + ): R = mutex.withLock { if (closed) default else block() } + + private suspend inline fun ifOpen(crossinline block: () -> Unit) = mutex.withLock { if (!closed) block() } + + suspend fun size(): Int = + withContext(Dispatchers.IO) { + ifOpen(0) { + val cursor = db.query("SELECT COUNT(*) FROM $tableName") + cursor.use { if (it.moveToFirst()) it.getInt(0) else 0 } + } + } + + /** + * Discards every table in [db] and recreates this index's own. + * + * The index is a cache of what the scanners produce, so a format change is handled by + * rebuilding rather than migrating: rows written by an older scanner are wrong, not just + * differently shaped. + */ + private fun rebuild(db: SupportSQLiteDatabase) { + val tables = + db.query(USER_TABLES_QUERY).use { + buildList { while (it.moveToNext()) add(it.getString(0)) } + } + for (table in tables) { + db.execSQL("DROP TABLE IF EXISTS \"$table\"") + } + createTable(db) + } + + private fun createTable(db: SupportSQLiteDatabase) { + val columns = + buildString { + append("_key TEXT NOT NULL, ") + append("_source_id TEXT NOT NULL, ") + + for (field in descriptor.fields) { + val col = fieldColumns[field.name]!! + append("$col TEXT, ") + + if (field.prefixSearchable) { + val lowerCol = prefixColumns[field.name]!! + append("$lowerCol TEXT, ") + } + } + + append("_payload BLOB NOT NULL, ") + append("PRIMARY KEY (_source_id, _key)") + } + + db.execSQL("CREATE TABLE IF NOT EXISTS $tableName ($columns)") + db.execSQL( + "CREATE TABLE IF NOT EXISTS $sourcesTableName (_source_id TEXT NOT NULL PRIMARY KEY, _fingerprint TEXT NOT NULL)", + ) + + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_key ON $tableName(_key, _source_id)", + ) + + for (field in descriptor.fields) { + val col = fieldColumns[field.name]!! + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_$col ON $tableName($col)", + ) + + if (field.prefixSearchable) { + val lowerCol = prefixColumns[field.name]!! + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_$lowerCol ON $tableName($lowerCol)", + ) + } + } + } + + private fun insertBatchLocked( + entries: List, + fingerprint: SourceFingerprint?, + ) { + db.beginTransaction() + try { + for (entry in entries) { + val cv = + ContentValues().apply { + put("_key", entry.key) + put("_source_id", entry.sourceId) + + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + val col = fieldColumns[fieldName] ?: continue + put(col, value) + + val lowerCol = prefixColumns[fieldName] + if (lowerCol != null) { + put(lowerCol, value?.lowercase()) + } + } + + put("_payload", descriptor.serialize(entry)) + } + + db.insert( + tableName, + SQLiteDatabase.CONFLICT_REPLACE, + cv, + ) + } + if (fingerprint != null) { + val cv = + ContentValues().apply { + put("_source_id", fingerprint.sourceId) + put("_fingerprint", fingerprint.value) + } + db.insert(sourcesTableName, SQLiteDatabase.CONFLICT_REPLACE, cv) + } + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + + private data class SourceFingerprint( + val sourceId: String, + val value: String, + ) + + private data class SqlQuery( + val sql: String, + val args: List, + ) + + private fun effectiveLimit(query: IndexQuery) = if (query.limit > 0) query.limit else Int.MAX_VALUE + + /** + * The smallest string greater than every string starting with [prefix], or `null` when no such + * bound exists because [prefix] consists only of the highest representable character, U+FFFF. + * + * Increments the trailing character by code point, not by UTF-16 code unit: a supplementary + * character (one stored as a surrogate pair) is incremented as the single code point it + * represents. Incrementing only the low surrogate, in isolation, would produce a code unit pair + * SQLite decodes as an unrelated, smaller code point, putting the bound below [prefix] itself and + * making the range it feeds ([SQLiteIndex.buildWhereClause]) match nothing. When an increment + * would land in the surrogate range (`0xD800`-`0xDFFF`, which by itself encodes no character), the + * result steps over it to the first ordinary character above it, `0xE000`. + */ + private fun exclusiveUpperBound(prefix: String): String? { + var end = prefix.length + while (end > 0) { + val isSupplementary = end >= 2 && Character.isSurrogatePair(prefix[end - 2], prefix[end - 1]) + val start = if (isSupplementary) end - 2 else end - 1 + val codePoint = + if (isSupplementary) { + Character.toCodePoint(prefix[end - 2], prefix[end - 1]) + } else { + prefix[end - 1].code + } + val maxAtThisPosition = if (isSupplementary) Character.MAX_CODE_POINT else Char.MAX_VALUE.code + + if (codePoint == maxAtThisPosition) { + end = start + continue + } + + val incremented = codePoint + 1 + val next = + if (incremented in Character.MIN_SURROGATE.code..Character.MAX_SURROGATE.code) { + Character.MAX_SURROGATE.code + 1 + } else { + incremented + } + return prefix.substring(0, start) + String(Character.toChars(next)) + } + return null + } + + /** Escapes the LIKE metacharacters in [literal] so it matches only itself. */ + private fun escapeLikeLiteral(literal: String) = + literal + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + + /** + * Splits [IndexQuery.sourceIds] into groups small enough for one `IN (...)` clause. + * + * Returns a single `null` chunk when the query is unscoped, and no chunks at all when it is + * scoped to an empty set -- the caller then runs no statement and yields nothing, which is the + * difference between "any source" and "none of them". + * + * Chunks come in ascending source id order. Together with the per-statement `ORDER BY` on key + * queries, that is what makes a limited key lookup return the smallest source id overall rather + * than the smallest within whichever chunk happened to run first. + */ + private fun sourceIdChunks(query: IndexQuery): List?> { + val sourceIds = query.sourceIds ?: return listOf(null) + if (sourceIds.isEmpty()) { + return emptyList() + } + return sourceIds.toSortedSet().chunked(sourceIdChunkSize) + } + + private fun buildSelectQuery( + query: IndexQuery, + sourceIdChunk: List?, + limit: Int, + ): SqlQuery { + val (where, args) = buildWhereClause(query, sourceIdChunk) + val sql = + buildString { + append("SELECT _payload FROM $tableName") + if (where.isNotEmpty()) { + append(" WHERE ") + append(where) + } + if (query.key != null) { + append(" ORDER BY _source_id") + } + if (limit != Int.MAX_VALUE) { + append(" LIMIT $limit") + } + } + + return SqlQuery(sql, args) + } + + private fun buildDistinctQuery( + column: String, + query: IndexQuery, + sourceIdChunk: List?, + limit: Int, + ): SqlQuery { + val (where, args) = buildWhereClause(query, sourceIdChunk) + val filter = filterOnlyMarker(query) + val sql = + buildString { + append("SELECT DISTINCT $column FROM $tableName WHERE $filter$column IS NOT NULL") + if (where.isNotEmpty()) { + append(" AND ") + append(where) + } + if (limit != Int.MAX_VALUE) { + append(" LIMIT $limit") + } + } + return SqlQuery(sql, args) + } + + /** Returns the plan of the first statement [query] runs, one plan row per line. */ + @VisibleForTesting + internal fun explainQuery(query: IndexQuery): String { + val select = buildSelectQuery(query, firstSourceIdChunk(query), effectiveLimit(query)) + return explain(select) + } + + /** Returns the plan of the first statement [distinctValues] runs for [fieldName] and [query]. */ + @VisibleForTesting + internal fun explainDistinctValues( + fieldName: String, + query: IndexQuery, + ): String { + val col = fieldColumns[fieldName] ?: throw IllegalArgumentException("Unknown field: $fieldName") + return explain(buildDistinctQuery(col, query, firstSourceIdChunk(query), effectiveLimit(query))) + } + + private fun firstSourceIdChunk(query: IndexQuery): List? { + val chunks = sourceIdChunks(query) + require(chunks.isNotEmpty()) { "A query scoped to no sources runs no statement" } + return chunks.first() + } + + private fun explain(query: SqlQuery): String = + runBlocking { + ifOpen("") { + db.query("EXPLAIN QUERY PLAN ${query.sql}", query.args.toTypedArray()).use { cursor -> + val detail = cursor.getColumnIndexOrThrow("detail") + buildList { while (cursor.moveToNext()) add(cursor.getString(detail)) }.joinToString("\n") + } + } + } + + /** + * Whether [query] has a predicate its own SQL index narrows to few rows: a key, a non-empty + * prefix, or a value match on a selective field. + */ + private fun hasSelectivePredicate(query: IndexQuery): Boolean = + query.key != null || + query.exactMatch.keys.any { it in selectiveFields } || + query.anyOf.any { (field, values) -> field in selectiveFields && values.isNotEmpty() } || + query.prefixMatch.any { (field, prefix) -> field in fieldColumns && prefix.isNotEmpty() } + + /** + * The unary `+` that keeps a term out of SQL index selection when [query] has a selective + * predicate, and nothing otherwise. The term is still evaluated against every candidate row. + * + * Without statistics SQLite rates an `IN` list or an equality on any indexed column as good as a + * range, so a scope over hundreds of sources, or a kind, would otherwise choose the index and + * leave the selective predicate to be checked row by row. A query with no selective predicate + * keeps every term eligible, so its scope can still be served by the primary key. + * + * The trade-off: any non-empty prefix wins over the scope, even a one-character prefix under a + * single-source scope, where the scope would be narrower. Production scopes span hundreds of + * sources, so the prefix is the right choice for the queries that matter. + */ + private fun filterOnlyMarker(query: IndexQuery) = if (hasSelectivePredicate(query)) "+" else "" + + /** + * Builds the shared `WHERE` body for [query], restricted to [sourceIdChunk] when the query is + * source-scoped. Returns the clause without the `WHERE` keyword so both the row select and the + * distinct projection can splice it in. + */ + private fun buildWhereClause( + query: IndexQuery, + sourceIdChunk: List?, + ): Pair> { + query.requireDeclaredFields(descriptor) + val where = StringBuilder() + val args = mutableListOf() + + fun and( + clause: String, + vararg values: String, + ) { + if (where.isNotEmpty()) where.append(" AND ") + where.append(clause) + args.addAll(values) + } + + val filter = filterOnlyMarker(query) + + fun filterUnlessSelective(field: String) = if (field in selectiveFields) "" else filter + + query.key?.let { and("_key = ?", it) } + query.sourceId?.let { and("${filter}_source_id = ?", it) } + + if (sourceIdChunk != null) { + val placeholders = sourceIdChunk.joinToString(",") { "?" } + and("${filter}_source_id IN ($placeholders)", *sourceIdChunk.toTypedArray()) + } + + for ((field, value) in query.exactMatch) { + val col = fieldColumns.getValue(field) + and("${filterUnlessSelective(field)}$col = ?", value) + } + + for ((field, values) in query.anyOf) { + val col = fieldColumns.getValue(field) + if (values.isEmpty()) { + // Scoped to nothing, as opposed to unscoped: no row can satisfy it. + and("0 = 1") + continue + } + val distinct = values.distinct() + val placeholders = distinct.joinToString(",") { "?" } + and("${filterUnlessSelective(field)}$col IN ($placeholders)", *distinct.toTypedArray()) + } + + for ((field, prefix) in query.prefixMatch) { + val lowerCol = prefixColumns[field] + // Prefix-searchable fields match case-insensitively through their pre-lowercased column; + // everything else matches the stored value as-is. + val col = lowerCol ?: fieldColumns.getValue(field) + val value = if (lowerCol != null) prefix.lowercase() else prefix + + if (value.isEmpty()) { + // An empty prefix means the field must simply be present. + and("$filter$col IS NOT NULL") + continue + } + + /* + * The range bounds are what make this use the column's index: SQLite only optimises LIKE + * into a range scan when case_sensitive_like is on or the column collates NOCASE, and + * neither holds here, so a bare LIKE scans the whole table. The range already rejects a + * literal '_' or '%' substituted for another character -- e.g. prefix "my_f" against a row + * "myXfield" -- because that changes the value's ordering, not just what a LIKE wildcard + * would match. The escaped LIKE stays as a second guard for the rare case the range alone + * is not tight: [exclusiveUpperBound] returns `null` for a prefix with no upper bound, or a + * looser bound when its last character carries into an earlier one, and either lets through + * a value that only differs from the prefix after the point the range stops constraining it. + */ + val upperBound = exclusiveUpperBound(value) + if (upperBound != null) { + and("$col >= ? AND $col < ?", value, upperBound) + } else { + and("$col >= ?", value) + } + and("$col LIKE ? ESCAPE '\\'", "${escapeLikeLiteral(value)}%") + } + + for ((field, mustExist) in query.presence) { + val col = fieldColumns.getValue(field) + if (mustExist) { + and("$filter$col IS NOT NULL") + } else { + and("$filter$col IS NULL") + } + } + + return where.toString() to args + } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt index b595e604ba..dc5ba74bcd 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt @@ -3,9 +3,10 @@ package org.appdevforall.codeonthego.indexing.api /** * Any object that can be stored in an index. * - * The only requirements are a unique key (for deduplication and - * point lookups) and a source identifier (for bulk operations - * when the source changes). + * The only requirements are a key (for deduplication and point + * lookups) and a source identifier (for bulk operations when the + * source changes). An entry is identified by the two together: the + * same key may appear once in each of several sources. * * What constitutes a "key" and "source" depends entirely on * the consumer: @@ -14,16 +15,15 @@ package org.appdevforall.codeonthego.indexing.api * - For Python symbols: key = qualified name, source = .py file path */ interface Indexable { + /** Identifies the entry within its source. */ + val key: String - /** Unique identifier within the index. */ - val key: String - - /** - * Identifies the origin of this entry. - * All entries sharing a [sourceId] can be removed atomically - * via [WritableIndex.removeBySource]. - */ - val sourceId: String + /** + * Identifies the origin of this entry. + * All entries sharing a [sourceId] can be removed atomically + * via [WritableIndex.removeBySource]. + */ + val sourceId: String } /** @@ -36,45 +36,44 @@ interface Indexable { * @param T The domain type being indexed. */ interface IndexDescriptor { + /** + * A unique name for this index type. Used as the table name + * in persistent storage and the namespace in composite indexes. + */ + val name: String - /** - * A unique name for this index type. Used as the table name - * in persistent storage and the namespace in composite indexes. - */ - val name: String - - /** - * The fields that should be queryable. - * Defines the "schema" for this index type. - * - * The persistent layer will create SQL columns and indexes - * for each declared field. - */ - val fields: List + /** + * The fields that should be queryable. + * Defines the "schema" for this index type. + * + * The persistent layer will create SQL columns and indexes + * for each declared field. + */ + val fields: List - /** - * Extract the queryable field values from an entry. - * - * The returned map's keys must be a subset of [fields]'s names. - * Null values mean the field is not applicable for this entry - * (e.g. receiverType is null for a non-extension function). - */ - fun fieldValues(entry: T): Map + /** + * Extract the queryable field values from an entry. + * + * The returned map's keys must be a subset of [fields]'s names. + * Null values mean the field is not applicable for this entry + * (e.g. receiverType is null for a non-extension function). + */ + fun fieldValues(entry: T): Map - /** - * Serialize an entry to bytes for persistent storage. - * - * Use whatever format is appropriate - protobuf, JSON, - * custom binary. Called once on insert; the bytes are - * stored opaquely. - */ - fun serialize(entry: T): ByteArray + /** + * Serialize an entry to bytes for persistent storage. + * + * Use whatever format is appropriate - protobuf, JSON, + * custom binary. Called once on insert; the bytes are + * stored opaquely. + */ + fun serialize(entry: T): ByteArray - /** - * Deserialize bytes back into an entry. - * Must be the inverse of [serialize]. - */ - fun deserialize(bytes: ByteArray): T + /** + * Deserialize bytes back into an entry. + * Must be the inverse of [serialize]. + */ + fun deserialize(bytes: ByteArray): T } /** @@ -84,8 +83,13 @@ interface IndexDescriptor { * @param prefixSearchable Whether this field supports prefix queries * (e.g. name prefix for completions). Affects how * the persistent layer creates SQL indexes. + * @param selective Whether matching this field to a value narrows a query to few entries. + * Pass `false` for a field with few distinct values, such as a kind or a + * flag: a match on it then only filters a query that has a selective + * predicate, instead of choosing the SQL index the query is served by. */ data class IndexField( - val name: String, - val prefixSearchable: Boolean = false, + val name: String, + val prefixSearchable: Boolean = false, + val selective: Boolean = true, ) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index ed354a7a1a..c405768a64 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -11,77 +11,96 @@ import java.io.Closeable * @param T The indexed type. */ interface ReadableIndex { - - /** - * Query the index. Returns a lazy [Sequence] of matching entries. - * - * Results are not guaranteed to be in any particular order - * unless the implementation specifies otherwise. - * - * If [IndexQuery.limit] is 0, all matches are emitted. - */ - fun query(query: IndexQuery): Sequence - - /** - * Point lookup by key. Returns null if not found. - */ - suspend fun get(key: String): T? - - /** - * Fast existence check for a source. - */ - suspend fun containsSource(sourceId: String): Boolean - - /** - * Returns distinct values for a given field across all entries. - * - * Useful for enumerating packages, kinds, etc. without - * deserializing full entries. - * - * @param fieldName Must be one of the fields declared in the - * [IndexDescriptor]. - */ - fun distinctValues(fieldName: String): Sequence + /** + * Query the index. Returns a lazy [Sequence] of matching entries. + * + * Results are not guaranteed to be in any particular order + * unless the implementation specifies otherwise. + * + * If [IndexQuery.limit] is 0, all matches are emitted. + */ + fun query(query: IndexQuery): Sequence + + /** + * Point lookup by key. Returns null if not found. + * + * When several sources hold an entry for [key], returns the one with the smallest source id, so + * the answer does not depend on insertion order. + */ + suspend fun get(key: String): T? + + /** + * Fast existence check for a source. + */ + suspend fun containsSource(sourceId: String): Boolean + + /** + * Returns distinct values for a given field across all entries. + * + * Useful for enumerating packages, kinds, etc. without + * deserializing full entries. + * + * @param fieldName Must be one of the fields declared in the + * [IndexDescriptor]. + */ + fun distinctValues(fieldName: String): Sequence = distinctValues(fieldName, IndexQuery(limit = 0)) + + /** + * Returns the distinct values of [fieldName] among the entries matching [query]. + * + * This projects a single column instead of materializing entries, which is what makes it usable + * for enumerating a large field -- every package on a classpath, say -- where fetching the + * matching entries and reducing them in memory would defeat the point. + * + * Values are deduplicated across the whole result, and their order is unspecified. A positive + * [IndexQuery.limit] caps how many distinct values are emitted; 0 or negative means unlimited. + * + * @param fieldName Must be one of the fields declared in the [IndexDescriptor]. + * @param query Restricts which entries contribute a value. + */ + fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence } /** * Write interface for mutating an index. */ interface WritableIndex { - - /** - * Insert entries from a [Sequence]. - * - * Entries are consumed lazily from the sequence and batched - * internally for throughput. If an entry with the same key - * already exists, it is replaced. - */ - suspend fun insertAll(entries: Sequence) - - /** - * Convenience: insert a single entry. - */ - suspend fun insert(entry: T) - - /** - * Remove all entries from the given source. - */ - suspend fun removeBySource(sourceId: String) - - /** - * Remove all entries from the given sources in a single transaction. - * - * Equivalent to calling [removeBySource] for each id, but issues the - * deletes as one batched, transactional operation instead of N - * sequential statements. Implementations should chunk the ids so the - * generated SQL stays within parameter limits. - */ - suspend fun removeBySources(sourceIds: Collection) - - /** - * Remove all entries. - */ - suspend fun clear() + /** + * Insert entries from a [Sequence]. + * + * Entries are consumed lazily from the sequence and batched + * internally for throughput. If an entry with the same source + * and key already exists, it is replaced. + */ + suspend fun insertAll(entries: Sequence) + + /** + * Convenience: insert a single entry. + */ + suspend fun insert(entry: T) + + /** + * Remove all entries from the given source, and its fingerprint if one is recorded. + */ + suspend fun removeBySource(sourceId: String) + + /** + * Remove all entries from the given sources in a single transaction. + * + * Equivalent to calling [removeBySource] for each id, but issues the + * deletes as one batched, transactional operation instead of N + * sequential statements. Implementations should chunk the ids so the + * generated SQL stays within parameter limits. + */ + suspend fun removeBySources(sourceIds: Collection) + + /** + * Remove all entries and fingerprints. + */ + suspend fun clear() } /** @@ -89,13 +108,38 @@ interface WritableIndex { * * @param T The indexed type. */ -interface Index : ReadableIndex, WritableIndex, Closeable { - - /** Human-readable name for logging. */ - val name: String - - /** The descriptor governing serialization and field extraction. */ - val descriptor: IndexDescriptor - - override fun close() {} +interface Index : + ReadableIndex, + WritableIndex, + Closeable { + /** Human-readable name for logging. */ + val name: String + + /** The descriptor governing serialization and field extraction. */ + val descriptor: IndexDescriptor + + /** + * Inserts [entries], all from [sourceId], and records [fingerprint] as that source's. + * + * The fingerprint is opaque to the index; it is what lets a caller tell whether the source has + * changed since it was indexed. It is committed with the last batch of rows, so it is present + * only once every entry is stored: an insert that fails or is cancelled part-way leaves none. + * Removing the source's entries removes it too. + */ + suspend fun insertSource( + sourceId: String, + fingerprint: String, + entries: Sequence, + ) + + /** Returns the fingerprint recorded for [sourceId] by [insertSource], or `null` if there is none. */ + suspend fun sourceFingerprint(sourceId: String): String? + + /** + * Refreshes whatever the index keeps to plan its queries, after a bulk change to its entries. + * Does nothing by default. + */ + suspend fun optimize() {} + + override fun close() {} } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt index 8b3e1a6a12..85745e5eb0 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -8,71 +8,140 @@ package org.appdevforall.codeonthego.indexing.api * for Kotlin symbols, Android resources, Python declarations, etc. */ data class IndexQuery( - /** Exact match predicates: field name → expected value. */ - val exactMatch: Map = emptyMap(), - - /** Prefix match predicates: field name → prefix (case-insensitive). */ - val prefixMatch: Map = emptyMap(), - - /** - * Presence predicates: field name → whether the field must be - * non-null (true) or null (false). - */ - val presence: Map = emptyMap(), - - /** Filter by source ID. */ - val sourceId: String? = null, - - /** Filter by key (exact). */ - val key: String? = null, - - /** Maximum number of results. 0 = unlimited (use with care). */ - val limit: Int = 200, + /** Exact match predicates: field name -> expected value. */ + val exactMatch: Map = emptyMap(), + /** + * Prefix match predicates: field name -> prefix. + * + * Case-insensitive on a field declared [prefixSearchable][IndexField.prefixSearchable]; + * case-sensitive on any other field. + */ + val prefixMatch: Map = emptyMap(), + /** + * Set-membership predicates: field name to the values that field may take. + * + * An empty collection matches nothing, on the same reasoning as [sourceIds]. This exists so a + * caller wanting several values of one field -- every classifier kind, say -- can say so in the + * query instead of fetching every value and discarding most of them. + */ + val anyOf: Map> = emptyMap(), + /** + * Presence predicates: field name -> whether the field must be + * non-null (true) or null (false). + */ + val presence: Map = emptyMap(), + /** Filter by source ID. */ + val sourceId: String? = null, + /** + * Filter by a set of source IDs. `null` imposes no restriction; an empty collection matches + * nothing, which is what makes "scoped to a set that happens to be empty" distinguishable from + * "unscoped". + * + * Implementations must apply this before any [limit], so a scoped query cannot lose matches to + * rows that were only going to be discarded. + */ + val sourceIds: Collection? = null, + /** + * Filter by key (exact). + * + * Several sources can hold the same key. A key query returns its matches in ascending source id + * order, so with a limit of 1 it yields the one with the smallest source id among the sources in + * scope. + */ + val key: String? = null, + /** Maximum number of results. 0 = unlimited (use with care). */ + val limit: Int = 200, ) { - companion object { - /** Match everything up to [limit]. */ - val ALL = IndexQuery() + companion object { + /** Match everything up to [limit]. */ + val ALL = IndexQuery() + + /** Exact key lookup: the entry with the smallest source id among those holding [key]. */ + fun byKey(key: String) = IndexQuery(key = key, limit = 1) - /** Exact key lookup. */ - fun byKey(key: String) = IndexQuery(key = key, limit = 1) + /** All entries from a specific source. */ + fun bySource(sourceId: String) = IndexQuery(sourceId = sourceId, limit = 0) + } +} - /** All entries from a specific source. */ - fun bySource(sourceId: String) = IndexQuery(sourceId = sourceId, limit = 0) - } +/** + * Throws [IllegalArgumentException] if this query has a predicate on a field [descriptor] does not + * declare. + * + * Every index implementation checks this, so a misspelt field fails the same way everywhere rather + * than matching nothing in one implementation and every row in another. + */ +internal fun IndexQuery.requireDeclaredFields(descriptor: IndexDescriptor<*>) { + val declared = descriptor.fields.mapTo(HashSet()) { it.name } + val named = exactMatch.keys + prefixMatch.keys + anyOf.keys + presence.keys + val unknown = named - declared + require(unknown.isEmpty()) { "Unknown field(s) $unknown in a query on ${descriptor.name}" } } /** * DSL builder for [IndexQuery]. */ class IndexQueryBuilder { - private val exact = mutableMapOf() - private val prefix = mutableMapOf() - private val pres = mutableMapOf() - var sourceId: String? = null - var key: String? = null - var limit: Int = 200 + private val exact = mutableMapOf() + private val prefix = mutableMapOf() + private val pres = mutableMapOf() + private val anyOfValues = mutableMapOf>() + var sourceId: String? = null + + /** See [IndexQuery.sourceIds]. */ + var sourceIds: Collection? = null + var key: String? = null + var limit: Int = 200 + + /** Exact match on a field. */ + fun eq( + field: String, + value: String, + ) { + exact[field] = value + } - /** Exact match on a field. */ - fun eq(field: String, value: String) { exact[field] = value } + /** See [IndexQuery.prefixMatch]. */ + fun prefix( + field: String, + value: String, + ) { + prefix[field] = value + } - /** Prefix match on a field (case-insensitive). */ - fun prefix(field: String, value: String) { prefix[field] = value } + /** + * Field must hold one of [values]. + * + * Copies [values] so a later mutation of the caller's collection cannot change the built query. + */ + fun anyOf( + field: String, + values: Collection, + ) { + anyOfValues[field] = values.toList() + } - /** Field must be non-null. */ - fun exists(field: String) { pres[field] = true } + /** Field must be non-null. */ + fun exists(field: String) { + pres[field] = true + } - /** Field must be null. */ - fun notExists(field: String) { pres[field] = false } + /** Field must be null. */ + fun notExists(field: String) { + pres[field] = false + } - fun build() = IndexQuery( - exactMatch = exact.toMap(), - prefixMatch = prefix.toMap(), - presence = pres.toMap(), - sourceId = sourceId, - key = key, - limit = limit, - ) + fun build() = + IndexQuery( + exactMatch = exact.toMap(), + prefixMatch = prefix.toMap(), + anyOf = anyOfValues.toMap(), + presence = pres.toMap(), + sourceId = sourceId, + sourceIds = sourceIds, + key = key, + limit = limit, + ) } -inline fun indexQuery(block: IndexQueryBuilder.() -> Unit): IndexQuery = - IndexQueryBuilder().apply(block).build() +inline fun indexQuery(block: IndexQueryBuilder.() -> Unit): IndexQuery = IndexQueryBuilder().apply(block).build() diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt index ff46835f3d..3ed300e028 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt @@ -2,6 +2,7 @@ package org.appdevforall.codeonthego.indexing.util import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob @@ -60,8 +61,9 @@ class BackgroundIndexer( private val index: Index, parentScope: CoroutineScope = CoroutineScope(Dispatchers.Default), ) : Closeable { - private val job = SupervisorJob(parentScope.coroutineContext[Job]) - private val scope = CoroutineScope(parentScope.coroutineContext + job) + /** Named apart from any per-source [Job] a call to [indexSource] launches, so it can't be shadowed. */ + private val supervisor = SupervisorJob(parentScope.coroutineContext[Job]) + private val scope = CoroutineScope(parentScope.coroutineContext + supervisor) companion object { private val log = LoggerFactory.getLogger(BackgroundIndexer::class.java) @@ -70,34 +72,72 @@ class BackgroundIndexer( var progressListener: IndexingProgressListener? = null - private val activeJobs = ConcurrentHashMap() + /** The job currently indexing a source, and the fingerprint it was submitted with. */ + private data class ActiveJob( + val job: Job, + val fingerprint: String?, + ) + + private val activeJobs = ConcurrentHashMap() /** * Index a single source. The [provider] returns a [Flow] that is * collected on [Dispatchers.IO] and inserted in batches through - * [Index.insertAll]. + * [Index.insertAll]; with a [fingerprint], the last batch goes through + * [Index.insertSource] so the fingerprint is recorded only on success. + * + * If [skipIfExists] is true and the source is already indexed, this is a + * no-op. With a [fingerprint], "already indexed" means the index holds + * that same fingerprint for the source, so a source whose content changed + * at the same id is re-indexed; without one, it means the index holds any + * entry from the source. * - * If [skipIfExists] is true and the source is already indexed, - * this is a no-op. + * A source already being indexed by a still-running job submitted with the same + * [fingerprint] is left alone rather than restarted: that job will produce the same + * result, so cancelling and resubmitting it would only throw away its progress. A + * source being indexed under a different fingerprint (its content changed again before + * the previous pass finished) is handled by cancelling and joining that job before this + * one deletes its rows, so the two passes' writes can never interleave. + * + * The check for an existing job and the recording of this call's own job in [activeJobs] are + * not atomic with each other, so two overlapping calls for the same [sourceId] could each + * decide independently to proceed. That's safe only because every caller of [indexSource] for + * a given index submits through its own single-submitter mutex (see the indexing services' + * `indexingMutex`), never concurrently with itself. * * @param sourceId Identifies the source. * @param skipIfExists Skip if already indexed. + * @param fingerprint Identifies the source's current content, or `null` to not track it. * @param provider Lambda returning a [Flow] of entries. Resources the flow opens * are released when collection completes, fails or is cancelled. - * @return The launched job. + * @return The launched job, or the already-running one this call was folded into. */ fun indexSource( sourceId: String, skipIfExists: Boolean = true, + fingerprint: String? = null, provider: (sourceId: String) -> Flow, ): Job { - // Cancel any in-flight job for this source - activeJobs[sourceId]?.cancel() + val existing = activeJobs[sourceId] + if (existing != null && fingerprint != null && fingerprint == existing.fingerprint && existing.job.isActive) { + log.debug("Skipping resubmit of already-running: {}", sourceId) + progressListener?.onProgress(sourceId, IndexingEvent.Skipped) + return existing.job + } + + val previousJob = existing?.job + // Signalled as soon as this call supersedes it, rather than only once this job's own + // coroutine gets scheduled, so a superseded pass starts winding down immediately. + previousJob?.cancel() val job = - scope.launch { + scope.launch(start = CoroutineStart.LAZY) { try { - if (skipIfExists && index.containsSource(sourceId)) { + // Let a superseded pass for this source finish (or actually stop) before + // this one touches its rows, so the two passes' writes never interleave. + previousJob?.join() + + if (skipIfExists && isIndexed(sourceId, fingerprint)) { log.debug("Skipping already-indexed: {}", sourceId) progressListener?.onProgress(sourceId, IndexingEvent.Skipped) return@launch @@ -139,7 +179,11 @@ class BackgroundIndexer( // also leaves a partial source that containsSource() treats as indexed. error = e } - if (batch.isNotEmpty()) index.insertAll(batch.asSequence()) + if (fingerprint != null && error == null) { + index.insertSource(sourceId, fingerprint, batch.asSequence()) + } else if (batch.isNotEmpty()) { + index.insertAll(batch.asSequence()) + } error?.let { throw it } } @@ -154,14 +198,28 @@ class BackgroundIndexer( } } - activeJobs[sourceId] = job + val active = ActiveJob(job, fingerprint) + // Recorded before the job starts, so a concurrent call for the same source always sees + // this job rather than racing to observe it mid-launch. + activeJobs[sourceId] = active // Untrack on completion, not in the body: a job cancelled before it runs never reaches a // finally, and a handler added after completion runs at once. Remove only this job: a // replacement for the same source may already be registered. - job.invokeOnCompletion { activeJobs.remove(sourceId, job) } + job.invokeOnCompletion { activeJobs.remove(sourceId, active) } + job.start() return job } + private suspend fun isIndexed( + sourceId: String, + fingerprint: String?, + ): Boolean = + if (fingerprint != null) { + index.sourceFingerprint(sourceId) == fingerprint + } else { + index.containsSource(sourceId) + } + /** * Index multiple sources sequentially in the background. * @@ -185,14 +243,14 @@ class BackgroundIndexer( * Cancel all in-flight indexing and wait for completion. */ suspend fun cancelAll() { - activeJobs.values.toList().forEach { it.cancelAndJoin() } + activeJobs.values.toList().forEach { it.job.cancelAndJoin() } } /** * Wait for all in-flight indexing to complete. */ suspend fun awaitAll() { - activeJobs.values.toList().joinAll() + activeJobs.values.map { it.job }.joinAll() } /** @@ -209,7 +267,7 @@ class BackgroundIndexer( ) } runBlocking { - job.cancelAndJoin() + supervisor.cancelAndJoin() } activeJobs.clear() } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt new file mode 100644 index 0000000000..e51a6d7262 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt @@ -0,0 +1,156 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * [FilteredIndex] against a real SQLite backing, where the limit is applied by the database rather + * than by the caller. + * + * These use SQLite rather than [InMemoryIndex] deliberately: the defect they pin only appears when + * the backing index applies [IndexQuery.limit] itself, which is what production does. + */ +@RunWith(RobolectricTestRunner::class) +class FilteredIndexLimitTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_filtered_limit" + override val fields = listOf(IndexField("value", prefixSearchable = true)) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val backing = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + ) + + @After + fun tearDown() { + backing.close() + } + + private fun activeOnly(vararg sourceIds: String) = FilteredIndex(backing).apply { setActiveSources(sourceIds.toSet()) } + + private fun widgetQuery(limit: Int) = IndexQuery(prefixMatch = mapOf("value" to "Widget"), limit = limit) + + @Test + fun `limited query does not lose active matches behind inactive rows`() = + runTest { + // Twenty matching rows from an inactive source are inserted first, so they occupy the whole + // of a five-row SQL limit, and the one active match sits beyond them. + repeat(20) { backing.insert(Entry("stale$it", "inactiveJar", "Widget$it")) } + backing.insert(Entry("live", "activeJar", "WidgetLive")) + + val found = activeOnly("activeJar").query(widgetQuery(limit = 5)) + + assertThat(found.map { it.key }.toList()).containsExactly("live") + } + + @Test + fun `limited query counts only visible rows towards the limit`() = + runTest { + repeat(20) { backing.insert(Entry("stale$it", "inactiveJar", "Widget$it")) } + repeat(5) { backing.insert(Entry("live$it", "activeJar", "Widget$it")) } + + val found = activeOnly("activeJar").query(widgetQuery(limit = 3)) + + assertThat(found.toList()).hasSize(3) + } + + @Test + fun `explicit source scope is intersected with the active set`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + backing.insert(Entry("b", "jarB", "Widget")) + + // jarB is asked for but is not active, so it must contribute nothing. + val found = + activeOnly("jarA") + .query(IndexQuery(sourceIds = listOf("jarA", "jarB"), limit = 0)) + .map { it.key } + .toList() + + assertThat(found).containsExactly("a") + } + + @Test + fun `a subclass that declares every source visible is not filtered`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + backing.insert(Entry("b", "jarB", "Widget")) + + // No source is ever activated. Overriding the scope, rather than a per-id test, is what + // makes that mean "everything" instead of "nothing" once scoping is pushed into the query. + val unfiltered = + object : FilteredIndex(backing) { + override fun visibleSourceIds(): Collection? = null + } + + assertThat(unfiltered.query(widgetQuery(limit = 0)).map { it.key }.toList()) + .containsExactly("a", "b") + } + + @Test + fun `a point lookup still refuses an entry from an inactive source`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + + assertThat(activeOnly("jarA").get("a")).isNotNull() + assertThat(activeOnly("otherJar").get("a")).isNull() + } + + @Test + fun `containsSource still refuses an inactive source`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + + assertThat(activeOnly("jarA").containsSource("jarA")).isTrue() + assertThat(activeOnly("otherJar").containsSource("jarA")).isFalse() + } + + @Test + fun `distinctValues excludes inactive sources`() = + runTest { + backing.insert(Entry("a", "jarA", "Alpha")) + backing.insert(Entry("b", "jarB", "Beta")) + + assertThat(activeOnly("jarA").distinctValues("value").toList()).containsExactly("Alpha") + } + + @Test + fun `a point lookup finds the active source's row when an inactive source shares the key`() = + runTest { + backing.insert(Entry("shared", "jarA", "Inactive")) + backing.insert(Entry("shared", "jarB", "Active")) + + assertThat(activeOnly("jarB").get("shared")!!.value).isEqualTo("Active") + assertThat(activeOnly("jarB").query(IndexQuery.byKey("shared")).single().value).isEqualTo("Active") + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt index 81168d8d74..f899530617 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt @@ -12,154 +12,188 @@ import org.junit.runners.JUnit4 @RunWith(JUnit4::class) class FilteredIndexTest { - - data class Entry( - override val key: String, - override val sourceId: String, - val value: String, - ) : Indexable - - private val descriptor = object : IndexDescriptor { - override val name = "test_filtered" - override val fields = listOf(IndexField("value")) - - override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) - - override fun serialize(entry: Entry) = - "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() - - override fun deserialize(bytes: ByteArray): Entry { - val parts = String(bytes).split("|") - return Entry(parts[0], parts[1], parts[2]) - } - } - - private suspend fun setupBackingAndFiltered(): Pair, FilteredIndex> { - val backing = InMemoryIndex(descriptor) - backing.insert(Entry("k1", "src1", "val1")) - backing.insert(Entry("k2", "src1", "val2")) - backing.insert(Entry("k3", "src2", "val3")) - return backing to FilteredIndex(backing) - } - - @Test - fun `query returns nothing when no sources active`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.query(IndexQuery.ALL).toList()).isEmpty() - } - - @Test - fun `query returns entries for active source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k1", "k2") - } - - @Test - fun `deactivateSource hides entries from that source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.activateSource("src2") - filtered.deactivateSource("src1") - - val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k3") - } - - @Test - fun `setActiveSources replaces entire active set`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.setActiveSources(setOf("src2")) - - val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k3") - } - - @Test - fun `get returns null when source is inactive`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.get("k1")).isNull() - } - - @Test - fun `get returns entry when source is active`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - assertThat(filtered.get("k1")).isNotNull() - assertThat(filtered.get("k1")!!.key).isEqualTo("k1") - } - - @Test - fun `get returns null for nonexistent key even with active source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - assertThat(filtered.get("missing")).isNull() - } - - @Test - fun `containsSource returns false when source inactive`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.containsSource("src1")).isFalse() - } - - @Test - fun `containsSource returns true only when source is active AND in backing`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - assertThat(filtered.containsSource("src1")).isTrue() - assertThat(filtered.containsSource("src999")).isFalse() - } - - @Test - fun `isCached returns true if source exists in backing regardless of active state`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.isCached("src1")).isTrue() - assertThat(filtered.isCached("src999")).isFalse() - } - - @Test - fun `activeSources returns current active set`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.activateSource("src2") - assertThat(filtered.activeSources()).containsExactly("src1", "src2") - } - - @Test - fun `isActive reflects current state`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.isActive("src1")).isFalse() - filtered.activateSource("src1") - assertThat(filtered.isActive("src1")).isTrue() - filtered.deactivateSource("src1") - assertThat(filtered.isActive("src1")).isFalse() - } - - @Test - fun `explicit sourceId query returns empty for inactive source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - // src1 has entries in backing but is not active - val results = filtered.query(IndexQuery.bySource("src1")).toList() - assertThat(results).isEmpty() - } - - @Test - fun `all sources active returns all entries from backing`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.activateSource("src2") - - val results = filtered.query(IndexQuery.ALL).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `close clears active sources`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.close() - assertThat(filtered.activeSources()).isEmpty() - } + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_filtered" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private suspend fun setupBackingAndFiltered(): Pair, FilteredIndex> { + val backing = InMemoryIndex(descriptor) + backing.insert(Entry("k1", "src1", "val1")) + backing.insert(Entry("k2", "src1", "val2")) + backing.insert(Entry("k3", "src2", "val3")) + return backing to FilteredIndex(backing) + } + + @Test + fun `query returns nothing when no sources active`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.query(IndexQuery.ALL).toList()).isEmpty() + } + + @Test + fun `query returns entries for active source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k1", "k2") + } + + @Test + fun `deactivateSource hides entries from that source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.activateSource("src2") + filtered.deactivateSource("src1") + + val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k3") + } + + @Test + fun `setActiveSources replaces entire active set`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.setActiveSources(setOf("src2")) + + val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k3") + } + + @Test + fun `get returns null when source is inactive`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.get("k1")).isNull() + } + + @Test + fun `get returns entry when source is active`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + assertThat(filtered.get("k1")).isNotNull() + assertThat(filtered.get("k1")!!.key).isEqualTo("k1") + } + + @Test + fun `get returns null for nonexistent key even with active source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + assertThat(filtered.get("missing")).isNull() + } + + @Test + fun `containsSource returns false when source inactive`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.containsSource("src1")).isFalse() + } + + @Test + fun `containsSource returns true only when source is active AND in backing`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + assertThat(filtered.containsSource("src1")).isTrue() + assertThat(filtered.containsSource("src999")).isFalse() + } + + @Test + fun `isCached returns true if source exists in backing regardless of active state`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.isCached("src1")).isTrue() + assertThat(filtered.isCached("src999")).isFalse() + } + + @Test + fun `activeSources returns current active set`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.activateSource("src2") + assertThat(filtered.activeSources()).containsExactly("src1", "src2") + } + + @Test + fun `isActive reflects current state`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.isActive("src1")).isFalse() + filtered.activateSource("src1") + assertThat(filtered.isActive("src1")).isTrue() + filtered.deactivateSource("src1") + assertThat(filtered.isActive("src1")).isFalse() + } + + @Test + fun `explicit sourceId query returns empty for inactive source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + // src1 has entries in backing but is not active + val results = filtered.query(IndexQuery.bySource("src1")).toList() + assertThat(results).isEmpty() + } + + @Test + fun `all sources active returns all entries from backing`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.activateSource("src2") + + val results = filtered.query(IndexQuery.ALL).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `close clears active sources`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.close() + assertThat(filtered.activeSources()).isEmpty() + } + + @Test + fun `byKey returns nothing for a key only in an inactive source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + + assertThat(filtered.query(IndexQuery.byKey("k3")).toList()).isEmpty() + } + + @Test + fun `get returns the active source's entry when an inactive source shares the key`() = + runTest { + val backing = InMemoryIndex(descriptor) + backing.insert(Entry("shared", "src1", "inactive")) + backing.insert(Entry("shared", "src2", "active")) + val filtered = FilteredIndex(backing).apply { activateSource("src2") } + + assertThat(filtered.get("shared")!!.value).isEqualTo("active") + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt index 45cff41032..a9fa04b3f4 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt @@ -7,243 +7,428 @@ import org.appdevforall.codeonthego.indexing.api.IndexField import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable import org.appdevforall.codeonthego.indexing.api.indexQuery +import org.junit.Assert.assertThrows import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 @RunWith(JUnit4::class) class InMemoryIndexTest { - - data class TestEntry( - override val key: String, - override val sourceId: String, - val name: String, - val category: String? = null, - ) : Indexable - - private val descriptor = object : IndexDescriptor { - override val name = "test" - override val fields = listOf( - IndexField("name", prefixSearchable = true), - IndexField("category"), - ) - - override fun fieldValues(entry: TestEntry) = mapOf( - "name" to entry.name, - "category" to entry.category, - ) - - override fun serialize(entry: TestEntry) = - "${entry.key}|${entry.sourceId}|${entry.name}|${entry.category}".toByteArray() - - override fun deserialize(bytes: ByteArray): TestEntry { - val parts = String(bytes).split("|") - return TestEntry(parts[0], parts[1], parts[2], parts[3].takeIf { it != "null" }) - } - } - - private fun makeIndex() = InMemoryIndex(descriptor) - - private fun entry(key: String, sourceId: String, name: String, category: String? = null) = - TestEntry(key, sourceId, name, category) - - @Test - fun `insert and get by key`() = runTest { - val index = makeIndex() - val e = entry("k1", "src1", "Foo") - index.insert(e) - assertThat(index.get("k1")).isEqualTo(e) - assertThat(index.get("missing")).isNull() - } - - @Test - fun `query returns all entries when no predicates`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - index.insert(entry("k3", "src2", "Gamma")) - - val results = index.query(IndexQuery.ALL).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `exact match on field`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta", "app")) - index.insert(entry("k3", "src2", "Gamma", "lib")) - - val results = index.query(indexQuery { eq("category", "lib") }).toList() - assertThat(results.map { it.key }).containsExactly("k1", "k3") - } - - @Test - fun `prefix match on field`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "ArrayList")) - index.insert(entry("k2", "src1", "ArrayDeque")) - index.insert(entry("k3", "src1", "String")) - - val results = index.query(indexQuery { prefix("name", "Array") }).toList() - assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) - } - - @Test - fun `prefix match is case-insensitive`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "ArrayList")) - index.insert(entry("k2", "src1", "ARRAYDEQUE")) - - val results = index.query(indexQuery { prefix("name", "array") }).toList() - assertThat(results).hasSize(2) - } - - @Test - fun `containsSource returns true for indexed source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Foo")) - assertThat(index.containsSource("src1")).isTrue() - assertThat(index.containsSource("src999")).isFalse() - } - - @Test - fun `removeBySource removes all entries from that source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - index.insert(entry("k3", "src2", "Gamma")) - - index.removeBySource("src1") - - assertThat(index.get("k1")).isNull() - assertThat(index.get("k2")).isNull() - assertThat(index.get("k3")).isNotNull() - assertThat(index.containsSource("src1")).isFalse() - assertThat(index.size).isEqualTo(1) - } - - @Test - fun `clear removes all entries`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src2", "Beta")) - - index.clear() - - assertThat(index.size).isEqualTo(0) - assertThat(index.sourceCount).isEqualTo(0) - } - - @Test - fun `insert replaces existing entry with same key`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Original")) - index.insert(entry("k1", "src1", "Updated")) - - assertThat(index.get("k1")!!.name).isEqualTo("Updated") - assertThat(index.size).isEqualTo(1) - } - - @Test - fun `query by key returns single match`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - - val results = index.query(IndexQuery.byKey("k1")).toList() - assertThat(results).hasSize(1) - assertThat(results[0].key).isEqualTo("k1") - } - - @Test - fun `query by source returns entries for that source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - index.insert(entry("k3", "src2", "Gamma")) - - val results = index.query(IndexQuery.bySource("src1")).toList() - assertThat(results.map { it.key }).containsExactly("k1", "k2") - } - - @Test - fun `query respects limit`() = runTest { - val index = makeIndex() - repeat(10) { i -> index.insert(entry("k$i", "src1", "Name$i")) } - - val results = index.query(IndexQuery(limit = 3)).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `presence filter - field exists`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta")) // no category - - val results = index.query(indexQuery { exists("category") }).toList() - assertThat(results.map { it.key }).containsExactly("k1") - } - - @Test - fun `presence filter - field absent`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta")) - - val results = index.query(indexQuery { notExists("category") }).toList() - assertThat(results.map { it.key }).containsExactly("k2") - } - - @Test - fun `distinctValues returns unique field values`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta", "lib")) - index.insert(entry("k3", "src2", "Gamma", "app")) - - val values = index.distinctValues("category").toSet() - assertThat(values).containsExactly("lib", "app") - } - - @Test - fun `multi-field combined query`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "ArrayList", "java")) - index.insert(entry("k2", "src1", "ArrayDeque", "java")) - index.insert(entry("k3", "src2", "ArrayBlockingQueue", "concurrent")) - - val results = index.query(indexQuery { - eq("category", "java") - prefix("name", "Array") - }).toList() - assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) - } - - @Test - fun `insertAll inserts multiple entries`() = runTest { - val index = makeIndex() - val entries = (1..5).map { i -> entry("k$i", "src1", "Name$i") } - index.insertAll(entries.asSequence()) - - assertThat(index.size).isEqualTo(5) - } - - @Test - fun `query with unknown field returns empty sequence`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Foo")) - - val results = index.query(indexQuery { eq("nonexistentField", "value") }).toList() - assertThat(results).isEmpty() - } - - @Test - fun `removeBySource is no-op for unknown source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Foo")) - index.removeBySource("unknownSource") - assertThat(index.size).isEqualTo(1) - } + data class TestEntry( + override val key: String, + override val sourceId: String, + val name: String, + val category: String? = null, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test" + override val fields = + listOf( + IndexField("name", prefixSearchable = true), + IndexField("category"), + ) + + override fun fieldValues(entry: TestEntry) = + mapOf( + "name" to entry.name, + "category" to entry.category, + ) + + override fun serialize(entry: TestEntry) = "${entry.key}|${entry.sourceId}|${entry.name}|${entry.category}".toByteArray() + + override fun deserialize(bytes: ByteArray): TestEntry { + val parts = String(bytes).split("|") + return TestEntry(parts[0], parts[1], parts[2], parts[3].takeIf { it != "null" }) + } + } + + private fun makeIndex() = InMemoryIndex(descriptor) + + private fun entry( + key: String, + sourceId: String, + name: String, + category: String? = null, + ) = TestEntry(key, sourceId, name, category) + + @Test + fun `insert and get by key`() = + runTest { + val index = makeIndex() + val e = entry("k1", "src1", "Foo") + index.insert(e) + assertThat(index.get("k1")).isEqualTo(e) + assertThat(index.get("missing")).isNull() + } + + @Test + fun `query returns all entries when no predicates`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + index.insert(entry("k3", "src2", "Gamma")) + + val results = index.query(IndexQuery.ALL).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `exact match on field`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta", "app")) + index.insert(entry("k3", "src2", "Gamma", "lib")) + + val results = index.query(indexQuery { eq("category", "lib") }).toList() + assertThat(results.map { it.key }).containsExactly("k1", "k3") + } + + @Test + fun `prefix match on field`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "ArrayList")) + index.insert(entry("k2", "src1", "ArrayDeque")) + index.insert(entry("k3", "src1", "String")) + + val results = index.query(indexQuery { prefix("name", "Array") }).toList() + assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) + } + + @Test + fun `prefix match is case-insensitive`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "ArrayList")) + index.insert(entry("k2", "src1", "ARRAYDEQUE")) + + val results = index.query(indexQuery { prefix("name", "array") }).toList() + assertThat(results).hasSize(2) + } + + @Test + fun `containsSource returns true for indexed source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + assertThat(index.containsSource("src1")).isTrue() + assertThat(index.containsSource("src999")).isFalse() + } + + @Test + fun `removeBySource removes all entries from that source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + index.insert(entry("k3", "src2", "Gamma")) + + index.removeBySource("src1") + + assertThat(index.get("k1")).isNull() + assertThat(index.get("k2")).isNull() + assertThat(index.get("k3")).isNotNull() + assertThat(index.containsSource("src1")).isFalse() + assertThat(index.size).isEqualTo(1) + } + + @Test + fun `clear removes all entries`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src2", "Beta")) + + index.clear() + + assertThat(index.size).isEqualTo(0) + assertThat(index.sourceCount).isEqualTo(0) + } + + @Test + fun `insert replaces existing entry with same key`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Original")) + index.insert(entry("k1", "src1", "Updated")) + + assertThat(index.get("k1")!!.name).isEqualTo("Updated") + assertThat(index.size).isEqualTo(1) + } + + @Test + fun `query by key returns single match`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + + val results = index.query(IndexQuery.byKey("k1")).toList() + assertThat(results).hasSize(1) + assertThat(results[0].key).isEqualTo("k1") + } + + @Test + fun `query by source returns entries for that source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + index.insert(entry("k3", "src2", "Gamma")) + + val results = index.query(IndexQuery.bySource("src1")).toList() + assertThat(results.map { it.key }).containsExactly("k1", "k2") + } + + @Test + fun `query respects limit`() = + runTest { + val index = makeIndex() + repeat(10) { i -> index.insert(entry("k$i", "src1", "Name$i")) } + + val results = index.query(IndexQuery(limit = 3)).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `presence filter - field exists`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta")) // no category + + val results = index.query(indexQuery { exists("category") }).toList() + assertThat(results.map { it.key }).containsExactly("k1") + } + + @Test + fun `presence filter - field absent`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta")) + + val results = index.query(indexQuery { notExists("category") }).toList() + assertThat(results.map { it.key }).containsExactly("k2") + } + + @Test + fun `distinctValues returns unique field values`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta", "lib")) + index.insert(entry("k3", "src2", "Gamma", "app")) + + val values = index.distinctValues("category").toSet() + assertThat(values).containsExactly("lib", "app") + } + + @Test + fun `multi-field combined query`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "ArrayList", "java")) + index.insert(entry("k2", "src1", "ArrayDeque", "java")) + index.insert(entry("k3", "src2", "ArrayBlockingQueue", "concurrent")) + + val results = + index + .query( + indexQuery { + eq("category", "java") + prefix("name", "Array") + }, + ).toList() + assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) + } + + @Test + fun `insertAll inserts multiple entries`() = + runTest { + val index = makeIndex() + val entries = (1..5).map { i -> entry("k$i", "src1", "Name$i") } + index.insertAll(entries.asSequence()) + + assertThat(index.size).isEqualTo(5) + } + + @Test + fun `query with unknown field is rejected`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + assertThrows(IllegalArgumentException::class.java) { + index.query(indexQuery { eq("nonexistentField", "value") }).toList() + } + } + + @Test + fun `removeBySource is no-op for unknown source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + index.removeBySource("unknownSource") + assertThat(index.size).isEqualTo(1) + } + + @Test + fun `query scoped to source ids returns only those sources`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + index.insert(entry("k2", "src2", "Bar")) + index.insert(entry("k3", "src3", "Baz")) + + val keys = index.query(IndexQuery(sourceIds = listOf("src1", "src3"), limit = 0)).map { it.key }.toList() + assertThat(keys).containsExactly("k1", "k3") + } + + @Test + fun `query scoped to an empty source set returns nothing`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + assertThat(index.query(IndexQuery(sourceIds = emptyList(), limit = 0)).toList()).isEmpty() + assertThat(index.query(IndexQuery(sourceIds = null, limit = 0)).toList()).hasSize(1) + } + + @Test + fun `source scope is combined with the other predicates`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + index.insert(entry("k2", "src2", "Foo")) + + val keys = + index + .query(IndexQuery(exactMatch = mapOf("name" to "Foo"), sourceIds = listOf("src2"), limit = 0)) + .map { it.key } + .toList() + assertThat(keys).containsExactly("k2") + } + + @Test + fun `distinctValues projects only the values of matching rows`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + index.insert(entry("k2", "src2", "Bar", category = "beta")) + + val categories = index.distinctValues("category", IndexQuery(sourceIds = listOf("src1"), limit = 0)).toList() + assertThat(categories).containsExactly("alpha") + } + + @Test + fun `distinctValues does not report values whose only entry was removed`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + index.insert(entry("k2", "src2", "Bar", category = "beta")) + index.removeBySource("src1") + + assertThat(index.distinctValues("category").toList()).containsExactly("beta") + } + + @Test + fun `the same key in two sources keeps an entry for each`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "FromOne")) + index.insert(entry("k1", "src2", "FromTwo")) + + val fromOne = index.query(IndexQuery(key = "k1", sourceIds = listOf("src1"), limit = 0)).single() + val fromTwo = index.query(IndexQuery(key = "k1", sourceIds = listOf("src2"), limit = 0)).single() + assertThat(fromOne.name).isEqualTo("FromOne") + assertThat(fromTwo.name).isEqualTo("FromTwo") + assertThat(index.size).isEqualTo(2) + } + + @Test + fun `get returns the entry with the smallest source id when several sources share the key`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src2", "FromTwo")) + index.insert(entry("k1", "src1", "FromOne")) + index.insert(entry("k1", "src3", "FromThree")) + + assertThat(index.get("k1")!!.sourceId).isEqualTo("src1") + assertThat(index.query(IndexQuery.byKey("k1")).single().sourceId).isEqualTo("src1") + } + + @Test + fun `removing one source keeps the same key in another source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "FromOne", category = "alpha")) + index.insert(entry("k1", "src2", "FromTwo", category = "alpha")) + index.removeBySource("src1") + + assertThat(index.get("k1")!!.sourceId).isEqualTo("src2") + assertThat(index.query(IndexQuery(exactMatch = mapOf("category" to "alpha"), limit = 0)).single().sourceId) + .isEqualTo("src2") + } + + @Test + fun `a key query still applies its source scope`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + assertThat(index.query(IndexQuery(key = "k1", sourceIds = listOf("src2"), limit = 0)).toList()).isEmpty() + assertThat(index.query(IndexQuery(key = "k1", sourceId = "src2", limit = 0)).toList()).isEmpty() + } + + @Test + fun `a key query still applies its field predicates`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + + assertThat(index.query(IndexQuery(key = "k1", exactMatch = mapOf("category" to "beta"), limit = 0)).toList()) + .isEmpty() + assertThat(index.query(IndexQuery(key = "k1", exactMatch = mapOf("category" to "alpha"), limit = 0)).toList()) + .hasSize(1) + } + + @Test + fun `an exact match on a value no entry has matches nothing`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + + assertThat(index.query(IndexQuery(exactMatch = mapOf("category" to "missing"), limit = 0)).toList()).isEmpty() + } + + @Test + fun `a query for a source with no entries matches nothing`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + assertThat(index.query(IndexQuery.bySource("src999")).toList()).isEmpty() + } + + @Test + fun `insertSource records the fingerprint and removing the source drops it`() = + runTest { + val index = makeIndex() + index.insertSource("src1", "100:1", sequenceOf(entry("k1", "src1", "Foo"))) + index.insertSource("src2", "200:2", emptySequence()) + + assertThat(index.sourceFingerprint("src1")).isEqualTo("100:1") + assertThat(index.sourceFingerprint("src2")).isEqualTo("200:2") + + index.removeBySource("src1") + index.removeBySource("src2") + + assertThat(index.sourceFingerprint("src1")).isNull() + assertThat(index.sourceFingerprint("src2")).isNull() + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt index d41c8ed03c..4086d75548 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt @@ -12,145 +12,160 @@ import org.junit.runners.JUnit4 @RunWith(JUnit4::class) class MergedIndexTest { - - data class Entry( - override val key: String, - override val sourceId: String, - val value: String, - ) : Indexable - - private val descriptor = object : IndexDescriptor { - override val name = "test_merged" - override val fields = listOf(IndexField("value")) - - override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) - - override fun serialize(entry: Entry) = - "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() - - override fun deserialize(bytes: ByteArray): Entry { - val parts = String(bytes).split("|") - return Entry(parts[0], parts[1], parts[2]) - } - } - - private fun makeIndex() = InMemoryIndex(descriptor) - - @Test - fun `merges results from multiple indexes`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "v1")) } - val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "v2")) } - val merged = MergedIndex(idx1, idx2) - - val keys = merged.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k1", "k2") - } - - @Test - fun `deduplicates by key - first index wins`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "from-index1")) } - val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "from-index2")) } - val merged = MergedIndex(idx1, idx2) - - val results = merged.query(IndexQuery.ALL).toList() - assertThat(results).hasSize(1) - assertThat(results[0].value).isEqualTo("from-index1") - } - - @Test - fun `respects limit across indexes`() = runTest { - val idx1 = makeIndex().also { e -> - (1..5).forEach { i -> e.insert(Entry("k$i", "s1", "v$i")) } - } - val idx2 = makeIndex().also { e -> - (6..10).forEach { i -> e.insert(Entry("k$i", "s2", "v$i")) } - } - val merged = MergedIndex(idx1, idx2) - - val results = merged.query(IndexQuery(limit = 3)).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `get returns first match in priority order`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "primary")) } - val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "secondary")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.get("k1")!!.value).isEqualTo("primary") - } - - @Test - fun `get returns from second index when not in first`() = runTest { - val idx1 = makeIndex() - val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "second")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.get("k2")!!.value).isEqualTo("second") - } - - @Test - fun `get returns null if key not in any index`() = runTest { - val merged = MergedIndex(makeIndex()) - assertThat(merged.get("missing")).isNull() - } - - @Test - fun `containsSource checks all indexes`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "src1", "v1")) } - val idx2 = makeIndex().also { it.insert(Entry("k2", "src2", "v2")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.containsSource("src1")).isTrue() - assertThat(merged.containsSource("src2")).isTrue() - assertThat(merged.containsSource("src999")).isFalse() - } - - @Test - fun `distinctValues deduplicates across indexes`() = runTest { - val idx1 = makeIndex().also { e -> - e.insert(Entry("k1", "s1", "alpha")) - e.insert(Entry("k2", "s1", "beta")) - } - val idx2 = makeIndex().also { e -> - e.insert(Entry("k3", "s2", "beta")) // duplicate - e.insert(Entry("k4", "s2", "gamma")) - } - val merged = MergedIndex(idx1, idx2) - - val values = merged.distinctValues("value").toSet() - assertThat(values).containsExactly("alpha", "beta", "gamma") - } - - @Test - fun `empty merged index returns empty results`() = runTest { - val merged = MergedIndex(emptyList()) - - assertThat(merged.query(IndexQuery.ALL).toList()).isEmpty() - assertThat(merged.get("k1")).isNull() - assertThat(merged.containsSource("s1")).isFalse() - assertThat(merged.distinctValues("value").toList()).isEmpty() - } - - @Test - fun `single index merged works identically to unmerged`() = runTest { - val idx = makeIndex().also { - it.insert(Entry("k1", "s1", "val1")) - it.insert(Entry("k2", "s1", "val2")) - } - val merged = MergedIndex(idx) - - assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) - assertThat(merged.get("k1")).isNotNull() - assertThat(merged.containsSource("s1")).isTrue() - } - - @Test - fun `vararg constructor builds merged index`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "a")) } - val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "b")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) - } + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_merged" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private fun makeIndex() = InMemoryIndex(descriptor) + + @Test + fun `merges results from multiple indexes`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "v1")) } + val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "v2")) } + val merged = MergedIndex(idx1, idx2) + + val keys = merged.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k1", "k2") + } + + @Test + fun `deduplicates by key - first index wins`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "from-index1")) } + val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "from-index2")) } + val merged = MergedIndex(idx1, idx2) + + val results = merged.query(IndexQuery.ALL).toList() + assertThat(results).hasSize(1) + assertThat(results[0].value).isEqualTo("from-index1") + } + + @Test + fun `respects limit across indexes`() = + runTest { + val idx1 = + makeIndex().also { e -> + (1..5).forEach { i -> e.insert(Entry("k$i", "s1", "v$i")) } + } + val idx2 = + makeIndex().also { e -> + (6..10).forEach { i -> e.insert(Entry("k$i", "s2", "v$i")) } + } + val merged = MergedIndex(idx1, idx2) + + val results = merged.query(IndexQuery(limit = 3)).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `get returns first match in priority order`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "primary")) } + val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "secondary")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.get("k1")!!.value).isEqualTo("primary") + } + + @Test + fun `get returns from second index when not in first`() = + runTest { + val idx1 = makeIndex() + val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "second")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.get("k2")!!.value).isEqualTo("second") + } + + @Test + fun `get returns null if key not in any index`() = + runTest { + val merged = MergedIndex(makeIndex()) + assertThat(merged.get("missing")).isNull() + } + + @Test + fun `containsSource checks all indexes`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "src1", "v1")) } + val idx2 = makeIndex().also { it.insert(Entry("k2", "src2", "v2")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.containsSource("src1")).isTrue() + assertThat(merged.containsSource("src2")).isTrue() + assertThat(merged.containsSource("src999")).isFalse() + } + + @Test + fun `distinctValues deduplicates across indexes`() = + runTest { + val idx1 = + makeIndex().also { e -> + e.insert(Entry("k1", "s1", "alpha")) + e.insert(Entry("k2", "s1", "beta")) + } + val idx2 = + makeIndex().also { e -> + e.insert(Entry("k3", "s2", "beta")) // duplicate + e.insert(Entry("k4", "s2", "gamma")) + } + val merged = MergedIndex(idx1, idx2) + + val values = merged.distinctValues("value").toSet() + assertThat(values).containsExactly("alpha", "beta", "gamma") + } + + @Test + fun `empty merged index returns empty results`() = + runTest { + val merged = MergedIndex(emptyList()) + + assertThat(merged.query(IndexQuery.ALL).toList()).isEmpty() + assertThat(merged.get("k1")).isNull() + assertThat(merged.containsSource("s1")).isFalse() + assertThat(merged.distinctValues("value").toList()).isEmpty() + } + + @Test + fun `single index merged works identically to unmerged`() = + runTest { + val idx = + makeIndex().also { + it.insert(Entry("k1", "s1", "val1")) + it.insert(Entry("k2", "s1", "val2")) + } + val merged = MergedIndex(idx) + + assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) + assertThat(merged.get("k1")).isNotNull() + assertThat(merged.containsSource("s1")).isTrue() + } + + @Test + fun `vararg constructor builds merged index`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "a")) } + val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "b")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexAnalysisLimitTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexAnalysisLimitTest.kt new file mode 100644 index 0000000000..d9d7c6479d --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexAnalysisLimitTest.kt @@ -0,0 +1,38 @@ +package org.appdevforall.codeonthego.indexing + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** Which SQLite versions [SQLiteIndex.optimize] analyzes on: those with `PRAGMA analysis_limit`, 3.32.0 and later. */ +@RunWith(JUnit4::class) +class SQLiteIndexAnalysisLimitTest { + @Test + fun `versions from 3_32_0 on support the analysis limit`() { + for (version in listOf("3.32.0", "3.32.2", "3.39.2", "3.51.2", "4.0")) { + assertWithMessage(version).that(SQLiteIndex.supportsAnalysisLimit(version)).isTrue() + } + } + + @Test + fun `versions before 3_32_0 do not`() { + for (version in listOf("3.31.1", "3.28.0", "3.22.0", "2.99.99")) { + assertWithMessage(version).that(SQLiteIndex.supportsAnalysisLimit(version)).isFalse() + } + } + + @Test + fun `versions compare by number, not as text`() { + // As text "3.9.0" sorts after "3.32.0"; as a version it is older. + assertThat(SQLiteIndex.supportsAnalysisLimit("3.9.0")).isFalse() + } + + @Test + fun `an unparseable version does not`() { + for (version in listOf("", "3.x", "unknown")) { + assertWithMessage(version).that(SQLiteIndex.supportsAnalysisLimit(version)).isFalse() + } + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexCancellationTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexCancellationTest.kt new file mode 100644 index 0000000000..afdb91aa02 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexCancellationTest.kt @@ -0,0 +1,94 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +/** A cancelled insert must stop committing batches rather than run its entries to completion. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexCancellationTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_cancellation" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + /** batchSize = 1 so every entry is its own transaction, isolating the per-batch check. */ + private val index = + SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1, batchSize = 1) + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + private fun CountDownLatch.awaitOrFail() { + assertThat(await(5, TimeUnit.SECONDS)).isTrue() + } + + @Test + fun `cancelling an insert stops it from committing batches produced after cancellation`() = + runTest { + val secondEntryReached = CountDownLatch(1) + val proceedWithSecondEntry = CountDownLatch(1) + val entries = + sequence { + yield(Entry("k1", "jarA", "v1")) + secondEntryReached.countDown() + proceedWithSecondEntry.awaitOrFail() + yield(Entry("k2", "jarA", "v2")) + } + + val job = CoroutineScope(Dispatchers.IO).launch { index.insertSource("jarA", "1:1", entries) } + + // The first batch (batchSize = 1) has already committed by the time the sequence + // asks to produce its second entry. + secondEntryReached.awaitOrFail() + + job.cancel() + proceedWithSecondEntry.countDown() + job.cancelAndJoin() + + assertThat(index.get("k1")!!.value).isEqualTo("v1") + assertThat(index.get("k2")).isNull() + // The fingerprint is only ever written alongside the source's last batch; a cancelled + // insert must not record one, or a later call would wrongly treat "jarA" as up to date. + assertThat(index.sourceFingerprint("jarA")).isNull() + } + + private companion object { + const val DB_NAME = "cancellation_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexFingerprintTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexFingerprintTest.kt new file mode 100644 index 0000000000..5c4c39fa96 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexFingerprintTest.kt @@ -0,0 +1,128 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** Per-source fingerprints on the SQLite-backed index, stored and dropped with the source's rows. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexFingerprintTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_fingerprint" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + private fun open() = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) + + private val index = open() + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + @Test + fun `insertSource records the fingerprint and it survives reopening`() = + runTest { + index.insertSource("jarA", "100:1", sequenceOf(Entry("k1", "jarA", "v"))) + index.close() + + open().use { assertThat(it.sourceFingerprint("jarA")).isEqualTo("100:1") } + } + + @Test + fun `a source with no entries still records its fingerprint`() = + runTest { + index.insertSource("jarA", "100:1", emptySequence()) + + assertThat(index.sourceFingerprint("jarA")).isEqualTo("100:1") + } + + @Test + fun `removing a source drops its fingerprint`() = + runTest { + index.insertSource("jarA", "100:1", sequenceOf(Entry("k1", "jarA", "v"))) + index.insertSource("jarB", "200:2", sequenceOf(Entry("k1", "jarB", "v"))) + index.insertSource("jarC", "300:3", sequenceOf(Entry("k1", "jarC", "v"))) + + index.removeBySource("jarA") + index.removeBySources(listOf("jarB")) + + assertThat(index.sourceFingerprint("jarA")).isNull() + assertThat(index.sourceFingerprint("jarB")).isNull() + assertThat(index.sourceFingerprint("jarC")).isEqualTo("300:3") + } + + @Test + fun `clear drops every fingerprint`() = + runTest { + index.insertSource("jarA", "100:1", sequenceOf(Entry("k1", "jarA", "v"))) + + index.clear() + + assertThat(index.sourceFingerprint("jarA")).isNull() + } + + @Test + fun `an insert that fails part-way records no fingerprint`() = + runTest { + // Past the first batch, so some rows are already committed when the source fails. + val failing = + sequence { + repeat(700) { yield(Entry("k$it", "jarA", "v")) } + throw IllegalStateException("corrupt JAR") + } + + runCatching { index.insertSource("jarA", "100:1", failing) } + + assertThat(index.containsSource("jarA")).isTrue() + assertThat(index.sourceFingerprint("jarA")).isNull() + } + + @Test + fun `a JAR rebuilt at the same path is re-indexed and an unchanged one is skipped`() = + runTest { + val indexer = BackgroundIndexer(index) + + indexer.indexSource("jarA", fingerprint = "100:1") { flowOf(Entry("k1", it, "v1")) }.join() + indexer.indexSource("jarA", fingerprint = "100:1") { flowOf(Entry("k1", it, "skipped")) }.join() + assertThat(index.get("k1")!!.value).isEqualTo("v1") + + indexer.indexSource("jarA", fingerprint = "140:2") { flowOf(Entry("k2", it, "v2")) }.join() + assertThat(index.get("k1")).isNull() + assertThat(index.get("k2")!!.value).isEqualTo("v2") + } + + private companion object { + const val DB_NAME = "fingerprint_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt new file mode 100644 index 0000000000..0c9b13764f --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt @@ -0,0 +1,138 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** Row identity on the SQLite-backed index: a row is identified by its source and key together. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexIdentityTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_identity" + override val fields = listOf(IndexField("value", prefixSearchable = true)) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + private val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + @Test + fun `the same key in two sources keeps a row for each`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + index.insert(Entry("com.example.Foo", "jarB", "fromB")) + + assertThat(valueOfKeyIn("com.example.Foo", "jarA")).isEqualTo("fromA") + assertThat(valueOfKeyIn("com.example.Foo", "jarB")).isEqualTo("fromB") + assertThat(index.size()).isEqualTo(2) + } + + @Test + fun `re-inserting the same source and key replaces the row`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "original")) + index.insert(Entry("com.example.Foo", "jarA", "updated")) + + assertThat(valueOfKeyIn("com.example.Foo", "jarA")).isEqualTo("updated") + assertThat(index.size()).isEqualTo(1) + } + + @Test + fun `get returns the row with the smallest source id when several sources share the key`() = + runTest { + index.insert(Entry("com.example.Foo", "jarB", "fromB")) + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + index.insert(Entry("com.example.Foo", "jarC", "fromC")) + + assertThat(index.get("com.example.Foo")!!.sourceId).isEqualTo("jarA") + assertThat(index.query(IndexQuery.byKey("com.example.Foo")).single().sourceId).isEqualTo("jarA") + } + + @Test + fun `a scoped key lookup returns the smallest source id across source chunks`() = + runTest { + // More sources than one IN clause holds, requested in descending order, so the smallest id + // sits in the last chunk the caller's order would produce. + val sourceIds = (0 until 2000).map { "jar%04d".format(it) } + index.insertAll(sourceIds.asSequence().map { Entry("com.example.Foo", it, it) }) + + val found = index.query(IndexQuery(key = "com.example.Foo", sourceIds = sourceIds.reversed(), limit = 1)) + + assertThat(found.single().sourceId).isEqualTo("jar0000") + } + + @Test + fun `a key lookup scoped to one source ignores the same key elsewhere`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + index.insert(Entry("com.example.Foo", "jarB", "fromB")) + + val found = index.query(IndexQuery(key = "com.example.Foo", sourceIds = listOf("jarB"), limit = 1)) + + assertThat(found.single().value).isEqualTo("fromB") + } + + @Test + fun `source id lookups are served by the primary key`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + + val plan = index.explainQuery(IndexQuery.bySource("jarA")) + + assertThat(plan).contains("sqlite_autoindex_test_identity_1") + } + + @Test + fun `key lookups are served by an index`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + + val plan = index.explainQuery(IndexQuery.byKey("com.example.Foo")) + + assertThat(plan).contains("USING INDEX") + assertThat(plan).doesNotContain("SCAN") + } + + private fun valueOfKeyIn( + key: String, + sourceId: String, + ) = index + .query(IndexQuery(key = key, sourceIds = listOf(sourceId), limit = 0)) + .single() + .value + + private companion object { + const val DB_NAME = "identity_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexJournalTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexJournalTest.kt new file mode 100644 index 0000000000..c7ab19a3d1 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexJournalTest.kt @@ -0,0 +1,71 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexJournalTest { + data class Entry( + override val key: String, + override val sourceId: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_journal" + override val fields = emptyList() + + override fun fieldValues(entry: Entry) = emptyMap() + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + @Test + fun `the database uses write-ahead logging`() { + val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) + + /* + * Probed over a read-only connection because a writable one would apply its own journal mode + * to the file on open, and WAL, once set, is what every later connection reports. + */ + val path = context.getDatabasePath(DB_NAME).path + val mode = + try { + SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> + db.rawQuery("PRAGMA journal_mode", null).use { cursor -> + cursor.moveToFirst() + cursor.getString(0) + } + } + } finally { + index.close() + } + + assertThat(mode).isEqualTo("wal") + } + + private companion object { + const val DB_NAME = "journal_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt new file mode 100644 index 0000000000..d5b7a04e10 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt @@ -0,0 +1,195 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * Prefix matching on the SQLite-backed index. + * + * `_` and `%` are the cases that matter: `_` is a legal identifier character, so a user typing one + * must not be handed matches that merely have some other character in that position. + */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexPrefixTest { + data class Entry( + override val key: String, + override val sourceId: String, + val name: String, + val pkg: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_prefix" + override val fields = + listOf( + IndexField("name", prefixSearchable = true), + IndexField("pkg"), + ) + + override fun fieldValues(entry: Entry) = mapOf("name" to entry.name, "pkg" to entry.pkg) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.name}|${entry.pkg}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2], parts[3]) + } + } + + private val index = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + ) + + @After + fun tearDown() { + index.close() + } + + private fun namesMatching(prefix: String) = + index + .query(IndexQuery(prefixMatch = mapOf("name" to prefix), limit = 0)) + .map { it.name } + .toList() + + @Test + fun `underscore in a prefix matches only a literal underscore`() = + runTest { + index.insert(Entry("k1", "s", "my_field", "p")) + index.insert(Entry("k2", "s", "myXfield", "p")) + + assertThat(namesMatching("my_f")).containsExactly("my_field") + } + + @Test + fun `percent in a prefix matches only a literal percent`() = + runTest { + index.insert(Entry("k1", "s", "pct%value", "p")) + index.insert(Entry("k2", "s", "pctOther", "p")) + + assertThat(namesMatching("pct%")).containsExactly("pct%value") + } + + @Test + fun `backslash in a prefix matches only a literal backslash`() = + runTest { + index.insert(Entry("k1", "s", "a\\b", "p")) + index.insert(Entry("k2", "s", "axb", "p")) + + assertThat(namesMatching("a\\")).containsExactly("a\\b") + } + + @Test + fun `prefix search on a prefix-searchable field ignores case`() = + runTest { + index.insert(Entry("k1", "s", "ArrayList", "p")) + index.insert(Entry("k2", "s", "arraydeque", "p")) + + assertThat(namesMatching("array")).containsExactly("ArrayList", "arraydeque") + } + + @Test + fun `prefix search on a plain field respects case`() = + runTest { + index.insert(Entry("k1", "s", "n1", "com.foo")) + index.insert(Entry("k2", "s", "n2", "COM.BAR")) + + // Matches InMemoryIndex, which has always compared these with startsWith. + val pkgs = + index + .query(IndexQuery(prefixMatch = mapOf("pkg" to "com."), limit = 0)) + .map { it.pkg } + .toList() + assertThat(pkgs).containsExactly("com.foo") + } + + @Test + fun `an empty prefix matches every entry that has the field`() = + runTest { + index.insert(Entry("k1", "s", "Alpha", "p")) + index.insert(Entry("k2", "s", "Beta", "p")) + + assertThat(namesMatching("")).containsExactly("Alpha", "Beta") + } + + @Test + fun `a prefix ending in the highest character still matches`() = + runTest { + // The trailing U+FFFF carries into the preceding character, giving an upper bound of "{". + val high = "z${Char.MAX_VALUE}" + index.insert(Entry("k1", "s", "${high}tail", "p")) + index.insert(Entry("k2", "s", "zzz", "p")) + + assertThat(namesMatching(high)).containsExactly("${high}tail") + } + + @Test + fun `a prefix consisting only of the highest character still matches`() = + runTest { + // Every character carries here, so no exclusive upper bound exists at all: only the lower + // bound is emitted. + val high = "${Char.MAX_VALUE}" + index.insert(Entry("k1", "s", "${high}tail", "p")) + index.insert(Entry("k2", "s", "notIt", "p")) + + assertThat(namesMatching(high)).containsExactly("${high}tail") + } + + @Test + fun `a prefix ending just below the surrogate range steps over it`() = + runTest { + // Naively incrementing 0xD7FF by one lands on 0xD800, the first high surrogate, which + // encodes no character by itself; the bound must step over the whole surrogate block. + val prefix = "a\uD7FF" + index.insert(Entry("k1", "s", "${prefix}tail", "p")) + index.insert(Entry("k2", "s", "b", "p")) + + assertThat(namesMatching(prefix)).containsExactly("${prefix}tail") + } + + @Test + fun `the LIKE guard rejects a row the carried range bound alone would admit`() = + runTest { + /* + * The trailing U+FFFF carries, giving the prefix an upper bound of "b": a row whose + * second character is any code point at or above U+FFFF -- including a supplementary + * character, which sorts above the whole basic plane -- falls inside that range without + * actually starting with the prefix. Only the LIKE clause (built through + * escapeLikeLiteral) rejects it. + */ + val prefix = "a${Char.MAX_VALUE}" + index.insert(Entry("k1", "s", "${prefix}match", "p")) + index.insert(Entry("k2", "s", "a\uD800\uDC00reject", "p")) + + assertThat(namesMatching(prefix)).containsExactly("${prefix}match") + } + + @Test + fun `a prefix ending in a supplementary character increments the whole code point`() = + runTest { + /* + * U+1D7FF is the surrogate pair \uD835\uDFFF. Incrementing only the trailing low surrogate + * (the old, per-code-unit bug) produces \uD835\uE000, which SQLite decodes as a different, + * smaller code point than the prefix itself, so the range would match nothing. + */ + val prefix = "a\uD835\uDFFF" + index.insert(Entry("k1", "s", "${prefix}tail", "p")) + index.insert(Entry("k2", "s", "b", "p")) + + assertThat(namesMatching(prefix)).containsExactly("${prefix}tail") + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt new file mode 100644 index 0000000000..9fda84f4f4 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt @@ -0,0 +1,209 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.indexQuery +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * Which SQL index serves each query shape the JVM symbol index issues. + * + * Every production query is scoped to the active sources, and without statistics SQLite rates an + * `IN` list or an equality on any indexed column as good as a range, so these pin that the + * selective predicate drives the plan and the scope is only a filter. + */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexQueryPlanTest { + data class Entry( + override val key: String, + override val sourceId: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_plan" + override val fields = + listOf( + IndexField(NAME, prefixSearchable = true), + IndexField(PACKAGE), + IndexField(KIND, selective = false), + IndexField(CONTAINING_CLASS), + ) + + override fun fieldValues(entry: Entry) = emptyMap() + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + private val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + @Test + fun `a scoped name prefix query is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + prefix(NAME, "get") + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(NAME_INDEX) + } + + @Test + fun `a scoped name prefix query restricted to several kinds is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + prefix(NAME, "get") + anyOf(KIND, listOf("CLASS", "INTERFACE")) + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(NAME_INDEX) + } + + @Test + fun `a scoped name prefix query restricted to one kind is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + prefix(NAME, "get") + eq(KIND, "CLASS") + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(NAME_INDEX) + } + + @Test + fun `a scoped exact name query restricted to one kind is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + eq(NAME, "Foo") + anyOf(KIND, listOf("CLASS")) + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains("$EXACT_NAME_INDEX (") + } + + @Test + fun `scoped distinct packages under a prefix are served by the package index`() { + val plan = + index.explainDistinctValues( + PACKAGE, + indexQuery { + prefix(PACKAGE, "com.example") + sourceIds = SOURCES + limit = 0 + }, + ) + + assertThat(plan).contains(PACKAGE_INDEX) + } + + @Test + fun `scoped top-level callables in a package are served by the package index`() { + val plan = + index.explainQuery( + indexQuery { + eq(PACKAGE, "com.example") + anyOf(KIND, listOf("FUNCTION", "PROPERTY")) + notExists(CONTAINING_CLASS) + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(PACKAGE_INDEX) + } + + @Test + fun `a scoped key lookup is served by the key index`() { + val plan = + index.explainQuery( + indexQuery { + key = "com.example.Foo" + sourceIds = SOURCES + limit = 1 + }, + ) + + assertThat(plan).contains(KEY_INDEX) + } + + @Test + fun `a query scoped by source alone is served by the primary key`() { + val plan = index.explainQuery(indexQuery { sourceIds = SOURCES }) + + assertThat(plan).contains(PRIMARY_KEY_INDEX) + } + + @Test + fun `optimize collects planner statistics for the table's indexes`() = + runTest { + index.insertAll((0 until 10).asSequence().map { Entry("com.example.Foo$it", "/libs/lib$it.jar") }) + + index.optimize() + + assertThat(indexesWithStatistics()).containsAtLeast(KEY_INDEX, NAME_INDEX, PACKAGE_INDEX) + } + + /** Reads `sqlite_stat1` over a second connection, since the index exposes no statistics. */ + private fun indexesWithStatistics(): List { + val path = context.getDatabasePath(DB_NAME).path + return SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> + val hasStatTable = + db.rawQuery("SELECT 1 FROM sqlite_master WHERE name = 'sqlite_stat1'", null).use { it.moveToFirst() } + if (!hasStatTable) { + return@use emptyList() + } + db.rawQuery("SELECT idx FROM sqlite_stat1 WHERE tbl = 'test_plan'", null).use { cursor -> + buildList { while (cursor.moveToNext()) add(cursor.getString(0)) } + } + } + } + + private companion object { + const val DB_NAME = "plan_test.db" + + const val NAME = "name" + const val PACKAGE = "package" + const val KIND = "kind" + const val CONTAINING_CLASS = "containingClass" + + const val NAME_INDEX = "idx_test_plan_f_name_lower" + const val EXACT_NAME_INDEX = "idx_test_plan_f_name" + const val PACKAGE_INDEX = "idx_test_plan_f_package" + const val KEY_INDEX = "idx_test_plan_key" + const val PRIMARY_KEY_INDEX = "sqlite_autoindex_test_plan_1" + + val SOURCES = (0 until 300).map { "/libs/lib$it.jar" } + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt new file mode 100644 index 0000000000..2225efe3b0 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt @@ -0,0 +1,338 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** Source scoping, chunked multi-statement queries, and column projection on the SQLite-backed index. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexScopeTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + val group: String = "", + val kind: String = "", + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_scope" + override val fields = + listOf( + IndexField("value", prefixSearchable = true), + IndexField("group"), + IndexField("kind", selective = false), + ) + + override fun fieldValues(entry: Entry) = + mapOf( + "value" to entry.value, + "group" to entry.group, + "kind" to entry.kind, + ) + + override fun serialize(entry: Entry) = + listOf(entry.key, entry.sourceId, entry.value, entry.group, entry.kind).joinToString("|").toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2], parts[3], parts[4]) + } + } + + private val index = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + ) + + @After + fun tearDown() { + index.close() + } + + private fun smallChunkIndex(chunkSize: Int) = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + sourceIdChunkSize = chunkSize, + ) + + @Test + fun `query scoped to source ids returns only those sources`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha")) + index.insert(Entry("k2", "jarB", "Beta")) + index.insert(Entry("k3", "jarC", "Gamma")) + + val keys = + index + .query(IndexQuery(sourceIds = listOf("jarA", "jarC"), limit = 0)) + .map { it.key } + .toList() + + assertThat(keys).containsExactly("k1", "k3") + } + + @Test + fun `query scoped to an empty source set returns nothing`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha")) + + // Distinct from sourceIds = null, which imposes no restriction at all. + assertThat(index.query(IndexQuery(sourceIds = emptyList(), limit = 0)).toList()).isEmpty() + assertThat(index.query(IndexQuery(sourceIds = null, limit = 0)).toList()).hasSize(1) + } + + @Test + fun `query scoped to more sources than one IN clause holds returns every match`() = + runTest { + // Over SQLite's 999 bound-parameter limit, so the scope has to be split across statements. + val sourceIds = (0 until 2100).map { "jar$it" } + index.insertAll(sourceIds.asSequence().mapIndexed { i, src -> Entry("k$i", src, "Cls$i") }) + + val found = index.query(IndexQuery(sourceIds = sourceIds, limit = 0)).map { it.key }.toList() + + assertThat(found).hasSize(2100) + assertThat(found.toSet()).hasSize(2100) + } + + @Test + fun `query scoped across multiple small chunks returns every match`() = + runTest { + /* + * Robolectric's native SQLite allows far more than 999 bound parameters, so the 2100-id + * test above passes even with chunking deleted. A small chunk size exercises the same + * multi-statement path on real hardware. + */ + smallChunkIndex(2).use { idx -> + val sourceIds = (0 until 5).map { "jar$it" } + idx.insertAll(sourceIds.asSequence().mapIndexed { i, src -> Entry("k$i", src, "Cls$i") }) + + val found = idx.query(IndexQuery(sourceIds = sourceIds, limit = 0)).map { it.key }.toList() + + assertThat(found.toSet()).containsExactly("k0", "k1", "k2", "k3", "k4") + } + } + + @Test + fun `limit is honored across chunk boundaries`() = + runTest { + // Three sources, one chunk each: the limit must be spent across chunks rather than per + // chunk, and the loop must stop querying once it is met. + smallChunkIndex(1).use { idx -> + idx.insert(Entry("k1", "s1", "Alpha")) + idx.insert(Entry("k2", "s1", "Alpha")) + idx.insert(Entry("k3", "s2", "Alpha")) + idx.insert(Entry("k4", "s2", "Alpha")) + idx.insert(Entry("k5", "s3", "Alpha")) + idx.insert(Entry("k6", "s3", "Alpha")) + + val found = + idx + .query(IndexQuery(sourceIds = listOf("s1", "s2", "s3"), limit = 3)) + .map { it.key } + .toList() + + assertThat(found).hasSize(3) + assertThat(found).containsAtLeast("k1", "k2") + assertThat(found).containsNoneOf("k5", "k6") + } + } + + @Test + fun `a source id repeated across chunks is not returned twice`() = + runTest { + smallChunkIndex(1).use { idx -> + idx.insert(Entry("ka", "a", "Alpha")) + idx.insert(Entry("kb", "b", "Beta")) + + val found = + idx + .query(IndexQuery(sourceIds = listOf("a", "a", "b"), limit = 0)) + .map { it.key } + .toList() + + assertThat(found).containsExactly("ka", "kb") + } + } + + @Test + fun `distinctValues does not starve a later chunk's new values on the shrinking limit`() = + runTest { + /* + * Old bug: each chunk was queried with `limit - values.size`, the remaining budget, so a + * value repeating across chunks could exhaust that budget before a later chunk's genuinely + * new values were ever read. Chunk one already contributes "g1"; chunk two repeats "g1" + * (ordered first, by source id and by value) ahead of the newly-seen "g3", so a + * budget-of-one second query returns the duplicate and misses "g3" entirely. + */ + smallChunkIndex(2).use { idx -> + idx.insert(Entry("k1", "s1", "V1", group = "g1")) + idx.insert(Entry("k2", "s2", "V2", group = "g2")) + idx.insert(Entry("k3", "s3", "V3", group = "g1")) + idx.insert(Entry("k4", "s4", "V4", group = "g3")) + + val groups = + idx + .distinctValues("group", IndexQuery(sourceIds = listOf("s1", "s2", "s3", "s4"), limit = 3)) + .toList() + + assertThat(groups).containsExactly("g1", "g2", "g3") + } + } + + @Test + fun `distinctValues stops adding once the limit is reached mid-chunk`() = + runTest { + /* + * Old bug: querying a chunk with the full limit fixed the starvation above, but nothing + * capped the total afterward -- the inner cursor loop added every row a chunk returned. A + * chunk holding more new distinct values than remain in the budget must stop partway + * through, not push the total past the limit. + */ + smallChunkIndex(2).use { idx -> + idx.insert(Entry("k1", "s1", "V1", group = "g1")) + idx.insert(Entry("k2", "s2", "V2", group = "g1")) + idx.insert(Entry("k3", "s3", "V3", group = "g2")) + idx.insert(Entry("k4", "s4", "V4", group = "g3")) + + val groups = + idx + .distinctValues("group", IndexQuery(sourceIds = listOf("s1", "s2", "s3", "s4"), limit = 2)) + .toList() + + assertThat(groups).hasSize(2) + } + } + + @Test + fun `scoped query applies the scope before the limit`() = + runTest { + repeat(20) { index.insert(Entry("noise$it", "jarNoise", "Widget$it")) } + index.insert(Entry("wanted", "jarWanted", "Widget99")) + + val found = + index + .query(IndexQuery(prefixMatch = mapOf("value" to "Widget"), sourceIds = listOf("jarWanted"), limit = 1)) + .map { it.key } + .toList() + + assertThat(found).containsExactly("wanted") + } + + @Test + fun `distinctValues projects only the values of matching rows`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + index.insert(Entry("k2", "jarA", "Beta", group = "one")) + index.insert(Entry("k3", "jarB", "Gamma", group = "two")) + + val groups = + index + .distinctValues("group", IndexQuery(sourceIds = listOf("jarA"), limit = 0)) + .toList() + + assertThat(groups).containsExactly("one") + } + + @Test + fun `distinctValues deduplicates across source chunks`() = + runTest { + // One shared value spread over more sources than a single IN clause can hold: DISTINCT is + // per statement, so without cross-chunk deduplication this value comes back repeatedly. + val sourceIds = (0 until 1500).map { "jar$it" } + index.insertAll( + sourceIds.asSequence().mapIndexed { i, src -> Entry("k$i", src, "Cls$i", group = "shared") }, + ) + + val groups = index.distinctValues("group", IndexQuery(sourceIds = sourceIds, limit = 0)).toList() + + assertThat(groups).containsExactly("shared") + } + + @Test + fun `distinctValues without a query still returns every value`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + index.insert(Entry("k2", "jarB", "Beta", group = "two")) + + assertThat(index.distinctValues("group").toList()).containsExactly("one", "two") + } + + @Test + fun `anyOf matches any of the listed values`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + index.insert(Entry("k2", "jarA", "Beta", group = "two")) + index.insert(Entry("k3", "jarA", "Gamma", group = "three")) + + val keys = + index + .query(IndexQuery(anyOf = mapOf("group" to listOf("one", "three")), limit = 0)) + .map { it.key } + .toList() + + assertThat(keys).containsExactly("k1", "k3") + } + + @Test + fun `an empty anyOf matches nothing`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + + // Same distinction as an empty source scope: restricted to nothing, not unrestricted. + assertThat(index.query(IndexQuery(anyOf = mapOf("group" to emptyList()), limit = 0)).toList()) + .isEmpty() + } + + @Test + fun `anyOf narrows the rows the limit is spent on`() = + runTest { + repeat(20) { index.insert(Entry("other$it", "jarA", "Cls$it", group = "unwanted")) } + index.insert(Entry("wanted", "jarA", "ClsWanted", group = "wanted")) + + val keys = + index + .query(IndexQuery(anyOf = mapOf("group" to listOf("wanted")), limit = 1)) + .map { it.key } + .toList() + + assertThat(keys).containsExactly("wanted") + } + + @Test + fun `scope and kind terms that only filter still exclude non-matching rows`() = + runTest { + index.insert(Entry("match", "jarA", "Widget1", kind = "CLASS")) + index.insert(Entry("otherKind", "jarA", "Widget2", kind = "METHOD")) + index.insert(Entry("otherSource", "jarB", "Widget3", kind = "CLASS")) + index.insert(Entry("otherName", "jarA", "Gadget", kind = "CLASS")) + + val byAnyOf = + IndexQuery( + prefixMatch = mapOf("value" to "Widget"), + anyOf = mapOf("kind" to listOf("CLASS")), + sourceIds = listOf("jarA"), + limit = 0, + ) + val byExact = byAnyOf.copy(anyOf = emptyMap(), exactMatch = mapOf("kind" to "CLASS")) + + assertThat(index.query(byAnyOf).map { it.key }.toList()).containsExactly("match") + assertThat(index.query(byExact).map { it.key }.toList()).containsExactly("match") + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexVersionTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexVersionTest.kt new file mode 100644 index 0000000000..1961ccfcc2 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexVersionTest.kt @@ -0,0 +1,99 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** The stored format version decides whether an existing database file's rows are kept. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexVersionTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_version" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + private fun open(version: Int) = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = version) + + @Test + fun `reopening at the same version keeps the rows`() = + runTest { + open(version = 1).use { it.insert(Entry("k1", "jarA", "v")) } + + open(version = 1).use { assertThat(it.containsSource("jarA")).isTrue() } + } + + @Test + fun `reopening at a newer version drops the old rows`() = + runTest { + open(version = 1).use { it.insert(Entry("k1", "jarA", "v")) } + + open(version = 2).use { + assertThat(it.containsSource("jarA")).isFalse() + assertThat(it.size()).isEqualTo(0) + } + } + + @Test + fun `reopening at an older version drops the newer rows`() = + runTest { + open(version = 2).use { it.insert(Entry("k1", "jarA", "v")) } + + open(version = 1).use { assertThat(it.size()).isEqualTo(0) } + } + + @Test + fun `a version change replaces a table created with the single-column key`() = + runTest { + val path = context.getDatabasePath(DB_NAME) + path.parentFile!!.mkdirs() + SQLiteDatabase.openOrCreateDatabase(path, null).use { db -> + db.execSQL( + "CREATE TABLE test_version (_key TEXT PRIMARY KEY, _source_id TEXT NOT NULL, f_value TEXT, _payload BLOB NOT NULL)", + ) + db.version = 1 + } + + open(version = 2).use { + it.insert(Entry("k1", "jarA", "fromA")) + it.insert(Entry("k1", "jarB", "fromB")) + + assertThat(it.size()).isEqualTo(2) + } + } + + private companion object { + const val DB_NAME = "version_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/UndeclaredFieldQueryTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/UndeclaredFieldQueryTest.kt new file mode 100644 index 0000000000..0c7e7d9ef8 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/UndeclaredFieldQueryTest.kt @@ -0,0 +1,75 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.indexQuery +import org.junit.After +import org.junit.Assert.assertThrows +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * A query naming a field the descriptor does not declare is a caller bug, which SQLite used to hide + * by dropping the predicate and returning every row in scope. InMemoryIndexTest covers the in-memory + * index, which rejects it the same way. + */ +@RunWith(RobolectricTestRunner::class) +class UndeclaredFieldQueryTest { + data class Entry( + override val key: String, + override val sourceId: String, + val kind: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_undeclared" + override val fields = listOf(IndexField("kind")) + + override fun fieldValues(entry: Entry) = mapOf("kind" to entry.kind) + + override fun serialize(entry: Entry) = listOf(entry.key, entry.sourceId, entry.kind).joinToString("|").toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val sqlite = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + ) + + @After + fun tearDown() { + sqlite.close() + } + + private fun assertRejects( + index: Index, + query: IndexQuery, + ) = runTest { + index.insert(Entry("k1", "jar", "CLASS")) + assertThrows(IllegalArgumentException::class.java) { index.query(query).toList() } + } + + @Test + fun `SQLite rejects an anyOf on an undeclared field`() = assertRejects(sqlite, indexQuery { anyOf("kinds", listOf("CLASS")) }) + + @Test + fun `SQLite rejects an exact match on an undeclared field`() = assertRejects(sqlite, indexQuery { eq("kinds", "CLASS") }) + + @Test + fun `SQLite rejects a presence check on an undeclared field`() = assertRejects(sqlite, indexQuery { notExists("kinds") }) +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/api/IndexQueryBuilderTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/api/IndexQueryBuilderTest.kt new file mode 100644 index 0000000000..90e76d6064 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/api/IndexQueryBuilderTest.kt @@ -0,0 +1,19 @@ +package org.appdevforall.codeonthego.indexing.api + +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class IndexQueryBuilderTest { + @Test + fun `anyOf is unaffected by later mutation of the caller's collection`() { + val values = mutableListOf("CLASS") + + val query = indexQuery { anyOf("kind", values) } + values.add("INTERFACE") + + assertThat(query.anyOf["kind"]).containsExactly("CLASS") + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt index fb06569b55..e5b426490b 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt @@ -26,6 +26,8 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import java.io.IOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger @RunWith(JUnit4::class) @@ -191,6 +193,19 @@ class BackgroundIndexerTest { assertThat(indexer.activeJobCount).isEqualTo(0) } + @Test + fun `activeJobCount drops to zero once a completed job's entry is removed`() = + runTest { + val (_, indexer) = makeIndexAndIndexer() + + indexer + .indexSource("src1", skipIfExists = false) { sourceId -> + flowOf(Entry("k1", sourceId, "v1")) + }.join() + + assertThat(indexer.activeJobCount).isEqualTo(0) + } + @Test fun `close cancels all active jobs`() = runTest { @@ -365,4 +380,158 @@ class BackgroundIndexerTest { assertThat(indexer.activeJobCount).isEqualTo(0) } + + @Test + fun `an unchanged fingerprint skips re-indexing`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "original")) }.join() + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "updated")) }.join() + + assertThat(index.get("k1")!!.value).isEqualTo("original") + } + + @Test + fun `a changed fingerprint re-indexes the source`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "original")) }.join() + indexer.indexSource("src1", fingerprint = "120:2") { flowOf(Entry("k1", it, "rebuilt")) }.join() + + assertThat(index.get("k1")!!.value).isEqualTo("rebuilt") + assertThat(index.sourceFingerprint("src1")).isEqualTo("120:2") + } + + @Test + fun `a fingerprint re-indexes a source that was indexed without one`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + indexer.indexSource("src1") { flowOf(Entry("k1", it, "original")) }.join() + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "fingerprinted")) }.join() + + assertThat(index.get("k1")!!.value).isEqualTo("fingerprinted") + } + + /** Waits for the latch, failing the test (rather than silently proceeding) on timeout. */ + private fun CountDownLatch.awaitOrFail() { + assertThat(await(5, TimeUnit.SECONDS)).isTrue() + } + + @Test + fun `a new job waits for the previous job to finish before deleting its rows`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + val job1Started = CountDownLatch(1) + val releaseJob1 = CountDownLatch(1) + + val job1 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + job1Started.countDown() + releaseJob1.awaitOrFail() + // Long enough that, without cancelAndJoin, job2's in-memory write below finishes first. + Thread.sleep(50) + flowOf(Entry("k1", sourceId, "old1"), Entry("k2", sourceId, "old2")) + } + + job1Started.awaitOrFail() + + val job2 = + indexer.indexSource("src1", fingerprint = "200:2") { sourceId -> + flowOf(Entry("k3", sourceId, "new1")) + } + + releaseJob1.countDown() + job2.join() + job1.join() + + assertThat(index.get("k1")).isNull() + assertThat(index.get("k2")).isNull() + assertThat(index.get("k3")!!.value).isEqualTo("new1") + assertThat(index.sourceFingerprint("src1")).isEqualTo("200:2") + } + + @Test + fun `indexSource skips resubmitting a source with an active job at the same fingerprint`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + val job1Started = CountDownLatch(1) + val releaseJob1 = CountDownLatch(1) + var providerInvocations = 0 + + val job1 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + providerInvocations++ + job1Started.countDown() + releaseJob1.awaitOrFail() + flowOf(Entry("k1", sourceId, "v1")) + } + + job1Started.awaitOrFail() + + val job2 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + providerInvocations++ + flowOf(Entry("k1", sourceId, "should-not-run")) + } + + assertThat(job2).isSameInstanceAs(job1) + + releaseJob1.countDown() + job1.join() + + assertThat(providerInvocations).isEqualTo(1) + assertThat(index.get("k1")!!.value).isEqualTo("v1") + } + + @Test + fun `a superseded job finishing does not clear the new job's active-job entry`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + val releaseJob1 = CountDownLatch(1) + val job2ProviderStarted = CountDownLatch(1) + val releaseJob2 = CountDownLatch(1) + + val job1 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + releaseJob1.awaitOrFail() + flowOf(Entry("k1", sourceId, "old")) + } + + var job2ProviderInvocations = 0 + val job2 = + indexer.indexSource("src1", fingerprint = "200:2") { sourceId -> + job2ProviderInvocations++ + job2ProviderStarted.countDown() + releaseJob2.awaitOrFail() + flowOf(Entry("k2", sourceId, "new")) + } + + // Let the superseded job1 finish (its cancellation is cooperative-only here, since + // InMemoryIndex never checks it) while job2 is still mid-flight. + releaseJob1.countDown() + job1.join() + job2ProviderStarted.awaitOrFail() + + var job3ProviderInvocations = 0 + val job3 = + indexer.indexSource("src1", fingerprint = "200:2") { sourceId -> + job3ProviderInvocations++ + flowOf(Entry("k2", sourceId, "should-not-run")) + } + + assertThat(job3).isSameInstanceAs(job2) + assertThat(job3ProviderInvocations).isEqualTo(0) + + releaseJob2.countDown() + job2.join() + + assertThat(job2ProviderInvocations).isEqualTo(1) + assertThat(index.get("k2")!!.value).isEqualTo("new") + } } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt index 1a6289d091..0ccfdd026d 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow import org.jetbrains.kotlin.analysis.api.KaImplementationDetail import org.jetbrains.kotlin.analysis.api.impl.base.util.LibraryUtils +import org.jetbrains.kotlin.com.intellij.openapi.vfs.VfsUtilCore import org.jetbrains.kotlin.com.intellij.openapi.vfs.VirtualFile import org.jetbrains.org.objectweb.asm.AnnotationVisitor import org.jetbrains.org.objectweb.asm.ClassReader @@ -30,8 +31,8 @@ object CombinedJarScanner { sequence { val allFiles = LibraryUtils.getAllVirtualFilesFromRoot(rootVf, includeRoot = true) for (vf in allFiles) { - if (!vf.name.endsWith(".class")) continue - if (vf.name == "module-info.class" || vf.name == "package-info.class") continue + val relativePath = VfsUtilCore.getRelativePath(vf, rootVf)?.takeIf { it.isNotEmpty() } ?: vf.name + if (!isIndexableClassEntry(relativePath)) continue try { val bytes = vf.contentsToByteArray() val symbols = @@ -65,8 +66,7 @@ object CombinedJarScanner { val entries = jar.entries() while (entries.hasMoreElements()) { val entry = entries.nextElement() - if (!entry.name.endsWith(".class")) continue - if (entry.name == "module-info.class" || entry.name == "package-info.class") continue + if (!isIndexableClassEntry(entry.name)) continue // emit stays outside the try so cancellation is not swallowed. val symbols = @@ -93,6 +93,18 @@ object CombinedJarScanner { } } + /* + * Everything under META-INF is skipped. A multi-release JAR's META-INF/versions// copy of a + * class shares the base class's key within one source, so indexing it would replace the base + * row. + */ + private fun isIndexableClassEntry(relativePath: String): Boolean { + if (!relativePath.endsWith(".class")) return false + if (relativePath.startsWith("META-INF/")) return false + val fileName = relativePath.substringAfterLast('/') + return fileName != "module-info.class" && fileName != "package-info.class" + } + private fun hasKotlinMetadata(classBytes: ByteArray): Boolean { var found = false try { diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt index e0054d0de8..ec3aa34599 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt @@ -10,14 +10,15 @@ import org.jetbrains.org.objectweb.asm.Type import java.io.InputStream /** - * Parses class files with ASM into [JvmSymbol]s. + * Parses a Java class file with ASM into [JvmSymbol]s. * - * For Java class files, this gives complete information. - * For Kotlin class files, use [KotlinMetadataScanner] or - * [CombinedJarScanner] instead — ASM cannot see Kotlin-specific - * semantics like extensions, suspend, or nullable types. + * [CombinedJarScanner] routes Kotlin class files to [KotlinMetadataScanner] instead: ASM cannot see + * Kotlin-specific semantics like extensions, suspend, or nullable types. */ object JarSymbolScanner { + private const val VISIBILITY_AND_STATIC_FLAGS = + Opcodes.ACC_PUBLIC or Opcodes.ACC_PRIVATE or Opcodes.ACC_PROTECTED or Opcodes.ACC_STATIC + internal fun parseClassFile( input: InputStream, sourceId: String, @@ -45,6 +46,7 @@ object JarSymbolScanner { private var superName: String? = null private var interfaces: Array? = null private var isInnerClass = false + private var outerClassName: String? = null private var classDeprecated = false override fun visit( @@ -56,7 +58,7 @@ object JarSymbolScanner { interfaces: Array?, ) { className = name - classFqName = name.replace('/', '.').replace('$', '.') + classFqName = name.replace('/', '.') classAccess = access this.superName = superName this.interfaces = interfaces @@ -66,9 +68,30 @@ object JarSymbolScanner { packageName = if (lastSlash >= 0) name.substring(0, lastSlash).replace('/', '.') else "" val afterPackage = if (lastSlash >= 0) name.substring(lastSlash + 1) else name - shortClassName = afterPackage.replace('$', '.') + shortClassName = afterPackage + isInnerClass = false + outerClassName = null + } - isInnerClass = name.contains('$') + /* + * JVMS 4.7.6 gives every nested class an InnerClasses entry for itself, and a top-level class + * none, so that entry, not a '$' in the name, decides nesting: `Foo$$ViewBinder` is top-level. + * JVMS 4.1 also forbids ACC_PRIVATE, ACC_PROTECTED and ACC_STATIC in a class file header, so + * the entry is the only record of a nested class's declared visibility and staticness. ASM + * visits it before any member, so the member gates below see the real flags. + */ + override fun visitInnerClass( + name: String, + outerName: String?, + innerName: String?, + access: Int, + ) { + if (name != className) return + isInnerClass = true + outerClassName = outerName + classFqName = className.replace('/', '.').replace('$', '.') + shortClassName = shortClassName.replace('$', '.') + classAccess = (classAccess and VISIBILITY_AND_STATIC_FLAGS.inv()) or (access and VISIBILITY_AND_STATIC_FLAGS) } override fun visitAnnotation( @@ -81,7 +104,14 @@ object JarSymbolScanner { } override fun visitEnd() { - if (!isPublicOrProtected(classAccess)) return + /* + * Package-private classes are indexed, not dropped: they are real classpath entries and + * same-package code may legitimately reference them, so the caller decides by visibility + * whether to offer one. Private classes stay out - nothing outside the declaring class can + * name them. Members remain gated on the class being public or protected, so a + * package-private class contributes its own name and nothing else. + */ + if (hasFlag(classAccess, Opcodes.ACC_PRIVATE)) return val isAnonymous = isInnerClass && @@ -105,7 +135,7 @@ object JarSymbolScanner { val containingClass = if (isInnerClass) { - className.substringBeforeLast('$') + outerClassName ?: className.substringBeforeLast('$') } else { "" } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt index 5a8c5ee4eb..937b4d8666 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt @@ -6,6 +6,7 @@ import com.itsaky.androidide.projects.api.ModuleProject import com.itsaky.androidide.tasks.cancelIfActive import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -13,6 +14,7 @@ import org.appdevforall.codeonthego.indexing.service.IndexKey import org.appdevforall.codeonthego.indexing.service.IndexRegistry import org.appdevforall.codeonthego.indexing.service.IndexingService import org.slf4j.LoggerFactory +import java.io.File import java.nio.file.Paths import kotlin.io.path.extension @@ -29,14 +31,15 @@ val JVM_GENERATED_SYMBOL_INDEX = IndexKey("jvm-generated-symbols * [IndexingService] that scans build-generated JARs (R.jar, etc.) and * maintains a dedicated [JvmSymbolIndex] for them. * - * Generated JARs are re-indexed unconditionally on every build completion - * because their contents change (new R-field values, new resource IDs) even - * when the set of JARs doesn't change. + * A build completion re-scans every generated JAR's size and modification time (see + * [jarFingerprint]) and only re-indexes those whose fingerprint changed, the same check + * [JvmLibraryIndexingService] uses. This also catches a generated JAR rewritten outside an + * in-app build, or left half-indexed by a killed process, which staying cached by path alone + * would otherwise miss until some later build happened to touch it again. */ class JvmGeneratedIndexingService( private val context: Context, ) : IndexingService { - companion object { const val ID = "jvm-generated-indexing-service" private const val DB_NAME = "jvm_generated_symbol_index.db" @@ -53,11 +56,12 @@ class JvmGeneratedIndexingService( private val coroutineScope = CoroutineScope(Dispatchers.Default) override suspend fun initialize(registry: IndexRegistry) { - val index = JvmSymbolIndex.createSqliteIndex( - context = context, - dbName = DB_NAME, - indexName = INDEX_NAME, - ) + val index = + JvmSymbolIndex.createSqliteIndex( + context = context, + dbName = DB_NAME, + indexName = INDEX_NAME, + ) this.generatedIndex = index registry.register(JVM_GENERATED_SYMBOL_INDEX, index) @@ -65,32 +69,31 @@ class JvmGeneratedIndexingService( // Kick off an initial index pass for any already-built JARs. coroutineScope.launch { - indexingMutex.withLock { - reindexGeneratedJars(forceReindex = false) - } + val jobs = indexingMutex.withLock { reindexGeneratedJars() } + generatedIndex?.optimizeAfter(jobs) } } override suspend fun onBuildCompleted() { - // Generated JARs (especially R.jar) always change after a build — - // their field values are regenerated. Force a full re-index. coroutineScope.launch { - indexingMutex.withLock { - reindexGeneratedJars(forceReindex = true) - } + val jobs = indexingMutex.withLock { reindexGeneratedJars() } + generatedIndex?.optimizeAfter(jobs) } } - private suspend fun reindexGeneratedJars(forceReindex: Boolean) { - val index = this.generatedIndex ?: run { - log.warn("Not indexing generated JARs — index not initialized.") - return - } + /** Submits every generated JAR whose fingerprint changed and returns the submitted jobs. */ + private suspend fun reindexGeneratedJars(): List { + val index = + this.generatedIndex ?: run { + log.warn("Not indexing generated JARs - index not initialized.") + return emptyList() + } - val workspace = ProjectManagerImpl.getInstance().workspace ?: run { - log.warn("Not indexing generated JARs — workspace model not available.") - return - } + val workspace = + ProjectManagerImpl.getInstance().workspace ?: run { + log.warn("Not indexing generated JARs - workspace model not available.") + return emptyList() + } val generatedJars = workspace.subProjects @@ -107,21 +110,23 @@ class JvmGeneratedIndexingService( // Make exactly these JARs visible; remove stale ones from scope. index.setActiveSources(generatedJars) - var submitted = 0 + val jobs = mutableListOf() for (jarPath in generatedJars) { - if (forceReindex || !index.isCached(jarPath)) { - submitted++ - index.indexSource(jarPath, skipIfExists = false) { sourceId -> - CombinedJarScanner.scan(Paths.get(jarPath), sourceId) - } + val fingerprint = jarFingerprint(File(jarPath)) + if (index.sourceFingerprint(jarPath) != fingerprint) { + jobs += + index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> + CombinedJarScanner.scan(Paths.get(jarPath), sourceId) + } } } - if (submitted > 0) { - log.info("{} generated JARs submitted for background indexing (force={})", submitted, forceReindex) + if (jobs.isNotEmpty()) { + log.info("{} generated JARs submitted for background indexing", jobs.size) } else { log.info("All generated JARs already cached, nothing to index") } + return jobs } override fun close() { diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt index 8b53c393dd..0a4b1d1be8 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt @@ -8,6 +8,7 @@ import com.itsaky.androidide.projects.models.bootClassPaths import com.itsaky.androidide.tasks.cancelIfActive import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -17,6 +18,7 @@ import org.appdevforall.codeonthego.indexing.service.IndexingService import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode import org.slf4j.LoggerFactory +import java.io.File import java.nio.file.Path import java.nio.file.Paths import kotlin.io.path.extension @@ -40,7 +42,6 @@ val JVM_LIBRARY_SYMBOL_INDEX = IndexKey("jvm-library-symbols") class JvmLibraryIndexingService( private val context: Context, ) : IndexingService { - companion object { const val ID = "jvm-indexing-service" private val log = LoggerFactory.getLogger(JvmLibraryIndexingService::class.java) @@ -55,11 +56,12 @@ class JvmLibraryIndexingService( private val coroutineScope = CoroutineScope(Dispatchers.Default) override suspend fun initialize(registry: IndexRegistry) { - val jvmIndex = JvmSymbolIndex.createSqliteIndex( - context = context, - dbName = JvmSymbolIndex.DB_NAME_DEFAULT, - indexName = JvmSymbolIndex.INDEX_NAME_LIBRARY - ) + val jvmIndex = + JvmSymbolIndex.createSqliteIndex( + context = context, + dbName = JvmSymbolIndex.DB_NAME_DEFAULT, + indexName = JvmSymbolIndex.INDEX_NAME_LIBRARY, + ) this.libraryIndex = jvmIndex registry.register(JVM_LIBRARY_SYMBOL_INDEX, jvmIndex) @@ -74,22 +76,24 @@ class JvmLibraryIndexingService( fun refresh() { coroutineScope.launch { - indexingMutex.withLock { - reindexLibraries() - } + val jobs = indexingMutex.withLock { reindexLibraries() } + libraryIndex?.optimizeAfter(jobs) } } - private suspend fun reindexLibraries() { - val index = this.libraryIndex ?: run { - log.warn("Not indexing libraries. Index not initialized.") - return - } + /** Submits every library JAR that needs indexing and returns the submitted jobs. */ + private suspend fun reindexLibraries(): List { + val index = + this.libraryIndex ?: run { + log.warn("Not indexing libraries. Index not initialized.") + return emptyList() + } - val workspace = ProjectManagerImpl.getInstance().workspace ?: run { - log.warn("Not indexing libraries. Workspace model not available.") - return - } + val workspace = + ProjectManagerImpl.getInstance().workspace ?: run { + log.warn("Not indexing libraries. Workspace model not available.") + return emptyList() + } val currentJars = workspace.subProjects @@ -104,38 +108,40 @@ class JvmLibraryIndexingService( addAll(project.getCompileClasspaths(excludeSourceGeneratedClassPath = true)) } - } - .filter { jar -> jar.exists() && isIndexableJar(jar.toPath()) } + }.filter { jar -> jar.exists() && isIndexableJar(jar.toPath()) } .map { jar -> jar.absolutePath } .toSet() log.info("{} JARs on classpath", currentJars.size) - // Step 1: Set the active set - this is instant. - // JARs not in the set become invisible to queries. - // JARs in the set that are already cached become - // visible immediately. + /* + * Step 1: Set the active set - this is instant. JARs not in the set become invisible to + * queries; JARs in the set that are already cached become visible immediately. + */ index.setActiveSources(currentJars) - // Step 2: Index any JARs not yet in the cache. - // Already-cached JARs are skipped (cheap existence check). - // Newly cached JARs are automatically visible because - // they're already in the active set. - var newCount = 0 + /* + * Step 2: Index any JAR that is not cached, or was cached with a different fingerprint: + * a JAR rebuilt at the same path (a snapshot, a local file dependency) is re-scanned. + * Newly cached JARs are automatically visible because they're already in the active set. + */ + val jobs = mutableListOf() for (jarPath in currentJars) { - if (!index.isCached(jarPath)) { - newCount++ - index.indexSource(jarPath, skipIfExists = true) { sourceId -> - CombinedJarScanner.scan(Paths.get(jarPath), sourceId) - } + val fingerprint = jarFingerprint(File(jarPath)) + if (index.sourceFingerprint(jarPath) != fingerprint) { + jobs += + index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> + CombinedJarScanner.scan(Paths.get(jarPath), sourceId) + } } } - if (newCount > 0) { - log.info("{} new JARs submitted for background indexing", newCount) + if (jobs.isNotEmpty()) { + log.info("{} new or changed JARs submitted for background indexing", jobs.size) } else { log.info("All JARs already cached, nothing to index") } + return jobs } override fun close() { @@ -148,4 +154,14 @@ class JvmLibraryIndexingService( val ext = path.extension.lowercase() return ext == "jar" || ext == "aar" } -} \ No newline at end of file +} + +/** + * Identifies the content of [jar] by its size and last-modified time, which a rewrite of the file + * changes without the cost of reading it. Stats the file, so never call it on the main thread. + * + * Accepted limitation: a rewrite that keeps the exact same size and lands within the + * filesystem's modification-time granularity (a second on some filesystems) produces the same + * fingerprint as the original, so that rewrite goes undetected. + */ +internal fun jarFingerprint(jar: File): String = "${jar.length()}:${jar.lastModified()}" diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt index aec88b542e..cef123e0d9 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt @@ -3,43 +3,112 @@ package org.appdevforall.codeonthego.indexing.jvm import org.appdevforall.codeonthego.indexing.api.Indexable enum class JvmSymbolKind { - CLASS, INTERFACE, ENUM, ENUM_ENTRY, ANNOTATION_CLASS, - OBJECT, COMPANION_OBJECT, DATA_CLASS, VALUE_CLASS, - SEALED_CLASS, SEALED_INTERFACE, - FUNCTION, EXTENSION_FUNCTION, CONSTRUCTOR, - PROPERTY, EXTENSION_PROPERTY, FIELD, - TYPE_ALIAS; - - val isCallable: Boolean - get() = this in CALLABLE_KINDS - - val isClassifier: Boolean - get() = this in CLASSIFIER_KINDS - - val isExtension: Boolean - get() = this == EXTENSION_FUNCTION || this == EXTENSION_PROPERTY - - companion object { - val CALLABLE_KINDS = setOf( - FUNCTION, EXTENSION_FUNCTION, CONSTRUCTOR, - PROPERTY, EXTENSION_PROPERTY, FIELD, - ) - val CLASSIFIER_KINDS = setOf( - CLASS, INTERFACE, ENUM, ANNOTATION_CLASS, - OBJECT, COMPANION_OBJECT, DATA_CLASS, - VALUE_CLASS, SEALED_CLASS, SEALED_INTERFACE, - TYPE_ALIAS, - ) - } + CLASS, + INTERFACE, + ENUM, + ENUM_ENTRY, + ANNOTATION_CLASS, + OBJECT, + COMPANION_OBJECT, + DATA_CLASS, + VALUE_CLASS, + SEALED_CLASS, + SEALED_INTERFACE, + FUNCTION, + EXTENSION_FUNCTION, + CONSTRUCTOR, + PROPERTY, + EXTENSION_PROPERTY, + FIELD, + TYPE_ALIAS, + + /** + * The synthetic class a Kotlin file's top-level declarations compile into, named after the file + * (`Foo.kt` -> `FooKt`). + * + * Deliberately not in [CLASSIFIER_KINDS]: Kotlin code names the declarations inside it and never + * the class, so offering it as a Kotlin type would be wrong. It is in [JVM_CLASS_KINDS] because a + * class file by that name really exists, and Java calls the top-level functions through it. + */ + FILE_FACADE, + ; + + val isCallable: Boolean + get() = this in CALLABLE_KINDS + + val isClassifier: Boolean + get() = this in CLASSIFIER_KINDS + + /** Whether a class file exists under this symbol's name. */ + val isJvmClass: Boolean + get() = this in JVM_CLASS_KINDS + + val isExtension: Boolean + get() = this == EXTENSION_FUNCTION || this == EXTENSION_PROPERTY + + companion object { + val CALLABLE_KINDS = + setOf( + FUNCTION, + EXTENSION_FUNCTION, + CONSTRUCTOR, + PROPERTY, + EXTENSION_PROPERTY, + FIELD, + ) + + /** Kinds that declare a type in the source language. */ + val CLASSIFIER_KINDS = + setOf( + CLASS, + INTERFACE, + ENUM, + ANNOTATION_CLASS, + OBJECT, + COMPANION_OBJECT, + DATA_CLASS, + VALUE_CLASS, + SEALED_CLASS, + SEALED_INTERFACE, + TYPE_ALIAS, + ) + + /** + * Kinds backed by an actual class file, which is what a classpath scan finds and what an + * import has to resolve to. + * + * This is not [CLASSIFIER_KINDS]: a type alias declares a type but compiles to no class, and a + * file facade is a class that declares no type. + */ + val JVM_CLASS_KINDS = + setOf( + CLASS, + INTERFACE, + ENUM, + ANNOTATION_CLASS, + OBJECT, + COMPANION_OBJECT, + DATA_CLASS, + VALUE_CLASS, + SEALED_CLASS, + SEALED_INTERFACE, + FILE_FACADE, + ) + } } enum class JvmSourceLanguage { JAVA, KOTLIN } enum class JvmVisibility { - PUBLIC, PROTECTED, INTERNAL, PRIVATE, PACKAGE_PRIVATE; - - val isAccessibleOutsideClass: Boolean - get() = this == PUBLIC || this == PROTECTED || this == INTERNAL + PUBLIC, + PROTECTED, + INTERNAL, + PRIVATE, + PACKAGE_PRIVATE, + ; + + val isAccessibleOutsideClass: Boolean + get() = this == PUBLIC || this == PROTECTED || this == INTERNAL } /** @@ -56,7 +125,6 @@ enum class JvmVisibility { data class JvmSymbol( override val key: String, override val sourceId: String, - val name: String, val shortName: String, val packageName: String, @@ -64,56 +132,88 @@ data class JvmSymbol( val language: JvmSourceLanguage, val visibility: JvmVisibility = JvmVisibility.PUBLIC, val isDeprecated: Boolean = false, - val data: JvmSymbolInfo, ) : Indexable { - val fqName: String get() = name.toFqName() - val isTopLevel: Boolean - get() = data.containingClassName.isEmpty() + val isTopLevel: Boolean + get() = data.containingClassName.isEmpty() - val isExtension: Boolean - get() = kind.isExtension + val isExtension: Boolean + get() = kind.isExtension - val receiverTypeName: String? - get() = when (val d = data) { - is JvmFunctionInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } - is JvmFieldInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } - else -> null - } + val receiverTypeName: String? + get() = + when (val d = data) { + is JvmFunctionInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } + is JvmFieldInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } + else -> null + } val receiverTypeFqName: String? get() = receiverTypeName?.toFqName() - val containingClassName: String - get() = data.containingClassName - - val returnTypeDisplay: String - get() = when (val d = data) { - is JvmFunctionInfo -> d.returnTypeDisplayName - is JvmFieldInfo -> d.typeDisplayName - else -> "" - } - - val signatureDisplay: String - get() = when (val d = data) { - is JvmFunctionInfo -> d.signatureDisplay - else -> "" - } + val containingClassName: String + get() = data.containingClassName + + val returnTypeDisplay: String + get() = + when (val d = data) { + is JvmFunctionInfo -> d.returnTypeDisplayName + is JvmFieldInfo -> d.typeDisplayName + else -> "" + } + + val signatureDisplay: String + get() = + when (val d = data) { + is JvmFunctionInfo -> d.signatureDisplay + else -> "" + } } +/** + * The identity two rows stand for "the same symbol" when deduplicating matches drawn from more than + * one source or index. + * + * A classifier's qualified name already identifies it uniquely, but a callable's does not -- two + * overloads share one -- so a callable is identified by its full [JvmSymbol.key] instead. This is why + * the same class indexed from two JARs on the active source set (composite-keyed by + * `(sourceId, key)`, so both rows exist) collapses to one match: both rows share this identity even + * though their `(sourceId, key)` pairs differ. + */ +val JvmSymbol.dedupeKey: String + get() = if (kind.isCallable) key else fqName + +/** + * Builds the [JvmSymbol.key] of a Kotlin or Java callable named [name]. + * + * [receiverTypeName] (an internal name, null for a non-extension) is part of the key because + * extension overloads can differ by receiver alone, such as `Iterable.first()` and + * `List.first()`, and rows sharing a key within one source replace each other. + * [parameterTypeNames] is null for a property, which has no parameter list. + */ +fun jvmCallableKey( + name: String, + receiverTypeName: String?, + parameterTypeNames: List?, +): String = + buildString { + append(name) + if (receiverTypeName != null) append('@').append(receiverTypeName.toFqName()) + parameterTypeNames?.joinTo(this, ",", "(", ")") + } + /** * Base for all type-specific symbol data. * Every variant provides [containingClassName] (empty for top-level). */ sealed interface JvmSymbolInfo { - /** * The internal name of the containing class. */ - val containingClassName: String + val containingClassName: String /** * The fully qualified name of the containing class, in dot format. @@ -135,15 +235,15 @@ data class JvmClassInfo( ) : JvmSymbolInfo data class KotlinClassInfo( - val isData: Boolean = false, - val isValue: Boolean = false, - val isSealed: Boolean = false, - val isFunInterface: Boolean = false, - val isExpect: Boolean = false, - val isActual: Boolean = false, - val isExternal: Boolean = false, - val sealedSubclasses: List = emptyList(), - val companionObjectName: String = "", + val isData: Boolean = false, + val isValue: Boolean = false, + val isSealed: Boolean = false, + val isFunInterface: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isExternal: Boolean = false, + val sealedSubclasses: List = emptyList(), + val companionObjectName: String = "", ) data class JvmFunctionInfo( @@ -164,30 +264,30 @@ data class JvmFunctionInfo( } data class JvmParameterInfo( - val name: String, - val typeName: String, - val typeDisplayName: String, - val hasDefaultValue: Boolean = false, - val isCrossinline: Boolean = false, - val isNoinline: Boolean = false, - val isVararg: Boolean = false, + val name: String, + val typeName: String, + val typeDisplayName: String, + val hasDefaultValue: Boolean = false, + val isCrossinline: Boolean = false, + val isNoinline: Boolean = false, + val isVararg: Boolean = false, ) { val typeFqName: String get() = typeName.toFqName() } data class KotlinFunctionInfo( - val receiverTypeName: String = "", - val receiverTypeDisplayName: String = "", - val isSuspend: Boolean = false, - val isInline: Boolean = false, - val isInfix: Boolean = false, - val isOperator: Boolean = false, - val isTailrec: Boolean = false, - val isExternal: Boolean = false, - val isExpect: Boolean = false, - val isActual: Boolean = false, - val isReturnTypeNullable: Boolean = false, + val receiverTypeName: String = "", + val receiverTypeDisplayName: String = "", + val isSuspend: Boolean = false, + val isInline: Boolean = false, + val isInfix: Boolean = false, + val isOperator: Boolean = false, + val isTailrec: Boolean = false, + val isExternal: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isReturnTypeNullable: Boolean = false, ) { val receiverTypeFqName: String get() = receiverTypeName.toFqName() @@ -207,17 +307,17 @@ data class JvmFieldInfo( } data class KotlinPropertyInfo( - val receiverTypeName: String = "", - val receiverTypeDisplayName: String = "", - val isConst: Boolean = false, - val isLateinit: Boolean = false, - val hasGetter: Boolean = false, - val hasSetter: Boolean = false, - val isDelegated: Boolean = false, - val isExpect: Boolean = false, - val isActual: Boolean = false, - val isExternal: Boolean = false, - val isTypeNullable: Boolean = false, + val receiverTypeName: String = "", + val receiverTypeDisplayName: String = "", + val isConst: Boolean = false, + val isLateinit: Boolean = false, + val hasGetter: Boolean = false, + val hasSetter: Boolean = false, + val isDelegated: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isExternal: Boolean = false, + val isTypeNullable: Boolean = false, ) { val receiverTypeFqName: String get() = receiverTypeName.toFqName() @@ -241,4 +341,4 @@ data class JvmTypeAliasInfo( private fun String.toFqName() = replace('/', '.') .replace('$', '.') - .replace('#', '.') \ No newline at end of file + .replace('#', '.') diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt index c414709759..0988a03a9f 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt @@ -19,7 +19,6 @@ import org.appdevforall.codeonthego.indexing.jvm.proto.JvmSymbolProtos.JvmSymbol * Blob serialization uses Protobuf with `oneof` for type-specific data. */ object JvmSymbolDescriptor : IndexDescriptor { - const val KEY_NAME = "name" const val KEY_PACKAGE = "package" const val KEY_KIND = "kind" @@ -27,390 +26,410 @@ object JvmSymbolDescriptor : IndexDescriptor { const val KEY_CONTAINING_CLASS = "containingClass" const val KEY_LANGUAGE = "language" - override val name: String = "jvm_symbols" - - override val fields: List = listOf( - IndexField(name = KEY_NAME, prefixSearchable = true), - IndexField(name = KEY_PACKAGE), - IndexField(name = KEY_KIND), - IndexField(name = KEY_RECEIVER_TYPE), - IndexField(name = KEY_CONTAINING_CLASS), - IndexField(name = KEY_LANGUAGE), - ) - - override fun fieldValues(entry: JvmSymbol): Map = mapOf( - KEY_NAME to entry.shortName, - KEY_PACKAGE to entry.packageName, - KEY_KIND to entry.kind.name, - KEY_RECEIVER_TYPE to entry.receiverTypeName, - KEY_CONTAINING_CLASS to entry.containingClassName.ifEmpty { null }, - KEY_LANGUAGE to entry.language.name, - ) - - override fun serialize(entry: JvmSymbol): ByteArray = - toProto(entry).toByteArray() - - override fun deserialize(bytes: ByteArray): JvmSymbol = - fromProto(JvmSymbolData.parseFrom(bytes)) - - private fun toProto(s: JvmSymbol): JvmSymbolData { - val builder = JvmSymbolData.newBuilder() - .setName(s.name) - .setShortName(s.shortName) - .setPackageName(s.packageName) - .setSourceId(s.sourceId) - .setKind(kindToProto(s.kind)) - .setLanguage(languageToProto(s.language)) - .setVisibility(visibilityToProto(s.visibility)) - .setIsDeprecated(s.isDeprecated) - - when (val data = s.data) { - is JvmClassInfo -> builder.setClassData(classInfoToProto(data)) - is JvmFunctionInfo -> builder.setFunctionData(functionInfoToProto(data)) - is JvmFieldInfo -> builder.setFieldData(fieldInfoToProto(data)) - is JvmEnumEntryInfo -> builder.setEnumEntryData(enumEntryToProto(data)) - is JvmTypeAliasInfo -> builder.setTypeAliasData(typeAliasToProto(data)) - } - - return builder.build() - } - - private fun classInfoToProto(d: JvmClassInfo): JvmSymbolProtos.ClassData { - val builder = JvmSymbolProtos.ClassData.newBuilder() - .setContainingClassName(d.containingClassName) - .addAllSupertypeNames(d.supertypeNames) - .addAllTypeParameters(d.typeParameters) - .setIsAbstract(d.isAbstract) - .setIsFinal(d.isFinal) - .setIsInner(d.isInner) - .setIsStatic(d.isStatic) - - d.kotlin?.let { kd -> - builder.setKotlin( - JvmSymbolProtos.KotlinClassData.newBuilder() - .setIsData(kd.isData) - .setIsValue(kd.isValue) - .setIsSealed(kd.isSealed) - .setIsFunInterface(kd.isFunInterface) - .setIsExpect(kd.isExpect) - .setIsActual(kd.isActual) - .setIsExternal(kd.isExternal) - .addAllSealedSubclasses(kd.sealedSubclasses) - .setCompanionObjectName(kd.companionObjectName) - ) - } - - return builder.build() - } - - private fun functionInfoToProto(d: JvmFunctionInfo): JvmSymbolProtos.FunctionData { - val builder = JvmSymbolProtos.FunctionData.newBuilder() - .setContainingClassName(d.containingClassName) - .setReturnTypeName(d.returnTypeName) - .setReturnTypeDisplayName(d.returnTypeDisplayName) - .setParameterCount(d.parameterCount) - .addAllParameters(d.parameters.map { paramToProto(it) }) - .setSignatureDisplay(d.signatureDisplay) - .addAllTypeParameters(d.typeParameters) - .setIsStatic(d.isStatic) - .setIsAbstract(d.isAbstract) - .setIsFinal(d.isFinal) - - d.kotlin?.let { kd -> - builder.setKotlin( - JvmSymbolProtos.KotlinFunctionData.newBuilder() - .setReceiverTypeName(kd.receiverTypeName) - .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) - .setIsSuspend(kd.isSuspend) - .setIsInline(kd.isInline) - .setIsInfix(kd.isInfix) - .setIsOperator(kd.isOperator) - .setIsTailrec(kd.isTailrec) - .setIsExternal(kd.isExternal) - .setIsExpect(kd.isExpect) - .setIsActual(kd.isActual) - .setIsReturnTypeNullable(kd.isReturnTypeNullable) - ) - } - - return builder.build() - } - - private fun paramToProto(p: JvmParameterInfo): JvmSymbolProtos.ParameterData = - JvmSymbolProtos.ParameterData.newBuilder() - .setName(p.name) - .setTypeName(p.typeName) - .setTypeDisplayName(p.typeDisplayName) - .setHasDefaultValue(p.hasDefaultValue) - .setIsCrossinline(p.isCrossinline) - .setIsNoinline(p.isNoinline) - .setIsVararg(p.isVararg) - .build() - - private fun fieldInfoToProto(d: JvmFieldInfo): JvmSymbolProtos.FieldData { - val builder = JvmSymbolProtos.FieldData.newBuilder() - .setContainingClassName(d.containingClassName) - .setTypeName(d.typeName) - .setTypeDisplayName(d.typeDisplayName) - .setIsStatic(d.isStatic) - .setIsFinal(d.isFinal) - .setConstantValue(d.constantValue) - - d.kotlin?.let { kd -> - builder.setKotlin( - JvmSymbolProtos.KotlinPropertyData.newBuilder() - .setReceiverTypeName(kd.receiverTypeName) - .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) - .setIsConst(kd.isConst) - .setIsLateinit(kd.isLateinit) - .setHasGetter(kd.hasGetter) - .setHasSetter(kd.hasSetter) - .setIsDelegated(kd.isDelegated) - .setIsExpect(kd.isExpect) - .setIsActual(kd.isActual) - .setIsExternal(kd.isExternal) - .setIsTypeNullable(kd.isTypeNullable) - ) - } - - return builder.build() - } - - private fun enumEntryToProto(d: JvmEnumEntryInfo): JvmSymbolProtos.EnumEntryData = - JvmSymbolProtos.EnumEntryData.newBuilder() - .setContainingEnumName(d.containingClassName) - .setOrdinal(d.ordinal) - .build() - - private fun typeAliasToProto(d: JvmTypeAliasInfo): JvmSymbolProtos.TypeAliasData = - JvmSymbolProtos.TypeAliasData.newBuilder() - .setExpandedTypeName(d.expandedTypeName) - .setExpandedTypeDisplayName(d.expandedTypeDisplayName) - .addAllTypeParameters(d.typeParameters) - .build() - - private fun fromProto(p: JvmSymbolData): JvmSymbol { - val kind = kindFromProto(p.kind) - val data = dataFromProto(p) - - val key = when { - kind.isCallable && kind != JvmSymbolKind.PROPERTY - && kind != JvmSymbolKind.EXTENSION_PROPERTY - && kind != JvmSymbolKind.FIELD -> { - val params = (data as? JvmFunctionInfo) - ?.parameters - ?.joinToString(",") { it.typeName } - ?: "" - "${p.name}($params)" - } - else -> p.name - } - - return JvmSymbol( - key = key, - sourceId = p.sourceId, - name = p.name, - shortName = p.shortName, - packageName = p.packageName, - kind = kind, - language = languageFromProto(p.language), - visibility = visibilityFromProto(p.visibility), - isDeprecated = p.isDeprecated, - data = data, - ) - } - - private fun dataFromProto(p: JvmSymbolData): JvmSymbolInfo = when (p.dataCase) { - JvmSymbolData.DataCase.CLASS_DATA -> classInfoFromProto(p.classData) - JvmSymbolData.DataCase.FUNCTION_DATA -> functionInfoFromProto(p.functionData) - JvmSymbolData.DataCase.FIELD_DATA -> fieldInfoFromProto(p.fieldData) - JvmSymbolData.DataCase.ENUM_ENTRY_DATA -> enumEntryFromProto(p.enumEntryData) - JvmSymbolData.DataCase.TYPE_ALIAS_DATA -> typeAliasFromProto(p.typeAliasData) - else -> JvmClassInfo() // fallback - } - - private fun classInfoFromProto(p: JvmSymbolProtos.ClassData): JvmClassInfo { - val kotlin = if (p.hasKotlin()) { - val kd = p.kotlin - KotlinClassInfo( - isData = kd.isData, - isValue = kd.isValue, - isSealed = kd.isSealed, - isFunInterface = kd.isFunInterface, - isExpect = kd.isExpect, - isActual = kd.isActual, - isExternal = kd.isExternal, - sealedSubclasses = kd.sealedSubclassesList.toList(), - companionObjectName = kd.companionObjectName, - ) - } else null - - return JvmClassInfo( - containingClassName = p.containingClassName, - supertypeNames = p.supertypeNamesList.toList(), - typeParameters = p.typeParametersList.toList(), - isAbstract = p.isAbstract, - isFinal = p.isFinal, - isInner = p.isInner, - isStatic = p.isStatic, - kotlin = kotlin, - ) - } - - private fun functionInfoFromProto(p: JvmSymbolProtos.FunctionData): JvmFunctionInfo { - val kotlin = if (p.hasKotlin()) { - val kd = p.kotlin - KotlinFunctionInfo( - receiverTypeName = kd.receiverTypeName, - receiverTypeDisplayName = kd.receiverTypeDisplayName, - isSuspend = kd.isSuspend, - isInline = kd.isInline, - isInfix = kd.isInfix, - isOperator = kd.isOperator, - isTailrec = kd.isTailrec, - isExternal = kd.isExternal, - isExpect = kd.isExpect, - isActual = kd.isActual, - isReturnTypeNullable = kd.isReturnTypeNullable, - ) - } else null - - return JvmFunctionInfo( - containingClassName = p.containingClassName, - returnTypeName = p.returnTypeName, - returnTypeDisplayName = p.returnTypeDisplayName, - parameterCount = p.parameterCount, - parameters = p.parametersList.map { paramFromProto(it) }, - signatureDisplay = p.signatureDisplay, - typeParameters = p.typeParametersList.toList(), - isStatic = p.isStatic, - isAbstract = p.isAbstract, - isFinal = p.isFinal, - kotlin = kotlin, - ) - } - - private fun paramFromProto(p: JvmSymbolProtos.ParameterData): JvmParameterInfo = - JvmParameterInfo( - name = p.name, - typeName = p.typeName, - typeDisplayName = p.typeDisplayName, - hasDefaultValue = p.hasDefaultValue, - isCrossinline = p.isCrossinline, - isNoinline = p.isNoinline, - isVararg = p.isVararg, - ) - - private fun fieldInfoFromProto(p: JvmSymbolProtos.FieldData): JvmFieldInfo { - val kotlin = if (p.hasKotlin()) { - val kd = p.kotlin - KotlinPropertyInfo( - receiverTypeName = kd.receiverTypeName, - receiverTypeDisplayName = kd.receiverTypeDisplayName, - isConst = kd.isConst, - isLateinit = kd.isLateinit, - hasGetter = kd.hasGetter, - hasSetter = kd.hasSetter, - isDelegated = kd.isDelegated, - isExpect = kd.isExpect, - isActual = kd.isActual, - isExternal = kd.isExternal, - isTypeNullable = kd.isTypeNullable, - ) - } else null - - return JvmFieldInfo( - containingClassName = p.containingClassName, - typeName = p.typeName, - typeDisplayName = p.typeDisplayName, - isStatic = p.isStatic, - isFinal = p.isFinal, - constantValue = p.constantValue, - kotlin = kotlin, - ) - } - - private fun enumEntryFromProto(p: JvmSymbolProtos.EnumEntryData): JvmEnumEntryInfo = - JvmEnumEntryInfo( - containingClassName = p.containingEnumName, - ordinal = p.ordinal, - ) - - private fun typeAliasFromProto(p: JvmSymbolProtos.TypeAliasData): JvmTypeAliasInfo = - JvmTypeAliasInfo( - expandedTypeName = p.expandedTypeName, - expandedTypeDisplayName = p.expandedTypeDisplayName, - typeParameters = p.typeParametersList.toList(), - ) - - private fun kindToProto(k: JvmSymbolKind) = when (k) { - JvmSymbolKind.CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_CLASS - JvmSymbolKind.INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE - JvmSymbolKind.ENUM -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM - JvmSymbolKind.ENUM_ENTRY -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY - JvmSymbolKind.ANNOTATION_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS - JvmSymbolKind.OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT - JvmSymbolKind.COMPANION_OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT - JvmSymbolKind.DATA_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS - JvmSymbolKind.VALUE_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS - JvmSymbolKind.SEALED_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS - JvmSymbolKind.SEALED_INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE - JvmSymbolKind.FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION - JvmSymbolKind.EXTENSION_FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION - JvmSymbolKind.CONSTRUCTOR -> JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR - JvmSymbolKind.PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY - JvmSymbolKind.EXTENSION_PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY - JvmSymbolKind.FIELD -> JvmSymbolProtos.JvmSymbolKind.KIND_FIELD - JvmSymbolKind.TYPE_ALIAS -> JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS - } - - private fun kindFromProto(k: JvmSymbolProtos.JvmSymbolKind) = when (k) { - JvmSymbolProtos.JvmSymbolKind.KIND_CLASS -> JvmSymbolKind.CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE -> JvmSymbolKind.INTERFACE - JvmSymbolProtos.JvmSymbolKind.KIND_ENUM -> JvmSymbolKind.ENUM - JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY -> JvmSymbolKind.ENUM_ENTRY - JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS -> JvmSymbolKind.ANNOTATION_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT -> JvmSymbolKind.OBJECT - JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT -> JvmSymbolKind.COMPANION_OBJECT - JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS -> JvmSymbolKind.DATA_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS -> JvmSymbolKind.VALUE_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS -> JvmSymbolKind.SEALED_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE -> JvmSymbolKind.SEALED_INTERFACE - JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION -> JvmSymbolKind.FUNCTION - JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION -> JvmSymbolKind.EXTENSION_FUNCTION - JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR -> JvmSymbolKind.CONSTRUCTOR - JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY -> JvmSymbolKind.PROPERTY - JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY -> JvmSymbolKind.EXTENSION_PROPERTY - JvmSymbolProtos.JvmSymbolKind.KIND_FIELD -> JvmSymbolKind.FIELD - JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS -> JvmSymbolKind.TYPE_ALIAS - else -> JvmSymbolKind.CLASS - } - - private fun languageToProto(l: JvmSourceLanguage) = when (l) { - JvmSourceLanguage.JAVA -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA - JvmSourceLanguage.KOTLIN -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN - } - - private fun languageFromProto(l: JvmSymbolProtos.JvmSourceLanguage) = when (l) { - JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA -> JvmSourceLanguage.JAVA - JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN -> JvmSourceLanguage.KOTLIN - else -> JvmSourceLanguage.JAVA - } - - private fun visibilityToProto(v: JvmVisibility) = when (v) { - JvmVisibility.PUBLIC -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC - JvmVisibility.PROTECTED -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED - JvmVisibility.INTERNAL -> JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL - JvmVisibility.PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE - JvmVisibility.PACKAGE_PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE - } - - private fun visibilityFromProto(v: JvmSymbolProtos.JvmVisibility) = when (v) { - JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC -> JvmVisibility.PUBLIC - JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED -> JvmVisibility.PROTECTED - JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL -> JvmVisibility.INTERNAL - JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE -> JvmVisibility.PRIVATE - JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE -> JvmVisibility.PACKAGE_PRIVATE - else -> JvmVisibility.PUBLIC - } + override val name: String = "jvm_symbols" + + override val fields: List = + listOf( + IndexField(name = KEY_NAME, prefixSearchable = true), + IndexField(name = KEY_PACKAGE), + IndexField(name = KEY_KIND, selective = false), + IndexField(name = KEY_RECEIVER_TYPE), + IndexField(name = KEY_CONTAINING_CLASS), + IndexField(name = KEY_LANGUAGE, selective = false), + ) + + override fun fieldValues(entry: JvmSymbol): Map = + mapOf( + KEY_NAME to entry.shortName, + KEY_PACKAGE to entry.packageName, + KEY_KIND to entry.kind.name, + KEY_RECEIVER_TYPE to entry.receiverTypeName, + KEY_CONTAINING_CLASS to entry.containingClassName.ifEmpty { null }, + KEY_LANGUAGE to entry.language.name, + ) + + override fun serialize(entry: JvmSymbol): ByteArray = toProto(entry).toByteArray() + + override fun deserialize(bytes: ByteArray): JvmSymbol = fromProto(JvmSymbolData.parseFrom(bytes)) + + private fun toProto(s: JvmSymbol): JvmSymbolData { + val builder = + JvmSymbolData + .newBuilder() + .setKey(s.key) + .setName(s.name) + .setShortName(s.shortName) + .setPackageName(s.packageName) + .setSourceId(s.sourceId) + .setKind(kindToProto(s.kind)) + .setLanguage(languageToProto(s.language)) + .setVisibility(visibilityToProto(s.visibility)) + .setIsDeprecated(s.isDeprecated) + + when (val data = s.data) { + is JvmClassInfo -> builder.setClassData(classInfoToProto(data)) + is JvmFunctionInfo -> builder.setFunctionData(functionInfoToProto(data)) + is JvmFieldInfo -> builder.setFieldData(fieldInfoToProto(data)) + is JvmEnumEntryInfo -> builder.setEnumEntryData(enumEntryToProto(data)) + is JvmTypeAliasInfo -> builder.setTypeAliasData(typeAliasToProto(data)) + } + + return builder.build() + } + + private fun classInfoToProto(d: JvmClassInfo): JvmSymbolProtos.ClassData { + val builder = + JvmSymbolProtos.ClassData + .newBuilder() + .setContainingClassName(d.containingClassName) + .addAllSupertypeNames(d.supertypeNames) + .addAllTypeParameters(d.typeParameters) + .setIsAbstract(d.isAbstract) + .setIsFinal(d.isFinal) + .setIsInner(d.isInner) + .setIsStatic(d.isStatic) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinClassData + .newBuilder() + .setIsData(kd.isData) + .setIsValue(kd.isValue) + .setIsSealed(kd.isSealed) + .setIsFunInterface(kd.isFunInterface) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsExternal(kd.isExternal) + .addAllSealedSubclasses(kd.sealedSubclasses) + .setCompanionObjectName(kd.companionObjectName), + ) + } + + return builder.build() + } + + private fun functionInfoToProto(d: JvmFunctionInfo): JvmSymbolProtos.FunctionData { + val builder = + JvmSymbolProtos.FunctionData + .newBuilder() + .setContainingClassName(d.containingClassName) + .setReturnTypeName(d.returnTypeName) + .setReturnTypeDisplayName(d.returnTypeDisplayName) + .setParameterCount(d.parameterCount) + .addAllParameters(d.parameters.map { paramToProto(it) }) + .setSignatureDisplay(d.signatureDisplay) + .addAllTypeParameters(d.typeParameters) + .setIsStatic(d.isStatic) + .setIsAbstract(d.isAbstract) + .setIsFinal(d.isFinal) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinFunctionData + .newBuilder() + .setReceiverTypeName(kd.receiverTypeName) + .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) + .setIsSuspend(kd.isSuspend) + .setIsInline(kd.isInline) + .setIsInfix(kd.isInfix) + .setIsOperator(kd.isOperator) + .setIsTailrec(kd.isTailrec) + .setIsExternal(kd.isExternal) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsReturnTypeNullable(kd.isReturnTypeNullable), + ) + } + + return builder.build() + } + + private fun paramToProto(p: JvmParameterInfo): JvmSymbolProtos.ParameterData = + JvmSymbolProtos.ParameterData + .newBuilder() + .setName(p.name) + .setTypeName(p.typeName) + .setTypeDisplayName(p.typeDisplayName) + .setHasDefaultValue(p.hasDefaultValue) + .setIsCrossinline(p.isCrossinline) + .setIsNoinline(p.isNoinline) + .setIsVararg(p.isVararg) + .build() + + private fun fieldInfoToProto(d: JvmFieldInfo): JvmSymbolProtos.FieldData { + val builder = + JvmSymbolProtos.FieldData + .newBuilder() + .setContainingClassName(d.containingClassName) + .setTypeName(d.typeName) + .setTypeDisplayName(d.typeDisplayName) + .setIsStatic(d.isStatic) + .setIsFinal(d.isFinal) + .setConstantValue(d.constantValue) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinPropertyData + .newBuilder() + .setReceiverTypeName(kd.receiverTypeName) + .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) + .setIsConst(kd.isConst) + .setIsLateinit(kd.isLateinit) + .setHasGetter(kd.hasGetter) + .setHasSetter(kd.hasSetter) + .setIsDelegated(kd.isDelegated) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsExternal(kd.isExternal) + .setIsTypeNullable(kd.isTypeNullable), + ) + } + + return builder.build() + } + + private fun enumEntryToProto(d: JvmEnumEntryInfo): JvmSymbolProtos.EnumEntryData = + JvmSymbolProtos.EnumEntryData + .newBuilder() + .setContainingEnumName(d.containingClassName) + .setOrdinal(d.ordinal) + .build() + + private fun typeAliasToProto(d: JvmTypeAliasInfo): JvmSymbolProtos.TypeAliasData = + JvmSymbolProtos.TypeAliasData + .newBuilder() + .setExpandedTypeName(d.expandedTypeName) + .setExpandedTypeDisplayName(d.expandedTypeDisplayName) + .addAllTypeParameters(d.typeParameters) + .build() + + private fun fromProto(p: JvmSymbolData): JvmSymbol { + val kind = kindFromProto(p.kind) + val data = dataFromProto(p) + + return JvmSymbol( + key = p.key, + sourceId = p.sourceId, + name = p.name, + shortName = p.shortName, + packageName = p.packageName, + kind = kind, + language = languageFromProto(p.language), + visibility = visibilityFromProto(p.visibility), + isDeprecated = p.isDeprecated, + data = data, + ) + } + + private fun dataFromProto(p: JvmSymbolData): JvmSymbolInfo = + when (p.dataCase) { + JvmSymbolData.DataCase.CLASS_DATA -> classInfoFromProto(p.classData) + JvmSymbolData.DataCase.FUNCTION_DATA -> functionInfoFromProto(p.functionData) + JvmSymbolData.DataCase.FIELD_DATA -> fieldInfoFromProto(p.fieldData) + JvmSymbolData.DataCase.ENUM_ENTRY_DATA -> enumEntryFromProto(p.enumEntryData) + JvmSymbolData.DataCase.TYPE_ALIAS_DATA -> typeAliasFromProto(p.typeAliasData) + else -> JvmClassInfo() // fallback + } + + private fun classInfoFromProto(p: JvmSymbolProtos.ClassData): JvmClassInfo { + val kotlin = + if (p.hasKotlin()) { + val kd = p.kotlin + KotlinClassInfo( + isData = kd.isData, + isValue = kd.isValue, + isSealed = kd.isSealed, + isFunInterface = kd.isFunInterface, + isExpect = kd.isExpect, + isActual = kd.isActual, + isExternal = kd.isExternal, + sealedSubclasses = kd.sealedSubclassesList.toList(), + companionObjectName = kd.companionObjectName, + ) + } else { + null + } + + return JvmClassInfo( + containingClassName = p.containingClassName, + supertypeNames = p.supertypeNamesList.toList(), + typeParameters = p.typeParametersList.toList(), + isAbstract = p.isAbstract, + isFinal = p.isFinal, + isInner = p.isInner, + isStatic = p.isStatic, + kotlin = kotlin, + ) + } + + private fun functionInfoFromProto(p: JvmSymbolProtos.FunctionData): JvmFunctionInfo { + val kotlin = + if (p.hasKotlin()) { + val kd = p.kotlin + KotlinFunctionInfo( + receiverTypeName = kd.receiverTypeName, + receiverTypeDisplayName = kd.receiverTypeDisplayName, + isSuspend = kd.isSuspend, + isInline = kd.isInline, + isInfix = kd.isInfix, + isOperator = kd.isOperator, + isTailrec = kd.isTailrec, + isExternal = kd.isExternal, + isExpect = kd.isExpect, + isActual = kd.isActual, + isReturnTypeNullable = kd.isReturnTypeNullable, + ) + } else { + null + } + + return JvmFunctionInfo( + containingClassName = p.containingClassName, + returnTypeName = p.returnTypeName, + returnTypeDisplayName = p.returnTypeDisplayName, + parameterCount = p.parameterCount, + parameters = p.parametersList.map { paramFromProto(it) }, + signatureDisplay = p.signatureDisplay, + typeParameters = p.typeParametersList.toList(), + isStatic = p.isStatic, + isAbstract = p.isAbstract, + isFinal = p.isFinal, + kotlin = kotlin, + ) + } + + private fun paramFromProto(p: JvmSymbolProtos.ParameterData): JvmParameterInfo = + JvmParameterInfo( + name = p.name, + typeName = p.typeName, + typeDisplayName = p.typeDisplayName, + hasDefaultValue = p.hasDefaultValue, + isCrossinline = p.isCrossinline, + isNoinline = p.isNoinline, + isVararg = p.isVararg, + ) + + private fun fieldInfoFromProto(p: JvmSymbolProtos.FieldData): JvmFieldInfo { + val kotlin = + if (p.hasKotlin()) { + val kd = p.kotlin + KotlinPropertyInfo( + receiverTypeName = kd.receiverTypeName, + receiverTypeDisplayName = kd.receiverTypeDisplayName, + isConst = kd.isConst, + isLateinit = kd.isLateinit, + hasGetter = kd.hasGetter, + hasSetter = kd.hasSetter, + isDelegated = kd.isDelegated, + isExpect = kd.isExpect, + isActual = kd.isActual, + isExternal = kd.isExternal, + isTypeNullable = kd.isTypeNullable, + ) + } else { + null + } + + return JvmFieldInfo( + containingClassName = p.containingClassName, + typeName = p.typeName, + typeDisplayName = p.typeDisplayName, + isStatic = p.isStatic, + isFinal = p.isFinal, + constantValue = p.constantValue, + kotlin = kotlin, + ) + } + + private fun enumEntryFromProto(p: JvmSymbolProtos.EnumEntryData): JvmEnumEntryInfo = + JvmEnumEntryInfo( + containingClassName = p.containingEnumName, + ordinal = p.ordinal, + ) + + private fun typeAliasFromProto(p: JvmSymbolProtos.TypeAliasData): JvmTypeAliasInfo = + JvmTypeAliasInfo( + expandedTypeName = p.expandedTypeName, + expandedTypeDisplayName = p.expandedTypeDisplayName, + typeParameters = p.typeParametersList.toList(), + ) + + private fun kindToProto(k: JvmSymbolKind) = + when (k) { + JvmSymbolKind.CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_CLASS + JvmSymbolKind.INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE + JvmSymbolKind.ENUM -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM + JvmSymbolKind.ENUM_ENTRY -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY + JvmSymbolKind.ANNOTATION_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS + JvmSymbolKind.OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT + JvmSymbolKind.COMPANION_OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT + JvmSymbolKind.DATA_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS + JvmSymbolKind.VALUE_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS + JvmSymbolKind.SEALED_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS + JvmSymbolKind.SEALED_INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE + JvmSymbolKind.FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION + JvmSymbolKind.EXTENSION_FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION + JvmSymbolKind.CONSTRUCTOR -> JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR + JvmSymbolKind.PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY + JvmSymbolKind.EXTENSION_PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY + JvmSymbolKind.FIELD -> JvmSymbolProtos.JvmSymbolKind.KIND_FIELD + JvmSymbolKind.TYPE_ALIAS -> JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS + JvmSymbolKind.FILE_FACADE -> JvmSymbolProtos.JvmSymbolKind.KIND_FILE_FACADE + } + + private fun kindFromProto(k: JvmSymbolProtos.JvmSymbolKind) = + when (k) { + JvmSymbolProtos.JvmSymbolKind.KIND_CLASS -> JvmSymbolKind.CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE -> JvmSymbolKind.INTERFACE + JvmSymbolProtos.JvmSymbolKind.KIND_ENUM -> JvmSymbolKind.ENUM + JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY -> JvmSymbolKind.ENUM_ENTRY + JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS -> JvmSymbolKind.ANNOTATION_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT -> JvmSymbolKind.OBJECT + JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT -> JvmSymbolKind.COMPANION_OBJECT + JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS -> JvmSymbolKind.DATA_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS -> JvmSymbolKind.VALUE_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS -> JvmSymbolKind.SEALED_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE -> JvmSymbolKind.SEALED_INTERFACE + JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION -> JvmSymbolKind.FUNCTION + JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION -> JvmSymbolKind.EXTENSION_FUNCTION + JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR -> JvmSymbolKind.CONSTRUCTOR + JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY -> JvmSymbolKind.PROPERTY + JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY -> JvmSymbolKind.EXTENSION_PROPERTY + JvmSymbolProtos.JvmSymbolKind.KIND_FIELD -> JvmSymbolKind.FIELD + JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS -> JvmSymbolKind.TYPE_ALIAS + JvmSymbolProtos.JvmSymbolKind.KIND_FILE_FACADE -> JvmSymbolKind.FILE_FACADE + else -> JvmSymbolKind.CLASS + } + + private fun languageToProto(l: JvmSourceLanguage) = + when (l) { + JvmSourceLanguage.JAVA -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA + JvmSourceLanguage.KOTLIN -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN + } + + private fun languageFromProto(l: JvmSymbolProtos.JvmSourceLanguage) = + when (l) { + JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA -> JvmSourceLanguage.JAVA + JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN -> JvmSourceLanguage.KOTLIN + else -> JvmSourceLanguage.JAVA + } + + private fun visibilityToProto(v: JvmVisibility) = + when (v) { + JvmVisibility.PUBLIC -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC + JvmVisibility.PROTECTED -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED + JvmVisibility.INTERNAL -> JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL + JvmVisibility.PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE + JvmVisibility.PACKAGE_PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE + } + + private fun visibilityFromProto(v: JvmSymbolProtos.JvmVisibility) = + when (v) { + JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC -> JvmVisibility.PUBLIC + JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED -> JvmVisibility.PROTECTED + JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL -> JvmVisibility.INTERNAL + JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE -> JvmVisibility.PRIVATE + JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE -> JvmVisibility.PACKAGE_PRIVATE + else -> JvmVisibility.PUBLIC + } } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index b9ac3170db..cfff6beae7 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -4,12 +4,14 @@ import android.content.Context import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.joinAll import org.appdevforall.codeonthego.indexing.FilteredIndex import org.appdevforall.codeonthego.indexing.SQLiteIndex import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.WritableIndex import org.appdevforall.codeonthego.indexing.api.indexQuery import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_CONTAINING_CLASS +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_KIND import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_NAME import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_PACKAGE import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_RECEIVER_TYPE @@ -28,9 +30,27 @@ open class JvmSymbolIndex( WritableIndex by backing, Closeable { companion object { + /** Kind names as the descriptor stores them, for set-membership predicates. */ + private val CLASSIFIER_KIND_NAMES = JvmSymbolKind.CLASSIFIER_KINDS.map { it.name } + private val CALLABLE_KIND_NAMES = JvmSymbolKind.CALLABLE_KINDS.map { it.name } + const val DB_NAME_DEFAULT = "jvm_symbol_index.db" const val INDEX_NAME_LIBRARY = "jvm-library-cache" + /** + * Storage format version of the SQLite-backed JVM symbol and Kotlin file metadata indexes. + * + * Bump it whenever the index schema or any scanner's output changes: an install holding a + * different version drops its rows and re-indexes every source, which is the only way rows + * produced by an older scanner get replaced. [KtFileMetadataIndex] shares it because a + * source file's symbols are re-indexed only when its metadata row says so; dropping the + * symbols alone would leave files recorded as indexed with no symbols. + * + * Version 3 keys extension callables by their receiver, stores the key in the payload, and + * stops treating a top-level class with '$' in its name as nested. + */ + const val FORMAT_VERSION = 3 + /** * Create (or get) a JVM symbol index backed by SQLite. * @@ -48,6 +68,7 @@ open class JvmSymbolIndex( descriptor = JvmSymbolDescriptor, context = context, dbName = dbName, + formatVersion = FORMAT_VERSION, name = indexName, ) @@ -59,38 +80,51 @@ open class JvmSymbolIndex( /** * Index a single source. The [provider] returns a [Flow] that is * collected on [Dispatchers.IO] and inserted in batches through - * [Index.insertAll]. + * [Index.insertAll], or [Index.insertSource] when a [fingerprint] is given. * - * If [skipIfExists] is true and the source is already indexed, - * this is a no-op. + * If [skipIfExists] is true and the source is already indexed (with the + * same [fingerprint], when one is given), this is a no-op. * * @param sourceId Identifies the source. * @param skipIfExists Skip if already indexed. + * @param fingerprint Identifies the source's current content, or `null` to not track it. * @param provider Lambda returning a [Flow] of entries. Resources the flow opens * (e.g. a JAR) are released when collection completes, fails or is * cancelled, so Flow operators such as `.filter {}` are safe. - * @return The launched job. + * @return The launched job, or the already-running job for the same fingerprint if one exists. + * @see BackgroundIndexer.indexSource */ fun indexSource( sourceId: String, skipIfExists: Boolean = true, + fingerprint: String? = null, provider: (sourceId: String) -> Flow, - ): Job = indexer.indexSource(sourceId, skipIfExists, provider) + ): Job = indexer.indexSource(sourceId, skipIfExists, fingerprint, provider) + + /** Returns the fingerprint [sourceId] was last indexed with, or `null` if it has none. */ + suspend fun sourceFingerprint(sourceId: String): String? = backing.sourceFingerprint(sourceId) /** * Find symbols matching the given prefix. * * @param prefix The prefix to search for. * @param limit The result limit. + * @param kinds When non-null, only symbols of these kinds, filtered in the query so that the + * limit is not spent on rows the caller would discard. + * @param topLevelOnly Whether to leave out symbols declared inside a class. * @see query */ fun findByPrefix( prefix: String, limit: Int = 200, + kinds: Set? = null, + topLevelOnly: Boolean = false, ): Sequence = query( indexQuery { prefix(KEY_NAME, prefix) + if (kinds != null) anyOf(KEY_KIND, kinds.map { it.name }) + if (topLevelOnly) notExists(KEY_CONTAINING_CLASS) this.limit = limit }, ) @@ -111,6 +145,14 @@ open class JvmSymbolIndex( }, ) + /** + * Top-level callables declared in [packageName]. + * + * The kind and top-level conditions are part of the query rather than a filter over its result: + * a package holds far more members than top-level callables, so filtering afterwards meant + * fetching every symbol in the package to keep a handful. Top-level is expressible because the + * descriptor stores a null containing class for it. + */ fun findTopLevelCallablesInPackage( packageName: String, namePrefix: String = "", @@ -119,11 +161,14 @@ open class JvmSymbolIndex( query( indexQuery { eq(KEY_PACKAGE, packageName) + anyOf(KEY_KIND, CALLABLE_KIND_NAMES) + notExists(KEY_CONTAINING_CLASS) if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) - this.limit = 0 + this.limit = limit }, - ).filter { it.kind.isCallable && it.isTopLevel }.take(limit) + ) + /** Classifiers declared in [packageName], filtered by kind in the query for the same reason. */ fun findClassifiersInPackage( packageName: String, namePrefix: String = "", @@ -132,10 +177,11 @@ open class JvmSymbolIndex( query( indexQuery { eq(KEY_PACKAGE, packageName) + anyOf(KEY_KIND, CLASSIFIER_KIND_NAMES) if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) - this.limit = 0 + this.limit = limit }, - ).filter { it.kind.isClassifier }.take(limit) + ) fun findMembersOf( classFqName: String, @@ -150,12 +196,23 @@ open class JvmSymbolIndex( }, ) + /** + * Symbols whose simple name is exactly [name], optionally restricted to [kinds]. + * + * Restricting by kind matters more than it looks: without it an exact-name lookup still returns + * every method and field sharing that name, and a caller wanting only classifiers pays for all + * of them. [topLevelOnly] likewise leaves out symbols declared inside a class. + */ fun findBySimpleName( name: String, limit: Int = 200, + kinds: Set? = null, + topLevelOnly: Boolean = false, ) = query( indexQuery { eq(KEY_NAME, name) + if (kinds != null) anyOf(KEY_KIND, kinds.map { it.name }) + if (topLevelOnly) notExists(KEY_CONTAINING_CLASS) this.limit = limit }, ) @@ -166,6 +223,20 @@ open class JvmSymbolIndex( suspend fun awaitIndexing() = indexer.awaitAll() + /** + * Optimizes the backing index once every job in [jobs] has finished, or does nothing if [jobs] + * is empty. + * + * Whoever submits a pass of sources calls this with the jobs it submitted, because only it knows + * when the pass is over: the indexer can sit idle between two submissions while the submitter + * works out what to submit next, so an idle indexer is not a finished pass. + */ + suspend fun optimizeAfter(jobs: Collection) { + if (jobs.isEmpty()) return + jobs.joinAll() + backing.optimize() + } + override fun close() { indexer.close() super.close() diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt index 3121ff9793..50ae6bdaac 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -34,10 +34,10 @@ import kotlin.metadata.modality import kotlin.metadata.visibility /** - * Parses Kotlin metadata from class files into [JvmSymbol]s - * with full Kotlin semantics (extensions, suspend, inline, etc.). + * Parses a Kotlin class file's `@Metadata` into [JvmSymbol]s with full Kotlin semantics + * (extensions, suspend, inline, etc.). * - * Skips non-Kotlin class files (no `@Metadata` annotation). + * Returns nothing for a class file without `@Metadata`. */ object KotlinMetadataScanner { private val log = LoggerFactory.getLogger(KotlinMetadataScanner::class.java) @@ -55,47 +55,130 @@ object KotlinMetadataScanner { val header = collector.metadataHeader ?: return null - val metadata = - try { - KotlinClassMetadata.readStrict(header) - } catch (e: Exception) { - log.debug("Failed to read Kotlin metadata: {}", e.message) - return null - } + val metadata = readMetadata(header) ?: return null return when (metadata) { is KotlinClassMetadata.Class -> { - extractFromClass(metadata.kmClass, sourceId) + extractFromClass(metadata.kmClass, collector.internalName, sourceId) } is KotlinClassMetadata.FileFacade -> { - extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) + facadeSymbol(collector, sourceId) + + extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) } + /* + * A part class is a compiler-generated fragment of a facade ("FooKt__Part"). Its + * declarations belong to the facade, so they are extracted, but neither language ever names + * the part class itself, so it is deliberately left out. + */ is KotlinClassMetadata.MultiFileClassPart -> { extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) } + /* + * The facade of a @JvmMultifileClass. It declares nothing of its own -- the parts hold the + * declarations -- but Java calls the top-level functions through it, so the class still has + * to be findable. + */ + is KotlinClassMetadata.MultiFileClassFacade -> { + facadeSymbol(collector, sourceId) + } + else -> { null } } } + /** + * Reads [header] strictly, falling back to a lenient read. + * + * A strict read rejects metadata written by a newer Kotlin than the bundled reader supports, which + * a dependency or the project's own output can carry. Dropping those classes would hide them from + * every lookup, while a lenient read still yields their declarations. + */ + private fun readMetadata(header: Metadata): KotlinClassMetadata? { + try { + return KotlinClassMetadata.readStrict(header) + } catch (e: Exception) { + log.debug("Reading Kotlin metadata leniently after a strict read failed: {}", e.message) + } + return try { + KotlinClassMetadata.readLenient(header) + } catch (e: Exception) { + log.debug("Failed to read Kotlin metadata: {}", e.message) + null + } + } + + /** + * The symbol for a facade class itself, as opposed to the declarations it carries. + * + * Metadata for a facade describes a package rather than a class, so without this the class is + * absent from the index entirely and a Java file importing `FooKt` has nothing to resolve + * against. + */ + private fun facadeSymbol( + collector: MetadataCollector, + sourceId: String, + ): List { + val internalName = collector.internalName.ifEmpty { return emptyList() } + return listOf( + JvmSymbol( + key = internalName, + sourceId = sourceId, + name = internalName, + shortName = internalName.substringAfterLast('/'), + packageName = collector.packageName, + kind = JvmSymbolKind.FILE_FACADE, + language = JvmSourceLanguage.KOTLIN, + data = JvmClassInfo(internalName = internalName), + ), + ) + } + + /** + * Symbols for [klass] and its members. + * + * [className] is the class file's internal name (`kotlin/text/Regex$Companion`), not the metadata + * name (`kotlin/text/Regex.Companion`): JarSymbolScanner and SourceFileIndexer key and nest by the + * former, and a containing-class lookup only matches when every scanner spells it the same way. + */ private fun extractFromClass( klass: KmClass, + className: String, sourceId: String, ): List { val symbols = mutableListOf() - val className = klass.name + val metadataName = klass.name val packageName = - className - .substringBeforeLast('/') + metadataName + .substringBeforeLast('/', "") .replace('/', '.') - val shortName = - className - .substringAfterLast('/') - .substringAfterLast('$') + /* + * A leading '.' marks a local or anonymous class. The classpath trie never held these -- it + * treated any name containing '$' as not top level -- and neither language names them, so + * offering one as a completion would be noise. JarSymbolScanner drops its equivalents too. + */ + if (metadataName.startsWith('.')) { + return symbols + } + + /* + * Metadata nests with '.', so here a '$' can only be part of a declared name. It is still split + * on because the classpath trie treated every '$' as nesting, and the index should hold the + * same top-level classes. + */ + val simpleNames = metadataName.substringAfterLast('/') + val lastSeparator = maxOf(simpleNames.lastIndexOf('.'), simpleNames.lastIndexOf('$')) + val shortName = if (lastSeparator >= 0) simpleNames.substring(lastSeparator + 1) else simpleNames + val containingClassName = + if (lastSeparator >= 0) { + className.substring(0, className.length - shortName.length - 1) + } else { + "" + } val kind = when (klass.kind) { @@ -128,6 +211,8 @@ object KotlinMetadataScanner { visibility = kmVisibility(klass.visibility), data = JvmClassInfo( + internalName = className, + containingClassName = containingClassName, supertypeNames = supertypes, typeParameters = klass.typeParameters.map { it.name }, isAbstract = klass.modality == Modality.ABSTRACT, @@ -244,7 +329,7 @@ object KotlinMetadataScanner { } else { "$packageName#${fn.name}" } - val key = "$name(${parameters.joinToString(",") { it.typeFqName }})" + val key = jvmCallableKey(name, receiverType?.let { kmTypeToName(it) }, parameters.map { it.typeFqName }) val signatureDisplay = buildString { @@ -315,7 +400,7 @@ object KotlinMetadataScanner { } return JvmSymbol( - key = name, + key = jvmCallableKey(name, receiverType?.let { kmTypeToName(it) }, parameterTypeNames = null), sourceId = sourceId, name = name, shortName = prop.name, @@ -375,6 +460,9 @@ object KotlinMetadataScanner { var metadataHeader: Metadata? = null var packageName = "" + /** The visited class's internal name, e.g. `com/example/FooKt`. */ + var internalName = "" + private var metadataKind: Int? = null private var metadataVersion: IntArray? = null private var data1: Array? = null @@ -391,6 +479,7 @@ object KotlinMetadataScanner { superName: String?, interfaces: Array?, ) { + internalName = name val lastSlash = name.lastIndexOf('/') packageName = if (lastSlash >= 0) name.substring(0, lastSlash).replace('/', '.') else "" } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt index e1ecee7391..44e5eed353 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt @@ -14,9 +14,7 @@ import java.io.Closeable class KtFileMetadataIndex( private val backing: Index, ) : Closeable { - companion object { - /** * Creates a [KtFileMetadataIndex] backed by a SQLite database. * @@ -25,23 +23,24 @@ class KtFileMetadataIndex( */ fun sqliteBacked( context: Context, - dbName: String? = null + dbName: String? = null, ): KtFileMetadataIndex = KtFileMetadataIndex( SQLiteIndex( descriptor = KtFileMetadataDescriptor, context = context, dbName = dbName, + formatVersion = JvmSymbolIndex.FORMAT_VERSION, name = "kt-file-metadata", - ) + ), ) } /** * Insert or replace the metadata record for a single file. * - * Because [KtFileMetadata.key] == [KtFileMetadata.filePath], the - * underlying `CONFLICT_REPLACE` strategy ensures this is a true upsert. + * Because [KtFileMetadata.key] and [KtFileMetadata.sourceId] are both the + * file path, a file has at most one row and an insert replaces it. */ suspend fun upsert(metadata: KtFileMetadata) = backing.insert(metadata) @@ -57,7 +56,7 @@ class KtFileMetadataIndex( * batched, transactional operation. * * Equivalent to calling [remove] once per path, but issues the deletes as one - * transaction instead of N — see [Index.removeBySources]. Paths not present in + * transaction instead of N - see [Index.removeBySources]. Paths not present in * the index are ignored. */ suspend fun removeAll(filePaths: Collection) = backing.removeBySources(filePaths) @@ -82,15 +81,14 @@ class KtFileMetadataIndex( indexQuery { eq(KEY_PACKAGE, packageFqName) limit = 0 - } + }, ) /** * Returns a [Sequence] of absolute file paths whose declared package exactly * matches [packageFqName]. */ - fun getFilePathsForPackage(packageFqName: String): Sequence = - getFilesForPackage(packageFqName).map { it.filePath } + fun getFilePathsForPackage(packageFqName: String): Sequence = getFilesForPackage(packageFqName).map { it.filePath } /** * Returns `true` if at least one file with package [packageFqName] is @@ -99,10 +97,13 @@ class KtFileMetadataIndex( * Pass an empty string for the root (default) package. */ fun packageExists(packageFqName: String): Boolean = - backing.query(indexQuery { - eq(KEY_PACKAGE, packageFqName) - limit = 1 - }).firstOrNull() != null + backing + .query( + indexQuery { + eq(KEY_PACKAGE, packageFqName) + limit = 1 + }, + ).firstOrNull() != null /** * Returns the simple names of the direct child packages of [packageFqName]. @@ -140,10 +141,13 @@ class KtFileMetadataIndex( * symbols have not yet been extracted ([KtFileMetadata.isIndexed] is `false`). */ fun getUnindexedFiles(): Sequence = - backing.query(indexQuery { - eq(KEY_IS_INDEXED, false.toString()) - limit = 0 - }).map { it.filePath } + backing + .query( + indexQuery { + eq(KEY_IS_INDEXED, false.toString()) + limit = 0 + }, + ).map { it.filePath } /** Remove all records from the index. */ suspend fun clear() = backing.clear() diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt new file mode 100644 index 0000000000..23e7f8f0c8 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt @@ -0,0 +1,120 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import com.itsaky.androidide.projects.classpath.JarFsClasspathReader +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.junit.Assume.assumeFalse +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized +import java.io.File +import java.nio.file.Paths + +/** + * The index must offer exactly the classes the classpath trie held. + * + * The oracle is built from the **old** path: `JarFsClasspathReader` filtered to top-level classes is + * literally what filled `ModuleProject.compileClasspathClasses`. Building it from the index instead + * would let the new code agree with itself and pin nothing. + * + * It runs against real JARs because the cases that matter do not occur in a hand-built fixture: + * Kotlin file facades, multi-file class parts and anonymous Kotlin classes in kotlin-stdlib, and + * package-private, anonymous and nested Java classes in JUnit. + */ +@RunWith(Parameterized::class) +class ClasspathTrieDifferentialTest( + private val corpus: String, + private val anchor: Class<*>, +) { + companion object { + @JvmStatic + @Parameterized.Parameters(name = "{0}") + fun corpora(): List> = + listOf( + arrayOf("kotlin-stdlib", Unit::class.java), + arrayOf("junit", Test::class.java), + ) + } + + private val jar: File by lazy { + val location = + File( + anchor.protectionDomain.codeSource.location + .toURI(), + ) + assertWithMessage("$corpus must resolve to a JAR on the test classpath, got $location") + .that(location.isFile && location.name.endsWith(".jar")) + .isTrue() + location + } + + private val isKotlinCorpus get() = anchor == Unit::class.java + + /** What the trie held: every top-level class the classpath reader found. */ + private fun trieNames(): Set = + JarFsClasspathReader() + .listClasses(listOf(jar)) + .asSequence() + .filter { it.isTopLevel } + .map { it.name } + .toSet() + + private fun indexClasses(): List = + runBlocking { + CombinedJarScanner + .scan(Paths.get(jar.absolutePath), "differential") + .filter { it.kind.isJvmClass && it.isTopLevel } + .toList() + } + + /** What the index offers for the same JAR. */ + private fun indexNames(): Set = indexClasses().map { it.fqName }.toSet() + + /** + * A fragment of a `@JvmMultifileClass` facade, e.g. `LazyKt__LazyJVMKt`. + * + * The only thing the index deliberately withholds: the declarations belong to the facade, which + * is indexed, and neither Java nor Kotlin names the part. + */ + private fun isMultiFilePart(qualifiedName: String) = qualifiedName.substringAfterLast('.').contains("__") + + @Test + fun `every class the trie held is offered by the index, bar multi-file parts`() { + val missing = trieNames() - indexNames() + + assertThat(missing.filterNot(::isMultiFilePart)).isEmpty() + } + + @Test + fun `the index offers nothing the trie did not hold`() { + assertThat(indexNames() - trieNames()).isEmpty() + } + + @Test + fun `the withheld classes really are multi-file parts, and a Kotlin corpus has some`() { + val missing = trieNames() - indexNames() + + // Guards the exception itself: if the corpus stopped containing parts, the first test would + // start passing for the wrong reason. + assertThat(missing.all(::isMultiFilePart)).isTrue() + if (isKotlinCorpus) assertThat(missing).isNotEmpty() + } + + @Test + fun `the corpus is large enough for the comparison to mean something`() { + assertThat(trieNames().size).isGreaterThan(if (isKotlinCorpus) 500 else 150) + } + + @Test + fun `a Java corpus holds the package-private, anonymous and nested classes it is here for`() { + assumeFalse("only the Java corpus is chosen for these shapes", isKotlinCorpus) + val classes = JarFsClasspathReader().listClasses(listOf(jar)) + + assertThat(indexClasses().any { it.visibility == JvmVisibility.PACKAGE_PRIVATE }).isTrue() + assertThat(classes.any { it.isAnonymous }).isTrue() + assertThat(classes.any { it.isInner }).isTrue() + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt new file mode 100644 index 0000000000..0d6240a785 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt @@ -0,0 +1,98 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.com.intellij.mock.MockApplication +import org.jetbrains.kotlin.com.intellij.openapi.application.ApplicationManager +import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer +import org.jetbrains.kotlin.com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.util.jar.Attributes +import java.util.jar.JarEntry +import java.util.jar.JarOutputStream +import java.util.jar.Manifest + +/** + * A multi-release JAR must contribute its base classes only. + * + * Its `META-INF/versions//` copy of a class shares the base class's key within the same source, + * so indexing both let the versioned copy overwrite the base row. + */ +@RunWith(JUnit4::class) +class CombinedJarScannerMultiReleaseTest { + @get:Rule + val temp = TemporaryFolder() + + private fun classWithMethod( + internalName: String, + methodName: String, + ): ByteArray = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, internalName, null, "java/lang/Object", null) + visitMethod(Opcodes.ACC_PUBLIC, methodName, "()V", null, null).visitEnd() + visitEnd() + }.toByteArray() + + private fun multiReleaseJar(): File { + val manifest = + Manifest().apply { + mainAttributes[Attributes.Name.MANIFEST_VERSION] = "1.0" + mainAttributes[Attributes.Name("Multi-Release")] = "true" + } + val jar = temp.newFile("multi-release.jar") + JarOutputStream(jar.outputStream(), manifest).use { out -> + fun put( + path: String, + bytes: ByteArray, + ) { + out.putNextEntry(JarEntry(path)) + out.write(bytes) + out.closeEntry() + } + put("com/example/Widget.class", classWithMethod("com/example/Widget", "base")) + put("META-INF/versions/11/com/example/Widget.class", classWithMethod("com/example/Widget", "versioned")) + put("META-INF/versions/11/com/example/Extra.class", classWithMethod("com/example/Extra", "extra")) + } + return jar + } + + private fun assertOnlyBaseClasses(symbols: List) { + assertThat(symbols.filter { it.kind.isJvmClass }.map { it.key }).containsExactly("com/example/Widget") + assertThat(symbols.filter { it.kind == JvmSymbolKind.FUNCTION }.map { it.shortName }).containsExactly("base") + } + + @Test + fun `a jar path scan skips versioned entries`() { + val jar = multiReleaseJar() + + assertOnlyBaseClasses(runBlocking { CombinedJarScanner.scan(jar.toPath(), "mr").toList() }) + } + + @Test + fun `a virtual file scan skips versioned entries`() { + val jar = multiReleaseJar() + val disposable = Disposer.newDisposable() + val jarFileSystem = CoreJarFileSystem() + try { + // Reading a jar entry's contents consults the application's file size limits. + ApplicationManager.setApplication(MockApplication(disposable), disposable) + val root = jarFileSystem.findFileByPath("${jar.absolutePath}!/")!! + + assertOnlyBaseClasses(CombinedJarScanner.scan(root, "mr").toList()) + } finally { + // The handler this test opened for the jar keeps its file handle open; releasing it here + // is what lets TemporaryFolder delete the jar afterwards. + jarFileSystem.clearHandlersCache() + Disposer.dispose(disposable) + } + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarFingerprintTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarFingerprintTest.kt new file mode 100644 index 0000000000..14d53d6d7e --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarFingerprintTest.kt @@ -0,0 +1,38 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class JarFingerprintTest { + @get:Rule + val temp = TemporaryFolder() + + @Test + fun `rewriting a JAR with a different size changes its fingerprint`() { + val jar = temp.newFile("lib.jar").apply { writeBytes(ByteArray(10)) } + val before = jarFingerprint(jar) + val mtime = jar.lastModified() + + jar.writeBytes(ByteArray(20)) + jar.setLastModified(mtime) + + assertThat(jarFingerprint(jar)).isNotEqualTo(before) + } + + @Test + fun `rewriting a JAR at the same size changes its fingerprint through the modification time`() { + val jar = temp.newFile("lib.jar").apply { writeBytes(ByteArray(10)) } + jar.setLastModified(1_000_000L) + val before = jarFingerprint(jar) + + jar.writeBytes(ByteArray(10) { 1 }) + jar.setLastModified(2_000_000L) + + assertThat(jarFingerprint(jar)).isNotEqualTo(before) + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerDollarNameTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerDollarNameTest.kt new file mode 100644 index 0000000000..89059ff5c8 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerDollarNameTest.kt @@ -0,0 +1,56 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * A '$' in a class file name does not make it nested: code generators emit top-level classes such + * as `Foo$$ViewBinder` and `$AutoValue_Foo`. Only the class's own InnerClasses entry does. + */ +@RunWith(JUnit4::class) +class JarSymbolScannerDollarNameTest { + private fun scanClass( + internalName: String, + outerName: String? = null, + innerName: String? = null, + ): JvmSymbol { + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, internalName, null, "java/lang/Object", null) + if (innerName != null) visitInnerClass(internalName, outerName, innerName, Opcodes.ACC_PUBLIC) + visitEnd() + }.toByteArray() + return JarSymbolScanner.parseClassFile(bytes.inputStream(), "test.jar").single() + } + + @Test + fun `a top-level class with a doubled dollar in its name is top level`() { + val binder = scanClass("com/example/Foo\$\$ViewBinder") + + assertThat(binder.isTopLevel).isTrue() + assertThat(binder.shortName).isEqualTo("Foo\$\$ViewBinder") + } + + @Test + fun `a top-level class whose name starts with a dollar is top level`() { + val autoValue = scanClass("com/example/\$AutoValue_Foo") + + assertThat(autoValue.isTopLevel).isTrue() + assertThat(autoValue.shortName).isEqualTo("\$AutoValue_Foo") + } + + @Test + fun `a class with its own InnerClasses entry is nested in its outer class`() { + val nested = scanClass("com/example/Outer\$Inner", outerName = "com/example/Outer", innerName = "Inner") + + assertThat(nested.isTopLevel).isFalse() + assertThat(nested.containingClassName).isEqualTo("com/example/Outer") + assertThat(nested.data.containingClassFqName).isEqualTo("com.example.Outer") + assertThat(nested.shortName).isEqualTo("Inner") + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt new file mode 100644 index 0000000000..5cd3693cb6 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt @@ -0,0 +1,170 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.InputStream +import java.io.OutputStream +import java.nio.file.Files + +/** + * Which classes [JarSymbolScanner] admits, by visibility. + * + * The classpath trie this index replaces held every top-level class in a JAR regardless of + * visibility, so dropping package-private ones would silently narrow completion. + */ +@RunWith(JUnit4::class) +class JarSymbolScannerVisibilityTest { + private fun classBytes( + internalName: String, + access: Int, + ): ByteArray = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, access, internalName, null, "java/lang/Object", null) + visitEnd() + }.toByteArray() + + private fun scan( + internalName: String, + access: Int, + ): List = scan(classBytes(internalName, access)) + + private fun scan(bytes: ByteArray): List = JarSymbolScanner.parseClassFile(bytes.inputStream(), "test.jar") + + private fun scanNested(simpleName: String): List = scan(nestedClasses.getValue("com/example/Outer\$$simpleName")) + + private fun nestedClass(simpleName: String): JvmSymbol = scanNested(simpleName).single { it.kind.isJvmClass } + + @Test + fun `a public class is indexed`() { + val symbols = scan("com/example/Pub", Opcodes.ACC_PUBLIC) + + assertThat(symbols.map { it.shortName }).containsExactly("Pub") + assertThat(symbols.single().visibility).isEqualTo(JvmVisibility.PUBLIC) + } + + @Test + fun `a package-private class is indexed and keeps its visibility`() { + val symbols = scan("com/example/Pkg", 0) + + assertThat(symbols.map { it.shortName }).containsExactly("Pkg") + assertThat(symbols.single().visibility).isEqualTo(JvmVisibility.PACKAGE_PRIVATE) + } + + @Test + fun `a private nested class is not indexed`() { + /* + * JVMS 4.1 forbids ACC_PRIVATE in a class file header, so javac writes ACC_SUPER there and + * records `private` only in the InnerClasses attribute. + */ + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_SUPER, "com/example/Outer\$Hidden", null, "java/lang/Object", null) + visitInnerClass( + "com/example/Outer\$Hidden", + "com/example/Outer", + "Hidden", + Opcodes.ACC_PRIVATE or Opcodes.ACC_STATIC, + ) + visitEnd() + }.toByteArray() + + assertThat(scan(bytes)).isEmpty() + } + + @Test + fun `a private nested class compiled by javac is not indexed`() { + assertThat(scanNested("Hidden")).isEmpty() + } + + @Test + fun `a protected nested class compiled by javac is indexed as protected`() { + // javac widens the header of a protected nested class to public. + assertThat(nestedClass("Guarded").visibility).isEqualTo(JvmVisibility.PROTECTED) + } + + @Test + fun `a static nested class compiled by javac is static, not inner`() { + val info = nestedClass("Open").data as JvmClassInfo + + assertThat(info.isStatic).isTrue() + assertThat(info.isInner).isFalse() + } + + @Test + fun `an inner class compiled by javac keeps package-private visibility and is inner`() { + val symbol = nestedClass("Pkg") + val info = symbol.data as JvmClassInfo + + assertThat(symbol.visibility).isEqualTo(JvmVisibility.PACKAGE_PRIVATE) + assertThat(info.isInner).isTrue() + assertThat(info.isStatic).isFalse() + } + + companion object { + /** + * JUnit4 builds a fresh test instance per `@Test` method, so an instance-level `by lazy` + * would recompile this fixture once per test. A companion `by lazy` is shared by every + * instance in the class, so javac runs once for the whole class. + */ + private val nestedClasses: Map by lazy { + compileJava( + "com.example.Outer", + """ + package com.example; + + public class Outer { + private static class Hidden {} + protected static class Guarded {} + public static class Open {} + class Pkg {} + } + """.trimIndent(), + ) + } + } +} + +/** + * Compiles one Java source file with the test JVM's own javac, returning each produced class file + * by internal name. + * + * Reached through reflection because this Android module compiles against `android.jar`, which has + * no `javax.tools`; the JDK running the tests does. + */ +private fun compileJava( + className: String, + source: String, +): Map { + val workDir = Files.createTempDirectory("javac-fixture") + try { + val sourceFile = workDir.resolve("${className.substringAfterLast('.')}.java") + sourceFile.toFile().writeText(source) + val outDir = Files.createDirectory(workDir.resolve("out")) + + val compiler = Class.forName("javax.tools.ToolProvider").getMethod("getSystemJavaCompiler").invoke(null) + assertWithMessage("the test JVM has no system Java compiler").that(compiler).isNotNull() + val run = + Class + .forName("javax.tools.Tool") + .getMethod("run", InputStream::class.java, OutputStream::class.java, OutputStream::class.java, Array::class.java) + val args = arrayOf("--release", "11", "-d", outDir.toString(), sourceFile.toString()) + val exitCode = run.invoke(compiler, null, null, null, args) as Int + assertWithMessage("javac exit code for $className").that(exitCode).isEqualTo(0) + + return Files.walk(outDir).use { paths -> + paths + .filter { it.toString().endsWith(".class") } + .toList() + .associate { outDir.relativize(it).toString().removeSuffix(".class") to Files.readAllBytes(it) } + } + } finally { + workDir.toFile().deleteRecursively() + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt new file mode 100644 index 0000000000..e51a84c955 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt @@ -0,0 +1,160 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * The package and simple-name lookups filter by kind in the query rather than over its result. + * + * A package holds far more members than top-level declarations, so a post-filter meant fetching + * every symbol in the package to keep a handful -- and the caller's limit was spent on rows it + * would then throw away. + */ +@RunWith(JUnit4::class) +class JvmSymbolIndexKindScopeTest { + private fun index(): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { + override fun visibleSourceIds(): Collection? = null + } + } + + private fun classifier( + simpleName: String, + pkg: String = "com.example", + ) = JvmSymbol( + key = "$pkg.$simpleName", + sourceId = "test.jar", + name = "$pkg.$simpleName", + shortName = simpleName, + packageName = pkg, + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(internalName = "$pkg/$simpleName"), + ) + + private fun member( + simpleName: String, + owner: String, + pkg: String = "com.example", + ) = JvmSymbol( + key = "$pkg.$owner#$simpleName", + sourceId = "test.jar", + name = "$pkg.$owner#$simpleName", + shortName = simpleName, + packageName = pkg, + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.JAVA, + data = JvmFunctionInfo(containingClassName = "$pkg/$owner"), + ) + + private fun topLevelFunction( + simpleName: String, + pkg: String = "com.example", + ) = JvmSymbol( + key = "$pkg#$simpleName", + sourceId = "test.jar", + name = "$pkg#$simpleName", + shortName = simpleName, + packageName = pkg, + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.KOTLIN, + data = JvmFunctionInfo(), + ) + + @Test + fun `classifiers in a package exclude that package's members`() = + runTest { + val index = index() + index.insert(classifier("Widget")) + repeat(50) { index.insert(member("helper$it", owner = "Widget")) } + + val found = index.findClassifiersInPackage("com.example").map { it.shortName }.toList() + + assertThat(found).containsExactly("Widget") + } + + @Test + fun `a classifier is still returned when members would have filled the limit`() = + runTest { + val index = index() + // Members are inserted first, so a post-filter would spend the limit on them. + repeat(50) { index.insert(member("helper$it", owner = "Widget")) } + index.insert(classifier("Widget")) + + val found = index.findClassifiersInPackage("com.example", limit = 5).map { it.shortName }.toList() + + assertThat(found).containsExactly("Widget") + } + + @Test + fun `top-level callables in a package exclude members of its classes`() = + runTest { + val index = index() + index.insert(topLevelFunction("launch")) + repeat(20) { index.insert(member("helper$it", owner = "Widget")) } + + val found = index.findTopLevelCallablesInPackage("com.example").map { it.shortName }.toList() + + assertThat(found).containsExactly("launch") + } + + @Test + fun `a simple-name lookup can be restricted to classifiers`() = + runTest { + val index = index() + index.insert(classifier("Widget")) + index.insert(member("Widget", owner = "Factory")) + + val all = index.findBySimpleName("Widget").toList() + val classifiersOnly = + index.findBySimpleName("Widget", kinds = JvmSymbolKind.CLASSIFIER_KINDS).toList() + + assertThat(all).hasSize(2) + assertThat(classifiersOnly.map { it.kind }).containsExactly(JvmSymbolKind.CLASS) + } + + @Test + fun `a prefix lookup can be restricted by kind`() = + runTest { + val index = index() + index.insert(classifier("Widget")) + index.insert(member("WidgetFactory", owner = "Factory")) + + val found = index.findByPrefix("Widget", kinds = JvmSymbolKind.CLASSIFIER_KINDS).toList() + + assertThat(found.map { it.shortName }).containsExactly("Widget") + } + + @Test + fun `a file facade is not offered as a classifier`() = + runTest { + val index = index() + index.insert( + JvmSymbol( + key = "com/example/WidgetKt", + sourceId = "test.jar", + name = "com/example/WidgetKt", + shortName = "WidgetKt", + packageName = "com.example", + kind = JvmSymbolKind.FILE_FACADE, + language = JvmSourceLanguage.KOTLIN, + data = JvmClassInfo(internalName = "com/example/WidgetKt"), + ), + ) + + // Kotlin's import action asks for classifiers, and a facade is not one. + assertThat( + index.findBySimpleName("WidgetKt", kinds = JvmSymbolKind.CLASSIFIER_KINDS).toList(), + ).isEmpty() + // It is still findable as a class, which is what Java needs. + assertThat( + index.findBySimpleName("WidgetKt", kinds = JvmSymbolKind.JVM_CLASS_KINDS).toList(), + ).hasSize(1) + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexOptimizeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexOptimizeTest.kt new file mode 100644 index 0000000000..fb93bf48a1 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexOptimizeTest.kt @@ -0,0 +1,72 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.util.concurrent.CopyOnWriteArrayList + +/** When an indexing pass optimizes the index behind a [JvmSymbolIndex]. */ +@RunWith(JUnit4::class) +class JvmSymbolIndexOptimizeTest { + /** Records which of the test's sources the index held each time it was optimized. */ + private class RecordingIndex( + private val backing: Index, + ) : Index by backing { + val sourcesAtOptimize = CopyOnWriteArrayList>() + + override suspend fun optimize() { + sourcesAtOptimize += listOf(FIRST, SECOND).filter { backing.containsSource(it) }.toSet() + } + } + + @Test + fun `a pass with a gap between submissions optimizes once, after every job`() = + runTest { + val backing = RecordingIndex(InMemoryIndex(JvmSymbolDescriptor)) + val index = JvmSymbolIndex(backing, BackgroundIndexer(backing)) + + val first = index.indexSource(FIRST, skipIfExists = false) { flowOf(classifier("Foo", it)) } + // The gap: the first job finishes before the second is submitted, leaving the indexer idle mid-pass. + first.join() + val second = index.indexSource(SECOND, skipIfExists = false) { flowOf(classifier("Bar", it)) } + index.optimizeAfter(listOf(first, second)) + + assertThat(backing.sourcesAtOptimize).isEqualTo(listOf(setOf(FIRST, SECOND))) + } + + @Test + fun `a pass that submits nothing does not optimize`() = + runTest { + val backing = RecordingIndex(InMemoryIndex(JvmSymbolDescriptor)) + val index = JvmSymbolIndex(backing, BackgroundIndexer(backing)) + + index.optimizeAfter(emptyList()) + + assertThat(backing.sourcesAtOptimize).isEmpty() + } + + private fun classifier( + simpleName: String, + sourceId: String, + ) = JvmSymbol( + key = "com.example.$simpleName", + sourceId = sourceId, + name = "com.example.$simpleName", + shortName = simpleName, + packageName = "com.example", + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(internalName = "com/example/$simpleName"), + ) + + private companion object { + const val FIRST = "/libs/first.jar" + const val SECOND = "/libs/second.jar" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerDollarNameTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerDollarNameTest.kt new file mode 100644 index 0000000000..7f9707e80f --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerDollarNameTest.kt @@ -0,0 +1,77 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import kotlin.metadata.ClassKind +import kotlin.metadata.KmClass +import kotlin.metadata.KmClassifier +import kotlin.metadata.KmType +import kotlin.metadata.Visibility +import kotlin.metadata.jvm.JvmMetadataVersion +import kotlin.metadata.jvm.KotlinClassMetadata +import kotlin.metadata.kind +import kotlin.metadata.visibility + +/** + * A '$' in a Kotlin class's metadata name is part of a declared name, never nesting, yet the index + * treats it as nesting so it holds the same top-level classes the classpath trie did. + */ +@RunWith(JUnit4::class) +class KotlinMetadataScannerDollarNameTest { + /** A class file for [name] carrying genuine `@kotlin.Metadata` written by kotlin-metadata-jvm. */ + private fun kotlinClassFile(name: String): ByteArray { + val kmClass = + KmClass().apply { + this.name = name + kind = ClassKind.CLASS + visibility = Visibility.PUBLIC + supertypes += KmType().apply { classifier = KmClassifier.Class("kotlin/Any") } + } + val metadata = KotlinClassMetadata.Class(kmClass, JvmMetadataVersion.LATEST_STABLE_SUPPORTED, 0).write() + + return ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, name, null, "java/lang/Object", null) + visitAnnotation("Lkotlin/Metadata;", true).apply { + visit("k", metadata.kind) + visit("mv", metadata.metadataVersion) + visit("xi", metadata.extraInt) + visitArray("d1").apply { + metadata.data1.forEach { visit(null, it) } + visitEnd() + } + visitArray("d2").apply { + metadata.data2.forEach { visit(null, it) } + visitEnd() + } + visitEnd() + } + visitEnd() + }.toByteArray() + } + + private fun classSymbol(name: String): JvmSymbol = + KotlinMetadataScanner + .parseKotlinClass(kotlinClassFile(name).inputStream(), "test.jar")!! + .single { it.kind.isClassifier } + + @Test + fun `a dollar in a Kotlin class name is treated as nesting`() { + val symbol = classSymbol("com/example/Odd\$Name") + + assertThat(symbol.shortName).isEqualTo("Name") + assertThat(symbol.containingClassName).isEqualTo("com/example/Odd") + } + + @Test + fun `a Kotlin class without a dollar is top level`() { + val symbol = classSymbol("com/example/Plain") + + assertThat(symbol.shortName).isEqualTo("Plain") + assertThat(symbol.isTopLevel).isTrue() + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt new file mode 100644 index 0000000000..ce6d1f17a4 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt @@ -0,0 +1,142 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.util.jar.JarFile + +/** + * [KotlinMetadataScanner] against the real Kotlin standard library, which is the only convenient + * source of genuine `@kotlin.Metadata` in the shapes that matter: a file facade, a multi-file facade + * and a nested class. + */ +@RunWith(JUnit4::class) +class KotlinMetadataScannerShapeTest { + private val stdlibJar: File by lazy { + val location = + File( + Unit::class.java.protectionDomain.codeSource.location + .toURI(), + ) + assertWithMessage("kotlin-stdlib must resolve to a JAR on the test classpath, got $location") + .that(location.isFile && location.name.endsWith(".jar")) + .isTrue() + location + } + + private fun symbolsFor(entryName: String): List = + JarFile(stdlibJar).use { file -> + val entry = file.getJarEntry(entryName) + assertWithMessage("$entryName in ${stdlibJar.name}").that(entry).isNotNull() + file.getInputStream(entry).use { KotlinMetadataScanner.parseKotlinClass(it, "stdlib") } + } ?: emptyList() + + private fun classifierFor(entryName: String): JvmSymbol = symbolsFor(entryName).single { it.kind.isClassifier } + + @Test + fun `a nested class reports its simple name, not the outer-qualified one`() { + val companion = symbolsFor("kotlin/text/Regex\$Companion.class").first { it.kind.isClassifier } + + // Kotlin metadata spells this "kotlin/text/Regex.Companion", so splitting on '$' alone left + // the short name as "Regex.Companion" and no prefix search could reach it. + assertThat(companion.shortName).isEqualTo("Companion") + } + + @Test + fun `a nested class is not reported as top level`() { + val companion = symbolsFor("kotlin/text/Regex\$Companion.class").first { it.kind.isClassifier } + + assertThat(companion.isTopLevel).isFalse() + assertThat(companion.data.containingClassFqName).isEqualTo("kotlin.text.Regex") + } + + @Test + fun `a nested class is keyed by its class file name, as the Java scanner keys it`() { + val companion = classifierFor("kotlin/text/Regex\$Companion.class") + + // Metadata spells it "kotlin/text/Regex.Companion"; the class file and JarSymbolScanner use '$'. + assertThat(companion.key).isEqualTo("kotlin/text/Regex\$Companion") + assertThat((companion.data as JvmClassInfo).internalName).isEqualTo("kotlin/text/Regex\$Companion") + } + + @Test + fun `a doubly nested class names its containing class in class file form`() { + val companion = classifierFor("kotlin/text/Regex\$Serialized\$Companion.class") + + assertThat(companion.containingClassName).isEqualTo("kotlin/text/Regex\$Serialized") + } + + @Test + fun `members of a nested class name it in class file form`() { + val members = symbolsFor("kotlin/text/Regex\$Companion.class").filter { it.kind.isCallable } + + assertThat(members).isNotEmpty() + assertThat(members.map { it.containingClassName }.toSet()).containsExactly("kotlin/text/Regex\$Companion") + } + + @Test + fun `a file facade is indexed as a class in its own right`() { + val symbols = symbolsFor("kotlin/io/CloseableKt.class") + + val facade = symbols.singleOrNull { it.kind == JvmSymbolKind.FILE_FACADE } + assertThat(facade).isNotNull() + assertThat(facade!!.shortName).isEqualTo("CloseableKt") + assertThat(facade.fqName).isEqualTo("kotlin.io.CloseableKt") + assertThat(facade.packageName).isEqualTo("kotlin.io") + } + + @Test + fun `a multi-file facade is indexed even though it declares nothing itself`() { + val symbols = symbolsFor("kotlin/collections/CollectionsKt.class") + + val facade = symbols.singleOrNull { it.kind == JvmSymbolKind.FILE_FACADE } + assertThat(facade).isNotNull() + assertThat(facade!!.fqName).isEqualTo("kotlin.collections.CollectionsKt") + } + + @Test + fun `a facade is a JVM class but not a classifier`() { + val facade = + symbolsFor("kotlin/io/CloseableKt.class").single { it.kind == JvmSymbolKind.FILE_FACADE } + + // Java calls top-level functions through the facade, so it must be findable as a class. + assertThat(facade.kind.isJvmClass).isTrue() + // Kotlin refers to the declarations, never the facade, so it must not be offered as a type. + assertThat(facade.kind.isClassifier).isFalse() + } + + @Test + fun `the declarations inside a facade are still indexed alongside it`() { + val symbols = symbolsFor("kotlin/io/CloseableKt.class") + + assertThat(symbols.any { it.kind.isCallable }).isTrue() + } + + @Test + fun `extension overloads that differ only by receiver get distinct keys`() { + // Iterable.first() and List.first(): same name, same (empty) parameter list. + val firsts = noArgFirstExtensions() + + assertThat(firsts.map { it.receiverTypeName }).containsAtLeast("kotlin/collections/Iterable", "kotlin/collections/List") + assertThat(firsts.map { it.key }.toSet()).hasSize(firsts.size) + } + + @Test + fun `a callable's key survives a round trip through the index payload`() { + val firsts = noArgFirstExtensions() + + val restored = firsts.map { JvmSymbolDescriptor.deserialize(JvmSymbolDescriptor.serialize(it)) } + + assertThat(restored.map { it.key }).containsExactlyElementsIn(firsts.map { it.key }).inOrder() + } + + private fun noArgFirstExtensions(): List = + symbolsFor("kotlin/collections/CollectionsKt___CollectionsKt.class").filter { + it.kind == JvmSymbolKind.EXTENSION_FUNCTION && + it.shortName == "first" && + (it.data as JvmFunctionInfo).parameters.isEmpty() + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerVersionTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerVersionTest.kt new file mode 100644 index 0000000000..58bd72755f --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerVersionTest.kt @@ -0,0 +1,62 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.AnnotationVisitor +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.util.jar.JarFile + +/** A class compiled by a newer Kotlin than the bundled metadata reader supports is still indexed. */ +@RunWith(JUnit4::class) +class KotlinMetadataScannerVersionTest { + private fun stdlibClass(entryName: String): ByteArray { + val stdlib = + File( + Unit::class.java.protectionDomain.codeSource.location + .toURI(), + ) + return JarFile(stdlib).use { jar -> jar.getInputStream(jar.getJarEntry(entryName)).use { it.readBytes() } } + } + + /** Rewrites the `mv` (metadata version) of [classBytes]'s `@kotlin.Metadata` to [version]. */ + private fun withMetadataVersion( + classBytes: ByteArray, + version: IntArray, + ): ByteArray { + val writer = ClassWriter(0) + ClassReader(classBytes).accept( + object : ClassVisitor(Opcodes.ASM9, writer) { + override fun visitAnnotation( + descriptor: String?, + visible: Boolean, + ): AnnotationVisitor? { + val delegate = super.visitAnnotation(descriptor, visible) + if (descriptor != "Lkotlin/Metadata;") return delegate + return object : AnnotationVisitor(Opcodes.ASM9, delegate) { + override fun visit( + name: String?, + value: Any?, + ) = super.visit(name, if (name == "mv") version else value) + } + } + }, + 0, + ) + return writer.toByteArray() + } + + @Test + fun `a class with metadata newer than the reader supports is still indexed`() { + val newer = withMetadataVersion(stdlibClass("kotlin/Pair.class"), intArrayOf(99, 0, 0)) + + val symbols = KotlinMetadataScanner.parseKotlinClass(newer.inputStream(), "stdlib").orEmpty() + + assertThat(symbols.filter { it.kind.isClassifier }.map { it.key }).containsExactly("kotlin/Pair") + } +} diff --git a/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto index 7f7e5517e6..742d870bb4 100644 --- a/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto +++ b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto @@ -18,6 +18,9 @@ message JvmSymbolData { JvmVisibility visibility = 7; bool is_deprecated = 8; + // The row key, stored because no other field can rebuild it. + string key = 9; + oneof data { ClassData class_data = 20; FunctionData function_data = 21; @@ -192,6 +195,9 @@ enum JvmSymbolKind { KIND_EXTENSION_PROPERTY = 16; KIND_FIELD = 17; KIND_TYPE_ALIAS = 18; + // The synthetic class a Kotlin file's top-level declarations compile into ("FooKt"). + // Java can name it; Kotlin never should. + KIND_FILE_FACADE = 19; } enum JvmSourceLanguage { diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt index 17bdd07f33..7525dcc0f4 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt @@ -27,6 +27,10 @@ import com.itsaky.androidide.utils.flashError import com.itsaky.androidide.utils.flashInfo import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.jvm.JvmVisibility +import org.appdevforall.codeonthego.indexing.jvm.dedupeKey import java.nio.file.Path class AddImportAction : BaseKotlinCodeAction() { @@ -48,9 +52,11 @@ class AddImportAction : BaseKotlinCodeAction() { return } - // Optimistic visibility: decide from the in-memory unresolved-reference marker only. The - // importable-classifier resolution runs in the background execAction; doing it here would be - // main-thread SQLite I/O, because fillMenu() calls prepare() synchronously on the UI thread. + /* + * Optimistic visibility: decide from the in-memory unresolved-reference marker only. The + * importable-classifier resolution runs in the background execAction; doing it here would be + * main-thread SQLite I/O, because fillMenu() calls prepare() synchronously on the UI thread. + */ val resolveReferenceActionDiagnostic = data.findDiagnosticExtra() if (resolveReferenceActionDiagnostic == null) { @@ -86,15 +92,24 @@ class AddImportAction : BaseKotlinCodeAction() { referenceName: String, ): ImportCandidates { /* - * Resolved before the file is pinned, not inside the pin: this is an unbounded SQLite scan that - * never reads the file, and a pin held across it freezes live-PSI refresh for the path - every + * Resolved before the file is pinned, not inside the pin: this is a SQLite read that never + * reads the file, and a pin held across it freezes live-PSI refresh for the path - every * concurrent acquirer joins the frozen instance and the refresh is only owed on release. * Materialized here too, so the index's lazy source-active filter cannot trail into the scope. + * + * Left unbounded deliberately: dropping a candidate here would hide an importable class from + * the chooser. It stays cheap because it is an exact-name lookup on an indexed column narrowed + * to classifier kinds, so the row count is the number of classes with that simple name, not a + * scan. The kind restriction also keeps Kotlin from being offered a file facade, which is not + * a classifier. + * + * The same class indexed from two JARs on the active source set is two rows here, possibly + * with different visibility, so they are deduplicated below only after the visibility filter: + * deduplicating first could keep a hidden copy and drop the importable one. */ val classifiers = env.ktSymbolIndex - .findSymbolBySimpleName(referenceName, limit = 0) - .filter { it.kind.isClassifier } + .findSymbolBySimpleName(referenceName, limit = 0, kinds = JvmSymbolKind.CLASSIFIER_KINDS) .toList() if (classifiers.isEmpty()) { @@ -111,7 +126,11 @@ class AddImportAction : BaseKotlinCodeAction() { val candidates = live.read { ktFile -> - classifiers.associate { it.fqName to insertImport(ktFile, it.fqName) } + val currentPackage = ktFile.packageFqName.asString().ifEmpty { null } + classifiers + .filter { it.isImportableFrom(currentPackage) } + .distinctBy { it.dedupeKey } + .associate { it.fqName to insertImport(ktFile, it.fqName) } } if (live.isStale) { @@ -233,6 +252,21 @@ class AddImportAction : BaseKotlinCodeAction() { } } +/** + * Whether [this] classifier is one a file in [currentPackage] could actually import. + * + * A private declaration is never importable, and a package-private one only from its own package. + * Public, protected, and internal candidates pass through unchanged: telling them apart needs the + * use-site module that completion's visibility checker resolves, which is not available here, so + * this only covers the visibility distinctions that a package name settles on its own. + */ +private fun JvmSymbol.isImportableFrom(currentPackage: String?): Boolean = + when (visibility) { + JvmVisibility.PRIVATE -> false + JvmVisibility.PACKAGE_PRIVATE -> packageName == currentPackage + JvmVisibility.PUBLIC, JvmVisibility.PROTECTED, JvmVisibility.INTERNAL -> true + } + /** * The outcome of resolving import candidates. * diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt index 7c3625d422..7ec5949de6 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt @@ -24,6 +24,7 @@ import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.future.await import kotlinx.coroutines.launch import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolIndex +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind import org.appdevforall.codeonthego.indexing.jvm.KtFileMetadataIndex import org.checkerframework.checker.index.qual.NonNegative import org.jetbrains.kotlin.analysis.api.KaSession @@ -458,9 +459,11 @@ internal class KtSymbolIndex( null } - // Applied on the way out rather than during the pin: the version bump that arrived while the path - // was frozen still has to reach the FIR session. Skipped once the document is gone, since - // invalidateCurrent already unregistered it. + /* + * Applied on the way out rather than during the pin: the version bump that arrived while the path + * was frozen still has to reach the FIR session. Skipped once the document is gone, since + * invalidateCurrent already unregistered it. + */ if (refreshOwed && FileManager.isActive(path)) { scope.launch { refreshCurrentKtFile(path) } } @@ -583,11 +586,13 @@ internal class KtSymbolIndex( scanningJob?.cancelAndJoin() indexingJob?.join() - // Cancel AND JOIN the index's own scope. Beyond the main worker loop drained above, the - // debounced modifiedFileIndexer and queueOnFileChangedAsync coroutines also run - // project.read { PsiManager … }. Joining guarantees none survive into the caller's - // Disposer.dispose(...), which would otherwise crash with "Project is already disposed" - // (APPDEVFORALL-17R). This index owns `scope`. + /* + * Cancel AND JOIN the index's own scope. Beyond the main worker loop drained above, the + * debounced modifiedFileIndexer and queueOnFileChangedAsync coroutines also run + * project.read { PsiManager ... }. Joining guarantees none survive into the caller's + * Disposer.dispose(...), which would otherwise crash with "Project is already disposed" once + * one of them touches the project after it is gone. This index owns `scope`. + */ scope.coroutineContext[Job]?.cancelAndJoin() // Drain the refresh pool before disposal: refreshToCurrent runs project.read/write on it (same @@ -610,9 +615,17 @@ internal fun KtSymbolIndex.subpackageNames(packageFqn: String) = fileIndex.getSu * [org.appdevforall.codeonthego.indexing.api.ReadableIndex.query] ("If IndexQuery.limit is 0, all * matches are emitted"). A plain `take(limit)` would turn the common `limit = 0` call into * `take(0)`, silently yielding no results. + * + * The limit is passed to each index rather than applied only to the concatenation. Applying it + * afterwards made every call fetch both indexes in full, so a caller asking for ten paid for all of + * them. [kinds] narrows the fetch for the same reason: an exact-name lookup otherwise returns every + * method and field that happens to share the name. */ internal fun KtSymbolIndex.findSymbolBySimpleName( name: String, limit: Int, -) = (sourceIndex.findBySimpleName(name, 0) + libraryIndex.findBySimpleName(name, 0)) - .let { if (limit <= 0) it else it.take(limit) } + kinds: Set? = null, +) = ( + sourceIndex.findBySimpleName(name, limit, kinds) + + libraryIndex.findBySimpleName(name, limit, kinds) +).let { if (limit <= 0) it else it.take(limit) } diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt index 313b9acc49..f6c9d6698d 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt @@ -27,6 +27,7 @@ import org.appdevforall.codeonthego.indexing.jvm.KotlinFunctionInfo import org.appdevforall.codeonthego.indexing.jvm.KotlinPropertyInfo import org.appdevforall.codeonthego.indexing.jvm.KtFileMetadata import org.appdevforall.codeonthego.indexing.jvm.KtFileMetadataIndex +import org.appdevforall.codeonthego.indexing.jvm.jvmCallableKey import org.jetbrains.kotlin.analysis.api.KaExperimentalApi import org.jetbrains.kotlin.analysis.api.KaImplementationDetail import org.jetbrains.kotlin.analysis.api.KaSession @@ -327,7 +328,8 @@ private fun KaSession.analyzeFunction( } else { "$pkg#$fnName" } - val key = "$qualifiedName(${parameters.joinToString(",") { it.typeFqName }})" + val key = + jvmCallableKey(qualifiedName, receiverType?.let { kaTypeInternalName(it) }, parameters.map { it.typeFqName }) val signatureDisplay = buildString { @@ -473,7 +475,7 @@ private fun KaSession.analyzeProperty( } return JvmSymbol( - key = qualifiedName, + key = jvmCallableKey(qualifiedName, receiverType?.let { kaTypeInternalName(it) }, parameterTypeNames = null), sourceId = filePath, name = qualifiedName, shortName = propName, diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 09906f6a6c..bf257190c7 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -31,8 +31,10 @@ import io.github.rosemoe.sora.lang.completion.CompletionCancelledException import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolIndex import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind import org.appdevforall.codeonthego.indexing.jvm.JvmTypeAliasInfo +import org.appdevforall.codeonthego.indexing.jvm.dedupeKey import org.jetbrains.kotlin.analysis.api.KaExperimentalApi import org.jetbrains.kotlin.analysis.api.KaIdeApi import org.jetbrains.kotlin.analysis.api.KaSession @@ -77,8 +79,25 @@ private const val KT_COMPLETION_PLACEHOLDER = "KT_COMPLETION_PLACEHOLDER" private val logger = LoggerFactory.getLogger("KotlinCompletions") -/** Max unimported symbols pulled from each index for scope completion (see [collectUnimportedSymbols]). */ -private const val UNIMPORTED_SYMBOL_LIMIT = 100 +/** + * Max unimported symbols [collectUnimportedSymbols] adds to one completion result, across every + * index rather than per index. + * + * This counts items the user is actually offered, not rows a query returned, so a prefix whose + * leading rows are all rejected by the package, visibility, or kind filters still yields whatever + * valid matches exist instead of coming up empty. + */ +private const val UNIMPORTED_SYMBOL_DISPLAY_LIMIT = 100 + +/** + * Rows each index's prefix query may return before filtering, sized so ordinary filtering cannot + * starve the result. + * + * The queries already leave out every row [isUnimportedSymbolCandidate] would reject, so the budget + * only has to cover the current-package and visibility filters, without being unbounded: an + * unrestricted prefix query over the symbol table runs on every keystroke. + */ +private const val UNIMPORTED_SYMBOL_FETCH_BUDGET = 3 * UNIMPORTED_SYMBOL_DISPLAY_LIMIT /** * The [ScheduledCancelChecker] for the completion running on this thread, set for the duration of @@ -412,10 +431,10 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) val useSiteModule = this.useSiteModule val visibilityChecker = env.symbolVisibilityChecker - fun addCompletionItem(symbol: JvmSymbol) { + fun addCompletionItem(symbol: JvmSymbol): Boolean { abortIfCancelled() - if (symbol.packageName == currentPackage) return + if (symbol.packageName == currentPackage) return false val isVisible = visibilityChecker.isVisible( @@ -424,31 +443,171 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) useSitePackage = currentPackage, ) - if (!isVisible) return + if (!isVisible) return false - buildUnimportedSymbolItem(symbol)?.let { to += it } + val item = buildUnimportedSymbolItem(symbol) ?: return false + to += item + return true } - env.libraryIndex - ?.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_LIMIT) - ?.forEach(::addCompletionItem) + /* + * Source first, library last. One shared cap across the indexes means whichever is queried last + * loses when the cap binds, and a symbol from the user's own project is likelier to be the one + * they are reaching for than one from a dependency. + */ + val indexes = listOfNotNull(env.sourceIndex, env.generatedIndex, env.libraryIndex) + + collectUnimportedSymbolMatches( + indexes = indexes, + partial = ctx.partial, + kinds = UNIMPORTED_SYMBOL_KINDS, + topLevelKinds = UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS, + limit = UNIMPORTED_SYMBOL_DISPLAY_LIMIT, + fetchBudget = UNIMPORTED_SYMBOL_FETCH_BUDGET, + accept = ::addCompletionItem, + ) +} + +/** + * The unimported-symbol matches [collectUnimportedSymbols] considers for [partial] across [indexes], + * handed to [accept] in query order and capped at [limit] accepted symbols. + * + * An exact simple-name match from every index is collected before any index's prefix match. The + * exact stage is a small, indexed equality query, cheap enough to run unconditionally, and running + * it first keeps a cap the prefix stage fills on its own from crowding out an exact match sitting in + * a later index -- typically the library index, the last one queried, where the class the user is + * actually typing is likeliest to live. Within each stage, indexes are queried in [indexes] order. + * + * A match is deduplicated by [JvmSymbol] identity ([dedupeKey]), keeping the first *accepted* + * occurrence in that order and discarding the rest: since a row's key is unique only per source, the + * same class indexed from two JARs on the active source set otherwise yields two rows for what is + * one class. A key is recorded only once [accept] has taken it, not merely seen it, so a row rejected + * by [accept] (for instance, a module that cannot reach that row's source) does not block a later + * duplicate from a source [accept] does allow through. A rejected row's own source and key are + * remembered too, so the same physical row surfacing again from the prefix stage is skipped without + * a second call to [accept]. + * + * [kinds] are matched wherever they are declared, [topLevelKinds] only at the top level. + */ +internal fun collectUnimportedSymbolMatches( + indexes: List, + partial: String, + kinds: Set, + topLevelKinds: Set = emptySet(), + limit: Int, + fetchBudget: Int, + accept: (JvmSymbol) -> Boolean, +): Int { + val seen = mutableSetOf() + val rejectedRows = mutableSetOf>() + + fun acceptFirstOccurrence(symbol: JvmSymbol): Boolean { + val key = symbol.dedupeKey + if (key in seen) return false - env.sourceIndex - ?.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_LIMIT) - ?.forEach(::addCompletionItem) + /* + * Identifies this exact row (source and key), not the class it names: the exact and prefix + * stages can both return the very same row for one index, and a row `accept` already turned + * down is not worth asking about again. A different source sharing `key`'s dedup identity is + * untouched by this and still gets its own `accept` call. + */ + val row = symbol.sourceId to symbol.key + if (row in rejectedRows) return false + + if (!accept(symbol)) { + rejectedRows += row + return false + } - env.generatedIndex - ?.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_LIMIT) - ?.forEach(::addCompletionItem) + seen += key + return true + } + + fun exactMatches(index: JvmSymbolIndex): Sequence = + index.findBySimpleName(partial, limit = limit, kinds = kinds) + + index.findBySimpleName(partial, limit = limit, kinds = topLevelKinds, topLevelOnly = true) + + fun prefixMatches(index: JvmSymbolIndex): Sequence = + index.findByPrefix(partial, limit = fetchBudget, kinds = kinds) + + index.findByPrefix(partial, limit = fetchBudget, kinds = topLevelKinds, topLevelOnly = true) + + val exactSources = indexes.map { index -> { exactMatches(index) } } + val prefixSources = indexes.map { index -> { prefixMatches(index) } } + + return collectUpToLimit(limit = limit, sources = exactSources + prefixSources, accept = ::acceptFirstOccurrence) } +/** + * Offers items from [sources] in order to [accept], stopping once [limit] of them have been + * accepted, and returns how many were. + * + * The limit counts *accepted* items, which is the whole point: capping the fetch instead lets + * rejected entries consume the budget, so a query whose leading rows are all filtered out yields + * nothing while valid matches sit just beyond the cap. + * + * Sources are suppliers, not sequences, because `SQLiteIndex.query` runs its query and collects the + * rows before returning -- the sequence it hands back is already a list. Taking a supplier is what + * keeps a source that the limit makes unnecessary from being queried at all; how many rows a source + * fetches when it is reached remains the job of the limit passed to the query itself. + * + * A [limit] of zero or less accepts nothing and returns 0 without querying any source. This is the + * opposite of an index query's own convention, where a limit of zero means unbounded -- this helper + * has no unbounded case, since every caller caps how many items a user is offered. + */ +internal fun collectUpToLimit( + limit: Int, + sources: List<() -> Sequence>, + accept: (T) -> Boolean, +): Int { + var accepted = 0 + for (source in sources) { + if (accepted >= limit) { + break + } + for (item in source()) { + if (accepted >= limit) { + break + } + if (accept(item)) { + accepted++ + } + } + } + return accepted +} + +/** + * The kinds unimported-symbol completion offers wherever they are declared: classifiers and + * extensions. + * + * A file facade is not a classifier, and Kotlin cannot name one. A companion object is reached + * through its class, so offering every `Companion` in the index by that bare name only adds noise. + */ +internal val UNIMPORTED_SYMBOL_KINDS: Set = + (JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT) + + JvmSymbolKind.EXTENSION_FUNCTION + + JvmSymbolKind.EXTENSION_PROPERTY + +/** + * The kinds unimported-symbol completion offers only when declared at the top level. + * + * A member is reached through its receiver, never imported by name. Constructors and Java fields + * are always members, so they are not here at all. + */ +internal val UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS: Set = + setOf(JvmSymbolKind.FUNCTION, JvmSymbolKind.PROPERTY) + +/** + * Whether [symbol] can be offered as an unimported completion, whose bare name Kotlin resolves once + * the item's auto-import is applied. + */ +internal fun isUnimportedSymbolCandidate(symbol: JvmSymbol): Boolean = + symbol.kind in UNIMPORTED_SYMBOL_KINDS || + (symbol.kind in UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS && symbol.isTopLevel) + context(ctx: AnalysisContext) private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionItem? { - if (symbol.kind.isCallable && !symbol.isTopLevel && !symbol.isExtension) { - // member-level, non-extension callable symbols should not be - // completed in scope completions - return null - } + if (!isUnimportedSymbolCandidate(symbol)) return null abortIfCancelled() @@ -480,32 +639,24 @@ private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionIt ) item.overrideTypeText = symbol.returnTypeDisplay + // Every item imports its symbol when chosen; setClassCompletionData swaps in an equivalent handler. + item.additionalEditHandler = + KotlinAutoImportEditHandler( + analysisContext = ctx, + symbolToImport = symbol, + ) when (symbol.kind) { - JvmSymbolKind.EXTENSION_FUNCTION, JvmSymbolKind.FUNCTION, JvmSymbolKind.CONSTRUCTOR -> { + JvmSymbolKind.EXTENSION_FUNCTION, JvmSymbolKind.FUNCTION -> { val data = symbol.data as JvmFunctionInfo item.detail = data.signatureDisplay item.setInsertTextForFunction( name = symbol.shortName, hasParams = data.parameterCount > 0, ) - - item.additionalEditHandler = - KotlinAutoImportEditHandler( - analysisContext = ctx, - symbolToImport = symbol, - ) - - if (symbol.kind == JvmSymbolKind.CONSTRUCTOR) { - item.overrideTypeText = symbol.shortName - } } in JvmSymbolKind.CALLABLE_KINDS -> { - item.additionalEditHandler = - KotlinAutoImportEditHandler( - analysisContext = ctx, - symbolToImport = symbol, - ) + Unit } JvmSymbolKind.TYPE_ALIAS -> { @@ -517,12 +668,14 @@ private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionIt item.detail = symbol.fqName item.setClassCompletionData( className = symbol.fqName, - isNested = classInfo.isInner, + isNested = !symbol.isTopLevel, topLevelClass = classInfo.containingClassFqName, ) } - else -> {} + else -> { + return null + } } return item @@ -861,23 +1014,44 @@ private fun KaSession.kindOf(symbol: KaSymbol): CompletionItemKind = private fun KaSession.kindOf(symbol: JvmSymbol): CompletionItemKind = when (symbol.kind) { JvmSymbolKind.CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.INTERFACE -> CompletionItemKind.INTERFACE + JvmSymbolKind.ENUM -> CompletionItemKind.ENUM + JvmSymbolKind.ENUM_ENTRY -> CompletionItemKind.ENUM_MEMBER + JvmSymbolKind.ANNOTATION_CLASS -> CompletionItemKind.ANNOTATION_TYPE + JvmSymbolKind.OBJECT -> CompletionItemKind.CLASS + JvmSymbolKind.COMPANION_OBJECT -> CompletionItemKind.CLASS + JvmSymbolKind.DATA_CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.VALUE_CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.SEALED_CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.SEALED_INTERFACE -> CompletionItemKind.INTERFACE + JvmSymbolKind.FUNCTION -> CompletionItemKind.FUNCTION + JvmSymbolKind.EXTENSION_FUNCTION -> CompletionItemKind.FUNCTION + JvmSymbolKind.CONSTRUCTOR -> CompletionItemKind.CONSTRUCTOR + JvmSymbolKind.PROPERTY -> CompletionItemKind.PROPERTY + JvmSymbolKind.EXTENSION_PROPERTY -> CompletionItemKind.PROPERTY + JvmSymbolKind.FIELD -> CompletionItemKind.FIELD + JvmSymbolKind.TYPE_ALIAS -> CompletionItemKind.CLASS + + // Unimported-symbol completion rejects facades before building an item, since Kotlin + // cannot name one; this branch only keeps the mapping total. + JvmSymbolKind.FILE_FACADE -> CompletionItemKind.CLASS } private fun partialIdentifier(prefix: String): String = prefix.takeLastWhile { char -> Character.isJavaIdentifierPart(char) } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt index bb00c5dbeb..f2d6eafe87 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt @@ -9,6 +9,7 @@ import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.jvm.JvmVisibility import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test @@ -47,6 +48,8 @@ class AddImportActionTest : KtLspTest() { private fun index(vararg symbols: JvmSymbol) = runBlocking { symbols.forEach { env.ktSymbolIndex.sourceIndex.insert(it) } } + private fun indexLibrary(vararg symbols: JvmSymbol) = runBlocking { symbols.forEach { env.ktSymbolIndex.libraryIndex.insert(it) } } + @Test fun `resolves a single classifier candidate by simple name`() { index(classSymbol("lib", "Foo")) @@ -80,6 +83,49 @@ class AddImportActionTest : KtLspTest() { assertEquals(setOf("a.Foo"), candidates.keys) } + /** + * Pins the contract, not a bug this test caught: `associate`'s own dedup-by-fqName already makes + * this pass without `computeImportCandidates`'s explicit `.distinctBy { it.dedupeKey }`, which + * exists as insurance against a future change to how candidates are folded into the chooser, not + * because this ever surfaced a duplicate. + */ + @Test + fun `import candidates list each qualified name once, even from two jars`() { + indexLibrary(classSymbol("lib", "Foo").copy(sourceId = "jarA.jar"), classSymbol("lib", "Foo").copy(sourceId = "jarB.jar")) + createSourceFile("Main.kt", "package p\nfun f(x: Foo) {}") + + val candidates = AddImportAction().computeImportCandidates(env, mainPath, "Foo").found() + + assertEquals(setOf("lib.Foo"), candidates.keys) + assertEquals(1, candidates.size) + } + + @Test + fun `a class hidden in one jar is still offered from a jar that makes it public`() { + indexLibrary( + classSymbol("lib", "Foo").copy(sourceId = "jarA.jar", visibility = JvmVisibility.PACKAGE_PRIVATE), + classSymbol("lib", "Foo").copy(sourceId = "jarB.jar"), + ) + createSourceFile("Main.kt", "package p\nfun f(x: Foo) {}") + + val candidates = AddImportAction().computeImportCandidates(env, mainPath, "Foo").found() + + assertEquals(setOf("lib.Foo"), candidates.keys) + } + + @Test + fun `drops a package-private library classifier when a public one shares the simple name`() { + indexLibrary( + classSymbol("androidx.recyclerview.widget", "AdapterHelper").copy(visibility = JvmVisibility.PACKAGE_PRIVATE), + classSymbol("com.example", "AdapterHelper"), + ) + createSourceFile("Main.kt", "package p\nfun f(x: AdapterHelper) {}") + + val candidates = AddImportAction().computeImportCandidates(env, mainPath, "AdapterHelper").found() + + assertEquals(setOf("com.example.AdapterHelper"), candidates.keys) + } + @Test fun `returns no candidates for an unknown reference`() { createSourceFile("Main.kt", "package p\nfun f() {}") diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt index 9fee622f41..097e4be316 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt @@ -26,7 +26,7 @@ import java.time.Instant * commands must collapse BOTH the symbol-index deletes AND the per-file metadata deletes * into ONE batched call each ([JvmSymbolIndex.removeBySources] / * [KtFileMetadataIndex.removeAll], a single SQLite transaction apiece) instead of N separate - * [Index.removeBySource] calls (N transactions — the Sentry N+1). + * [Index.removeBySource] calls (N transactions - the Sentry N+1). */ @RunWith(JUnit4::class) class IndexWorkerBatchRemovalTest { @@ -88,7 +88,7 @@ class IndexWorkerBatchRemovalTest { val counting = CountingIndex(backing) val index = object : JvmSymbolIndex(counting, BackgroundIndexer(counting)) { - override fun isActive(sourceId: String): Boolean = true + override fun visibleSourceIds(): Collection? = null } return counting to index } @@ -124,7 +124,7 @@ class IndexWorkerBatchRemovalTest { pushBack = { pushedBack.add(it) }, ) - // The fix: exactly one batched transaction, zero per-source deletes — for BOTH indexes. + // The fix: exactly one batched transaction, zero per-source deletes - for BOTH indexes. assertThat(counting.removeBySourcesBatches).isEqualTo(1) assertThat(counting.removeBySourceCalls).isEqualTo(0) assertThat(counting.removedInLargestBatch).containsExactlyElementsIn(paths) diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt index d84ffc2b83..dfabdb5c78 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt @@ -23,43 +23,47 @@ import org.junit.Test * index worker is still running and about to call `PsiManager.findFile(project)`. */ class IndexWorkerDisposalTest : KtLspTest() { - private fun jvmIndex(): JvmSymbolIndex { val backing = InMemoryIndex(JvmSymbolDescriptor) return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { - override fun isActive(sourceId: String) = true + override fun visibleSourceIds(): Collection? = null } } - private fun fileIndex(): KtFileMetadataIndex = - KtFileMetadataIndex(InMemoryIndex(KtFileMetadataDescriptor)) + private fun fileIndex(): KtFileMetadataIndex = KtFileMetadataIndex(InMemoryIndex(KtFileMetadataDescriptor)) @Test - fun `worker stops without crashing when project is already disposed`(): Unit = runBlocking { - // Capture a real VirtualFile while the project is still alive. - val vf = createSourceFile("Sample.kt", "fun foo() = 1").virtualFile + fun `worker stops without crashing when project is already disposed`(): Unit = + runBlocking { + // Capture a real VirtualFile while the project is still alive. + val vf = createSourceFile("Sample.kt", "fun foo() = 1").virtualFile - val worker = IndexWorker( - project = env.project, - queue = WorkerQueue(), - fileIndex = fileIndex(), - sourceIndex = jvmIndex(), - scope = CoroutineScope(Dispatchers.Default + SupervisorJob()), - ) + val worker = + IndexWorker( + project = env.project, + queue = WorkerQueue(), + fileIndex = fileIndex(), + sourceIndex = jvmIndex(), + scope = CoroutineScope(Dispatchers.Default + SupervisorJob()), + ) - // Dispose the project out from under the worker, exactly as happens on LSP shutdown - // (env.close() disposes the project via its parent Disposable, the production path). - // IntelliJ requires model teardown to run inside a write action. - ApplicationManager.getApplication().runWriteAction { env.close() } - assertThat(env.project.isDisposed).isTrue() + /* + * Dispose the project out from under the worker, exactly as happens on LSP shutdown + * (env.close() disposes the project via its parent Disposable, the production path). + * IntelliJ requires model teardown to run inside a write action. + */ + ApplicationManager.getApplication().runWriteAction { env.close() } + assertThat(env.project.isDisposed).isTrue() - // Pre-fix, processing either command calls PsiManager.findFile on the disposed project - // and throws AssertionError("Project is already disposed"). Post-fix, the disposal guard - // breaks the loop, so start() returns cleanly instead of throwing. - worker.submitCommand(IndexCommand.ScanSourceFile(vf)) - worker.submitCommand(IndexCommand.IndexSourceFile(vf)) - worker.submitCommand(IndexCommand.Stop) + /* + * Pre-fix, processing either command calls PsiManager.findFile on the disposed project + * and throws AssertionError("Project is already disposed"). Post-fix, the disposal guard + * breaks the loop, so start() returns cleanly instead of throwing. + */ + worker.submitCommand(IndexCommand.ScanSourceFile(vf)) + worker.submitCommand(IndexCommand.IndexSourceFile(vf)) + worker.submitCommand(IndexCommand.Stop) - withTimeout(5_000) { worker.start() } - } + withTimeout(5_000) { worker.start() } + } } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt index 263d5aa4d2..641cd6484a 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt @@ -60,7 +60,7 @@ class IndexWorkerPhaseLifecycleTest { val symbolBacking = InMemoryIndex(JvmSymbolDescriptor) val sourceIndex = object : JvmSymbolIndex(symbolBacking, BackgroundIndexer(symbolBacking)) { - override fun isActive(sourceId: String) = true + override fun visibleSourceIds(): Collection? = null } return IndexWorker( @@ -116,9 +116,11 @@ class IndexWorkerPhaseLifecycleTest { Memprof.sink = sink val worker = worker() - // The second SourceScanningStarted simulates KtSymbolIndex.refreshSources() restarting - // a scan whose first attempt was cancelled before it ever reached SourceScanningComplete, - // leaving the first scanPhase stale and open. + /* + * The second SourceScanningStarted simulates KtSymbolIndex.refreshSources() restarting + * a scan whose first attempt was cancelled before it ever reached SourceScanningComplete, + * leaving the first scanPhase stale and open. + */ worker.submitCommand(IndexCommand.SourceScanningStarted(pass = 1)) worker.submitCommand(IndexCommand.SourceScanningStarted(pass = 2)) worker.submitCommand(IndexCommand.Stop) diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt index 3c912de6bb..4654c603e5 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt @@ -17,7 +17,7 @@ class SourceFileIndexerTest : KtLspTest() { private fun buildSymbolIndex(): JvmSymbolIndex { val backing = InMemoryIndex(JvmSymbolDescriptor) return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { - override fun isActive(sourceId: String): Boolean = true + override fun visibleSourceIds(): Collection? = null } } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt new file mode 100644 index 0000000000..a4b3193c3f --- /dev/null +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt @@ -0,0 +1,259 @@ +package com.itsaky.androidide.lsp.kotlin.completion + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolIndex +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * [collectUnimportedSymbolMatches] is the ordering and deduplication [collectUnimportedSymbols] + * runs on top of the raw index queries. + */ +@RunWith(JUnit4::class) +class CollectUnimportedSymbolMatchesTest { + /** A [JvmSymbolIndex] over a fresh, unfiltered in-memory backing store. */ + private fun newIndex(): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { + // The default filters to explicitly activated sources; these tests care about query + // ordering and dedup, not source activation, so admit everything inserted. + override fun visibleSourceIds(): Collection? = null + } + } + + private fun classSymbol( + sourceId: String, + shortName: String, + packageName: String = "com.example", + ): JvmSymbol { + val internalName = "${packageName.replace('.', '/')}/$shortName" + return JvmSymbol( + key = internalName, + sourceId = sourceId, + name = internalName, + shortName = shortName, + packageName = packageName, + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(internalName = internalName), + ) + } + + private fun funSymbol( + sourceId: String, + key: String, + shortName: String, + packageName: String = "com.example", + ): JvmSymbol { + val internalName = "${packageName.replace('.', '/')}/Foo#$shortName" + return JvmSymbol( + key = key, + sourceId = sourceId, + name = internalName, + shortName = shortName, + packageName = packageName, + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.JAVA, + data = JvmFunctionInfo(), + ) + } + + private fun JvmSymbolIndex.insertBlocking(symbol: JvmSymbol) = runBlocking { insert(symbol) } + + @Test + fun `an exact library match survives a cap already filled by an earlier index's prefix matches`() { + val sourceIndex = newIndex() + val libraryIndex = newIndex() + + // A prefix match that is not an exact match, sitting in the index queried first. + sourceIndex.insertBlocking(classSymbol(sourceId = "proj", shortName = "FooBar")) + // The exact match the user is typing, sitting in the index queried last. + libraryIndex.insertBlocking(classSymbol(sourceId = "lib.jar", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex, libraryIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 1, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + // With a cap of one, a prefix-only scan (source, then library) would spend the cap on + // FooBar and never reach the library index at all. + assertThat(accepted.map { it.shortName }).containsExactly("Foo") + } + + @Test + fun `the same class indexed from two jars is offered once`() { + val libraryIndex = newIndex() + libraryIndex.insertBlocking(classSymbol(sourceId = "libA.jar", shortName = "Foo")) + libraryIndex.insertBlocking(classSymbol(sourceId = "libB.jar", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(libraryIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + assertThat(accepted).hasSize(1) + assertThat(accepted.single().fqName).isEqualTo("com.example.Foo") + } + + @Test + fun `a single exact match is not also offered via the prefix stage`() { + val sourceIndex = newIndex() + sourceIndex.insertBlocking(classSymbol(sourceId = "proj", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + assertThat(accepted).hasSize(1) + } + + @Test + fun `a duplicate rejected on its first occurrence does not block a later acceptable one`() { + val sourceIndex = newIndex() + val libraryIndex = newIndex() + + /* + * Same class, indexed from two JARs. The querying module can only reach one of them -- + * modelled here as `accept` rejecting rows from "unreachable.jar" -- and that source happens + * to be queried first. + */ + sourceIndex.insertBlocking(classSymbol(sourceId = "unreachable.jar", shortName = "Foo")) + libraryIndex.insertBlocking(classSymbol(sourceId = "reachable.jar", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex, libraryIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { symbol -> + if (symbol.sourceId == "unreachable.jar") { + false + } else { + accepted += symbol + true + } + }, + ) + + // Marking the dedup key "seen" on the rejected row would silently swallow the reachable + // module's otherwise-valid completion. + assertThat(accepted.map { it.sourceId }).containsExactly("reachable.jar") + } + + @Test + fun `a row accept rejects in the exact stage is not asked about again in the prefix stage`() { + val sourceIndex = newIndex() + sourceIndex.insertBlocking(classSymbol(sourceId = "proj", shortName = "Foo")) + + var acceptCalls = 0 + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { + acceptCalls++ + false + }, + ) + + // The same row (same source and key) is a match in both the exact and prefix stages; once + // `accept` has turned it down, asking again wastes the visibility check and item-building + // work it would otherwise repeat. + assertThat(acceptCalls).isEqualTo(1) + } + + @Test + fun `two overloads sharing a qualified name are both offered`() { + val sourceIndex = newIndex() + sourceIndex.insertBlocking(funSymbol(sourceId = "proj", key = "com/example/Foo#foo()", shortName = "foo")) + sourceIndex.insertBlocking(funSymbol(sourceId = "proj", key = "com/example/Foo#foo(int)", shortName = "foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex), + partial = "foo", + kinds = setOf(JvmSymbolKind.FUNCTION), + limit = 10, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + assertThat(accepted).hasSize(2) + } + + @Test + fun `member functions do not use up the fetch budget before a top-level match`() { + val libraryIndex = newIndex() + for (suffix in listOf("A", "B", "C", "D", "E")) { + libraryIndex.insertBlocking(memberFunSymbol(shortName = "get$suffix")) + } + libraryIndex.insertBlocking(funSymbol(sourceId = "lib.jar", key = "com/example#getTop()", shortName = "getTop")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(libraryIndex), + partial = "get", + kinds = UNIMPORTED_SYMBOL_KINDS, + topLevelKinds = UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS, + limit = 10, + fetchBudget = 3, + accept = { symbol -> isUnimportedSymbolCandidate(symbol).also { if (it) accepted += symbol } }, + ) + + assertThat(accepted.map { it.shortName }).containsExactly("getTop") + } + + private fun memberFunSymbol(shortName: String): JvmSymbol = + JvmSymbol( + key = "com/example/Foo#$shortName()", + sourceId = "lib.jar", + name = "com/example/Foo#$shortName", + shortName = shortName, + packageName = "com.example", + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.JAVA, + data = JvmFunctionInfo(containingClassName = "com/example/Foo"), + ) +} diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUpToLimitTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUpToLimitTest.kt new file mode 100644 index 0000000000..9d88189263 --- /dev/null +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUpToLimitTest.kt @@ -0,0 +1,117 @@ +package com.itsaky.androidide.lsp.kotlin.completion + +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * [collectUpToLimit] is the cap that unimported-symbol completion runs on. + * + * The behaviour under test is that the cap counts items the user is offered, not rows the index + * returned. Capping the fetch instead is what made a prefix whose leading rows are all rejected -- + * by the current-package, visibility or member-kind filters -- produce nothing while valid matches + * sat just past the cap. + */ +@RunWith(JUnit4::class) +class CollectUpToLimitTest { + private fun source(vararg items: String): () -> Sequence = { items.asSequence() } + + @Test + fun `rejected items do not consume the limit`() { + val offered = (1..200).map { "reject$it" } + listOf("keepA", "keepB", "keepC") + val kept = mutableListOf() + + val accepted = + collectUpToLimit(limit = 3, sources = listOf(source(*offered.toTypedArray()))) { item -> + if (item.startsWith("keep")) { + kept += item + true + } else { + false + } + } + + assertThat(accepted).isEqualTo(3) + assertThat(kept).containsExactly("keepA", "keepB", "keepC") + } + + @Test + fun `collection stops once the limit is reached`() { + val kept = mutableListOf() + + val accepted = + collectUpToLimit(limit = 2, sources = listOf(source("a", "b", "c", "d"))) { item -> + kept += item + true + } + + assertThat(accepted).isEqualTo(2) + assertThat(kept).containsExactly("a", "b") + } + + @Test + fun `a later source is never queried once the limit is reached`() { + var secondSourceQueried = false + val second: () -> Sequence = { + secondSourceQueried = true + sequenceOf("x") + } + + collectUpToLimit(limit = 2, sources = listOf(source("a", "b"), second)) { true } + + // The supplier matters: an index query runs eagerly, so taking a sequence here would have + // issued the query regardless of the limit. + assertThat(secondSourceQueried).isFalse() + } + + @Test + fun `later sources fill the remainder when earlier ones fall short`() { + val kept = mutableListOf() + + val accepted = + collectUpToLimit( + limit = 4, + sources = listOf(source("a"), source("b", "c"), source("d", "e")), + ) { item -> + kept += item + true + } + + assertThat(accepted).isEqualTo(4) + assertThat(kept).containsExactly("a", "b", "c", "d").inOrder() + } + + @Test + fun `every source is drained when the limit is never reached`() { + val kept = mutableListOf() + + val accepted = + collectUpToLimit(limit = 100, sources = listOf(source("a"), source("b"))) { item -> + kept += item + true + } + + assertThat(accepted).isEqualTo(2) + assertThat(kept).containsExactly("a", "b").inOrder() + } + + @Test + fun `a source that yields nothing acceptable does not block the next one`() { + val kept = mutableListOf() + + collectUpToLimit( + limit = 2, + sources = listOf(source("reject1", "reject2"), source("keepA", "keepB")), + ) { item -> + if (item.startsWith("keep")) { + kept += item + true + } else { + false + } + } + + assertThat(kept).containsExactly("keepA", "keepB") + } +} diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt new file mode 100644 index 0000000000..634b8f4dbd --- /dev/null +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt @@ -0,0 +1,104 @@ +package com.itsaky.androidide.lsp.kotlin.completion + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmEnumEntryInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * Which index symbols unimported-symbol completion may offer. + * + * Accepting an item inserts its bare name, so only a symbol Kotlin can resolve after the + * auto-import is worth offering. + */ +@RunWith(JUnit4::class) +class UnimportedSymbolCandidateTest { + private fun symbol( + shortName: String, + kind: JvmSymbolKind, + data: JvmSymbolInfo = JvmClassInfo(internalName = "kotlin/text/$shortName"), + ) = JvmSymbol( + key = "kotlin/text/$shortName", + sourceId = "stdlib.jar", + name = "kotlin/text/$shortName", + shortName = shortName, + packageName = "kotlin.text", + kind = kind, + language = JvmSourceLanguage.KOTLIN, + data = data, + ) + + @Test + fun `a file facade is not offered`() { + // Kotlin cannot name `StringsKt`, so inserting it would not resolve. + assertThat(isUnimportedSymbolCandidate(symbol("StringsKt", JvmSymbolKind.FILE_FACADE))).isFalse() + } + + @Test + fun `a companion object is not offered by its bare name`() { + val companion = + symbol( + "Companion", + JvmSymbolKind.COMPANION_OBJECT, + JvmClassInfo(internalName = "kotlin/text/Regex\$Companion", containingClassName = "kotlin/text/Regex"), + ) + + assertThat(isUnimportedSymbolCandidate(companion)).isFalse() + } + + @Test + fun `a class is offered`() { + assertThat(isUnimportedSymbolCandidate(symbol("Regex", JvmSymbolKind.CLASS))).isTrue() + } + + @Test + fun `a top-level function is offered`() { + assertThat(isUnimportedSymbolCandidate(symbol("trimIndent", JvmSymbolKind.FUNCTION, JvmFunctionInfo()))).isTrue() + } + + @Test + fun `a member function is not offered`() { + val member = + symbol("find", JvmSymbolKind.FUNCTION, JvmFunctionInfo(containingClassName = "kotlin/text/Regex")) + + assertThat(isUnimportedSymbolCandidate(member)).isFalse() + } + + @Test + fun `an enum entry is not offered`() { + // Inserting a bare enum entry name with no import does not resolve; it must be qualified by + // its enum class. + val entry = + symbol( + "REGEX_OPTION", + JvmSymbolKind.ENUM_ENTRY, + JvmEnumEntryInfo(containingClassName = "kotlin/text/RegexOption"), + ) + + assertThat(isUnimportedSymbolCandidate(entry)).isFalse() + } + + @Test + fun `a constructor is not offered`() { + val constructor = + symbol("Regex", JvmSymbolKind.CONSTRUCTOR, JvmFunctionInfo(containingClassName = "kotlin/text/Regex")) + + assertThat(isUnimportedSymbolCandidate(constructor)).isFalse() + } + + @Test + fun `the queries ask for exactly the kinds that can be offered`() { + assertThat(UNIMPORTED_SYMBOL_KINDS + UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS) + .containsExactlyElementsIn( + JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT + + JvmSymbolKind.CALLABLE_KINDS - JvmSymbolKind.CONSTRUCTOR - JvmSymbolKind.FIELD, + ) + } +} diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt index f68d0d1d4e..eebbc340f1 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt @@ -100,10 +100,12 @@ internal class KtLspTestEnvironment( try { initialize(::buildModules, ::buildKtSymbolIndex) } catch (failure: Throwable) { - // A throwing constructor never hands the instance to KtLspTestRule, so its finally has - // nothing to close and the refcounted, process-wide application environment leaks for the - // rest of the suite - one bad TestSourceModuleSpec would reproduce the suite-wide OOM. - // Release it here, where the half-built instance is still reachable. + /* + * A throwing constructor never hands the instance to KtLspTestRule, so its finally has + * nothing to close and the refcounted, process-wide application environment leaks for the + * rest of the suite - one bad TestSourceModuleSpec would reproduce the suite-wide OOM. + * Release it here, where the half-built instance is still reachable. + */ runCatching { closeInWriteAction() }.exceptionOrNull()?.let(failure::addSuppressed) throw failure } @@ -222,7 +224,7 @@ internal class KtLspTestEnvironment( val inMemoryJvmSymbolIndex = object : JvmSymbolIndex(inMemoryJvmBackingIndex, BackgroundIndexer(inMemoryJvmBackingIndex)) { // ensure we're not filtering out anything - override fun isActive(sourceId: String) = true + override fun visibleSourceIds(): Collection? = null override fun query(query: IndexQuery): Sequence { onSymbolIndexQuery?.invoke(query) diff --git a/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt b/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt index 8edfaf87a0..e049984017 100644 --- a/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt +++ b/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt @@ -33,67 +33,66 @@ import kotlin.io.path.pathString * @author Akash Yadav */ object CachingJarFileSystemProvider : JarFileSystemProvider() { - private val cachedFs = ConcurrentHashMap() + private val cachedFs = ConcurrentHashMap() - private val log = LoggerFactory.getLogger(CachingJarFileSystemProvider::class.java) + private val log = LoggerFactory.getLogger(CachingJarFileSystemProvider::class.java) - override fun createFs(path: Path, env: MutableMap?): ZipFileSystem { - val cached = cachedFs[path.normalize().pathString] - if (cached != null) { - return cached - } - return createAndCache(path, env) - } + override fun createFs( + path: Path, + env: MutableMap?, + ): ZipFileSystem { + /* + * computeIfAbsent, not get-then-put: project setup opens the same JAR from many threads at + * once, and a check-then-act would build a file system per racing caller. Only the last would + * reach the map, so the others are never closed while a walk still holds them -- and the JARs + * most exposed to it are the most shared ones. + */ + return cachedFs.computeIfAbsent(path.normalize().pathString) { + CachedJarFileSystem(this, path, env) + } + } - fun newFileSystem(path: Path): FileSystem? { - return newFileSystem(path, mutableMapOf()) - } + fun newFileSystem(path: Path): FileSystem? = newFileSystem(path, mutableMapOf()) - fun clearCache() { - cachedFs.values.forEach(this::closeFs) - cachedFs.clear() - } + fun clearCache() { + cachedFs.values.forEach(this::closeFs) + cachedFs.clear() + } - fun clearCaches(predicate: (Path) -> Boolean) { - return clearCachesForPaths { predicate(Paths.get(it)) } - } + fun clearCaches(predicate: (Path) -> Boolean) = clearCachesForPaths { predicate(Paths.get(it)) } - fun clearCachesForPaths(predicate: (String) -> Boolean) { - val toRemove = - this.cachedFs.keys.mapNotNull { - return@mapNotNull if (predicate(it)) { - it - } else null - } + fun clearCachesForPaths(predicate: (String) -> Boolean) { + val toRemove = + this.cachedFs.keys.mapNotNull { + return@mapNotNull if (predicate(it)) { + it + } else { + null + } + } - if (toRemove.isNotEmpty()) { - toRemove.forEach(this::clearCache) - } - } + if (toRemove.isNotEmpty()) { + toRemove.forEach(this::clearCache) + } + } - fun clearCache(path: Path) { - clearCache(path.normalize().pathString) - } + fun clearCache(path: Path) { + clearCache(path.normalize().pathString) + } - fun clearCache(path: String) { - val fs = cachedFs.remove(path) - if (fs != null) { - log.debug("Clearing cached JAR file system for path: {}", path) - closeFs(fs) - } - } + fun clearCache(path: String) { + val fs = cachedFs.remove(path) + if (fs != null) { + log.debug("Clearing cached JAR file system for path: {}", path) + closeFs(fs) + } + } - private fun closeFs(fs: CachedJarFileSystem) { - try { - fs.doClose() - } catch (err: Throwable) { - log.error("Failed to close cached zip file system: {}", fs, err) - } - } - - private fun createAndCache(path: Path, env: MutableMap?): CachedJarFileSystem { - val fs = CachedJarFileSystem(this, path, env) - cachedFs[path.normalize().pathString] = fs - return fs - } + private fun closeFs(fs: CachedJarFileSystem) { + try { + fs.doClose() + } catch (err: Throwable) { + log.error("Failed to close cached zip file system: {}", fs, err) + } + } } diff --git a/subprojects/javac-services/src/test/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProviderConcurrencyTest.kt b/subprojects/javac-services/src/test/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProviderConcurrencyTest.kt new file mode 100644 index 0000000000..28e88b0086 --- /dev/null +++ b/subprojects/javac-services/src/test/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProviderConcurrencyTest.kt @@ -0,0 +1,100 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.javac.services.fs + +import com.google.common.truth.Truth.assertThat +import org.junit.After +import org.junit.Before +import org.junit.Test +import java.io.File +import java.nio.file.FileSystem +import java.nio.file.Files +import java.nio.file.Path +import java.util.Collections +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** + * Concurrent [CachingJarFileSystemProvider.createFs] calls for the same JAR path must open that + * JAR's file system exactly once. + */ +class CachingJarFileSystemProviderConcurrencyTest { + private lateinit var tmpDir: File + private lateinit var jarPath: Path + + @Before + fun setUp() { + // The provider caches by normalized path; start clean so prior runs cannot mask the race. + CachingJarFileSystemProvider.clearCache() + + tmpDir = Files.createTempDirectory("createfs-race").toFile() + jarPath = File(tmpDir, "test.jar").toPath() + writeMinimalJar(jarPath) + } + + @After + fun tearDown() { + CachingJarFileSystemProvider.clearCache() + tmpDir.deleteRecursively() + } + + @Test + fun concurrentCreateFsOpensTheSameJarOnce() { + val threadCount = 32 + val ready = CountDownLatch(threadCount) + val start = CountDownLatch(1) + val results = Collections.synchronizedList(mutableListOf()) + val executor = Executors.newFixedThreadPool(threadCount) + + repeat(threadCount) { + executor.submit { + ready.countDown() + start.await() + results.add(checkNotNull(CachingJarFileSystemProvider.newFileSystem(jarPath))) + } + } + + assertThat(ready.await(5, TimeUnit.SECONDS)).isTrue() + start.countDown() + executor.shutdown() + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue() + + assertThat(results).hasSize(threadCount) + /* + * Referential identity is the point: computeIfAbsent must hand every racing caller the + * same CachedJarFileSystem instance instead of each caller building and briefly holding + * its own, only for all but one to be overwritten and never closed. + */ + assertThat(results.toSet()).hasSize(1) + } +} + +private fun writeMinimalJar(path: Path) { + ZipOutputStream(Files.newOutputStream(path)).use { zip -> + zip.putNextEntry(ZipEntry("META-INF/MANIFEST.MF")) + zip.write("Manifest-Version: 1.0\n".toByteArray()) + zip.closeEntry() + + zip.putNextEntry(ZipEntry("com/example/Placeholder.class")) + zip.write(ByteArray(16)) + zip.closeEntry() + } +}