diff --git a/app/src/main/java/com/itsaky/androidide/activities/editor/BaseEditorActivity.kt b/app/src/main/java/com/itsaky/androidide/activities/editor/BaseEditorActivity.kt index a9a33c1e87..37962d11ef 100644 --- a/app/src/main/java/com/itsaky/androidide/activities/editor/BaseEditorActivity.kt +++ b/app/src/main/java/com/itsaky/androidide/activities/editor/BaseEditorActivity.kt @@ -167,6 +167,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.service.IndexingState import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode.MAIN import org.slf4j.Logger @@ -404,6 +405,7 @@ abstract class BaseEditorActivity : @UiThread set(value) { field = value onUpdateProgressBarVisibility() + postIndexingStatusUpdate() } private var debuggerService: DebuggerService? = null @@ -1608,17 +1610,47 @@ abstract class BaseEditorActivity : private fun onUpdateProgressBarVisibility() { log.debug( - "onBuildStatusChanged: isInitializing: ${editorViewModel.isInitializing}, isBuildInProgress: ${editorViewModel.isBuildInProgress}", + "onBuildStatusChanged: isInitializing: ${editorViewModel.isInitializing}, isBuildInProgress: ${editorViewModel.isBuildInProgress}, isIndexing: ${editorViewModel.isIndexing}", ) - // An internal build owns the same Gradle slot, so it shows the same progress bar. It does - // NOT relabel the Run button: the cancel affordance stays keyed off isBuildInProgress. - val visible = + val visible = isStatusOwnedElsewhere || editorViewModel.isIndexing + content.progressIndicator.visibility = if (visible) View.VISIBLE else View.GONE + invalidateOptionsMenu() + } + + /* + * Whether a build, project initialization or debugger start owns the progress bar and status slot. + * An internal build owns the same Gradle slot as a user build, but it does NOT relabel the Run + * button: the cancel affordance stays keyed off isBuildInProgress. + */ + private val isStatusOwnedElsewhere: Boolean + get() = editorViewModel.isBuildInProgress || editorViewModel.isInternalBuildInProgress || editorViewModel.isInitializing || isDebuggerStarting - content.progressIndicator.visibility = if (visible) View.VISIBLE else View.GONE - invalidateOptionsMenu() + + private fun onIndexingStateChanged(state: IndexingState) { + val wasIndexing = editorViewModel.isIndexing + editorViewModel.onIndexingStateChanged(state) + if (editorViewModel.isIndexing != wasIndexing) { + onUpdateProgressBarVisibility() + } + onUpdateIndexingStatus() + } + + private fun onUpdateIndexingStatus() { + editorViewModel.updateIndexingStatus(isStatusOwnedElsewhere) { done, total -> + getString(string.status_indexing_classes, done, total) + } + } + + /* + * Deferred to the next main-loop message: the callbacks that clear a slot-owning flag often + * write their final status (e.g. "Build was cancelled") after clearing it, and the indexing + * text must replace that write rather than be overwritten by it. + */ + private fun postIndexingStatusUpdate() { + lifecycleScope.launch(Dispatchers.Main) { onUpdateIndexingStatus() } } private fun setupStateObservers() { @@ -1656,17 +1688,31 @@ abstract class BaseEditorActivity : } } } + + launch { + ProjectManagerImpl + .getInstance() + .indexingServiceManager.state + .collect { state -> onIndexingStateChanged(state) } + } } } editorViewModel._isBuildInProgress.observe(this) { inProgress -> onUpdateProgressBarVisibility() + postIndexingStatusUpdate() // The bottom sheet's collapsed header follows the status line's height; while the // build narrates one line per task it only grows, or the peek jumps on every task. content.bottomSheet.setBuildNarrating(inProgress == true) } - editorViewModel._isInternalBuildInProgress.observe(this) { onUpdateProgressBarVisibility() } - editorViewModel._isInitializing.observe(this) { onUpdateProgressBarVisibility() } + editorViewModel._isInternalBuildInProgress.observe(this) { + onUpdateProgressBarVisibility() + postIndexingStatusUpdate() + } + editorViewModel._isInitializing.observe(this) { + onUpdateProgressBarVisibility() + postIndexingStatusUpdate() + } editorViewModel._statusText.observe(this) { content.bottomSheet.setStatus( it.first, diff --git a/app/src/main/java/com/itsaky/androidide/viewmodel/EditorViewModel.kt b/app/src/main/java/com/itsaky/androidide/viewmodel/EditorViewModel.kt index 5cc47ba654..2c22f22a5f 100644 --- a/app/src/main/java/com/itsaky/androidide/viewmodel/EditorViewModel.kt +++ b/app/src/main/java/com/itsaky/androidide/viewmodel/EditorViewModel.kt @@ -41,6 +41,7 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.service.IndexingState import java.io.File import java.io.IOException import java.util.Collections @@ -250,6 +251,50 @@ class EditorViewModel : ViewModel() { _statusText.value = (_statusText.value?.first ?: "") to value } + /** The latest class indexing state the editor has observed. */ + var indexingState: IndexingState = IndexingState.Idle + private set + + /** Whether class indexing is in flight, derived from [indexingState]. */ + val isIndexing: Boolean + get() = indexingState is IndexingState.Indexing + + /** The indexing text last written to the status slot, or null if none is shown. */ + private var shownIndexingStatus: CharSequence? = null + + /** The message the indexing text replaced in the status slot, restored when indexing ends. */ + private var replacedStatus: CharSequence? = null + + /** Records [state], as reported by the project's indexing service manager. */ + internal fun onIndexingStateChanged(state: IndexingState) { + indexingState = state + } + + /** + * Updates the status slot for [indexingState], formatting its progress with [formatProgress]. + * + * [isSlotOwnedElsewhere] is true while a build, project initialization or debugger start owns + * the slot; see [resolveIndexingStatus]. + */ + fun updateIndexingStatus( + isSlotOwnedElsewhere: Boolean, + formatProgress: (done: Int, total: Int) -> CharSequence, + ) { + val indexingStatus = (indexingState as? IndexingState.Indexing)?.let { formatProgress(it.done, it.total) } + val current = statusText + val next = + resolveIndexingStatus(isSlotOwnedElsewhere, indexingStatus, current, shownIndexingStatus, replacedStatus) + ?: return + if (indexingStatus != null && shownIndexingStatus?.contentEquals(current) != true) { + replacedStatus = current + } + if (indexingStatus == null) { + replacedStatus = null + } + shownIndexingStatus = indexingStatus + _statusText.value = next to CENTER + } + var displayedFileIndex: Int get() = _displayedFile.value!! set(value) { @@ -489,3 +534,27 @@ class EditorViewModel : ViewModel() { return manager.projectDir.name } } + +/** + * Decides what the status slot shows next for class indexing, or returns null to leave it alone. + * + * While [isSlotOwnedElsewhere] (a build, project initialization or debugger start is running) the + * slot is left to that operation. Otherwise [indexingStatus], the progress text or null when + * indexing is idle, takes it. When indexing ends the slot goes back to [replacedStatus], the message + * the indexing text replaced (a build result, say), or is cleared if there was none. It does so + * only if the slot still shows [shownIndexingStatus], the indexing text last written there, so a + * newer message survives. + */ +internal fun resolveIndexingStatus( + isSlotOwnedElsewhere: Boolean, + indexingStatus: CharSequence?, + currentStatus: CharSequence, + shownIndexingStatus: CharSequence?, + replacedStatus: CharSequence? = null, +): CharSequence? = + when { + isSlotOwnedElsewhere -> null + indexingStatus != null -> indexingStatus.takeUnless { it.contentEquals(currentStatus) } + shownIndexingStatus?.contentEquals(currentStatus) == true -> replacedStatus ?: "" + else -> null + } diff --git a/app/src/test/java/com/itsaky/androidide/viewmodel/EditorViewModelIndexingStatusTest.kt b/app/src/test/java/com/itsaky/androidide/viewmodel/EditorViewModelIndexingStatusTest.kt new file mode 100644 index 0000000000..685875c615 --- /dev/null +++ b/app/src/test/java/com/itsaky/androidide/viewmodel/EditorViewModelIndexingStatusTest.kt @@ -0,0 +1,246 @@ +package com.itsaky.androidide.viewmodel + +import androidx.arch.core.executor.testing.InstantTaskExecutorRule +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.service.IndexingState +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TestRule + +class EditorViewModelIndexingStatusTest { + @get:Rule + var rule: TestRule = InstantTaskExecutorRule() + + private lateinit var viewModel: EditorViewModel + + @Before + fun setUp() { + viewModel = EditorViewModel() + } + + @Test + fun whileIndexingWithNoBuild_theSlotShowsTheIndexingText() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = false, + indexingStatus = INDEXING, + currentStatus = "", + shownIndexingStatus = null, + ) + + assertThat(next).isEqualTo(INDEXING) + } + + @Test + fun whileABuildRuns_theBuildKeepsTheSlot() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = true, + indexingStatus = INDEXING, + currentStatus = BUILDING, + shownIndexingStatus = null, + ) + + assertThat(next).isNull() + } + + @Test + fun whenTheBuildEndsWhileIndexing_theIndexingTextReturns() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = false, + indexingStatus = INDEXING, + currentStatus = BUILD_SUCCESSFUL, + shownIndexingStatus = "old", + ) + + assertThat(next).isEqualTo(INDEXING) + } + + @Test + fun whenTheSlotAlreadyShowsTheIndexingText_nothingIsWritten() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = false, + indexingStatus = INDEXING, + currentStatus = INDEXING, + shownIndexingStatus = INDEXING, + ) + + assertThat(next).isNull() + } + + @Test + fun whenIndexingEndsWhileTheSlotShowsIndexingText_theSlotIsCleared() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = false, + indexingStatus = null, + currentStatus = INDEXING, + shownIndexingStatus = INDEXING, + ) + + assertThat(next.toString()).isEmpty() + } + + @Test + fun whenIndexingEndsWhileTheSlotShowsIndexingText_theMessageItReplacedReturns() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = false, + indexingStatus = null, + currentStatus = INDEXING, + shownIndexingStatus = INDEXING, + replacedStatus = BUILD_SUCCESSFUL, + ) + + assertThat(next.toString()).isEqualTo(BUILD_SUCCESSFUL) + } + + @Test + fun whenAnIndexingPassAfterABuildEnds_theBuildResultReturns() { + viewModel.statusText = BUILD_SUCCESSFUL + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 0, total = 2)) + update(isSlotOwnedElsewhere = false) + assertThat(viewModel.statusText.toString()).isEqualTo("Indexing classes (0/2)") + + viewModel.onIndexingStateChanged(IndexingState.Idle) + update(isSlotOwnedElsewhere = false) + + assertThat(viewModel.statusText.toString()).isEqualTo(BUILD_SUCCESSFUL) + } + + @Test + fun whenIndexingEndsWhileTheSlotShowsAnotherMessage_theMessageStays() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = false, + indexingStatus = null, + currentStatus = BUILD_SUCCESSFUL, + shownIndexingStatus = INDEXING, + ) + + assertThat(next).isNull() + } + + @Test + fun whenIndexingEndsWithNoIndexingTextEverShown_theSlotIsLeftAlone() { + val next = + resolveIndexingStatus( + isSlotOwnedElsewhere = false, + indexingStatus = null, + currentStatus = "", + shownIndexingStatus = null, + ) + + assertThat(next).isNull() + } + + @Test + fun isIndexingFollowsTheObservedState() { + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 0, total = 3)) + assertThat(viewModel.isIndexing).isTrue() + + viewModel.onIndexingStateChanged(IndexingState.Idle) + assertThat(viewModel.isIndexing).isFalse() + } + + @Test + fun theIndexingTextIsFormattedFromDoneThenTotal() { + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 123, total = 538)) + + update(isSlotOwnedElsewhere = false) + + assertThat(viewModel.statusText.toString()).isEqualTo("Indexing classes (123/538)") + } + + @Test + fun whenIndexingEnds_theIndexingTextTheViewModelWroteIsCleared() { + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 1, total = 2)) + update(isSlotOwnedElsewhere = false) + + viewModel.onIndexingStateChanged(IndexingState.Idle) + update(isSlotOwnedElsewhere = false) + + assertThat(viewModel.statusText.toString()).isEmpty() + } + + @Test + fun whenIndexingEndsDuringABuild_theBuildStatusIsKeptAfterTheBuildEnds() { + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 1, total = 2)) + update(isSlotOwnedElsewhere = false) + viewModel.statusText = BUILDING + + viewModel.onIndexingStateChanged(IndexingState.Idle) + update(isSlotOwnedElsewhere = true) + viewModel.statusText = BUILD_SUCCESSFUL + update(isSlotOwnedElsewhere = false) + + assertThat(viewModel.statusText.toString()).isEqualTo(BUILD_SUCCESSFUL) + } + + @Test + fun whenACancelledBuildWritesItsStatusAfterClearingTheFlag_theDeferredUpdateRestoresTheIndexingText() { + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 1, total = 2)) + viewModel.statusText = BUILDING + + // The flag is already clear when the cancel text lands; the deferred update runs after it. + viewModel.statusText = BUILD_CANCELLED + update(isSlotOwnedElsewhere = false) + + assertThat(viewModel.statusText.toString()).isEqualTo(INDEXING) + } + + @Test + fun anIndexingTickWhileTheProjectInitializes_leavesTheInitializationTextAlone() { + viewModel.statusText = INITIALIZING + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 1, total = 2)) + + update(isSlotOwnedElsewhere = true) + + assertThat(viewModel.statusText.toString()).isEqualTo(INITIALIZING) + } + + @Test + fun anIndexingTickWhileInitializationFailureIsShown_leavesTheFailureTextAlone() { + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 1, total = 2)) + update(isSlotOwnedElsewhere = false) + viewModel.statusText = INIT_FAILED + + update(isSlotOwnedElsewhere = true) + + assertThat(viewModel.statusText.toString()).isEqualTo(INIT_FAILED) + } + + @Test + fun whenTheDebuggerStopsStarting_theIndexingTextReturns() { + viewModel.onIndexingStateChanged(IndexingState.Indexing(done = 1, total = 2)) + viewModel.statusText = DEBUGGER_STARTING + update(isSlotOwnedElsewhere = true) + assertThat(viewModel.statusText.toString()).isEqualTo(DEBUGGER_STARTING) + + update(isSlotOwnedElsewhere = false) + + assertThat(viewModel.statusText.toString()).isEqualTo(INDEXING) + } + + private fun update(isSlotOwnedElsewhere: Boolean) { + viewModel.updateIndexingStatus(isSlotOwnedElsewhere, ::format) + } + + private fun format( + done: Int, + total: Int, + ): CharSequence = "Indexing classes ($done/$total)" + + private companion object { + const val INDEXING = "Indexing classes (1/2)" + const val BUILDING = "Building..." + const val BUILD_SUCCESSFUL = "BUILD SUCCESSFUL in 3s" + const val BUILD_CANCELLED = "Build was cancelled by the user." + const val INITIALIZING = "Initializing project..." + const val INIT_FAILED = "Failed to initialize project" + const val DEBUGGER_STARTING = "Starting debugger..." + } +} diff --git a/docs/adr/0010-navigation-resolves-via-analysis-api.md b/docs/adr/0010-navigation-resolves-via-analysis-api.md index 78c8bcbafd..ba91c88df0 100644 --- a/docs/adr/0010-navigation-resolves-via-analysis-api.md +++ b/docs/adr/0010-navigation-resolves-via-analysis-api.md @@ -31,6 +31,7 @@ It cannot. The index stores names, kinds, visibility, and containing-class metad **Negative / costs** - **Navigation requires a live analysis session.** Before one exists, go-to-definition returns nothing; it cannot degrade to an index-only answer. The user-facing gap - no way to say "still indexing" rather than "not found" - is cross-cutting across every LSP feature and remains unsolved. + - *Note, 2026-09-23:* indexing state is now exposed. `IndexingServiceManager.state` reports whether the JVM symbol indexes are being built and how far they got, and the editor shows it in its status text and progress bar, so a user can tell that an empty answer came while indexing was still running. A feature's own empty result still does not say why it is empty, and navigation still needs a live analysis session; this decision is unchanged. - Resolving a cross-module target means building PSI for the target file, which is more work than an index row lookup. Acceptable for a user-initiated, cancellable, one-at-a-time request; it would not be acceptable for a per-keystroke feature. - Symbols with no source PSI - stdlib, framework, any library jar - are simply unreachable. Library navigation would need decompilation or source-jar extraction, neither of which exists in the tree. diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt index 51ab70ec1d..b3e372da80 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -4,9 +4,9 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.PackageTree import org.appdevforall.codeonthego.indexing.api.ReadableIndex import java.io.Closeable -import java.util.concurrent.ConcurrentHashMap /** * A read-only view over an index that only exposes entries @@ -16,24 +16,34 @@ import java.util.concurrent.ConcurrentHashMap * This view controls which subset is visible based on which * sources (JAR paths, etc.) are currently "active." * + * It is a [PackageTree] of the active sources' packages when [backing] is one, and an empty tree + * otherwise. + * * @param T The indexed type. * @param backing The underlying index that holds all data. */ open class FilteredIndex( private val backing: ReadableIndex, ) : ReadableIndex, + PackageTree, Closeable { /** * The set of source IDs whose entries are visible. - * Uses a concurrent set for thread-safe reads during queries. + * + * An immutable set replaced in one assignment, so a query running during [setActiveSources] sees + * either the old set or the new one, never an emptied or half-filled one. Writers take [writeLock] + * so that concurrent single-source changes do not lose each other's update. */ - private val activeSources = ConcurrentHashMap.newKeySet() + @Volatile + private var activeSources: Set = emptySet() + + private val writeLock = Any() /** * Make a source's entries visible in query results. */ open fun activateSource(sourceId: String) { - activeSources.add(sourceId) + synchronized(writeLock) { activeSources = activeSources + sourceId } } /** @@ -41,7 +51,7 @@ open class FilteredIndex( * The data remains in the backing index. */ open fun deactivateSource(sourceId: String) { - activeSources.remove(sourceId) + synchronized(writeLock) { activeSources = activeSources - sourceId } } /** @@ -52,14 +62,14 @@ open class FilteredIndex( * current classpath JAR paths. */ open fun setActiveSources(sourceIds: Set) { - activeSources.clear() - activeSources.addAll(sourceIds) + val snapshot = sourceIds.toSet() + synchronized(writeLock) { activeSources = snapshot } } /** * Returns the current set of active source IDs. */ - fun activeSources(): Set = activeSources.toSet() + fun activeSources(): Set = activeSources /** * The source IDs whose entries are visible, or `null` if every source is. @@ -99,11 +109,31 @@ open class FilteredIndex( return if (query.sourceId in visible) query else query.copy(sourceIds = emptyList()) } - val requested = query.sourceIds - val scoped = requested?.filter { it in visible } ?: visible - return query.copy(sourceIds = scoped) + return query.copy(sourceIds = scopedToActive(query.sourceIds, visible)) + } + + /** Narrows a requested source scope to [visible]; an unscoped request becomes [visible] itself. */ + private fun scopedToActive( + requested: Collection?, + visible: Set, + ): Collection = requested?.filter { it in visible } ?: visible + + /** The requested source scope narrowed to the active sources, as [scopedToActive] does for a query. */ + private fun activeScope(requested: Collection?): Collection? { + val visible = visibleSourceIds()?.toSet() ?: return requested + return scopedToActive(requested, visible) } + override fun subpackages( + parent: String, + sourceIds: Collection?, + ): Set = (backing as? PackageTree)?.subpackages(parent, activeScope(sourceIds)) ?: emptySet() + + override fun containsPackage( + name: String, + sourceIds: Collection?, + ): Boolean = (backing as? PackageTree)?.containsPackage(name, activeScope(sourceIds)) ?: false + /** * Returns the visible entry for [key], the one with the smallest source id among the visible * sources that have it. @@ -125,7 +155,7 @@ open class FilteredIndex( ): Sequence = backing.distinctValues(fieldName, scopedToActive(query)) override fun close() { - activeSources.clear() + synchronized(writeLock) { activeSources = emptySet() } if (backing is Closeable) backing.close() } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index 01c8d19ec2..3b96728735 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -4,6 +4,9 @@ import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.PackageTree +import org.appdevforall.codeonthego.indexing.api.packageWithAncestors +import org.appdevforall.codeonthego.indexing.api.parentPackage import org.appdevforall.codeonthego.indexing.api.requireDeclaredFields import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantReadWriteLock @@ -28,6 +31,8 @@ import kotlin.concurrent.write * - [prefixBuckets]: fieldName -> (lowercased first char -> list of (value, row)) * Provides a ~36-way partition for prefix search. * - [fingerprints]: sourceId -> fingerprint recorded by [insertSource] + * - [packages]: sourceId -> (parent package -> full names of its subpackages), the [PackageTree]. + * Like [SQLiteIndex], a source's packages are only removed with the source. * * All mutations go through [lock] in write mode for consistency * across the multiple maps. Reads use read mode. @@ -38,13 +43,15 @@ import kotlin.concurrent.write class InMemoryIndex( override val descriptor: IndexDescriptor, override val name: String = "memory:${descriptor.name}", -) : Index { +) : Index, + PackageTree { private val rows = ConcurrentHashMap(256) private val sourceMap = ConcurrentHashMap>(32) private val keyMap = ConcurrentHashMap>(256) private val fieldMaps = ConcurrentHashMap>>() private val prefixBuckets = ConcurrentHashMap>>() private val fingerprints = ConcurrentHashMap() + private val packages = HashMap>>() private val lock = ReentrantReadWriteLock() @@ -124,20 +131,53 @@ class InMemoryIndex( override suspend fun insert(entry: T) = lock.write { insertSingleLocked(entry) } - /** Inserts under one write lock, so readers never see the fingerprint without every entry. */ + /** + * Replaces [sourceId]'s entries under one write lock, so readers see the old entries or the new + * ones and never the fingerprint without every entry. [entries] is read before the lock is taken, + * so one that fails part-way leaves the old entries untouched. + */ override suspend fun insertSource( sourceId: String, fingerprint: String, entries: Sequence, - ) = lock.write { - for (entry in entries) { - insertSingleLocked(entry) + ) { + val replacement = entries.toList() + lock.write { + removeBySourceLocked(sourceId) + for (entry in replacement) { + insertSingleLocked(entry) + } + fingerprints[sourceId] = fingerprint } - fingerprints[sourceId] = fingerprint } override suspend fun sourceFingerprint(sourceId: String): String? = fingerprints[sourceId] + override fun subpackages( + parent: String, + sourceIds: Collection?, + ): Set = + lock.read { + packagesIn(sourceIds).flatMapTo(HashSet()) { it[parent].orEmpty() } + } + + override fun containsPackage( + name: String, + sourceIds: Collection?, + ): Boolean = + lock.read { + val parent = parentPackage(name) + packagesIn(sourceIds).any { it[parent]?.contains(name) == true } + } + + /** The package trees of [sourceIds], or of every source when it is `null`. Caller holds the lock. */ + private fun packagesIn(sourceIds: Collection?): List>> = + if (sourceIds == null) { + packages.values.toList() + } else { + sourceIds.mapNotNull { packages[it] } + } + override suspend fun removeBySource(sourceId: String) = lock.write { removeBySourceLocked(sourceId) @@ -163,6 +203,7 @@ class InMemoryIndex( */ private fun removeBySourceLocked(sourceId: String) { fingerprints.remove(sourceId) + packages.remove(sourceId) val sourceRows = sourceMap.remove(sourceId) ?: return for (row in sourceRows) { val entry = rows.remove(row) ?: continue @@ -180,6 +221,7 @@ class InMemoryIndex( sourceMap.clear() keyMap.clear() fingerprints.clear() + packages.clear() fieldMaps.values.forEach { it.clear() } prefixBuckets.values.forEach { it.clear() } } @@ -297,6 +339,7 @@ class InMemoryIndex( rows[row] = entry sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(row) keyMap.getOrPut(entry.key) { mutableSetOf() }.add(row) + insertPackagesLocked(entry) val fields = descriptor.fieldValues(entry) for ((fieldName, value) in fields) { @@ -317,6 +360,16 @@ class InMemoryIndex( } } + private fun insertPackagesLocked(entry: T) { + val name = descriptor.packageOf(entry) ?: return + val sourcePackages = packages.getOrPut(entry.sourceId) { HashMap() } + for (pkg in packageWithAncestors(name)) { + if (!sourcePackages.getOrPut(parentPackage(pkg)) { HashSet() }.add(pkg)) { + break + } + } + } + /** Drops [row] from the field and prefix maps; [sourceMap] and [keyMap] are the caller's job. */ private fun removeFromSecondaryIndexes( row: RowId, diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index e44d05936e..a812e44a48 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -18,6 +18,9 @@ import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.PackageTree +import org.appdevforall.codeonthego.indexing.api.packageWithAncestors +import org.appdevforall.codeonthego.indexing.api.parentPackage import org.appdevforall.codeonthego.indexing.api.requireDeclaredFields import org.slf4j.LoggerFactory import kotlin.collections.iterator @@ -35,6 +38,7 @@ import kotlin.collections.iterator * f_{field2} TEXT, * ... * _payload BLOB NOT NULL, + * _gen INTEGER NOT NULL DEFAULT 0, * PRIMARY KEY (_source_id, _key) * ); * ``` @@ -43,11 +47,31 @@ import kotlin.collections.iterator * JARs, say) has one row per source. The primary key leads with `_source_id`, which is what serves * source-scoped reads and bulk removal. * + * `_gen` is the [insertSource] pass that wrote the row. Re-indexing a source overwrites its rows in + * place and only then deletes the ones the pass did not write, so a reader never finds the source + * missing or half-loaded while it is re-indexed: it sees the old rows, the new ones, or both. + * * SQL indexes are created on: * - `(_key, _source_id)` (for key lookups, already ordered by source) * - Each `f_{field}` (for equality filter) * - Each `f_{field}_lower` (for prefix search) * + * The [PackageTree] lives in a second table, filled from [IndexDescriptor.packageOf]: + * ``` + * CREATE TABLE IF NOT EXISTS {name}_packages ( + * _source_id TEXT NOT NULL, + * name TEXT NOT NULL, -- full dotted name + * parent TEXT NOT NULL, -- '' for a root package + * _gen INTEGER NOT NULL DEFAULT 0, + * PRIMARY KEY (_source_id, name) + * ) WITHOUT ROWID; + * ``` + * with an index on `(parent, _source_id)`, which serves both lookups. A batch writes its entries' + * packages and their ancestors in the same transaction as the entries, and removing a source + * removes its packages in the same transaction as its entries. A package row is only removed with + * its source or by an [insertSource] pass that no longer has it, so replacing a single entry with + * one in another package leaves the old package in place. + * * When a query has a selective predicate (a key, a prefix, or a match on a * [selective][org.appdevforall.codeonthego.indexing.api.IndexField.selective] field), its source * scope and its other terms only filter rows and never choose the index. [optimize] collects the @@ -84,7 +108,8 @@ class SQLiteIndex( override val name: String = "sqlite:${descriptor.name}", private val batchSize: Int = 500, @param:VisibleForTesting internal val sourceIdChunkSize: Int = SOURCE_ID_CHUNK_SIZE, -) : Index { +) : Index, + PackageTree { companion object { private val log = LoggerFactory.getLogger(SQLiteIndex::class.java) @@ -140,6 +165,9 @@ class SQLiteIndex( /** One row per source that was indexed with a fingerprint, see [insertSource]. */ private val sourcesTableName = "${tableName}_sources" + /** One row per package per source, see [PackageTree]. */ + private val packagesTableName = "${tableName}_packages" + /** Field column names: `f_{fieldName}`. */ private val fieldColumns = descriptor.fields.associate { field -> @@ -294,13 +322,24 @@ class SQLiteIndex( } } - override suspend fun insertAll(entries: Sequence) = insertBatched(entries, fingerprint = null) + override suspend fun insertAll(entries: Sequence) = insertBatched(entries, pass = null) + /** + * Replaces [sourceId]'s rows with [entries] without ever removing the source from view. + * + * The source's fingerprint is dropped first, then each batch overwrites rows in place under a + * new generation, and the last transaction deletes the rows and packages of older generations and + * records [fingerprint]. A pass that fails or is cancelled part-way leaves the old rows mixed with + * the new and no fingerprint, so the next pass re-indexes the source. + */ override suspend fun insertSource( sourceId: String, fingerprint: String, entries: Sequence, - ) = insertBatched(entries, SourceFingerprint(sourceId, fingerprint)) + ) = withContext(Dispatchers.IO) { + val generation = ifOpen(null) { beginPassLocked(sourceId) } ?: return@withContext + insertBatched(entries, SourcePass(sourceId, fingerprint, generation)) + } override suspend fun sourceFingerprint(sourceId: String): String? = withContext(Dispatchers.IO) { @@ -313,10 +352,42 @@ class SQLiteIndex( } } + override fun subpackages( + parent: String, + sourceIds: Collection?, + ): Set = + runBlocking { + ifOpen(emptySet()) { + val names = HashSet() + for (chunk in sourceIdChunks(sourceIds)) { + val (sql, args) = buildSubpackagesQuery(parent, chunk) + db.query(sql, args.toTypedArray()).use { + while (it.moveToNext()) { + names.add(it.getString(0)) + } + } + } + names + } + } + + override fun containsPackage( + name: String, + sourceIds: Collection?, + ): Boolean = + runBlocking { + ifOpen(false) { + sourceIdChunks(sourceIds).any { chunk -> + val (sql, args) = buildContainsPackageQuery(name, chunk) + db.query(sql, args.toTypedArray()).use { it.moveToFirst() } + } + } + } + /** * Inserts [entries] in transactions of [batchSize] rows, taking the lock per batch so reads can - * interleave with a long insert. A [fingerprint] goes into the last transaction, which runs - * even when there are no entries, so an empty source is still recorded as indexed. + * interleave with a long insert. A [pass] completes in the last transaction, which runs even + * when there are no entries, so an empty source is still recorded as indexed. * * Checks for cancellation before each transaction: [Mutex.withLock] only checks it while * actually suspended waiting for the lock, and its uncontended fast path never suspends, so a @@ -325,26 +396,27 @@ class SQLiteIndex( */ private suspend fun insertBatched( entries: Sequence, - fingerprint: SourceFingerprint?, + pass: SourcePass?, ) = withContext(Dispatchers.IO) { + val generation = pass?.generation ?: 0L val batch = mutableListOf() for (entry in entries) { batch.add(entry) if (batch.size >= batchSize) { ensureActive() - ifOpen { insertBatchLocked(batch, fingerprint = null) } + ifOpen { insertBatchLocked(batch, generation, completing = null) } batch.clear() } } - if (batch.isNotEmpty() || fingerprint != null) { + if (batch.isNotEmpty() || pass != null) { ensureActive() - ifOpen { insertBatchLocked(batch, fingerprint) } + ifOpen { insertBatchLocked(batch, generation, completing = pass) } } } override suspend fun insert(entry: T) = withContext(Dispatchers.IO) { - ifOpen { insertBatchLocked(listOf(entry), fingerprint = null) } + ifOpen { insertBatchLocked(listOf(entry), generation = 0L, completing = null) } } override suspend fun removeBySource(sourceId: String) = removeBySources(listOf(sourceId)) @@ -374,6 +446,10 @@ class SQLiteIndex( "DELETE FROM $sourcesTableName WHERE _source_id IN ($placeholders)", chunk.toTypedArray(), ) + db.execSQL( + "DELETE FROM $packagesTableName WHERE _source_id IN ($placeholders)", + chunk.toTypedArray(), + ) } db.setTransactionSuccessful() } finally { @@ -389,6 +465,7 @@ class SQLiteIndex( try { db.execSQL("DELETE FROM $tableName") db.execSQL("DELETE FROM $sourcesTableName") + db.execSQL("DELETE FROM $packagesTableName") db.setTransactionSuccessful() } finally { db.endTransaction() @@ -493,6 +570,7 @@ class SQLiteIndex( } append("_payload BLOB NOT NULL, ") + append("_gen INTEGER NOT NULL DEFAULT 0, ") append("PRIMARY KEY (_source_id, _key)") } @@ -501,6 +579,15 @@ class SQLiteIndex( "CREATE TABLE IF NOT EXISTS $sourcesTableName (_source_id TEXT NOT NULL PRIMARY KEY, _fingerprint TEXT NOT NULL)", ) + db.execSQL( + "CREATE TABLE IF NOT EXISTS $packagesTableName " + + "(_source_id TEXT NOT NULL, name TEXT NOT NULL, parent TEXT NOT NULL, _gen INTEGER NOT NULL DEFAULT 0, " + + "PRIMARY KEY (_source_id, name)) WITHOUT ROWID", + ) + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${packagesTableName}_parent ON $packagesTableName(parent, _source_id)", + ) + db.execSQL( "CREATE INDEX IF NOT EXISTS idx_${tableName}_key ON $tableName(_key, _source_id)", ) @@ -520,9 +607,36 @@ class SQLiteIndex( } } + /** + * Drops [sourceId]'s fingerprint and returns the generation for a new pass over it: one past any + * its rows or packages carry. + */ + private fun beginPassLocked(sourceId: String): Long { + db.beginTransaction() + try { + db.execSQL("DELETE FROM $sourcesTableName WHERE _source_id = ?", arrayOf(sourceId)) + val generation = + db + .query( + "SELECT MAX(IFNULL((SELECT MAX(_gen) FROM $tableName WHERE _source_id = ?1), 0), " + + "IFNULL((SELECT MAX(_gen) FROM $packagesTableName WHERE _source_id = ?1), 0)) + 1", + arrayOf(sourceId), + ).use { if (it.moveToFirst()) it.getLong(0) else 1L } + db.setTransactionSuccessful() + return generation + } finally { + db.endTransaction() + } + } + + /** + * Writes [entries] under [generation]. When [completing] is set, also deletes that source's rows + * and packages from older generations and records its fingerprint, in the same transaction. + */ private fun insertBatchLocked( entries: List, - fingerprint: SourceFingerprint?, + generation: Long, + completing: SourcePass?, ) { db.beginTransaction() try { @@ -544,6 +658,7 @@ class SQLiteIndex( } put("_payload", descriptor.serialize(entry)) + put("_gen", generation) } db.insert( @@ -552,13 +667,9 @@ class SQLiteIndex( cv, ) } - if (fingerprint != null) { - val cv = - ContentValues().apply { - put("_source_id", fingerprint.sourceId) - put("_fingerprint", fingerprint.value) - } - db.insert(sourcesTableName, SQLiteDatabase.CONFLICT_REPLACE, cv) + insertPackagesLocked(entries, generation) + if (completing != null) { + completePassLocked(completing) } db.setTransactionSuccessful() } finally { @@ -566,9 +677,50 @@ class SQLiteIndex( } } - private data class SourceFingerprint( + private fun completePassLocked(pass: SourcePass) { + val args = arrayOf(pass.sourceId, pass.generation) + db.execSQL("DELETE FROM $tableName WHERE _source_id = ? AND _gen <> ?", args) + db.execSQL("DELETE FROM $packagesTableName WHERE _source_id = ? AND _gen <> ?", args) + val cv = + ContentValues().apply { + put("_source_id", pass.sourceId) + put("_fingerprint", pass.fingerprint) + } + db.insert(sourcesTableName, SQLiteDatabase.CONFLICT_REPLACE, cv) + } + + /** + * Records the packages of [entries] and their ancestors under [generation]. Must run inside the + * entries' transaction. A package already recorded for the source takes the new generation. + */ + private fun insertPackagesLocked( + entries: List, + generation: Long, + ) { + val seen = HashSet>() + for (entry in entries) { + val name = descriptor.packageOf(entry) ?: continue + for (pkg in packageWithAncestors(name)) { + if (!seen.add(entry.sourceId to pkg)) { + break + } + val cv = + ContentValues().apply { + put("_source_id", entry.sourceId) + put("name", pkg) + put("parent", parentPackage(pkg)) + put("_gen", generation) + } + db.insert(packagesTableName, SQLiteDatabase.CONFLICT_REPLACE, cv) + } + } + } + + /** One [insertSource] pass: the source, the fingerprint it records, and the generation it writes. */ + private data class SourcePass( val sourceId: String, - val value: String, + val fingerprint: String, + val generation: Long, ) private data class SqlQuery( @@ -638,8 +790,13 @@ class SQLiteIndex( * queries, that is what makes a limited key lookup return the smallest source id overall rather * than the smallest within whichever chunk happened to run first. */ - private fun sourceIdChunks(query: IndexQuery): List?> { - val sourceIds = query.sourceIds ?: return listOf(null) + private fun sourceIdChunks(query: IndexQuery): List?> = sourceIdChunks(query.sourceIds) + + /** Splits a source scope into `IN (...)`-sized chunks, as [sourceIdChunks] does for a query's. */ + private fun sourceIdChunks(sourceIds: Collection?): List?> { + if (sourceIds == null) { + return listOf(null) + } if (sourceIds.isEmpty()) { return emptyList() } @@ -692,6 +849,52 @@ class SQLiteIndex( return SqlQuery(sql, args) } + private fun buildSubpackagesQuery( + parent: String, + sourceIdChunk: List?, + ): SqlQuery { + val (scope, scopeArgs) = sourceScopeClause(sourceIdChunk) + return SqlQuery("SELECT DISTINCT name FROM $packagesTableName WHERE parent = ?$scope", listOf(parent) + scopeArgs) + } + + /** + * Matches the parent as well as the name: the name alone has no index, while the parent is + * implied by the name and leads the `(parent, _source_id)` index. + */ + private fun buildContainsPackageQuery( + name: String, + sourceIdChunk: List?, + ): SqlQuery { + val (scope, scopeArgs) = sourceScopeClause(sourceIdChunk) + return SqlQuery( + "SELECT 1 FROM $packagesTableName WHERE parent = ? AND name = ?$scope LIMIT 1", + listOf(parentPackage(name), name) + scopeArgs, + ) + } + + /** An ` AND _source_id IN (...)` clause for [sourceIdChunk], or nothing when it is unscoped. */ + private fun sourceScopeClause(sourceIdChunk: List?): Pair> { + if (sourceIdChunk == null) { + return "" to emptyList() + } + val placeholders = sourceIdChunk.joinToString(",") { "?" } + return " AND _source_id IN ($placeholders)" to sourceIdChunk + } + + /** Returns the plan of the first statement [subpackages] runs for [parent] and [sourceIds]. */ + @VisibleForTesting + internal fun explainSubpackages( + parent: String, + sourceIds: Collection?, + ): String = explain(buildSubpackagesQuery(parent, firstSourceIdChunk(sourceIds))) + + /** Returns the plan of the first statement [containsPackage] runs for [name] and [sourceIds]. */ + @VisibleForTesting + internal fun explainContainsPackage( + name: String, + sourceIds: Collection?, + ): String = explain(buildContainsPackageQuery(name, firstSourceIdChunk(sourceIds))) + /** Returns the plan of the first statement [query] runs, one plan row per line. */ @VisibleForTesting internal fun explainQuery(query: IndexQuery): String { @@ -709,8 +912,10 @@ class SQLiteIndex( return explain(buildDistinctQuery(col, query, firstSourceIdChunk(query), effectiveLimit(query))) } - private fun firstSourceIdChunk(query: IndexQuery): List? { - val chunks = sourceIdChunks(query) + private fun firstSourceIdChunk(query: IndexQuery): List? = firstSourceIdChunk(query.sourceIds) + + private fun firstSourceIdChunk(sourceIds: Collection?): List? { + val chunks = sourceIdChunks(sourceIds) require(chunks.isNotEmpty()) { "A query scoped to no sources runs no statement" } return chunks.first() } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt index dc5ba74bcd..27ab257c68 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt @@ -60,6 +60,14 @@ interface IndexDescriptor { */ fun fieldValues(entry: T): Map + /** + * Returns the package [entry] makes exist, or `null` if it contributes none. + * + * An index that is a [PackageTree] records the package and its ancestors for the entry's source. + * The default package (`""`) contributes nothing, the same as `null`. + */ + fun packageOf(entry: T): String? = null + /** * Serialize an entry to bytes for persistent storage. * diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index c405768a64..cede513694 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -119,12 +119,14 @@ interface Index : val descriptor: IndexDescriptor /** - * Inserts [entries], all from [sourceId], and records [fingerprint] as that source's. + * Replaces [sourceId]'s entries with [entries], all from [sourceId], and records [fingerprint] as + * that source's. * - * The fingerprint is opaque to the index; it is what lets a caller tell whether the source has - * changed since it was indexed. It is committed with the last batch of rows, so it is present - * only once every entry is stored: an insert that fails or is cancelled part-way leaves none. - * Removing the source's entries removes it too. + * Readers never find the source missing while it is replaced: they see its old entries, its new + * ones, or both. The fingerprint is opaque to the index; it is what lets a caller tell whether the + * source has changed since it was indexed. It is committed with the last batch of rows, so it is + * present only once every entry is stored: a replacement that fails or is cancelled part-way + * leaves none. Removing the source's entries removes it too. */ suspend fun insertSource( sourceId: String, diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/PackageTree.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/PackageTree.kt new file mode 100644 index 0000000000..6cd6c5e7db --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/PackageTree.kt @@ -0,0 +1,43 @@ +package org.appdevforall.codeonthego.indexing.api + +/** + * The packages an index's entries make exist, as a tree of dotted names. + * + * An entry contributes the package [IndexDescriptor.packageOf] returns for it, and with it every + * ancestor: an entry in `a.b.c` makes `a.b.c`, `a.b` and `a` exist. The default package (`""`) is + * the root of the tree and is never itself an entry. A source's packages are removed with the source. + * + * `sourceIds` follows [IndexQuery.sourceIds]: `null` admits every source, an empty collection none. + * + * Both lookups block the calling thread until they return, and a persistent implementation reads + * from disk to answer them. The caller chooses the thread; never call them on the main thread. + */ +interface PackageTree { + /** + * Returns the full names of the direct subpackages of [parent] (`""` for the root packages) in + * any of [sourceIds], each once however many sources have it. + * + * Blocks on disk I/O in a persistent implementation, so never call it on the main thread. + */ + fun subpackages( + parent: String, + sourceIds: Collection?, + ): Set + + /** + * Returns whether any of [sourceIds] has the package [name]; `false` for the default package. + * + * Blocks on disk I/O in a persistent implementation, so never call it on the main thread. + */ + fun containsPackage( + name: String, + sourceIds: Collection?, + ): Boolean +} + +/** Returns the package enclosing [name], `""` for a root package. */ +internal fun parentPackage(name: String): String = name.substringBeforeLast('.', missingDelimiterValue = "") + +/** Returns [name] and every package enclosing it, innermost first; nothing for the default package. */ +internal fun packageWithAncestors(name: String): Sequence = + generateSequence(name.ifEmpty { null }) { parentPackage(it).ifEmpty { null } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingProgressTracker.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingProgressTracker.kt new file mode 100644 index 0000000000..5745316ebf --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingProgressTracker.kt @@ -0,0 +1,119 @@ +package org.appdevforall.codeonthego.indexing.service + +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import java.io.Closeable + +/** + * Aggregates the indexing passes of several services into one [IndexingState]. + * + * A service calls [openPass] when a pass starts, [Pass.track] for each source job it submits, and + * closes the pass in a `finally`, so a pass that throws or is cancelled cannot hold the state out of + * [IndexingState.Idle]. A source id already pending is counted once; a job that supersedes it takes + * its place without adding to the total. Sources are keyed by id alone, across every pass and + * service, so a JAR path two services submit is counted once, and the replaced job's completion is + * ignored. Each tracked job adds to `done` when it completes, whatever + * its outcome. The state is [IndexingState.Indexing] while any tracked job is pending or any open + * pass has tracked a job, and returns to [IndexingState.Idle], with both counts reset, otherwise. + * + * Thread-safe. + */ +class IndexingProgressTracker { + private val lock = Any() + private val mutableState = MutableStateFlow(IndexingState.Idle) + private val openPasses = mutableSetOf() + private val pending = mutableMapOf() + private var done = 0 + private var total = 0 + + /** The current indexing state; see [IndexingProgressTracker]. */ + val state: StateFlow = mutableState.asStateFlow() + + private val mutableFinishedPasses = MutableStateFlow(0L) + + /** + * How many times the state has returned to [IndexingState.Idle] from [IndexingState.Indexing]. + * + * [state] is conflated, so a collector can miss a pass that starts and finishes between two of + * its collections. This count only grows, so a collector that sees it change knows at least one + * pass finished, however short. + */ + val finishedPasses: StateFlow = mutableFinishedPasses.asStateFlow() + + /** Opens a pass; the caller must close it in a `finally`. */ + fun openPass(): Pass = Pass().also { pass -> synchronized(lock) { openPasses += pass } } + + /** Forgets every open pass and pending job and returns the state to [IndexingState.Idle]. */ + fun reset() { + synchronized(lock) { + openPasses.clear() + pending.clear() + publish() + } + } + + /** One service's indexing pass, reporting its submitted jobs until it is closed. */ + inner class Pass internal constructor() : Closeable { + internal var hasTracked = false + private set + + /** + * How many sources this pass added to the total: those it tracked while not already pending, + * so not a job it merely folded into another pass's running one. + */ + var newlyCounted = 0 + get() = synchronized(lock) { field } + private set + + /** Counts [job], which indexes [sourceId], toward the state until it completes. */ + fun track( + sourceId: String, + job: Job, + ) { + synchronized(lock) { + if (this !in openPasses) return + hasTracked = true + if (pending.put(sourceId, job) == null) { + total++ + newlyCounted++ + } + publish() + } + job.invokeOnCompletion { onCompleted(sourceId, job) } + } + + override fun close() { + synchronized(lock) { + if (openPasses.remove(this)) { + publish() + } + } + } + } + + private fun onCompleted( + sourceId: String, + job: Job, + ) { + synchronized(lock) { + if (!pending.remove(sourceId, job)) return + done++ + publish() + } + } + + private fun publish() { + if (pending.isNotEmpty() || openPasses.any { it.hasTracked }) { + mutableState.value = IndexingState.Indexing(done, total) + } else { + done = 0 + total = 0 + if (mutableState.value is IndexingState.Indexing) { + mutableFinishedPasses.value++ + } + mutableState.value = IndexingState.Idle + } + } +} diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManager.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManager.kt index 537c53066e..c6dacbd9b9 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManager.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManager.kt @@ -7,11 +7,13 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancelChildren +import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeoutOrNull import org.slf4j.LoggerFactory import java.io.Closeable +import java.util.Collections import java.util.concurrent.ConcurrentHashMap import kotlin.time.Duration.Companion.seconds @@ -62,9 +64,36 @@ class IndexingServiceManager( */ val registry = IndexRegistry() + /** Collects the progress of every service's indexing passes; supplied to a service when it is registered. */ + val progressTracker = IndexingProgressTracker() + + /** Whether indexing is in flight and how far it got, across every service reporting to [progressTracker]. */ + val state: StateFlow get() = progressTracker.state + + /** How many indexing passes have finished; see [IndexingProgressTracker.finishedPasses]. */ + val finishedPasses: StateFlow get() = progressTracker.finishedPasses + private val services = ConcurrentHashMap() private var initialized = false + /** + * JAR paths already reported unreadable this project session (see [filterNewlyUnreadableJars]). + * Cleared in [close], so a new session reports the same JAR again. + */ + private val reportedUnreadableJars = Collections.synchronizedSet(mutableSetOf()) + + /** + * Returns the subset of [jarPaths] not yet reported unreadable this session, recording every one + * of [jarPaths] as reported. + * + * Shared by every [IndexingService] built against this manager, so a JAR that stays unreadable + * across many passes - a library JAR whose scan two language servers both trigger at project + * open, or a generated JAR re-scanned after every build - is reported to the user once, not once + * per pass. It is still retried every pass regardless: this only dedupes the report, since no + * fingerprint is ever written for a JAR whose scan failed. + */ + fun filterNewlyUnreadableJars(jarPaths: Collection): List = jarPaths.filter { reportedUnreadableJars.add(it) } + /** * Register an [IndexingService]. * @@ -158,6 +187,8 @@ class IndexingServiceManager( val toClose = services.values.toList() services.clear() initialized = false + progressTracker.reset() + reportedUnreadableJars.clear() val previous = pendingClose pendingClose = diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingState.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingState.kt new file mode 100644 index 0000000000..69a65c7c04 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingState.kt @@ -0,0 +1,21 @@ +package org.appdevforall.codeonthego.indexing.service + +/** + * Whether source indexing is in flight, as [IndexingServiceManager.state] reports it. + * + * Every source (a JAR) is counted once while it is pending, across all services reporting to the + * same [IndexingProgressTracker]. + */ +sealed interface IndexingState { + /** Nothing is being indexed. */ + data object Idle : IndexingState + + /** + * [done] of the [total] sources submitted since the state last left [Idle] have finished, + * whether they were indexed, failed or were cancelled. + */ + data class Indexing( + val done: Int, + val total: Int, + ) : IndexingState +} diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt index 19782b36a6..185002f398 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt @@ -94,7 +94,7 @@ class BackgroundIndexer( * result, so cancelling and resubmitting it would only throw away its progress. A * source being indexed under a different fingerprint (its content changed again before * the previous pass finished) is handled by cancelling and joining that job before this - * one deletes its rows, so the two passes' writes can never interleave. + * one writes its rows, so the two passes' writes can never interleave. * * The check for an existing job and the recording of this call's own job in [activeJobs] are * not atomic with each other, so two overlapping calls for the same [sourceId] could each @@ -105,7 +105,9 @@ class BackgroundIndexer( * @param sourceId Identifies the source. * @param skipIfExists Skip if already indexed. * @param fingerprint Identifies the source's current content, or `null` to not track it. - * @param provider Lambda returning a [Sequence] of entries. + * @param provider Lambda returning a [Sequence] of entries. One that is also [Closeable], + * such as a [CloseableSequence], is closed once the pass ends, whether it + * finished, failed or was cancelled. * @return The launched job, or the already-running one this call was folded into. */ fun indexSource( @@ -141,27 +143,34 @@ class BackgroundIndexer( log.info("Indexing: {}", sourceId) - // Remove stale entries first - index.removeBySource(sourceId) + // insertSource replaces the source's entries itself, without a window where it has none. + if (fingerprint == null) { + index.removeBySource(sourceId) + } if (!isActive) return@launch progressListener?.onProgress(sourceId, IndexingEvent.Started) var count = 0 - val tracked = - provider(sourceId).map { entry -> - count++ - if (count % 1000 == 0) { - progressListener?.onProgress(sourceId, IndexingEvent.Progress(count)) + val entries = provider(sourceId) + try { + val tracked = + entries.map { entry -> + count++ + if (count % 1000 == 0) { + progressListener?.onProgress(sourceId, IndexingEvent.Progress(count)) + } + entry } - entry - } - if (fingerprint != null) { - index.insertSource(sourceId, fingerprint, tracked) - } else { - index.insertAll(tracked) + if (fingerprint != null) { + index.insertSource(sourceId, fingerprint, tracked) + } else { + index.insertAll(tracked) + } + } finally { + (entries as? Closeable)?.close() } progressListener?.onProgress(sourceId, IndexingEvent.Completed(count)) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/CloseableSequence.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/CloseableSequence.kt new file mode 100644 index 0000000000..cd88a487e7 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/CloseableSequence.kt @@ -0,0 +1,23 @@ +package org.appdevforall.codeonthego.indexing.util + +import java.io.Closeable +import java.util.concurrent.atomic.AtomicBoolean + +/** + * A [Sequence] that holds a resource open while it is iterated, and releases it on [close]. + * + * A `sequence {}` builder that is abandoned part-way, because its consumer failed or was cancelled, + * is never resumed, so a `use` or `finally` inside it never runs. The consumer closes this instead. + * [close] runs [onClose] at most once and is safe after the sequence was fully iterated. + */ +class CloseableSequence( + private val delegate: Sequence, + private val onClose: () -> Unit, +) : Sequence by delegate, + Closeable { + private val closed = AtomicBoolean(false) + + override fun close() { + if (closed.compareAndSet(false, true)) onClose() + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexConcurrencyTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexConcurrencyTest.kt new file mode 100644 index 0000000000..f5d3c25cbc --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexConcurrencyTest.kt @@ -0,0 +1,64 @@ +package org.appdevforall.codeonthego.indexing + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.concurrent.thread + +@RunWith(JUnit4::class) +class FilteredIndexConcurrencyTest { + data class Entry( + override val key: String, + override val sourceId: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_filtered_concurrency" + override val fields = listOf(IndexField("key")) + + override fun fieldValues(entry: Entry) = mapOf("key" to entry.key) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1]) + } + } + + @Test(timeout = 30_000) + fun `a reader never sees a half-replaced active set`() { + val filtered = FilteredIndex(InMemoryIndex(descriptor)) + val first = (1..50).map { "/libs/a$it.jar" }.toSet() + val second = (1..50).map { "/libs/b$it.jar" }.toSet() + filtered.setActiveSources(first) + + val stop = AtomicBoolean(false) + val writer = + thread { + while (!stop.get()) { + filtered.setActiveSources(second) + filtered.setActiveSources(first) + } + } + + val torn = + try { + (1..200_000).count { + val seen = filtered.activeSources() + seen != first && seen != second + } + } finally { + stop.set(true) + writer.join() + } + + assertThat(torn).isEqualTo(0) + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexPackageTreeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexPackageTreeTest.kt new file mode 100644 index 0000000000..3ae2a2e179 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexPackageTreeTest.kt @@ -0,0 +1,69 @@ +package org.appdevforall.codeonthego.indexing + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.ReadableIndex +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** A filtered view's package tree admits only the packages of its visible sources. */ +@RunWith(JUnit4::class) +class FilteredIndexPackageTreeTest { + private val backing = InMemoryIndex(packagedEntryDescriptor) + + @Before + fun setUp() = + runTest { + backing.insert(PackagedEntry("k1", "jarA", "a.fromA")) + backing.insert(PackagedEntry("k2", "jarB", "a.fromB")) + backing.insert(PackagedEntry("k3", "jarC", "a.fromC")) + } + + private fun activeOnly(vararg sourceIds: String) = FilteredIndex(backing).apply { setActiveSources(sourceIds.toSet()) } + + @Test + fun `an unscoped lookup sees only the active sources' packages`() { + val filtered = activeOnly("jarA", "jarB") + + assertThat(filtered.subpackages("a", sourceIds = null)).containsExactly("a.fromA", "a.fromB") + assertThat(filtered.containsPackage("a.fromC", sourceIds = null)).isFalse() + } + + @Test + fun `a scoped lookup sees only the requested sources that are active`() { + val filtered = activeOnly("jarA", "jarB") + + assertThat(filtered.subpackages("a", sourceIds = listOf("jarB", "jarC"))).containsExactly("a.fromB") + assertThat(filtered.containsPackage("a.fromC", sourceIds = listOf("jarC"))).isFalse() + } + + @Test + fun `no active source means no packages`() { + val filtered = FilteredIndex(backing) + + assertThat(filtered.subpackages("", sourceIds = null)).isEmpty() + assertThat(filtered.containsPackage("a", sourceIds = null)).isFalse() + } + + @Test + fun `a subclass that declares every source visible passes the requested scope through`() { + val unfiltered = + object : FilteredIndex(backing) { + override fun visibleSourceIds(): Collection? = null + } + + assertThat(unfiltered.subpackages("a", sourceIds = null)).containsExactly("a.fromA", "a.fromB", "a.fromC") + assertThat(unfiltered.subpackages("a", sourceIds = listOf("jarC"))).containsExactly("a.fromC") + } + + @Test + fun `a backing index without a package tree has no packages`() { + val withoutTree = object : ReadableIndex by backing {} + val filtered = FilteredIndex(withoutTree).apply { setActiveSources(setOf("jarA")) } + + assertThat(filtered.subpackages("a", sourceIds = null)).isEmpty() + assertThat(filtered.containsPackage("a", sourceIds = null)).isFalse() + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/IndexReplaceSourceTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/IndexReplaceSourceTest.kt new file mode 100644 index 0000000000..8c72ecd73a --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/IndexReplaceSourceTest.kt @@ -0,0 +1,64 @@ +package org.appdevforall.codeonthego.indexing + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.ParameterizedRobolectricTestRunner + +/** Re-inserting a source replaces its entries, pinned identically for both index implementations. */ +@RunWith(ParameterizedRobolectricTestRunner::class) +class IndexReplaceSourceTest( + private val implementation: String, +) { + private val index: PackageTreeIndex = openPackageTreeIndex(implementation) + + @After + fun tearDown() { + index.close() + } + + private fun keysOf(sourceId: String) = index.query(IndexQuery.bySource(sourceId)).map { it.key }.toSet() + + @Test + fun `inserting a source again replaces its previous entries`() = + runTest { + index.insertSource("jarA", "f1", sequenceOf(PackagedEntry("k1", "jarA", "a.old"), PackagedEntry("k2", "jarA", "a.old"))) + + index.insertSource("jarA", "f2", sequenceOf(PackagedEntry("k1", "jarA", "a.new"), PackagedEntry("k3", "jarA", "a.new"))) + + assertThat(keysOf("jarA")).containsExactly("k1", "k3") + assertThat(index.sourceFingerprint("jarA")).isEqualTo("f2") + } + + @Test + fun `inserting a source again drops the packages only its previous entries had`() = + runTest { + index.insertSource("jarA", "f1", sequenceOf(PackagedEntry("k1", "jarA", "a.old"))) + + index.insertSource("jarA", "f2", sequenceOf(PackagedEntry("k1", "jarA", "a.new"))) + + assertThat(index.subpackages("a", sourceIds = listOf("jarA"))).containsExactly("a.new") + } + + @Test + fun `inserting a source again leaves other sources alone`() = + runTest { + index.insertSource("jarA", "f1", sequenceOf(PackagedEntry("k1", "jarA", "a.b"))) + index.insertSource("jarB", "f1", sequenceOf(PackagedEntry("k1", "jarB", "a.b"))) + + index.insertSource("jarA", "f2", emptySequence()) + + assertThat(keysOf("jarA")).isEmpty() + assertThat(keysOf("jarB")).containsExactly("k1") + assertThat(index.subpackages("a", sourceIds = listOf("jarB"))).containsExactly("a.b") + } + + companion object { + @JvmStatic + @ParameterizedRobolectricTestRunner.Parameters(name = "{0}") + fun implementations() = listOf(arrayOf(SQLITE), arrayOf(MEMORY)) + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/PackageTreeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/PackageTreeTest.kt new file mode 100644 index 0000000000..e5cf37360e --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/PackageTreeTest.kt @@ -0,0 +1,207 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.PackageTree +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.ParameterizedRobolectricTestRunner + +/** The package tree an index derives from its entries, pinned identically for both index implementations. */ +@RunWith(ParameterizedRobolectricTestRunner::class) +class PackageTreeTest( + private val implementation: String, +) { + private val index: PackageTreeIndex = openPackageTreeIndex(implementation) + + @After + fun tearDown() { + index.close() + } + + @Test + fun `an entry's package makes every ancestor package exist`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b.c")) + + assertThat(index.containsPackage("a.b.c", sourceIds = null)).isTrue() + assertThat(index.containsPackage("a.b", sourceIds = null)).isTrue() + assertThat(index.containsPackage("a", sourceIds = null)).isTrue() + assertThat(index.containsPackage("a.b.c.d", sourceIds = null)).isFalse() + } + + @Test + fun `subpackages of the empty parent are the root packages`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b.c")) + index.insert(PackagedEntry("k2", "jarA", "x.y")) + + assertThat(index.subpackages("", sourceIds = null)).containsExactly("a", "x") + } + + @Test + fun `subpackages are the direct children only`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b.c")) + + assertThat(index.subpackages("a", sourceIds = null)).containsExactly("a.b") + assertThat(index.subpackages("a.b", sourceIds = null)).containsExactly("a.b.c") + assertThat(index.subpackages("a.b.c", sourceIds = null)).isEmpty() + } + + @Test + fun `a package in two sources is listed once`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b")) + index.insert(PackagedEntry("k2", "jarB", "a.b")) + index.insert(PackagedEntry("k3", "jarB", "a.c")) + + assertThat(index.subpackages("a", sourceIds = null)).containsExactly("a.b", "a.c") + } + + @Test + fun `a scoped lookup excludes packages only another source has`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b")) + index.insert(PackagedEntry("k2", "jarB", "a.c")) + + assertThat(index.subpackages("a", sourceIds = listOf("jarA"))).containsExactly("a.b") + assertThat(index.containsPackage("a.c", sourceIds = listOf("jarA"))).isFalse() + assertThat(index.containsPackage("a.c", sourceIds = listOf("jarA", "jarB"))).isTrue() + } + + @Test + fun `a lookup scoped to no sources finds nothing`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b")) + + assertThat(index.subpackages("", sourceIds = emptyList())).isEmpty() + assertThat(index.containsPackage("a", sourceIds = emptyList())).isFalse() + } + + @Test + fun `the default package has no package entry`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "")) + index.insert(PackagedEntry("k2", "jarA", null)) + + assertThat(index.subpackages("", sourceIds = null)).isEmpty() + assertThat(index.containsPackage("", sourceIds = null)).isFalse() + } + + @Test + fun `removing a source removes the packages only it had`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b")) + index.insert(PackagedEntry("k2", "jarA", "x")) + index.insert(PackagedEntry("k3", "jarB", "a.b")) + + index.removeBySource("jarA") + + assertThat(index.subpackages("", sourceIds = null)).containsExactly("a") + assertThat(index.containsPackage("a.b", sourceIds = null)).isTrue() + assertThat(index.containsPackage("a.b", sourceIds = listOf("jarA"))).isFalse() + assertThat(index.containsPackage("x", sourceIds = null)).isFalse() + } + + @Test + fun `removing several sources removes their packages`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a")) + index.insert(PackagedEntry("k2", "jarB", "b")) + index.insert(PackagedEntry("k3", "jarC", "c")) + + index.removeBySources(listOf("jarA", "jarB")) + + assertThat(index.subpackages("", sourceIds = null)).containsExactly("c") + } + + @Test + fun `clearing the index removes every package`() = + runTest { + index.insert(PackagedEntry("k1", "jarA", "a.b")) + + index.clear() + + assertThat(index.subpackages("", sourceIds = null)).isEmpty() + } + + @Test + fun `inserting a source records its packages`() = + runTest { + index.insertSource( + "jarA", + fingerprint = "f1", + entries = sequenceOf(PackagedEntry("k1", "jarA", "a.b"), PackagedEntry("k2", "jarA", "a.c")), + ) + + assertThat(index.subpackages("a", sourceIds = listOf("jarA"))).containsExactly("a.b", "a.c") + } + + companion object { + @JvmStatic + @ParameterizedRobolectricTestRunner.Parameters(name = "{0}") + fun implementations() = listOf(arrayOf(SQLITE), arrayOf(MEMORY)) + } +} + +/** An index whose package tree tests can query directly. */ +internal interface PackageTreeIndex : + Index, + PackageTree + +internal const val SQLITE = "sqlite" +internal const val MEMORY = "memory" + +internal data class PackagedEntry( + override val key: String, + override val sourceId: String, + val pkg: String?, +) : Indexable + +internal val packagedEntryDescriptor = + object : IndexDescriptor { + override val name = "test_packages" + override val fields = listOf(IndexField("pkg")) + + override fun fieldValues(entry: PackagedEntry) = mapOf("pkg" to entry.pkg) + + override fun packageOf(entry: PackagedEntry) = entry.pkg + + override fun serialize(entry: PackagedEntry) = "${entry.key}|${entry.sourceId}|${entry.pkg.orEmpty()}".toByteArray() + + override fun deserialize(bytes: ByteArray): PackagedEntry { + val parts = String(bytes).split("|") + return PackagedEntry(parts[0], parts[1], parts[2]) + } + } + +internal fun openPackageTreeIndex(implementation: String): PackageTreeIndex = + when (implementation) { + SQLITE -> { + val sqlite = + SQLiteIndex( + descriptor = packagedEntryDescriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + ) + object : PackageTreeIndex, Index by sqlite, PackageTree by sqlite {} + } + + MEMORY -> { + val memory = InMemoryIndex(packagedEntryDescriptor) + object : PackageTreeIndex, Index by memory, PackageTree by memory {} + } + + else -> { + error("Unknown implementation: $implementation") + } + } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPackageTreeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPackageTreeTest.kt new file mode 100644 index 0000000000..a3da931b85 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPackageTreeTest.kt @@ -0,0 +1,120 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** Package tree storage details only the SQLite-backed index has: chunked scopes, plans and rebuilds. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexPackageTreeTest { + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + private fun open( + formatVersion: Int = 1, + dbName: String? = null, + ) = SQLiteIndex( + descriptor = packagedEntryDescriptor, + context = context, + dbName = dbName, + formatVersion = formatVersion, + sourceIdChunkSize = 2, + ) + + @Test + fun `a scope spanning several chunks sees every chunk's packages once`() = + runTest { + open().use { index -> + val sources = (0 until 5).map { "jar$it" } + for (source in sources) { + index.insert(PackagedEntry("shared", source, "a.shared")) + index.insert(PackagedEntry("own", source, "a.$source")) + } + + val children = index.subpackages("a", sourceIds = sources) + + assertThat(children).containsExactly("a.shared", "a.jar0", "a.jar1", "a.jar2", "a.jar3", "a.jar4") + assertThat(index.containsPackage("a.jar4", sourceIds = sources)).isTrue() + } + } + + @Test + fun `a scoped subpackage lookup is served by the parent index`() { + open().use { index -> + val plan = index.explainSubpackages("a", sourceIds = listOf("jarA", "jarB")) + + assertThat(plan).contains("idx_test_packages_packages_parent") + } + } + + @Test + fun `an unscoped subpackage lookup is served by the parent index`() { + open().use { index -> + val plan = index.explainSubpackages("a", sourceIds = null) + + assertThat(plan).contains("idx_test_packages_packages_parent") + } + } + + @Test + fun `an unscoped package existence check is served by the parent index`() { + open().use { index -> + val plan = index.explainContainsPackage("a.b", sourceIds = null) + + assertThat(plan).contains("idx_test_packages_packages_parent") + } + } + + @Test + fun `reopening at a newer version drops the packages`() = + runTest { + open(formatVersion = 1, dbName = DB_NAME).use { it.insert(PackagedEntry("k1", "jarA", "a.b")) } + + open(formatVersion = 2, dbName = DB_NAME).use { + assertThat(it.subpackages("", sourceIds = null)).isEmpty() + } + } + + @Test + fun `the in-memory index lists the same packages`() = + runTest { + val entries = + listOf( + PackagedEntry("k1", "jarA", "android.app"), + PackagedEntry("k2", "jarA", "android.view.inputmethod"), + PackagedEntry("k3", "jarB", "android.view"), + PackagedEntry("k4", "jarB", "androidx.core.app"), + PackagedEntry("k5", "jarC", "kotlin"), + PackagedEntry("k6", "jarC", ""), + ) + val parents = listOf("", "android", "android.view", "androidx", "androidx.core", "kotlin", "missing") + val scopes = listOf(null, emptyList(), listOf("jarA"), listOf("jarB", "jarC"), listOf("jarA", "jarB", "jarC")) + + open().use { sqlite -> + val memory = InMemoryIndex(packagedEntryDescriptor) + sqlite.insertAll(entries.asSequence()) + memory.insertAll(entries.asSequence()) + + for (scope in scopes) { + for (parent in parents) { + assertThat(sqlite.subpackages(parent, scope)).isEqualTo(memory.subpackages(parent, scope)) + assertThat(sqlite.containsPackage(parent, scope)).isEqualTo(memory.containsPackage(parent, scope)) + } + } + assertThat(sqlite.subpackages("android", sourceIds = null)).containsExactly("android.app", "android.view") + } + } + + private companion object { + const val DB_NAME = "package_tree_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexReindexVisibilityTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexReindexVisibilityTest.kt new file mode 100644 index 0000000000..8cd78c6ab5 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexReindexVisibilityTest.kt @@ -0,0 +1,81 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** A source's rows stay readable while a background pass re-indexes it, and after one that fails. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexReindexVisibilityTest { + private val context = ApplicationProvider.getApplicationContext() + + private val index = + SQLiteIndex( + descriptor = packagedEntryDescriptor, + context = context, + dbName = DB_NAME, + formatVersion = 1, + batchSize = 1, + ) + + private val indexer = BackgroundIndexer(index) + + @After + fun tearDown() { + indexer.close() + index.close() + context.deleteDatabase(DB_NAME) + } + + private fun keysOf(sourceId: String) = index.query(IndexQuery.bySource(sourceId)).map { it.key }.toSet() + + private suspend fun indexJarA( + fingerprint: String, + entries: () -> Sequence, + ) = indexer.indexSource("jarA", skipIfExists = false, fingerprint = fingerprint) { entries() }.join() + + @Test(timeout = 30_000) + fun `a source's previous rows stay visible while it is re-indexed`() = + runTest { + indexJarA("f1") { sequenceOf(PackagedEntry("k1", "jarA", "a.b"), PackagedEntry("k2", "jarA", "a.b")) } + + var seenMidPass = emptySet() + indexJarA("f2") { + sequence { + yield(PackagedEntry("k1", "jarA", "a.b")) + seenMidPass = keysOf("jarA") + yield(PackagedEntry("k3", "jarA", "a.b")) + } + } + + assertThat(seenMidPass).containsAtLeast("k1", "k2") + assertThat(keysOf("jarA")).containsExactly("k1", "k3") + } + + @Test(timeout = 30_000) + fun `a failed re-index keeps the previous rows and records no fingerprint`() = + runTest { + indexJarA("f1") { sequenceOf(PackagedEntry("k1", "jarA", "a.b"), PackagedEntry("k2", "jarA", "a.b")) } + + indexJarA("f2") { + sequence { + yield(PackagedEntry("k1", "jarA", "a.b")) + error("jar unreadable") + } + } + + assertThat(keysOf("jarA")).containsAtLeast("k1", "k2") + assertThat(index.sourceFingerprint("jarA")).isNull() + } + + private companion object { + const val DB_NAME = "reindex_visibility_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingProgressTrackerTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingProgressTrackerTest.kt new file mode 100644 index 0000000000..e5548c5307 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingProgressTrackerTest.kt @@ -0,0 +1,200 @@ +package org.appdevforall.codeonthego.indexing.service + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.CompletableJob +import kotlinx.coroutines.Job +import org.appdevforall.codeonthego.indexing.service.IndexingState.Idle +import org.appdevforall.codeonthego.indexing.service.IndexingState.Indexing +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class IndexingProgressTrackerTest { + private val tracker = IndexingProgressTracker() + + @Test + fun `each tracked source counts once toward the total`() { + val pass = tracker.openPass() + + pass.track("a", Job()) + pass.track("b", Job()) + + assertThat(tracker.state.value).isEqualTo(Indexing(done = 0, total = 2)) + } + + @Test + fun `a source resubmitted while pending is counted once`() { + val pass = tracker.openPass() + val first = Job() + + pass.track("a", first) + pass.track("a", first) + pass.track("a", Job()) + + assertThat(tracker.state.value).isEqualTo(Indexing(done = 0, total = 1)) + } + + @Test + fun `a superseded job's completion is not counted as done`() { + val pass = tracker.openPass() + val superseded = Job() + pass.track("a", superseded) + pass.track("a", Job()) + + superseded.cancel() + + assertThat(tracker.state.value).isEqualTo(Indexing(done = 0, total = 1)) + } + + @Test + fun `a pass that tracks a source another pass has pending counts nothing new`() { + val job = Job() + tracker.openPass().track("a", job) + val folding = tracker.openPass() + + folding.track("a", job) + + assertThat(folding.newlyCounted).isEqualTo(0) + } + + @Test + fun `a pass counts each source it adds to the total`() { + val pass = tracker.openPass() + + pass.track("a", Job()) + pass.track("b", Job()) + + assertThat(pass.newlyCounted).isEqualTo(2) + } + + @Test + fun `a completed job counts toward done`() { + val pass = tracker.openPass() + val first = Job() + pass.track("a", first) + pass.track("b", Job()) + + first.complete() + + assertThat(tracker.state.value).isEqualTo(Indexing(done = 1, total = 2)) + } + + @Test + fun `the state returns to Idle once the pass closes and every job completed`() { + val job = Job() + tracker.openPass().use { it.track("a", job) } + + job.complete() + + assertThat(tracker.state.value).isEqualTo(Idle) + } + + @Test + fun `a cancelled job still returns the state to Idle`() { + val job = Job() + tracker.openPass().use { it.track("a", job) } + + job.cancel() + + assertThat(tracker.state.value).isEqualTo(Idle) + } + + @Test + fun `a failed job still returns the state to Idle`() { + val job = Job() + tracker.openPass().use { it.track("a", job) } + + job.completeExceptionally(IllegalStateException("scan failed")) + + assertThat(tracker.state.value).isEqualTo(Idle) + } + + @Test + fun `a pass that throws between submissions still returns the state to Idle`() { + val job = Job() + + runCatching { + tracker.openPass().use { pass -> + pass.track("a", job) + error("jarsToIndex failed") + } + } + job.complete() + + assertThat(tracker.state.value).isEqualTo(Idle) + } + + @Test + fun `an open pass stays Indexing after its only tracked job completes`() { + val pass = tracker.openPass() + + pass.track("a", completedJob()) + + assertThat(tracker.state.value).isEqualTo(Indexing(done = 1, total = 1)) + } + + @Test + fun `jobs still pending after their pass closes keep the state Indexing`() { + tracker.openPass().use { it.track("a", Job()) } + + assertThat(tracker.state.value).isEqualTo(Indexing(done = 0, total = 1)) + } + + @Test + fun `an open pass that tracked nothing leaves the state Idle`() { + tracker.openPass() + + tracker.openPass().use { it.track("a", completedJob()) } + + assertThat(tracker.state.value).isEqualTo(Idle) + } + + @Test + fun `counts restart from zero once the state returns to Idle`() { + tracker.openPass().use { it.track("a", completedJob()) } + + tracker.openPass().track("b", Job()) + + assertThat(tracker.state.value).isEqualTo(Indexing(done = 0, total = 1)) + } + + @Test + fun `reset returns the state to Idle while work is pending`() { + tracker.openPass().track("a", Job()) + + tracker.reset() + + assertThat(tracker.state.value).isEqualTo(Idle) + } + + @Test + fun `a pass left open across reset no longer counts its submissions`() { + val stale = tracker.openPass() + tracker.reset() + + stale.track("a", Job()) + + assertThat(tracker.state.value).isEqualTo(Idle) + } + + private fun completedJob(): CompletableJob = Job().apply { complete() } + + @Test + fun `a pass counts as finished once the state returns to idle`() { + val tracker = IndexingProgressTracker() + + tracker.openPass().use { pass -> pass.track("/libs/a.jar", Job().apply { complete() }) } + + assertThat(tracker.finishedPasses.value).isEqualTo(1L) + } + + @Test + fun `a pass that tracked nothing does not count as finished`() { + val tracker = IndexingProgressTracker() + + tracker.openPass().close() + + assertThat(tracker.finishedPasses.value).isEqualTo(0L) + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManagerStateTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManagerStateTest.kt new file mode 100644 index 0000000000..1dfaf4f4c3 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManagerStateTest.kt @@ -0,0 +1,30 @@ +package org.appdevforall.codeonthego.indexing.service + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.Job +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class IndexingServiceManagerStateTest { + @Test + fun `the state reports the tracker's progress`() { + val manager = IndexingServiceManager() + + manager.progressTracker.openPass().track("a", Job()) + + assertThat(manager.state.value).isEqualTo(IndexingState.Indexing(done = 0, total = 1)) + manager.close() + } + + @Test + fun `close leaves the state Idle while a pass is open and a job pending`() { + val manager = IndexingServiceManager() + manager.progressTracker.openPass().track("a", Job()) + + manager.close() + + assertThat(manager.state.value).isEqualTo(IndexingState.Idle) + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManagerUnreadableJarTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManagerUnreadableJarTest.kt new file mode 100644 index 0000000000..6ba246fedb --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManagerUnreadableJarTest.kt @@ -0,0 +1,37 @@ +package org.appdevforall.codeonthego.indexing.service + +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * [IndexingServiceManager.filterNewlyUnreadableJars] dedupes an unreadable JAR path across every + * call for the life of one manager, so a JAR that stays unreadable across many indexing passes is + * reported once rather than once per pass - and reports it again once a new project session starts + * a new manager (or this one is [closed][IndexingServiceManager.close] and reused). + */ +@RunWith(JUnit4::class) +class IndexingServiceManagerUnreadableJarTest { + @Test + fun `a jar path already reported is filtered out of a later call`() { + val manager = IndexingServiceManager() + + val first = manager.filterNewlyUnreadableJars(listOf("a.jar", "b.jar")) + val second = manager.filterNewlyUnreadableJars(listOf("a.jar", "c.jar")) + + assertThat(first).containsExactly("a.jar", "b.jar") + assertThat(second).containsExactly("c.jar") + manager.close() + } + + @Test + fun `close clears the reported set so a new session reports the same jar again`() { + val manager = IndexingServiceManager() + manager.filterNewlyUnreadableJars(listOf("a.jar")) + + manager.close() + + assertThat(manager.filterNewlyUnreadableJars(listOf("a.jar"))).containsExactly("a.jar") + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt index 86e6b9d7f8..1024dd1a38 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt @@ -58,6 +58,46 @@ class BackgroundIndexerTest { assertThat(index.get("k2")).isNotNull() } + /** A provider sequence holding a resource open, as a JAR scan does, recording whether it was closed. */ + private class ClosingEntries( + private val entries: Sequence, + ) : Sequence by entries, + java.io.Closeable { + var closed = false + + override fun close() { + closed = true + } + } + + @Test + fun `a provider's sequence is closed when the pass fails part-way`() = + runTest { + val (_, indexer) = makeIndexAndIndexer() + val entries = + ClosingEntries( + sequence { + yield(Entry("k1", "src1", "v1")) + error("jar unreadable") + }, + ) + + indexer.indexSource("src1", skipIfExists = false, fingerprint = "f1") { entries }.join() + + assertThat(entries.closed).isTrue() + } + + @Test + fun `a provider's sequence is closed when the pass completes`() = + runTest { + val (_, indexer) = makeIndexAndIndexer() + val entries = ClosingEntries(sequenceOf(Entry("k1", "src1", "v1"))) + + indexer.indexSource("src1", skipIfExists = false) { entries }.join() + + assertThat(entries.closed).isTrue() + } + @Test fun `skipIfExists=true skips re-indexing of existing source`() = runTest { diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt index fc94ebbc84..0130ac0bbb 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt @@ -73,17 +73,23 @@ import com.itsaky.androidide.utils.DocumentUtils import com.itsaky.androidide.utils.VMUtils import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.jvm.JarIndexingService import org.appdevforall.codeonthego.indexing.jvm.JvmGeneratedIndexingService import org.appdevforall.codeonthego.indexing.jvm.JvmLibraryIndexingService +import org.appdevforall.codeonthego.indexing.jvm.JvmModuleOutputIndexingService import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode import org.slf4j.LoggerFactory import java.nio.file.Files import java.nio.file.Path -import java.util.Objects +import java.util.concurrent.atomic.AtomicLong class JavaLanguageServer : ILanguageServer { private val completionProvider: CompletionProvider = CompletionProvider() @@ -94,7 +100,15 @@ class JavaLanguageServer : ILanguageServer { private var _settings: IServerSettings? = null private var selectedFile: Path? = null private val timer = AnalyzeTimer { analyzeSelected() } - private var cachedCompletion: CachedCompletion + + /** + * The last completion result, guarded against a reset racing its own store. Read from + * completion requests and reset from the indexing-state collector, so it crosses threads. + */ + private val completionCache = CompletionCacheGuard() + + private val serverScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private var indexingStateCollectorJob: Job? = null val settings: IServerSettings get() { @@ -113,20 +127,39 @@ class JavaLanguageServer : ILanguageServer { } init { - cachedCompletion = CachedCompletion.EMPTY - applySettings(JavaServerSettings.getInstance()) if (!EventBus.getDefault().isRegistered(this)) { EventBus.getDefault().register(this) } - val projectManager = ProjectManagerImpl.getInstance() - projectManager.indexingServiceManager.register( - service = JvmLibraryIndexingService(context = BaseApplication.baseInstance) + val indexingServiceManager = ProjectManagerImpl.getInstance().indexingServiceManager + val context = BaseApplication.baseInstance + val tracker = indexingServiceManager.progressTracker + val unreadableJarFilter = indexingServiceManager::filterNewlyUnreadableJars + indexingServiceManager.register( + service = + JvmLibraryIndexingService( + context = context, + progressTracker = tracker, + unreadableJarFilter = unreadableJarFilter, + ), + ) + indexingServiceManager.register( + service = + JvmGeneratedIndexingService( + context = context, + progressTracker = tracker, + unreadableJarFilter = unreadableJarFilter, + ), ) - projectManager.indexingServiceManager.register( - service = JvmGeneratedIndexingService(context = BaseApplication.baseInstance) + indexingServiceManager.register( + service = + JvmModuleOutputIndexingService( + context = context, + progressTracker = tracker, + unreadableJarFilter = unreadableJarFilter, + ), ) JavaSnippetRepository.init() @@ -140,6 +173,7 @@ class JavaLanguageServer : ILanguageServer { clearCache() EventBus.getDefault().unregister(this) timer.cancel() + serverScope.cancel() } override fun connectClient(client: ILanguageClient?) { @@ -163,10 +197,24 @@ class JavaLanguageServer : ILanguageServer { override fun setupWithProject(workspace: Workspace) { LSPEditorActions.ensureActionsMenuRegistered(JavaCodeActionsMenu) - (ProjectManagerImpl.getInstance() - .indexingServiceManager - .getService(JvmLibraryIndexingService.ID) as? JvmLibraryIndexingService?) - ?.refresh() + // A sync can change the modules and their dependencies; the generated scope follows builds instead. + val indexingServiceManager = ProjectManagerImpl.getInstance().indexingServiceManager + for (serviceId in listOf(JvmLibraryIndexingService.ID, JvmModuleOutputIndexingService.ID)) { + (indexingServiceManager.getService(serviceId) as? JarIndexingService)?.refresh() + } + + /* + * Closing a project nulls ProjectManagerImpl's indexing service manager and a later + * access recreates it, so the collector is restarted here against whichever instance is + * current rather than assumed to still be the one it was started against. + */ + indexingStateCollectorJob?.cancel() + indexingStateCollectorJob = + serverScope.launch { + collectIndexingPassResets(indexingServiceManager.finishedPasses) { + completionCache.reset() + } + } // Once we have project initialized // Destory the NO_MODULE_COMPILER instance @@ -196,8 +244,7 @@ class JavaLanguageServer : ILanguageServer { override fun complete(params: CompletionParams?): CompletionResult { val compiler = getCompiler(params!!.file) - if (!settings.completionsEnabled() || !completionProvider.canComplete(params.file) - ) { + if (!settings.completionsEnabled() || !completionProvider.canComplete(params.file)) { return CompletionResult.EMPTY } @@ -206,12 +253,13 @@ class JavaLanguageServer : ILanguageServer { diagnosticProvider.cancel() } + val generationAtStart = completionCache.currentGeneration() completionProvider.reset( compiler, settings, - cachedCompletion, - ) { cachedCompletion: CachedCompletion -> - updateCachedCompletion(cachedCompletion) + completionCache.cachedCompletion, + ) { candidate: CachedCompletion -> + completionCache.store(candidate, generationAtStart) } return completionProvider.complete(params) @@ -265,8 +313,7 @@ class JavaLanguageServer : ILanguageServer { } } - override fun formatCode(params: FormatCodeParams?): CodeFormatResult = - CodeFormatProvider(settings).format(params) + override fun formatCode(params: FormatCodeParams?): CodeFormatResult = CodeFormatProvider(settings).format(params) override fun handleFailure(failure: LSPFailure?): Boolean { return when (failure!!.type) { @@ -291,11 +338,6 @@ class JavaLanguageServer : ILanguageServer { return JavaCompilerProvider.get(module) } - private fun updateCachedCompletion(cachedCompletion: CachedCompletion) { - Objects.requireNonNull(cachedCompletion) - this.cachedCompletion = cachedCompletion - } - private fun startOrRestartAnalyzeTimer() { if (VMUtils.isJvm) { return @@ -364,3 +406,67 @@ class JavaLanguageServer : ILanguageServer { } } } + +/** + * Collects [finishedPasses], invoking [onIndexingFinished] every time it changes, which is every + * time at least one indexing pass has finished. + * + * A cached completion computed before or while a pass ran can be missing symbols the index has + * since gained, so the cache must be dropped when a pass finishes, not on every intermediate + * progress update. The count is collected rather than the indexing state because the state is + * conflated: a short pass, such as the one after a build, can start and finish between two + * collections and leave no trace in it. + */ +internal suspend fun collectIndexingPassResets( + finishedPasses: StateFlow, + onIndexingFinished: () -> Unit, +) { + var seen = finishedPasses.value + finishedPasses.collect { current -> + if (current != seen) { + seen = current + onIndexingFinished() + } + } +} + +/** + * A completion cache guarded by a generation counter, so a reset that happens while a completion + * is in flight cannot be clobbered by that completion's own (now-stale) result. + * + * [reset] bumps the generation before clearing the cache. [store] only takes effect if the + * generation captured by [currentGeneration] at the start of that completion is still current; + * [reset] and [store] are mutually exclusive, so there is no window in which a [store] can read + * the pre-reset generation and still land after [reset] has cleared the cache. + */ +internal class CompletionCacheGuard { + private val generation = AtomicLong(0) + private val lock = Any() + + @Volatile + var cachedCompletion: CachedCompletion = CachedCompletion.EMPTY + private set + + /** The generation to capture at the start of a completion, for a later [store] call. */ + fun currentGeneration(): Long = generation.get() + + /** Caches [candidate], unless a [reset] has run since [capturedGeneration] was captured. */ + fun store( + candidate: CachedCompletion, + capturedGeneration: Long, + ) { + synchronized(lock) { + if (generation.get() == capturedGeneration) { + cachedCompletion = candidate + } + } + } + + /** Bumps the generation and drops the cache, so no in-flight [store] can land after this. */ + fun reset() { + synchronized(lock) { + generation.incrementAndGet() + cachedCompletion = CachedCompletion.EMPTY + } + } +} diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/BaseJavaCodeAction.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/BaseJavaCodeAction.kt index fd9c7647b8..88906a07a5 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/BaseJavaCodeAction.kt +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/BaseJavaCodeAction.kt @@ -1,117 +1,124 @@ -/* - * This file is part of AndroidIDE. - * - * AndroidIDE is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * AndroidIDE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with AndroidIDE. If not, see . - */ - -package com.itsaky.androidide.lsp.java.actions - -import android.content.Context -import android.graphics.drawable.Drawable -import com.itsaky.androidide.actions.ActionData -import com.itsaky.androidide.actions.ActionItem -import com.itsaky.androidide.actions.EditorActionItem -import com.itsaky.androidide.actions.hasRequiredData -import com.itsaky.androidide.actions.markInvisible -import com.itsaky.androidide.actions.requireFile -import com.itsaky.androidide.lsp.api.ILanguageClient -import com.itsaky.androidide.lsp.api.ILanguageServerRegistry -import com.itsaky.androidide.lsp.java.JavaCompilerProvider -import com.itsaky.androidide.lsp.java.JavaLanguageServer -import com.itsaky.androidide.lsp.java.R -import com.itsaky.androidide.lsp.java.compiler.JavaCompilerService -import com.itsaky.androidide.lsp.java.rewrite.Rewrite -import com.itsaky.androidide.projects.IProjectManager -import com.itsaky.androidide.utils.DocumentUtils -import com.itsaky.androidide.utils.ILogger -import com.itsaky.androidide.utils.flashError -import java.io.File - -/** - * Base class for java code actions - * - * @author Akash Yadav - */ -abstract class BaseJavaCodeAction : EditorActionItem { - - override var visible: Boolean = true - override var enabled: Boolean = true - override var icon: Drawable? = null - override var requiresUIThread: Boolean = false - override var location: ActionItem.Location = ActionItem.Location.EDITOR_CODE_ACTIONS - - protected abstract val titleTextRes: Int - - override fun prepare(data: ActionData) { - super.prepare(data) - if ( - !data.hasRequiredData(Context::class.java, JavaLanguageServer::class.java, File::class.java) - ) { - markInvisible() - return - } - - if (titleTextRes != -1) { - label = data[Context::class.java]!!.getString(titleTextRes) - } - - val file = data.requireFile() - val isJava = DocumentUtils.isJavaFile(file.toPath()) - val module = IProjectManager.getInstance().findModuleForFile(file, false) - - visible = isJava - enabled = isJava && module != null - } - - fun performCodeAction(data: ActionData, result: Rewrite) { - val compiler = data.requireCompiler() - - val actions = - try { - result.asCodeActions(compiler, label) - } catch (e: Exception) { - flashError(e.cause?.message ?: e.message) - ILogger.ROOT.error(e.cause?.message ?: e.message, e) - return - } - - if (actions == null) { - onPerformCodeActionFailed(data) - return - } - - data.getLanguageClient()?.performCodeAction(actions) - } - - protected open fun onPerformCodeActionFailed(data: ActionData) { - flashError(R.string.msg_codeaction_failed) - } - - protected fun ActionData.requireLanguageServer(): JavaLanguageServer { - return ILanguageServerRegistry.default.getServer(JavaLanguageServer.SERVER_ID) - as JavaLanguageServer - } - - protected fun ActionData.getLanguageClient(): ILanguageClient? { - return requireLanguageServer().client - } - - protected fun ActionData.requireCompiler(): JavaCompilerService { - val module = IProjectManager.getInstance().findModuleForFile(requireFile(), false) - requireNotNull(module) { - "Cannot get compiler instance. Unable to find module for file: ${requireFile().name}" - } - return JavaCompilerProvider.get(module) - } -} +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.java.actions + +import android.content.Context +import android.graphics.drawable.Drawable +import com.itsaky.androidide.actions.ActionData +import com.itsaky.androidide.actions.ActionItem +import com.itsaky.androidide.actions.EditorActionItem +import com.itsaky.androidide.actions.hasRequiredData +import com.itsaky.androidide.actions.markInvisible +import com.itsaky.androidide.actions.requireFile +import com.itsaky.androidide.lsp.api.ILanguageClient +import com.itsaky.androidide.lsp.api.ILanguageServerRegistry +import com.itsaky.androidide.lsp.java.JavaCompilerProvider +import com.itsaky.androidide.lsp.java.JavaLanguageServer +import com.itsaky.androidide.lsp.java.R +import com.itsaky.androidide.lsp.java.compiler.JavaCompilerService +import com.itsaky.androidide.lsp.java.rewrite.Rewrite +import com.itsaky.androidide.projects.IProjectManager +import com.itsaky.androidide.utils.ILogger +import com.itsaky.androidide.utils.flashError +import java.io.File + +/** + * Base class for java code actions + * + * @author Akash Yadav + */ +abstract class BaseJavaCodeAction : EditorActionItem { + override var visible: Boolean = true + override var enabled: Boolean = true + override var icon: Drawable? = null + override var requiresUIThread: Boolean = false + override var location: ActionItem.Location = ActionItem.Location.EDITOR_CODE_ACTIONS + + protected abstract val titleTextRes: Int + + override fun prepare(data: ActionData) { + super.prepare(data) + if ( + !data.hasRequiredData(Context::class.java, JavaLanguageServer::class.java, File::class.java) + ) { + markInvisible() + return + } + + if (titleTextRes != -1) { + label = data[Context::class.java]!!.getString(titleTextRes) + } + + val file = data.requireFile() + val isJava = isJavaFileName(file.name) + val module = IProjectManager.getInstance().findModuleForFile(file, false) + + visible = isJava + enabled = isJava && module != null + } + + fun performCodeAction( + data: ActionData, + result: Rewrite, + ) { + val compiler = data.requireCompiler() + + val actions = + try { + result.asCodeActions(compiler, label) + } catch (e: Exception) { + flashError(e.cause?.message ?: e.message) + ILogger.ROOT.error(e.cause?.message ?: e.message, e) + return + } + + if (actions == null) { + onPerformCodeActionFailed(data) + return + } + + data.getLanguageClient()?.performCodeAction(actions) + } + + protected open fun onPerformCodeActionFailed(data: ActionData) { + flashError(R.string.msg_codeaction_failed) + } + + protected fun ActionData.requireLanguageServer(): JavaLanguageServer = + ILanguageServerRegistry.default.getServer(JavaLanguageServer.SERVER_ID) + as JavaLanguageServer + + protected fun ActionData.getLanguageClient(): ILanguageClient? = requireLanguageServer().client + + protected fun ActionData.requireCompiler(): JavaCompilerService { + val module = IProjectManager.getInstance().findModuleForFile(requireFile(), false) + requireNotNull(module) { + "Cannot get compiler instance. Unable to find module for file: ${requireFile().name}" + } + return JavaCompilerProvider.get(module) + } +} + +/** + * Returns whether [name] names a regular Java source file, excluding `module-info.java` and + * `package-info.java`, which declare no importable type. + * + * Decided by name alone, with no file-system access -- [BaseJavaCodeAction.prepare] runs on the UI + * thread. + */ +internal fun isJavaFileName(name: String): Boolean = name.endsWith(".java") && name != "module-info.java" && name != "package-info.java" diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AddImportAction.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AddImportAction.kt index 00e1e31549..90db5b1c6f 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AddImportAction.kt +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AddImportAction.kt @@ -35,14 +35,19 @@ import com.itsaky.androidide.lsp.java.JavaCompilerProvider import com.itsaky.androidide.lsp.java.actions.BaseJavaCodeAction import com.itsaky.androidide.lsp.java.models.DiagnosticCode import com.itsaky.androidide.lsp.java.rewrite.AddImport -import com.itsaky.androidide.lsp.java.rewrite.Rewrite import com.itsaky.androidide.lsp.models.CodeActionItem import com.itsaky.androidide.lsp.models.DiagnosticItem import com.itsaky.androidide.projects.IProjectManager +import com.itsaky.androidide.projects.ProjectManagerImpl +import com.itsaky.androidide.projects.util.StringSearch import com.itsaky.androidide.resources.R import com.itsaky.androidide.utils.applyLongPressRecursively +import com.itsaky.androidide.utils.flashError import jdkx.tools.Diagnostic import jdkx.tools.JavaFileObject +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.service.IndexingState import org.slf4j.LoggerFactory /** @author Akash Yadav */ @@ -72,16 +77,12 @@ class AddImportAction : BaseJavaCodeAction() { return } - val file = data.requireFile() - val module = - IProjectManager.getInstance().findModuleForFile(file, false) - ?: run { - markInvisible() - return - } - - val compiler = JavaCompilerProvider.get(module) - + /* + * Optimistic visibility: decide from the diagnostic alone. Resolving actual candidates queries + * the index, which is SQLite-backed I/O that must not run here -- this runs synchronously on + * the UI thread when the code-action menu is filled. execAction resolves candidates in the + * background instead. + */ @Suppress("UNCHECKED_CAST") val jcDiagnostic = JavaDiagnosticUtils.asJCDiagnostic(diagnostic.extra as Diagnostic) @@ -90,12 +91,10 @@ class AddImportAction : BaseJavaCodeAction() { return } - val found = - jcDiagnostic.args[1]?.toString()?.let { compiler.findQualifiedNames(it, true).isNotEmpty() } - ?: false - - visible = found - enabled = found + // BaseJavaCodeAction disables it outside any module, where execAction has nothing to offer. + if (!enabled) { + markInvisible() + } } override suspend fun execAction(data: ActionData): Any { @@ -104,66 +103,68 @@ class AddImportAction : BaseJavaCodeAction() { JavaDiagnosticUtils.asUnwrapper( data.get(DiagnosticItem::class.java)!!.extra as Diagnostic, )!! - val file = data.requireFile() - val module = - IProjectManager.getInstance().findModuleForFile(file, false) - ?: run { - markInvisible() - return Any() - } + val simpleName = diagnostic.d.args[1].toString() + val nioPath = data.requirePath() - val compiler = JavaCompilerProvider.get(module) - - val titles = mutableListOf() - val rewrites = mutableListOf() - val simpleName = diagnostic.d.args[1] - for (name in compiler.publicTopLevelTypes()) { - var klass = name - if (klass.contains('/')) { - klass = klass.replace('/', '.') + return withContext(Dispatchers.IO) { + val module = IProjectManager.getInstance().findModuleForFile(data.requireFile(), false) + if (module == null) { + return@withContext ImportCandidates.None(simpleName) } - if (!klass.endsWith(".$simpleName")) { - continue - } + val compiler = JavaCompilerProvider.get(module) + val importingPackage = StringSearch.packageName(nioPath) + val candidates = compiler.findImportableQualifiedNames(simpleName, importingPackage) - titles.add(klass) - rewrites.add(AddImport(data.requirePath(), klass)) - } + if (candidates.isEmpty()) { + return@withContext ImportCandidates.None(simpleName) + } - if (rewrites.isEmpty()) { - return false + ImportCandidates.Found( + titles = candidates, + rewrites = candidates.map { AddImport(nioPath, it) }, + ) } - - return Pair(titles, rewrites) } - @Suppress("UNCHECKED_CAST") override fun postExec( data: ActionData, result: Any, ) { - if (result !is Pair<*, *>) { + if (result is ImportCandidates.None) { + val context = data.requireContext() + val state = + ProjectManagerImpl + .getInstance() + .indexingServiceManager.state.value + flashError( + noImportableClassMessage( + simpleName = result.simpleName, + state = state, + template = context.getString(R.string.msg_no_importable_class), + indexingTemplate = context.getString(R.string.msg_no_importable_class_indexing), + ), + ) + return + } + + if (result !is ImportCandidates.Found) { return } - val file = data.requireFile() - val module = - IProjectManager.getInstance().findModuleForFile(file, false) - ?: run { - markInvisible() - return - } + val module = IProjectManager.getInstance().findModuleForFile(data.requireFile(), false) + if (module == null) { + markInvisible() + return + } - val compiler = JavaCompilerProvider.get(module) val client = data.getLanguageClient() ?: return + val compiler = JavaCompilerProvider.get(module) val actions = mutableListOf() - val titles = result.first as List - val rewrites = result.second as List - for (index in rewrites.indices) { - val name = titles[index] - val rewrite = rewrites[index] + for (index in result.rewrites.indices) { + val name = result.titles[index] + val rewrite = result.rewrites[index] rewrite.asCodeActions(compiler, name)?.let { actions.add(it) } } @@ -177,7 +178,7 @@ class AddImportAction : BaseJavaCodeAction() { } else -> { - showImportChooser(data, titles, actions, client) + showImportChooser(data, result.titles, actions, client) } } } @@ -231,3 +232,34 @@ class AddImportAction : BaseJavaCodeAction() { ) } } + +/** + * The outcome of resolving import candidates for an unresolved symbol. + */ +internal sealed interface ImportCandidates { + /** A candidate class per title, each mapped to the rewrite that imports it. */ + data class Found( + val titles: List, + val rewrites: List, + ) : ImportCandidates + + /** No importable class named [simpleName] was found. */ + data class None( + val simpleName: String, + ) : ImportCandidates +} + +/** + * The message [AddImportAction.postExec] flashes when no importable class is named [simpleName]: + * [template] normally, or [indexingTemplate] while [state] says library indexing is still in + * progress, since a class matching [simpleName] can still turn up once indexing finishes. + */ +internal fun noImportableClassMessage( + simpleName: String, + state: IndexingState, + template: String, + indexingTemplate: String, +): String { + val chosen = if (state is IndexingState.Indexing) indexingTemplate else template + return String.format(chosen, simpleName) +} diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AutoFixImportsAction.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AutoFixImportsAction.kt index e89567953e..afbdb2f6bd 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AutoFixImportsAction.kt +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/actions/diagnostics/AutoFixImportsAction.kt @@ -35,6 +35,7 @@ import com.itsaky.androidide.lsp.models.CodeActionKind import com.itsaky.androidide.lsp.models.DocumentChange import com.itsaky.androidide.lsp.models.TextEdit import com.itsaky.androidide.models.Range +import com.itsaky.androidide.projects.util.StringSearch import com.itsaky.androidide.utils.DialogUtils import com.itsaky.androidide.utils.applyLongPressRecursively import com.itsaky.androidide.utils.flashInfo @@ -59,29 +60,23 @@ class AutoFixImportsAction : BaseJavaCodeAction() { override suspend fun execAction(data: ActionData): Result { val path = data.requirePath() val compiler = data.requireCompiler() - return compiler.compile(path).get { task -> - val classes = mutableMapOf>() - - // find all unresolved simple names - unresolvedNames(path, task).forEach { simpleName -> - - // if we have already looked for this simple name - // we do not need to look it up again - if (classes[simpleName] != null) return@forEach - - // find classes with those names - compiler.findQualifiedNames(simpleName).let { names -> - - // if we find classes with that specific simple name, map them to the simple name - if (names.isNotEmpty()) { - classes[simpleName] = names - } - } + val importingPackage = StringSearch.packageName(path) + /* + * Only the compile-tree reads run inside get(), which holds the compiler's task: Java completion + * returns nothing while it is held, and the class lookups below wait on the index's disk I/O. + */ + val (fileImports, unresolved) = + compiler.compile(path).get { task -> + getFileImports(task, path) to unresolvedNames(path, task).distinct() } - // return the result - Result(getFileImports(task, path), classes) - } + // Importable classes per unresolved simple name, leaving out those this file's package cannot see. + val classes = + unresolved + .associateWith { simpleName -> compiler.findImportableQualifiedNames(simpleName, importingPackage) } + .filterValues { it.isNotEmpty() } + .toMutableMap() + return Result(fileImports, classes) } override fun postExec( diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/ClasspathTypeLookup.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/ClasspathTypeLookup.kt new file mode 100644 index 0000000000..76fa397856 --- /dev/null +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/ClasspathTypeLookup.kt @@ -0,0 +1,174 @@ +package com.itsaky.androidide.lsp.java.compiler + +import com.itsaky.androidide.projects.ProjectManagerImpl +import com.itsaky.androidide.projects.api.ModuleProject +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmVisibility +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup + +/** The packages and top-level classes of one module's compile classpath, as import paths name them. */ +interface ClasspathPackages { + /** Returns whether [qualifiedName] names a top-level class of the classpath. */ + fun isClass(qualifiedName: String): Boolean + + /** + * Returns the direct subpackages of [packageName], then its top-level classes. [packageName] `""` + * is the default package, whose children are the root packages and the default-package classes. + */ + fun children(packageName: String): List +} + +/** The top-level classes of one module's compile classpath, as the JVM symbol indexes hold them. */ +internal interface ClasspathClassNames : ClasspathPackages { + /** + * Returns the qualified names of top-level classes whose simple name starts with [prefix], ignoring + * case: the exact simple-name matches first, then up to [limit] more from each index. + */ + fun qualifiedNamesByPrefix( + prefix: String, + limit: Int, + ): List + + /** + * Returns the top-level classes whose simple name is exactly [simpleName], each carrying its + * visibility and package. + */ + fun classesNamed(simpleName: String): List +} + +/** + * Top-level class name lookups for a Java compiler: its module's Java source classes, compile + * classpath and boot classpath, deduplicated by qualified name. + * + * Construction does no I/O. Every lookup asks [classpath] for a fresh classpath lookup, which blocks + * on the index's disk I/O, so never call a lookup on the main thread. A classpath lookup is never held + * across requests: one held across a re-sync or an index re-registration answers from a stale + * classpath or a closed index. While the index is being built, lookups answer with what it holds so + * far. + */ +class ClasspathTypeLookup internal constructor( + private val sourceClasses: () -> Collection, + private val classpath: () -> ClasspathClassNames?, + private val bootClasses: () -> Collection, +) { + /** + * Returns the qualified names of top-level classes whose simple name starts with [partial], + * ignoring case: up to [limitPerSource] from each of the sources, the classpath and the boot + * classpath, exact simple-name matches first. + * + * Each source has its own limit so that one with many matches cannot crowd out another's: a + * library index holding dozens of `View*` classes must still leave room for `android.view.View`. + * The classpath applies the limit to each of its indexes, so it can return more than + * [limitPerSource] names, and all of them are kept. A name already taken from an earlier source + * does not count against a later source's limit. The caller ranks and trims the union. A caller + * that shows at most N names should pass N + 1, so a source with more matches than fit yields more + * names than it can show and it can tell the result is incomplete. There is no fuzzy matching. + */ + fun findTypeNamesMatching( + partial: String, + limitPerSource: Int, + ): List { + require(limitPerSource > 0) { "limitPerSource must be positive, was $limitPerSource" } + + val sources = sourceClasses() + val boot = bootClasses() + val (classpathNamed, classpathPrefixed) = + classpath() + ?.qualifiedNamesByPrefix(partial, limitPerSource) + .orEmpty() + .partition { simpleNameOf(it) == partial } + val names = LinkedHashSet() + + fun add(candidates: Sequence) = candidates.filterNot(names::contains).take(limitPerSource).forEach(names::add) + + fun Collection.named() = asSequence().filter { simpleNameOf(it) == partial } + + fun Collection.prefixed() = asSequence().filter { simpleNameOf(it).startsWith(partial, ignoreCase = true) } + + add(sources.named()) + names += classpathNamed + add(boot.named()) + add(sources.prefixed()) + names += classpathPrefixed + add(boot.prefixed()) + return names.toList() + } + + /** + * Returns the qualified names of the top-level classes whose simple name is exactly [simpleName] + * that a file in [importingPackage] may import. + * + * Source and boot classes are returned unconditionally. A classpath (index-backed) class is + * excluded when it is package-private or file-private outside [importingPackage], since an import + * of it from there does not compile. A Kotlin file-private top-level class compiles to + * package-private bytecode, but the index records its declared Kotlin visibility as + * [JvmVisibility.PRIVATE], so it needs the same package check. + */ + fun findImportableQualifiedNames( + simpleName: String, + importingPackage: String, + ): List { + val names = + sequence { + yieldAll(sourceClasses().filter { simpleNameOf(it) == simpleName }) + classpath()?.classesNamed(simpleName)?.forEach { symbol -> + val isPackageScoped = + symbol.visibility == JvmVisibility.PACKAGE_PRIVATE || symbol.visibility == JvmVisibility.PRIVATE + if (!isPackageScoped || symbol.packageName == importingPackage) { + yield(symbol.fqName) + } + } + yieldAll(bootClasses().filter { simpleNameOf(it) == simpleName }) + }.distinct() + return names.sorted().toList() + } + + /** + * Returns the packages and top-level classes of the module's compile classpath, or `null` without a + * module. + * + * Source and boot classes are not included. Like every lookup here, the result answers from the + * classpath and indexes as they are when it is created, so use it for one request only. + */ + fun classpathPackages(): ClasspathPackages? = classpath() + + private fun simpleNameOf(qualifiedName: String) = qualifiedName.substringAfterLast('.') + + companion object { + /** + * Creates the lookup for [module] over the indexes registered with the project manager, and the + * boot classes [bootClasses] returns. A `null` [module] has no source or classpath classes. + */ + @JvmStatic + fun forModule( + module: ModuleProject?, + bootClasses: () -> Collection, + ) = ClasspathTypeLookup( + sourceClasses = { + module + ?.compileJavaSourceClasses + ?.allSources() + ?.map { it.qualifiedName } + .orEmpty() + }, + classpath = { module?.let(::indexedClassNames) }, + bootClasses = bootClasses, + ) + + private fun indexedClassNames(module: ModuleProject): ClasspathClassNames { + val lookup = ModuleClasspathLookup.of(module, ProjectManagerImpl.getInstance().indexingServiceManager.registry) + return object : ClasspathClassNames { + override fun qualifiedNamesByPrefix( + prefix: String, + limit: Int, + ) = lookup.qualifiedNamesByPrefix(prefix, limit) + + override fun classesNamed(simpleName: String) = lookup.classesNamed(simpleName) + + override fun isClass(qualifiedName: String) = lookup.isClass(qualifiedName) + + override fun children(packageName: String) = lookup.children(packageName) + } + } + } +} diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/CompilerProvider.java b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/CompilerProvider.java index db23b4ee77..6772145cea 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/CompilerProvider.java +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/CompilerProvider.java @@ -23,44 +23,35 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Collection; -import java.util.List; import java.util.Optional; import java.util.TreeSet; import java.util.stream.Collectors; import jdkx.tools.JavaFileObject; public interface CompilerProvider { - Path NOT_FOUND = Paths.get(""); + Path NOT_FOUND = Paths.get(""); - TreeSet publicTopLevelTypes(); + default SynchronizedTask compile(Collection sources) { + return compile(new CompilationRequest(sources)); + } - TreeSet packagePrivateTopLevelTypes(String packageName); + SynchronizedTask compile(CompilationRequest request); - Optional findAnywhere(String className); + default SynchronizedTask compile(Path... files) { + return compile(Arrays.stream(files).map(SourceFileObject::new).collect(Collectors.toList())); + } - Path findTypeDeclaration(String className); + Optional findAnywhere(String className); - Path[] findTypeReferences(String className); + Path[] findMemberReferences(String className, String memberName); - Path[] findMemberReferences(String className, String memberName); + Path findTypeDeclaration(String className); - default List findQualifiedNames(String simpleName) { - return findQualifiedNames(simpleName, false); - } + Path[] findTypeReferences(String className); - List findQualifiedNames(String simpleName, boolean onlyOne); + TreeSet packagePrivateTopLevelTypes(String packageName); - ParseTask parse(Path file); + ParseTask parse(JavaFileObject file); - ParseTask parse(JavaFileObject file); - - default SynchronizedTask compile(Path... files) { - return compile(Arrays.stream(files).map(SourceFileObject::new).collect(Collectors.toList())); - } - - default SynchronizedTask compile(Collection sources) { - return compile(new CompilationRequest(sources)); - } - - SynchronizedTask compile(CompilationRequest request); + ParseTask parse(Path file); } diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/JavaCompilerService.java b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/JavaCompilerService.java index 8e7d5e2735..447381f678 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/JavaCompilerService.java +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/compiler/JavaCompilerService.java @@ -19,6 +19,7 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; import androidx.core.util.Pair; import com.itsaky.androidide.eventbus.events.editor.DocumentChangeEvent; import com.itsaky.androidide.javac.services.compiler.ReusableCompiler; @@ -76,13 +77,13 @@ public class JavaCompilerService implements CompilerProvider { private static final Cache cacheContainsWord = new Cache<>(); private static final Cache> cacheContainsType = new Cache<>(); private static final Logger LOG = LoggerFactory.getLogger(JavaCompilerService.class); - protected final Set classPathClasses; protected final List> diagnostics = new ArrayList<>(); protected final Map cachedModified = new HashMap<>(); protected final Cache> cacheFileImports = new Cache<>(); protected final SynchronizedTask synchronizedTask = new SynchronizedTask(); protected final SourceFileManager fileManager; protected final ModuleProject module; + private final ClasspathTypeLookup types; public ReusableCompiler compiler = new JCReusableCompiler(); protected Set bootClasspathClasses = BootClasspathProvider.getTopLevelClasses( Collections.singleton(Environment.ANDROID_JAR.getAbsolutePath())); @@ -98,23 +99,34 @@ public JavaCompilerService(@Nullable ModuleProject module) { this.module = module; if (module == null) { this.fileManager = SourceFileManager.NO_MODULE; - this.classPathClasses = Collections.emptySet(); } else { this.fileManager = SourceFileManager.forModule(module); - this.classPathClasses = Collections.unmodifiableSet(module.compileClasspathClasses.allClassNames()); this.bootClasspathClasses = Collections.unmodifiableSet(getBootclasspathClasses()); } + this.types = ClasspathTypeLookup.forModule(module, () -> this.bootClasspathClasses); } - private JavaCompilerService( + @VisibleForTesting + JavaCompilerService( @Nullable ModuleProject module, SourceFileManager fileManager, Set bootClasspathClasses, - Set classPathClasses) { + ClasspathTypeLookup types) { this.module = module; this.fileManager = fileManager; this.bootClasspathClasses = bootClasspathClasses; - this.classPathClasses = classPathClasses; + this.types = types; + } + + /** + * Returns the packages and top-level classes of the module's compile classpath, or {@code null} without a module. + * + *

+ * Use the result for one request only. Its lookups block on the class index's disk I/O, so never call them on the main thread. + */ + @Nullable + public ClasspathPackages classpathPackages() { + return types.classpathPackages(); } public synchronized void close() { @@ -131,7 +143,7 @@ public SynchronizedTask compile(final CompilationRequest request) { public JavaCompilerService copy() { final JavaCompilerService compiler = new JavaCompilerService( - this.module, this.fileManager, this.bootClasspathClasses, this.classPathClasses); + this.module, this.fileManager, this.bootClasspathClasses, this.types); compiler.cachedCompile = null; compiler.newCursorPosition = Position.NONE; compiler.lastReparsePosition = Position.NONE; @@ -161,6 +173,16 @@ public Optional findAnywhere(String className) { return Optional.empty(); } + /** + * Returns the qualified names of the top-level classes whose simple name is exactly {@code simpleName} that a file in {@code importingPackage} may import. + * + *

+ * Blocks on the class index's disk I/O, so never call it on the main thread. + */ + public List findImportableQualifiedNames(String simpleName, String importingPackage) { + return types.findImportableQualifiedNames(simpleName, importingPackage); + } + @Override public Path[] findMemberReferences(String className, String memberName) { List candidates = new ArrayList<>(); @@ -174,25 +196,6 @@ public Path[] findMemberReferences(String className, String memberName) { return candidates.toArray(new Path[0]); } - @Override - public List findQualifiedNames(String simpleName, boolean onlyOne) { - final var names = new ArrayList(); - for (var name : publicTopLevelTypes()) { - // This will be true in a test environment - if (name.contains("/")) { - name = name.replace('/', '.'); - } - - if (name.endsWith("." + simpleName)) { - names.add(name); - if (onlyOne) { - break; - } - } - } - return names; - } - @Override public Path findTypeDeclaration(String className) { Path fastFind = findPublicTypeDeclaration(className); @@ -214,6 +217,16 @@ public Path findTypeDeclaration(String className) { return NOT_FOUND; } + /** + * Returns the qualified names of top-level classes whose simple name starts with {@code partial}, ignoring case: up to {@code limitPerSource} from each of the sources, the classpath and the boot classpath, exact simple-name matches first. + * + *

+ * Blocks on the class index's disk I/O, so never call it on the main thread. See {@link ClasspathTypeLookup#findTypeNamesMatching} for why the limit is per source. + */ + public List findTypeNamesMatching(String partial, int limitPerSource) { + return types.findTypeNamesMatching(partial, limitPerSource); + } + @Override public Path[] findTypeReferences(String className) { String packageName = Extractors.packageName(className); @@ -262,18 +275,6 @@ public ParseTask parse(Path file) { return new ParseTask(parser.task, parser.root); } - @Override - public TreeSet publicTopLevelTypes() { - TreeSet all = new TreeSet<>(); - List sourceClasses = module != null ? module.compileJavaSourceClasses.allSources() : Collections.emptyList(); - for (SourceClassTrie.SourceNode node : sourceClasses) { - all.add(node.getQualifiedName()); - } - all.addAll(classPathClasses); - all.addAll(bootClasspathClasses); - return all; - } - @Nullable private Pair binarySearchCurrentMethod( @NonNull final List> positions, final long cursor) { diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/CompletionProvider.java b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/CompletionProvider.java index 6e1bb5110a..3774118440 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/CompletionProvider.java +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/CompletionProvider.java @@ -44,6 +44,7 @@ import com.itsaky.androidide.lsp.java.visitors.FindCompletionsAt; import com.itsaky.androidide.lsp.models.CompletionParams; import com.itsaky.androidide.lsp.models.CompletionResult; +import com.itsaky.androidide.projects.ProjectManagerImpl; import com.itsaky.androidide.utils.DocumentUtils; import io.github.rosemoe.sora.lang.completion.snippet.CodeSnippet; import java.nio.file.Path; @@ -54,6 +55,7 @@ import java.util.function.Consumer; import openjdk.source.tree.Tree; import openjdk.source.util.TreePath; +import org.appdevforall.codeonthego.indexing.service.IndexingState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +63,21 @@ public class CompletionProvider extends AbstractServiceProvider implements IComp public static final int MAX_COMPLETION_ITEMS = CompletionResult.MAX_ITEMS; private static final Logger LOG = LoggerFactory.getLogger(CompletionProvider.class); + + /** + * Whether a completion result computed while the indexing state moved between {@code stateAtStart} and {@code stateAtEnd} is safe to cache. + * + * A result computed while indexing was in flight at either end of the request may be missing symbols the index has or will soon have, so it must not be reused for later requests. + */ + static boolean canCacheCompletion(IndexingState stateAtStart, IndexingState stateAtEnd) { + return !(stateAtStart instanceof IndexingState.Indexing) && !(stateAtEnd instanceof IndexingState.Indexing); + } + + /** The indexing state at the moment this is called, read fresh (never cached). */ + private static IndexingState currentIndexingState() { + return ProjectManagerImpl.getInstance().getIndexingServiceManager().getState().getValue(); + } + private final AtomicBoolean completing = new AtomicBoolean(false); private JavaCompilerService compiler; private CachedCompletion cache; @@ -205,6 +222,8 @@ private CompletionResult completeInternal(final @NonNull CompletionParams params LOG.info("...cannot use cached completions"); } + final IndexingState stateAtStart = currentIndexingState(); + abortCompletionIfCancelled(); final long cursor = params.getPosition().requireIndex(); final var sourceObject = new SourceFileObject(file); @@ -238,7 +257,7 @@ private CompletionResult completeInternal(final @NonNull CompletionParams params logCompletionDuration(started, result); abortCompletionIfCancelled(); - if (this.nextCacheConsumer != null) { + if (this.nextCacheConsumer != null && canCacheCompletion(stateAtStart, currentIndexingState())) { this.nextCacheConsumer.accept(CachedCompletion.cache(params, result)); } diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ClassNamesCompletionProvider.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ClassNamesCompletionProvider.kt index 0b2f57916e..7b58e53bd1 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ClassNamesCompletionProvider.kt +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ClassNamesCompletionProvider.kt @@ -1,111 +1,107 @@ -/* - * This file is part of AndroidIDE. - * - * AndroidIDE is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * AndroidIDE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with AndroidIDE. If not, see . - */ - -package com.itsaky.androidide.lsp.java.providers.completion - -import com.itsaky.androidide.lsp.api.IServerSettings -import com.itsaky.androidide.lsp.java.compiler.CompileTask -import com.itsaky.androidide.lsp.java.compiler.JavaCompilerService -import com.itsaky.androidide.lsp.java.providers.CompletionProvider -import com.itsaky.androidide.lsp.models.CompletionResult -import com.itsaky.androidide.lsp.models.MatchLevel.NO_MATCH -import openjdk.source.tree.ClassTree -import openjdk.source.tree.CompilationUnitTree -import openjdk.source.util.TreePath -import java.nio.file.Path -import java.nio.file.Paths -import java.util.Objects - -/** - * Completes class names. - * - * @author Akash Yadav - */ -class ClassNamesCompletionProvider( - completingFile: Path, - cursor: Long, - compiler: JavaCompilerService, - settings: IServerSettings, - val root: CompilationUnitTree, -) : IJavaCompletionProvider(cursor, completingFile, compiler, settings) { - - override fun doComplete( - task: CompileTask, - path: TreePath, - partial: String, - endsWithParen: Boolean, - ): CompletionResult { - val list = mutableListOf() - val packageName = Objects.toString(root.packageName, "") - val uniques: MutableSet = HashSet() - - val file: Path = Paths.get(root.sourceFile.toUri()) - val imports: Set = - root.imports.map { it.qualifiedIdentifier }.mapNotNull { it.toString() }.toSet() - - abortCompletionIfCancelled() - for (className in compiler.packagePrivateTopLevelTypes(packageName)) { - val matchLevel = matchLevel(className, partial) - if (matchLevel == NO_MATCH) { - continue - } - - list.add(classItem(imports, file, className, matchLevel)) - uniques.add(className) - } - - abortCompletionIfCancelled() - - val topLevelTypes = compiler.publicTopLevelTypes() - for (className in topLevelTypes) { - val matchLevel = matchLevel(simpleName(className), partial) - if (matchLevel == NO_MATCH) { - continue - } - - if (uniques.contains(className)) { - continue - } - - list.add(classItem(imports, file, className, matchLevel)) - uniques.add(className) - } - abortCompletionIfCancelled() - for (t in root.typeDecls) { - if (t !is ClassTree) { - continue - } - val candidate = if (t.simpleName == null) "" else t.simpleName - - val matchLevel = matchLevel(candidate, partial) - if (matchLevel == NO_MATCH) { - continue - } - - val name = packageName + "." + t.simpleName - list.add(classItem(name, matchLevel)) - - if (list.size > CompletionProvider.MAX_COMPLETION_ITEMS) { - break - } - } - - log.info("...found {} class names", list.size) - - return CompletionResult(list) - } -} +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.java.providers.completion + +import com.itsaky.androidide.lsp.api.IServerSettings +import com.itsaky.androidide.lsp.java.compiler.CompileTask +import com.itsaky.androidide.lsp.java.compiler.JavaCompilerService +import com.itsaky.androidide.lsp.java.providers.CompletionProvider +import com.itsaky.androidide.lsp.models.CompletionResult +import com.itsaky.androidide.lsp.models.MatchLevel.NO_MATCH +import openjdk.source.tree.ClassTree +import openjdk.source.tree.CompilationUnitTree +import openjdk.source.util.TreePath +import java.nio.file.Path +import java.nio.file.Paths +import java.util.Objects + +/** + * Completes class names. + * + * @author Akash Yadav + */ +class ClassNamesCompletionProvider( + completingFile: Path, + cursor: Long, + compiler: JavaCompilerService, + settings: IServerSettings, + val root: CompilationUnitTree, +) : IJavaCompletionProvider(cursor, completingFile, compiler, settings) { + override fun doComplete( + task: CompileTask, + path: TreePath, + partial: String, + endsWithParen: Boolean, + ): CompletionResult { + val list = mutableListOf() + val packageName = Objects.toString(root.packageName, "") + val uniques: MutableSet = HashSet() + + val file: Path = Paths.get(root.sourceFile.toUri()) + val imports: Set = + root.imports + .map { it.qualifiedIdentifier } + .mapNotNull { it.toString() } + .toSet() + + abortCompletionIfCancelled() + for (className in compiler.packagePrivateTopLevelTypes(packageName)) { + val matchLevel = matchLevel(className, partial) + if (matchLevel == NO_MATCH) { + continue + } + + list.add(classItem(imports, file, className, matchLevel)) + uniques.add(className) + } + + abortCompletionIfCancelled() + + // One more than can be shown, so a source with too many matches marks the result incomplete. + for (className in compiler.findTypeNamesMatching(partial, CompletionProvider.MAX_COMPLETION_ITEMS + 1)) { + if (!uniques.add(className)) { + continue + } + + list.add(classItem(imports, file, className, matchLevel(simpleName(className), partial))) + } + abortCompletionIfCancelled() + for (t in root.typeDecls) { + if (t !is ClassTree) { + continue + } + val candidate = if (t.simpleName == null) "" else t.simpleName + + val matchLevel = matchLevel(candidate, partial) + if (matchLevel == NO_MATCH) { + continue + } + + val name = packageName + "." + t.simpleName + list.add(classItem(name, matchLevel)) + + if (list.size > CompletionProvider.MAX_COMPLETION_ITEMS) { + break + } + } + + log.info("...found {} class names", list.size) + + return CompletionResult(list) + } +} diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ImportCompletionProvider.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ImportCompletionProvider.kt index b8434e9b6c..4c3abf0e46 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ImportCompletionProvider.kt +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ImportCompletionProvider.kt @@ -1,446 +1,280 @@ -/* - * This file is part of AndroidIDE. - * - * AndroidIDE is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * AndroidIDE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with AndroidIDE. If not, see . - */ - -package com.itsaky.androidide.lsp.java.providers.completion - -import com.itsaky.androidide.lsp.api.IServerSettings -import com.itsaky.androidide.lsp.java.compiler.CompileTask -import com.itsaky.androidide.lsp.java.compiler.JavaCompilerService -import com.itsaky.androidide.lsp.java.providers.CompletionProvider.MAX_COMPLETION_ITEMS -import com.itsaky.androidide.lsp.models.CompletionItem -import com.itsaky.androidide.lsp.models.CompletionResult -import com.itsaky.androidide.lsp.models.MatchLevel.CASE_SENSITIVE_EQUAL -import com.itsaky.androidide.lsp.models.MatchLevel.NO_MATCH -import com.itsaky.androidide.projects.api.ModuleProject -import com.itsaky.androidide.projects.util.BootClasspathProvider -import com.itsaky.androidide.utils.ClassTrie -import com.itsaky.androidide.utils.ClassTrie.Node -import jdkx.lang.model.element.Element -import jdkx.lang.model.element.ElementKind -import jdkx.lang.model.element.ElementKind.ANNOTATION_TYPE -import jdkx.lang.model.element.ElementKind.CLASS -import jdkx.lang.model.element.ElementKind.CONSTRUCTOR -import jdkx.lang.model.element.ElementKind.ENUM -import jdkx.lang.model.element.ElementKind.ENUM_CONSTANT -import jdkx.lang.model.element.ElementKind.FIELD -import jdkx.lang.model.element.ElementKind.INSTANCE_INIT -import jdkx.lang.model.element.ElementKind.INTERFACE -import jdkx.lang.model.element.ElementKind.METHOD -import jdkx.lang.model.element.ElementKind.STATIC_INIT -import jdkx.lang.model.element.Modifier.STATIC -import jdkx.lang.model.element.TypeElement -import openjdk.source.util.TreePath -import openjdk.tools.javac.api.JavacTrees -import openjdk.tools.javac.code.Symbol.MethodSymbol -import openjdk.tools.javac.model.JavacTypes -import openjdk.tools.javac.tree.JCTree.JCImport -import java.nio.file.Path - -/** - * Provides completions for imports. - * - * @author Akash Yadav - */ -class ImportCompletionProvider( - completingFile: Path, - cursor: Long, - compiler: JavaCompilerService, - settings: IServerSettings, -) : IJavaCompletionProvider(cursor, completingFile, compiler, settings) { - - lateinit var importPath: String - - // TODO add tests for this - override fun doComplete( - task: CompileTask, - path: TreePath, - partial: String, - endsWithParen: Boolean, - ): CompletionResult { - - val importTree = path.leaf - if (importTree !is JCImport) { - return CompletionResult.EMPTY - } - - log.info("...complete import for path: {}", importPath) - - val names: MutableSet = HashSet() - val list = mutableListOf() - - var pkgName = importPath - val incomplete: String - if (!pkgName.contains(".")) { - pkgName = "" - incomplete = importPath - } else if (pkgName.endsWith(".")) { - pkgName = pkgName.substring(0, pkgName.lastIndex) - incomplete = "" - } else { - incomplete = pkgName.substringAfterLast(delimiter = '.') - pkgName = pkgName.substringBeforeLast(delimiter = '.') - } - - abortCompletionIfCancelled() - run { - val match = matchLevel("static", incomplete) - if (match != NO_MATCH && !importTree.isStatic && pkgName.isEmpty()) { - list.add(keyword("static", incomplete, match)) - } - } - - abortCompletionIfCancelled() - val module = compiler.module - if (module == null) { - legacyImportPathCompletion(partial, names, list) - return CompletionResult(list) - } - - if (pkgName.isEmpty() || pkgName.isBlank()) { - // User is typing first segment of package name - // Javac APIs will not work here - tryCompleteImport(pkgName, incomplete, list, names, module) - return CompletionResult(list) - } - - try { - val packages = collectPackageNodes(module, pkgName) - abortCompletionIfCancelled() - if (packages.isNotEmpty()) { - for (node in packages) { - addDirectChildNodes(node, incomplete, list, names, false) - } - } - } catch (err: RequireMemberCompletionException) { - // If pkgName is not an existing package name, check if it is a qualified classname - // A user might be trying to acess members of a member class. So, we keep replacing last '.' - // until we find a valid qualified name of a class - if (completeTypeMembers(task, path, pkgName, incomplete, list)) { - return CompletionResult(list) - } - } - - try { - // This maybe reached only in some rare cases - tryCompleteImport(pkgName, incomplete, list, names, module) - } catch (e: RequireMemberCompletionException) { - // User is trying to access members of a class - if (completeTypeMembers(task, path, pkgName, incomplete, list)) { - return CompletionResult(list) - } - } - - return CompletionResult(list) - } - - private fun completeTypeMembers( - task: CompileTask, - path: TreePath, - pkgName: String, - incomplete: String, - list: MutableList - ): Boolean { - abortCompletionIfCancelled() - val elements = task.task.elements - var typesForPkg: Set = setOf() - val maybeInnerName = StringBuilder(pkgName) - while (true) { - val types = elements.getAllTypeElements(maybeInnerName) - if (types.isNotEmpty()) { - typesForPkg = types - break - } - - if (!maybeInnerName.contains(".")) { - break - } - maybeInnerName.setCharAt(maybeInnerName.lastIndexOf('.'), '$') - } - - abortCompletionIfCancelled() - if (typesForPkg.isNotEmpty()) { - // We found a valid class name - // Add the accessible class items - for (type in typesForPkg) { - val result = completeTypeMembers(task, type, path, incomplete) - if (result.isNotEmpty()) { - list.addAll(result) - } - } - return true - } - return false - } - - /** - * Collects package nodes for [pkgName] in source paths, classpaths and bootclasspaths. If any - * segment of [pkgName] is a class, [RequireMemberCompletionException] is thrown to indicate that - * class members must be completed. - * - * @param module The project module - * @param pkgName The package name to collect nodes for. - */ - private fun collectPackageNodes(module: ModuleProject, pkgName: String): List { - abortCompletionIfCancelled() - val result = mutableListOf() - val fromSource = collectPackageNode(module.compileJavaSourceClasses, pkgName) - if (fromSource != null) { - result.add(fromSource) - } - - abortCompletionIfCancelled() - val fromClasspath = collectPackageNode(module.compileClasspathClasses, pkgName) - if (fromClasspath != null) { - result.add(fromClasspath) - } - - BootClasspathProvider.getAllEntries().forEach { - abortCompletionIfCancelled() - val fromBootclasspath = collectPackageNode(it, pkgName) - if (fromBootclasspath != null) { - result.add(fromBootclasspath) - } - } - return result - } - - /** - * Collect package nodes from the [trie] for the given [pkgName]. If any segment of [pkgName] is a - * class, [RequireMemberCompletionException] is thrown to indicate that class members must be - * completed. - * - * @param trie The [ClassTrie] to find package names from. - * @param pkgName The package name of the package to find node for. - * @return The found package name. Or `null` if no package can be found. - */ - private fun collectPackageNode(trie: com.itsaky.androidide.utils.ClassTrie, pkgName: String): Node? { - val segments = trie.segments(pkgName) - var node: Node? = trie.root - for (segment in segments) { - abortCompletionIfCancelled() - if (node == null) { - break - } - - if (node.isClass) { - // If any of the segment in pkgName is a class - // We need to complete memebers of a class - throw RequireMemberCompletionException() - } - - node = node.children[segment] - } - - return node - } - - private fun completeTypeMembers( - task: CompileTask, - type: TypeElement, - path: TreePath, - partial: String - ): MutableList { - - abortCompletionIfCancelled() - - val list = mutableListOf() - val elements = task.task.elements - val trees = JavacTrees.instance(task.task.context) - val jcTypes = JavacTypes.instance(task.task.context) - val scope = trees.getScope(path) - val isStatic = (path.leaf as JCImport).isStatic - if (!trees.isAccessible(scope, type)) { - // Type not accessible - return list - } - - val members = elements.getAllMembers(type) - for (member in members) { - abortCompletionIfCancelled() - if ( - member.kind == CONSTRUCTOR || member.kind == STATIC_INIT || member.kind == INSTANCE_INIT - ) { - continue - } - - val match = matchLevel(member.simpleName, partial) - if (match == NO_MATCH) { - continue - } - - if (isType(member)) { - list.add(classItem(member.simpleName.toString(), match)) - continue - } - - if (!isStatic) { - continue - } - - val mods = member.modifiers - if (!mods.contains(STATIC)) { - continue - } - - if (!trees.isAccessible(scope, member, jcTypes.getDeclaredType(type))) { - continue - } - - if (member.kind == METHOD) { - list.add(method(task, listOf(member as MethodSymbol), false, match, partial)) - continue - } - - if (member.kind == FIELD || member.kind == ENUM_CONSTANT) { - list.add(item(task, member, match)) - } - } - - return list - } - - @Throws(RequireMemberCompletionException::class) - private fun tryCompleteImport( - pkgName: String, - incomplete: String, - list: MutableList, - names: MutableSet, - module: ModuleProject, - packageOnly: Boolean = false - ) { - abortCompletionIfCancelled() - val sourceNode = - if (pkgName.isEmpty()) module.compileJavaSourceClasses.root - else module.compileJavaSourceClasses.findNode(pkgName) - if (sourceNode != null) { - if (sourceNode.isClass) { - throw RequireMemberCompletionException() - } - - addDirectChildNodes(sourceNode, incomplete, list, names, packageOnly) - } - - abortCompletionIfCancelled() - val classpathNode = - if (pkgName.isEmpty()) module.compileClasspathClasses.root - else module.compileClasspathClasses.findNode(pkgName) - if (classpathNode != null) { - if (classpathNode.isClass) { - throw RequireMemberCompletionException() - } - - addDirectChildNodes(classpathNode, incomplete, list, names, packageOnly) - } - - BootClasspathProvider.getAllEntries().forEach { - abortCompletionIfCancelled() - val node = - if (pkgName.isEmpty()) { - it.root - } else it.findNode(pkgName) - if (node != null) { - if (node.isClass) { - throw RequireMemberCompletionException() - } - addDirectChildNodes(node, incomplete, list, names, packageOnly) - } - } - } - - private fun addDirectChildNodes( - sourceNode: Node, - incomplete: String, - list: MutableList, - names: MutableSet, - packageOnly: Boolean - ) { - for (child in sourceNode.children.values) { - abortCompletionIfCancelled() - val match = - if (incomplete.isEmpty()) { - CASE_SENSITIVE_EQUAL - } else { - matchLevel(child.name, incomplete) - } - - if (match == NO_MATCH || names.contains(child.name)) { - continue - } - - if (packageOnly && child.isClass) { - continue - } - - if (child.isClass) { - list.add(classItem(child.qualifiedName, match)) - } else { - list.add(packageItem(child.qualifiedName, match)) - } - - names.add(child.name) - } - } - - private fun legacyImportPathCompletion( - partial: String, - names: MutableSet, - list: MutableList - ) { - abortCompletionIfCancelled() - for (className in compiler.publicTopLevelTypes()) { - val matchLevel = matchLevel(className, partial) - if (matchLevel == NO_MATCH) { - continue - } - - val start = importPath.lastIndexOf('.') - var end = className.indexOf('.', importPath.length) - if (end == -1) { - end = className.length - } - val segment = className.substring(start + 1, end) - if (names.contains(segment)) { - continue - } - names.add(segment) - val isClass = end == importPath.length - if (isClass) { - list.add(classItem(className, matchLevel)) - } else { - list.add(packageItem(segment, matchLevel)) - } - - if (list.size > MAX_COMPLETION_ITEMS) { - break - } - } - } - - internal fun isType(element: Element): Boolean { - return isType(element.kind) - } - - internal fun isType(kind: ElementKind): Boolean { - return kind == ANNOTATION_TYPE || kind == CLASS || kind == INTERFACE || kind == ENUM - } - - /** - * Internal exception to indicate that members of a class must be completed. This is thrown and - * caught internally when completing imports. - */ - internal class RequireMemberCompletionException : IllegalStateException() -} +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.java.providers.completion + +import com.itsaky.androidide.lsp.api.IServerSettings +import com.itsaky.androidide.lsp.java.compiler.CompileTask +import com.itsaky.androidide.lsp.java.compiler.JavaCompilerService +import com.itsaky.androidide.lsp.models.CompletionItem +import com.itsaky.androidide.lsp.models.CompletionResult +import com.itsaky.androidide.lsp.models.MatchLevel.CASE_SENSITIVE_EQUAL +import com.itsaky.androidide.lsp.models.MatchLevel.NO_MATCH +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.util.BootClasspathProvider +import com.itsaky.androidide.utils.ClassTrie +import jdkx.lang.model.element.Element +import jdkx.lang.model.element.ElementKind +import jdkx.lang.model.element.ElementKind.ANNOTATION_TYPE +import jdkx.lang.model.element.ElementKind.CLASS +import jdkx.lang.model.element.ElementKind.CONSTRUCTOR +import jdkx.lang.model.element.ElementKind.ENUM +import jdkx.lang.model.element.ElementKind.ENUM_CONSTANT +import jdkx.lang.model.element.ElementKind.FIELD +import jdkx.lang.model.element.ElementKind.INSTANCE_INIT +import jdkx.lang.model.element.ElementKind.INTERFACE +import jdkx.lang.model.element.ElementKind.METHOD +import jdkx.lang.model.element.ElementKind.STATIC_INIT +import jdkx.lang.model.element.Modifier.STATIC +import jdkx.lang.model.element.TypeElement +import openjdk.source.util.TreePath +import openjdk.tools.javac.api.JavacTrees +import openjdk.tools.javac.code.Symbol.MethodSymbol +import openjdk.tools.javac.model.JavacTypes +import openjdk.tools.javac.tree.JCTree.JCImport +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child +import java.nio.file.Path + +/** + * Provides completions for imports. + * + * @author Akash Yadav + */ +class ImportCompletionProvider( + completingFile: Path, + cursor: Long, + compiler: JavaCompilerService, + settings: IServerSettings, +) : IJavaCompletionProvider(cursor, completingFile, compiler, settings) { + lateinit var importPath: String + + override fun doComplete( + task: CompileTask, + path: TreePath, + partial: String, + endsWithParen: Boolean, + ): CompletionResult { + val importTree = path.leaf + if (importTree !is JCImport) { + return CompletionResult.EMPTY + } + + log.info("...complete import for path: {}", importPath) + + val list = mutableListOf() + + var pkgName = importPath + val incomplete: String + if (!pkgName.contains(".")) { + pkgName = "" + incomplete = importPath + } else if (pkgName.endsWith(".")) { + pkgName = pkgName.substring(0, pkgName.lastIndex) + incomplete = "" + } else { + incomplete = pkgName.substringAfterLast(delimiter = '.') + pkgName = pkgName.substringBeforeLast(delimiter = '.') + } + + abortCompletionIfCancelled() + run { + val match = matchLevel("static", incomplete) + if (match != NO_MATCH && !importTree.isStatic && pkgName.isEmpty()) { + list.add(keyword("static", incomplete, match)) + } + } + + abortCompletionIfCancelled() + val children = importPathChildren(compiler.module) + if (pkgName.isBlank()) { + // User is typing first segment of package name + // Javac APIs will not work here + addChildItems(children.of(pkgName), incomplete, list) + return CompletionResult(list) + } + + try { + addChildItems(children.ofPackage(pkgName), incomplete, list) + } catch (err: RequireMemberCompletionException) { + // If pkgName is not an existing package name, check if it is a qualified classname + // A user might be trying to acess members of a member class. So, we keep replacing last '.' + // until we find a valid qualified name of a class + if (completeTypeMembers(task, path, pkgName, incomplete, list)) { + return CompletionResult(list) + } + } + + try { + // pkgName may itself be a class name; its children were already offered by ofPackage above. + children.requireNotClass(pkgName) + } catch (e: RequireMemberCompletionException) { + // User is trying to access members of a class + if (completeTypeMembers(task, path, pkgName, incomplete, list)) { + return CompletionResult(list) + } + } + + return CompletionResult(list) + } + + private fun completeTypeMembers( + task: CompileTask, + path: TreePath, + pkgName: String, + incomplete: String, + list: MutableList, + ): Boolean { + abortCompletionIfCancelled() + val elements = task.task.elements + var typesForPkg: Set = setOf() + val maybeInnerName = StringBuilder(pkgName) + while (true) { + val types = elements.getAllTypeElements(maybeInnerName) + if (types.isNotEmpty()) { + typesForPkg = types + break + } + + if (!maybeInnerName.contains(".")) { + break + } + maybeInnerName.setCharAt(maybeInnerName.lastIndexOf('.'), '$') + } + + abortCompletionIfCancelled() + if (typesForPkg.isNotEmpty()) { + // We found a valid class name + // Add the accessible class items + for (type in typesForPkg) { + val result = completeTypeMembers(task, type, path, incomplete) + if (result.isNotEmpty()) { + list.addAll(result) + } + } + return true + } + return false + } + + private fun completeTypeMembers( + task: CompileTask, + type: TypeElement, + path: TreePath, + partial: String, + ): MutableList { + abortCompletionIfCancelled() + + val list = mutableListOf() + val elements = task.task.elements + val trees = JavacTrees.instance(task.task.context) + val jcTypes = JavacTypes.instance(task.task.context) + val scope = trees.getScope(path) + val isStatic = (path.leaf as JCImport).isStatic + if (!trees.isAccessible(scope, type)) { + // Type not accessible + return list + } + + val members = elements.getAllMembers(type) + for (member in members) { + abortCompletionIfCancelled() + if ( + member.kind == CONSTRUCTOR || member.kind == STATIC_INIT || member.kind == INSTANCE_INIT + ) { + continue + } + + val match = matchLevel(member.simpleName, partial) + if (match == NO_MATCH) { + continue + } + + if (isType(member)) { + list.add(classItem(member.simpleName.toString(), match)) + continue + } + + if (!isStatic) { + continue + } + + val mods = member.modifiers + if (!mods.contains(STATIC)) { + continue + } + + if (!trees.isAccessible(scope, member, jcTypes.getDeclaredType(type))) { + continue + } + + if (member.kind == METHOD) { + list.add(method(task, listOf(member as MethodSymbol), false, match, partial)) + continue + } + + if (member.kind == FIELD || member.kind == ENUM_CONSTANT) { + list.add(item(task, member, match)) + } + } + + return list + } + + // A file in no module has no sources or classpath of its own, but still completes boot classpath imports. + private fun importPathChildren(module: ModuleProject?) = + ImportPathChildren( + module?.compileJavaSourceClasses ?: ClassTrie(), + compiler.classpathPackages(), + BootClasspathProvider.getAllEntries(), + ) + + private fun addChildItems( + children: Sequence, + incomplete: String, + list: MutableList, + ) { + for (child in children) { + abortCompletionIfCancelled() + val match = + if (incomplete.isEmpty()) { + CASE_SENSITIVE_EQUAL + } else { + matchLevel(child.name, incomplete) + } + + if (match == NO_MATCH) { + continue + } + + if (child.isClass) { + list.add(classItem(child.qualifiedName, match)) + } else { + list.add(packageItem(child.qualifiedName, match)) + } + } + } + + internal fun isType(element: Element): Boolean = isType(element.kind) + + internal fun isType(kind: ElementKind): Boolean = kind == ANNOTATION_TYPE || kind == CLASS || kind == INTERFACE || kind == ENUM + + /** + * Internal exception to indicate that members of a class must be completed. This is thrown and + * caught internally when completing imports. + */ + internal class RequireMemberCompletionException : IllegalStateException() +} diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ImportPathChildren.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ImportPathChildren.kt new file mode 100644 index 0000000000..d69bbd2e72 --- /dev/null +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/providers/completion/ImportPathChildren.kt @@ -0,0 +1,128 @@ +package com.itsaky.androidide.lsp.java.providers.completion + +import com.itsaky.androidide.lsp.java.compiler.ClasspathPackages +import com.itsaky.androidide.lsp.java.providers.completion.ImportCompletionProvider.RequireMemberCompletionException +import com.itsaky.androidide.utils.ClassTrie +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child + +/** + * The packages and top-level classes that can continue an import path: those of a module's Java + * [sourceClasses], its compile [classpath] and the [bootClasses], in that order, each name offered + * once. + * + * It remembers the names it has offered, so build one per completion request. The classpath lookups + * block on disk I/O, so never use one on the main thread. + */ +internal class ImportPathChildren( + private val sourceClasses: ClassTrie, + private val classpath: ClasspathPackages?, + private val bootClasses: Collection, +) { + private val offered = HashSet() + + /** + * Returns the children of the package [packageName] not offered yet. + * + * @throws RequireMemberCompletionException when a segment of [packageName] before the last one is + * a class in any source, so the path continues into that class's members. + */ + fun ofPackage(packageName: String): Sequence { + val sourceNode = walkToNode(sourceClasses, packageName) + requireNoClasspathClassBefore(packageName) + val bootNodes = bootClasses.mapNotNull { walkToNode(it, packageName) } + return sequence { + sourceNode?.let { yieldAll(childrenOf(it)) } + classpath?.let { yieldAll(it.children(packageName)) } + bootNodes.forEach { yieldAll(childrenOf(it)) } + }.filter { offered.add(it.name) } + } + + /** + * Returns the children of [qualifiedName], the default package when it is empty, not offered yet. + * + * The sequence throws [RequireMemberCompletionException] when it reaches a source in which + * [qualifiedName] is a class; the children of the sources before that one have been yielded by + * then. + */ + fun of(qualifiedName: String): Sequence = + sequence { + yieldAll(childrenOfNode(sourceClasses, qualifiedName)) + classpath?.let { + if (qualifiedName.isNotEmpty() && it.isClass(qualifiedName)) { + throw RequireMemberCompletionException() + } + yieldAll(it.children(qualifiedName)) + } + bootClasses.forEach { yieldAll(childrenOfNode(it, qualifiedName)) } + }.filter { offered.add(it.name) } + + /** + * Throws [RequireMemberCompletionException] when [qualifiedName] itself names a class in the + * sources, the classpath or the boot classpath, so the path continues into that class's members. + * + * Unlike [of], this offers no children: use it to recheck a name whose children [ofPackage] + * already offered, where asking the classpath for them again would only be discarded by the + * offered-name dedup. + */ + fun requireNotClass(qualifiedName: String) { + if (isClassNode(sourceClasses, qualifiedName)) { + throw RequireMemberCompletionException() + } + classpath?.let { + if (qualifiedName.isNotEmpty() && it.isClass(qualifiedName)) { + throw RequireMemberCompletionException() + } + } + if (bootClasses.any { isClassNode(it, qualifiedName) }) { + throw RequireMemberCompletionException() + } + } + + private fun walkToNode( + trie: ClassTrie, + packageName: String, + ): ClassTrie.Node? { + var node: ClassTrie.Node? = trie.root + for (segment in trie.segments(packageName)) { + if (node == null) { + break + } + if (node.isClass) { + throw RequireMemberCompletionException() + } + node = node.children[segment] + } + return node + } + + private fun requireNoClasspathClassBefore(packageName: String) { + val classpath = classpath ?: return + val properPrefixes = packageName.indices.filter { packageName[it] == '.' }.map { packageName.substring(0, it) } + if (properPrefixes.any(classpath::isClass)) { + throw RequireMemberCompletionException() + } + } + + private fun findNode( + trie: ClassTrie, + qualifiedName: String, + ): ClassTrie.Node? = if (qualifiedName.isEmpty()) trie.root else trie.findNode(qualifiedName) + + private fun childrenOfNode( + trie: ClassTrie, + qualifiedName: String, + ): List { + val node = findNode(trie, qualifiedName) ?: return emptyList() + if (node.isClass) { + throw RequireMemberCompletionException() + } + return childrenOf(node) + } + + private fun isClassNode( + trie: ClassTrie, + qualifiedName: String, + ): Boolean = findNode(trie, qualifiedName)?.isClass == true + + private fun childrenOf(node: ClassTrie.Node) = node.children.values.map { Child(it.name, it.qualifiedName, it.isClass) } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/CollectIndexingPassResetsTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/CollectIndexingPassResetsTest.kt new file mode 100644 index 0000000000..f16fe85dd8 --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/CollectIndexingPassResetsTest.kt @@ -0,0 +1,79 @@ +package com.itsaky.androidide.lsp.java + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** @author Akash Yadav */ +@RunWith(JUnit4::class) +class CollectIndexingPassResetsTest { + @Test + fun `fires once per finished pass, across repeated passes`() = + runBlocking { + val finishedPasses = MutableStateFlow(0L) + var resets = 0 + val job = launch { collectIndexingPassResets(finishedPasses) { resets++ } } + yield() + + finishedPasses.value = 1 + yield() + assertThat(resets).isEqualTo(1) + + finishedPasses.value = 2 + yield() + assertThat(resets).isEqualTo(2) + + job.cancel() + } + + @Test + fun `does not fire for the passes finished before it started collecting`() = + runBlocking { + val finishedPasses = MutableStateFlow(3L) + var resets = 0 + val job = launch { collectIndexingPassResets(finishedPasses) { resets++ } } + yield() + + assertThat(resets).isEqualTo(0) + job.cancel() + } + + @Test + fun `does not fire when indexing starts`() = + runBlocking { + val tracker = IndexingProgressTracker() + var resets = 0 + val job = launch { collectIndexingPassResets(tracker.finishedPasses) { resets++ } } + yield() + + val pass = tracker.openPass() + pass.track("/libs/app.jar", Job()) + yield() + + assertThat(resets).isEqualTo(0) + pass.close() + job.cancel() + } + + @Test + fun `a pass that starts and finishes before the collector runs still resets`() = + runBlocking { + val tracker = IndexingProgressTracker() + var resets = 0 + val job = launch { collectIndexingPassResets(tracker.finishedPasses) { resets++ } } + yield() + + tracker.openPass().use { pass -> pass.track("/libs/app.jar", Job().apply { complete() }) } + yield() + + assertThat(resets).isEqualTo(1) + job.cancel() + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/CompletionCacheGuardTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/CompletionCacheGuardTest.kt new file mode 100644 index 0000000000..949c41d588 --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/CompletionCacheGuardTest.kt @@ -0,0 +1,61 @@ +package com.itsaky.androidide.lsp.java + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.lsp.internal.model.CachedCompletion +import com.itsaky.androidide.lsp.models.CompletionParams +import com.itsaky.androidide.lsp.models.CompletionResult +import com.itsaky.androidide.models.Position +import com.itsaky.androidide.progress.ICancelChecker +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.nio.file.Paths + +/** @author Akash Yadav */ +@RunWith(JUnit4::class) +class CompletionCacheGuardTest { + @Test + fun `store succeeds when the generation has not moved`() { + val guard = CompletionCacheGuard() + val generationAtStart = guard.currentGeneration() + val candidate = newCachedCompletion() + + guard.store(candidate, generationAtStart) + + assertThat(guard.cachedCompletion).isSameInstanceAs(candidate) + } + + @Test + fun `a reset between start and store blocks the store`() { + val guard = CompletionCacheGuard() + val generationAtStart = guard.currentGeneration() + + // Simulates a full Idle -> Indexing -> Idle cycle completing while the completion that + // captured generationAtStart is still computing. + guard.reset() + + val staleResult = newCachedCompletion() + guard.store(staleResult, generationAtStart) + + assertThat(guard.cachedCompletion).isNotSameInstanceAs(staleResult) + assertThat(guard.cachedCompletion).isSameInstanceAs(CachedCompletion.EMPTY) + } + + @Test + fun `store after a reset with the current generation still succeeds`() { + val guard = CompletionCacheGuard() + guard.reset() + val generationAfterReset = guard.currentGeneration() + val candidate = newCachedCompletion() + + guard.store(candidate, generationAfterReset) + + assertThat(guard.cachedCompletion).isSameInstanceAs(candidate) + } + + private fun newCachedCompletion(): CachedCompletion = + CachedCompletion.cache( + CompletionParams(Position.NONE, Paths.get("Test.java"), ICancelChecker.CANCELLED), + CompletionResult.EMPTY, + ) +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/AddImportTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/AddImportTest.kt index a0bd8e7e86..f2e87276f3 100644 --- a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/AddImportTest.kt +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/AddImportTest.kt @@ -1,69 +1,134 @@ -/* - * This file is part of AndroidIDE. - * - * AndroidIDE is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * AndroidIDE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with AndroidIDE. If not, see . - */ - -package com.itsaky.androidide.lsp.java.actions - -import com.google.common.truth.Truth.assertThat -import com.itsaky.androidide.lsp.java.JavaLSPTest -import com.itsaky.androidide.lsp.java.actions.diagnostics.AddImportAction -import kotlinx.coroutines.runBlocking -import org.junit.Before -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.annotation.Config - -/** @author Akash Yadav */ -@RunWith(RobolectricTestRunner::class) -@Config(manifest = Config.DEFAULT_VALUE_STRING) -class AddImportTest { - - @Before - fun setup() { - JavaLSPTest.setup() - } - - @Suppress("UNCHECKED_CAST") - @Test - fun addImport() { - JavaLSPTest.apply { - openFile("actions/AddImportAction") - val diagnostic = - runBlocking { - server.analyze(file!!).diagnostics.firstOrNull { - it.code == "compiler.err.cant.resolve.location" - } - } - - assertThat(diagnostic).isNotNull() - - val file = this.file!!.toFile() - val data = createActionData(diagnostic!!, file, this.file!!, this.server) - - val action = AddImportAction() - action.prepare(data) - assertThat(action.visible).isTrue() - assertThat(action.enabled).isTrue() - - val execResult = runBlocking { action.execAction(data) } - assertThat(execResult::class.java).isAssignableTo(Pair::class.java) - - val result = execResult as Pair, *> - assertThat(result.first).contains("java.util.stream.Stream") - } - } -} +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.java.actions + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.lsp.java.JavaCompilerProvider +import com.itsaky.androidide.lsp.java.JavaLSPTest +import com.itsaky.androidide.lsp.java.actions.diagnostics.AddImportAction +import com.itsaky.androidide.lsp.java.actions.diagnostics.ImportCandidates +import io.mockk.mockkStatic +import io.mockk.unmockkStatic +import io.mockk.verify +import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +/** @author Akash Yadav */ +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.DEFAULT_VALUE_STRING) +class AddImportTest { + @Before + fun setup() { + JavaLSPTest.setup() + } + + @Test + fun addImport() { + JavaLSPTest.apply { + openFile("actions/AddImportAction") + val diagnostic = unresolvedLocationDiagnostic() + + val file = this.file!!.toFile() + val data = createActionData(diagnostic, file, this.file!!, this.server) + + val action = AddImportAction() + action.prepare(data) + assertThat(action.visible).isTrue() + assertThat(action.enabled).isTrue() + + val execResult = runBlocking { action.execAction(data) } + assertThat(execResult).isInstanceOf(ImportCandidates.Found::class.java) + + val result = execResult as ImportCandidates.Found + assertThat(result.titles).contains("java.util.stream.Stream") + } + } + + @Test + fun `execAction returns None when nothing is importable`() { + JavaLSPTest.apply { + openFile("actions/AddImportActionNone") + val diagnostic = unresolvedLocationDiagnostic() + + val file = this.file!!.toFile() + val data = createActionData(diagnostic, file, this.file!!, this.server) + + val action = AddImportAction() + action.prepare(data) + assertThat(action.visible).isTrue() + + val execResult = runBlocking { action.execAction(data) } + assertThat(execResult).isInstanceOf(ImportCandidates.None::class.java) + assertThat((execResult as ImportCandidates.None).simpleName).isEqualTo("NoSuchClassAnywhere") + } + } + + @Test + fun `prepare hides the action for a Java file in no module`() { + JavaLSPTest.apply { + openFile("actions/AddImportAction") + val diagnostic = unresolvedLocationDiagnostic() + + val outside = java.io.File("/outside-any-project/Outside.java") + val data = createActionData(diagnostic, outside, outside.toPath(), this.server) + + val action = AddImportAction() + action.prepare(data) + + assertThat(action.visible).isFalse() + } + } + + @Test + fun `prepare does not touch the compiler provider`() { + JavaLSPTest.apply { + openFile("actions/AddImportAction") + val diagnostic = unresolvedLocationDiagnostic() + + val file = this.file!!.toFile() + val data = createActionData(diagnostic, file, this.file!!, this.server) + + // A call to the compiler provider from prepare() would be main-thread SQLite I/O once the + // index backs class lookups, so record every call to it and assert none happened. + mockkStatic(JavaCompilerProvider::class) + try { + val action = AddImportAction() + action.prepare(data) + + assertThat(action.visible).isTrue() + assertThat(action.enabled).isTrue() + verify(exactly = 0) { JavaCompilerProvider.get(any()) } + } finally { + unmockkStatic(JavaCompilerProvider::class) + } + } + } + + private fun unresolvedLocationDiagnostic() = + runBlocking { + checkNotNull( + JavaLSPTest.server.analyze(JavaLSPTest.file!!).diagnostics.firstOrNull { + it.code == "compiler.err.cant.resolve.location" + }, + ) { "no unresolved-location diagnostic found in ${JavaLSPTest.file}" } + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/IsJavaFileNameTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/IsJavaFileNameTest.kt new file mode 100644 index 0000000000..4bee41ddae --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/IsJavaFileNameTest.kt @@ -0,0 +1,46 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ +package com.itsaky.androidide.lsp.java.actions + +import com.google.common.truth.Truth.assertThat +import org.junit.Test + +/** + * [isJavaFileName] decides [BaseJavaCodeAction.prepare]'s visibility from the file name alone, with + * no [java.nio.file.Files] call, since `prepare()` runs on the UI thread. + */ +class IsJavaFileNameTest { + @Test + fun `a dot-java file is a java file`() { + assertThat(isJavaFileName("Foo.java")).isTrue() + } + + @Test + fun `a non-java file is not a java file`() { + assertThat(isJavaFileName("Foo.kt")).isFalse() + } + + @Test + fun `module-info java is not a java file`() { + assertThat(isJavaFileName("module-info.java")).isFalse() + } + + @Test + fun `package-info java is not a java file`() { + assertThat(isJavaFileName("package-info.java")).isFalse() + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/diagnostics/NoImportableClassMessageTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/diagnostics/NoImportableClassMessageTest.kt new file mode 100644 index 0000000000..ca6927eaee --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/actions/diagnostics/NoImportableClassMessageTest.kt @@ -0,0 +1,36 @@ +package com.itsaky.androidide.lsp.java.actions.diagnostics + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.service.IndexingState +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class NoImportableClassMessageTest { + @Test + fun `idle state formats only the template`() { + val message = + noImportableClassMessage( + simpleName = "Stream", + state = IndexingState.Idle, + template = "No importable class named '%1\$s'.", + indexingTemplate = "No importable class named '%1\$s'. Libraries are still being indexed.", + ) + + assertThat(message).isEqualTo("No importable class named 'Stream'.") + } + + @Test + fun `indexing state formats the indexing template instead of the plain one`() { + val message = + noImportableClassMessage( + simpleName = "Stream", + state = IndexingState.Indexing(done = 1, total = 5), + template = "No importable class named '%1\$s'.", + indexingTemplate = "No importable class named '%1\$s'. Libraries are still being indexed.", + ) + + assertThat(message).isEqualTo("No importable class named 'Stream'. Libraries are still being indexed.") + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/ClasspathTypeLookupTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/ClasspathTypeLookupTest.kt new file mode 100644 index 0000000000..d1cf87cdb6 --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/ClasspathTypeLookupTest.kt @@ -0,0 +1,297 @@ +package com.itsaky.androidide.lsp.java.compiler + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.jvm.JvmVisibility +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class ClasspathTypeLookupTest { + /** A stand-in for the index-backed classpath lookup: exact matches first, then case-insensitive prefix. */ + private class FakeClasspath( + private val names: List, + private val visibilityOf: (String) -> JvmVisibility = { JvmVisibility.PUBLIC }, + ) : ClasspathClassNames { + fun qualifiedNamesOf(simpleName: String) = names.filter { it.substringAfterLast('.') == simpleName } + + override fun qualifiedNamesByPrefix( + prefix: String, + limit: Int, + ): List { + val prefixed = names.filter { it.substringAfterLast('.').startsWith(prefix, ignoreCase = true) } + return (qualifiedNamesOf(prefix) + prefixed).distinct().take(limit) + } + + override fun classesNamed(simpleName: String): List = + qualifiedNamesOf(simpleName).map { fqName -> + JvmSymbol( + key = fqName, + sourceId = "fake", + name = fqName.replace('.', '/'), + shortName = fqName.substringAfterLast('.'), + packageName = fqName.substringBeforeLast('.', missingDelimiterValue = ""), + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + visibility = visibilityOf(fqName), + data = JvmClassInfo(), + ) + } + + override fun isClass(qualifiedName: String) = qualifiedName in names + + override fun children(packageName: String) = emptyList() + } + + private fun lookup( + sources: List = emptyList(), + classpath: ClasspathClassNames? = null, + boot: List = emptyList(), + ) = ClasspathTypeLookup({ sources }, { classpath }, { boot }) + + @Test + fun `importable qualified names union source, classpath and boot classes`() { + val types = + lookup( + sources = listOf("com.app.Widget"), + classpath = FakeClasspath(listOf("com.lib.Widget")), + boot = listOf("android.widget.Widget"), + ) + + assertThat(types.findImportableQualifiedNames("Widget", importingPackage = "com.app")) + .containsExactly("android.widget.Widget", "com.app.Widget", "com.lib.Widget") + .inOrder() + } + + @Test + fun `a class present in several sources is returned once`() { + val types = + lookup( + sources = listOf("com.shared.Thing"), + classpath = FakeClasspath(listOf("com.shared.Thing")), + boot = listOf("com.shared.Thing"), + ) + + assertThat(types.findImportableQualifiedNames("Thing", importingPackage = "com.app")).containsExactly("com.shared.Thing") + assertThat(types.findTypeNamesMatching("Th", 10)).containsExactly("com.shared.Thing") + } + + @Test + fun `importable qualified names match the whole simple name only`() { + val types = lookup(boot = listOf("java.util.ArrayList", "java.util.List")) + + assertThat(types.findImportableQualifiedNames("List", importingPackage = "com.app")).containsExactly("java.util.List") + } + + @Test + fun `type names match a prefix ignoring case`() { + val types = + lookup( + sources = listOf("com.app.ArrayAdapterX"), + classpath = FakeClasspath(listOf("com.lib.ArrayMap")), + boot = listOf("java.util.ArrayList", "java.util.List"), + ) + + assertThat(types.findTypeNamesMatching("array", 10)) + .containsExactly("com.app.ArrayAdapterX", "com.lib.ArrayMap", "java.util.ArrayList") + } + + @Test + fun `exact simple-name matches come first and survive the limit`() { + val prefixed = (1..20).map { "com.lib.ViewThing$it" } + val types = + lookup( + sources = listOf("com.app.ViewModelX"), + classpath = FakeClasspath(prefixed), + boot = listOf("android.view.View"), + ) + + val names = types.findTypeNamesMatching("View", 3) + + assertThat(names.first()).isEqualTo("android.view.View") + } + + @Test + fun `each source contributes at most the limit of prefix matches`() { + val types = + lookup( + sources = (1..5).map { "com.app.Foo$it" }, + classpath = FakeClasspath((1..5).map { "com.lib.Foo$it" }), + boot = (1..5).map { "java.foo.Foo$it" }, + ) + + assertThat(types.findTypeNamesMatching("Foo", 3)).hasSize(9) + } + + @Test + fun `boot prefix matches are offered when the classpath has more prefix matches than the limit`() { + val types = + lookup( + classpath = FakeClasspath((1..60).map { "com.lib.ViewThing$it" }), + boot = listOf("android.view.View"), + ) + + assertThat(types.findTypeNamesMatching("Vie", 50)).contains("android.view.View") + } + + @Test + fun `a class the classpath repeats from the sources does not cost the limit a slot`() { + val shared = (1..3).map { "com.shared.Bar$it" } + val types = + lookup( + sources = shared, + classpath = FakeClasspath(shared + (4..6).map { "com.lib.Bar$it" }), + ) + + assertThat(types.findTypeNamesMatching("Bar", 6)).hasSize(6) + } + + @Test + fun `a later classpath index's prefix matches survive a full first index`() { + val library = (1..3).map { "com.lib.Ma$it" } + val classpath = + object : ClasspathClassNames by FakeClasspath(library) { + // The real lookup caps each index separately, so it returns more than the limit. + override fun qualifiedNamesByPrefix( + prefix: String, + limit: Int, + ) = library + "com.module.MainViewModel" + } + val types = lookup(classpath = classpath) + + assertThat(types.findTypeNamesMatching("Ma", 3)).contains("com.module.MainViewModel") + } + + @Test + fun `an exact match does not cost its source a prefix slot`() { + val types = lookup(sources = listOf("com.app.Foo") + (1..3).map { "com.app.Foo$it" }) + + assertThat(types.findTypeNamesMatching("Foo", 3)).hasSize(4) + } + + @Test + fun `a fuzzy near-miss is not a type name match`() { + val types = lookup(boot = listOf("java.util.ArrayList")) + + assertThat(types.findTypeNamesMatching("ArgList", 50)).isEmpty() + } + + @Test + fun `no classpath lookup contributes nothing`() { + val types = lookup(sources = listOf("com.app.Widget"), classpath = null) + + assertThat(types.findImportableQualifiedNames("Widget", importingPackage = "com.app")).containsExactly("com.app.Widget") + assertThat(types.findTypeNamesMatching("Wid", 10)).containsExactly("com.app.Widget") + } + + @Test + fun `the classpath lookup is created for every request`() { + var created = 0 + val types = + ClasspathTypeLookup({ emptyList() }, { + created++ + FakeClasspath(emptyList()) + }, { emptyList() }) + + types.findImportableQualifiedNames("A", importingPackage = "com.app") + types.findTypeNamesMatching("A", 5) + + assertThat(created).isEqualTo(2) + } + + @Test + fun `construction does not create the classpath lookup`() { + var created = 0 + ClasspathTypeLookup({ emptyList() }, { + created++ + null + }, { emptyList() }) + + assertThat(created).isEqualTo(0) + } + + @Test + fun `a package-private classpath class from another package is not importable`() { + val types = + lookup( + classpath = + FakeClasspath( + listOf("com.lib.Hidden"), + visibilityOf = { JvmVisibility.PACKAGE_PRIVATE }, + ), + ) + + assertThat(types.findImportableQualifiedNames("Hidden", importingPackage = "com.app")).isEmpty() + } + + @Test + fun `a package-private classpath class in the importing package is importable`() { + val types = + lookup( + classpath = + FakeClasspath( + listOf("com.lib.Hidden"), + visibilityOf = { JvmVisibility.PACKAGE_PRIVATE }, + ), + ) + + assertThat(types.findImportableQualifiedNames("Hidden", importingPackage = "com.lib")) + .containsExactly("com.lib.Hidden") + } + + @Test + fun `a file-private classpath class from another package is not importable`() { + val types = + lookup( + classpath = + FakeClasspath( + listOf("com.lib.Hidden"), + visibilityOf = { JvmVisibility.PRIVATE }, + ), + ) + + assertThat(types.findImportableQualifiedNames("Hidden", importingPackage = "com.app")).isEmpty() + } + + @Test + fun `a file-private classpath class in the importing package is importable`() { + val types = + lookup( + classpath = + FakeClasspath( + listOf("com.lib.Hidden"), + visibilityOf = { JvmVisibility.PRIVATE }, + ), + ) + + assertThat(types.findImportableQualifiedNames("Hidden", importingPackage = "com.lib")) + .containsExactly("com.lib.Hidden") + } + + @Test + fun `source and boot classes are importable regardless of visibility`() { + val types = + lookup( + sources = listOf("com.app.PackagePrivateSource"), + boot = listOf("android.PackagePrivateBoot"), + ) + + assertThat(types.findImportableQualifiedNames("PackagePrivateSource", importingPackage = "com.other")) + .containsExactly("com.app.PackagePrivateSource") + assertThat(types.findImportableQualifiedNames("PackagePrivateBoot", importingPackage = "com.other")) + .containsExactly("android.PackagePrivateBoot") + } + + @Test + fun `a public classpath class is importable from any package`() { + val types = lookup(classpath = FakeClasspath(listOf("com.lib.Widget"))) + + assertThat(types.findImportableQualifiedNames("Widget", importingPackage = "com.app")) + .containsExactly("com.lib.Widget") + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/JavaCompilerServiceClassLookupTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/JavaCompilerServiceClassLookupTest.kt new file mode 100644 index 0000000000..70bb97e99a --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/JavaCompilerServiceClassLookupTest.kt @@ -0,0 +1,80 @@ +package com.itsaky.androidide.lsp.java.compiler + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.utils.Environment +import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File + +@RunWith(JUnit4::class) +class JavaCompilerServiceClassLookupTest { + companion object { + private var previousAndroidJar: File? = null + + @JvmStatic + @BeforeClass + fun setUpEnvironment() { + // JavaCompilerService's static initializer reads it; only a device sets it. + previousAndroidJar = Environment.ANDROID_JAR + if (Environment.ANDROID_JAR == null) { + Environment.ANDROID_JAR = File("android.jar") + } + } + + @JvmStatic + @AfterClass + fun tearDownEnvironment() { + Environment.ANDROID_JAR = previousAndroidJar + } + } + + private val indexed = + object : ClasspathClassNames { + fun qualifiedNamesOf(simpleName: String) = listOf("com.indexed.$simpleName").filter { simpleName == "Indexed" } + + override fun qualifiedNamesByPrefix( + prefix: String, + limit: Int, + ) = emptyList() + + override fun classesNamed(simpleName: String): List = + qualifiedNamesOf(simpleName).map { fqName -> + JvmSymbol( + key = fqName, + sourceId = "fake", + name = fqName.replace('.', '/'), + shortName = simpleName, + packageName = fqName.substringBeforeLast('.', missingDelimiterValue = ""), + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(), + ) + } + + override fun isClass(qualifiedName: String) = qualifiedName == "com.indexed.Indexed" + + override fun children(packageName: String) = emptyList() + } + + @Test + fun `importable qualified names come from the index`() { + val compiler = compilerWithClasspath(null, indexed) + + assertThat(compiler.findImportableQualifiedNames("Indexed", "com.app")).containsExactly("com.indexed.Indexed") + } + + @Test + fun `a copy answers from the same class lookup`() { + val compiler = compilerWithClasspath(null, indexed).copy() + + assertThat(compiler.findImportableQualifiedNames("Indexed", "com.app")).containsExactly("com.indexed.Indexed") + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/TestCompilers.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/TestCompilers.kt new file mode 100644 index 0000000000..89a99a115a --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/compiler/TestCompilers.kt @@ -0,0 +1,18 @@ +package com.itsaky.androidide.lsp.java.compiler + +import com.itsaky.androidide.projects.api.ModuleProject + +/** + * Creates a compiler for [module] whose classpath lookups answer from [classpath], with the boot + * classes [bootClasses] and no source classes. + */ +internal fun compilerWithClasspath( + module: ModuleProject?, + classpath: ClasspathClassNames?, + bootClasses: Set = emptySet(), +) = JavaCompilerService( + module, + SourceFileManager.NO_MODULE, + bootClasses, + ClasspathTypeLookup({ emptyList() }, { classpath }, { bootClasses }), +) diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/CompletionProviderCacheDecisionTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/CompletionProviderCacheDecisionTest.kt new file mode 100644 index 0000000000..8b4c9e9b6e --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/CompletionProviderCacheDecisionTest.kt @@ -0,0 +1,40 @@ +package com.itsaky.androidide.lsp.java.providers + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.service.IndexingState +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** @author Akash Yadav */ +@RunWith(JUnit4::class) +class CompletionProviderCacheDecisionTest { + @Test + fun `idle at both ends can be cached`() { + assertThat(CompletionProvider.canCacheCompletion(IndexingState.Idle, IndexingState.Idle)).isTrue() + } + + @Test + fun `indexing at start cannot be cached`() { + assertThat( + CompletionProvider.canCacheCompletion(IndexingState.Indexing(done = 0, total = 1), IndexingState.Idle), + ).isFalse() + } + + @Test + fun `indexing at end cannot be cached`() { + assertThat( + CompletionProvider.canCacheCompletion(IndexingState.Idle, IndexingState.Indexing(done = 1, total = 2)), + ).isFalse() + } + + @Test + fun `indexing at both ends cannot be cached`() { + assertThat( + CompletionProvider.canCacheCompletion( + IndexingState.Indexing(done = 0, total = 1), + IndexingState.Indexing(done = 1, total = 1), + ), + ).isFalse() + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/FakeClasspathPackages.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/FakeClasspathPackages.kt new file mode 100644 index 0000000000..9699505bea --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/FakeClasspathPackages.kt @@ -0,0 +1,32 @@ +package com.itsaky.androidide.lsp.java.providers.completion + +import com.itsaky.androidide.lsp.java.compiler.ClasspathClassNames +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child + +/** A compile classpath holding the top-level [classes], answering package queries the way the class index does. */ +internal class FakeClasspathPackages( + private val classes: List, +) : ClasspathClassNames { + override fun isClass(qualifiedName: String) = qualifiedName in classes + + override fun children(packageName: String): List { + val prefix = if (packageName.isEmpty()) "" else "$packageName." + val below = classes.filter { it.startsWith(prefix) }.map { it.removePrefix(prefix) } + val subpackages = + below + .filter { '.' in it } + .map { it.substringBefore('.') } + .distinct() + .map { Child(it, prefix + it, isClass = false) } + val topLevelClasses = below.filterNot { '.' in it }.map { Child(it, prefix + it, isClass = true) } + return subpackages + topLevelClasses + } + + override fun qualifiedNamesByPrefix( + prefix: String, + limit: Int, + ) = emptyList() + + override fun classesNamed(simpleName: String) = emptyList() +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/ImportCompletionProviderTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/ImportCompletionProviderTest.kt new file mode 100644 index 0000000000..7bf3999389 --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/ImportCompletionProviderTest.kt @@ -0,0 +1,144 @@ +package com.itsaky.androidide.lsp.java.providers.completion + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.lsp.java.compiler.ClasspathClassNames +import com.itsaky.androidide.lsp.java.compiler.CompileTask +import com.itsaky.androidide.lsp.java.compiler.compilerWithClasspath +import com.itsaky.androidide.lsp.java.refactor.JavacFixture +import com.itsaky.androidide.lsp.models.CompletionItem +import com.itsaky.androidide.lsp.models.CompletionItemKind +import com.itsaky.androidide.lsp.util.DefaultServerSettings +import com.itsaky.androidide.project.GradleModels +import com.itsaky.androidide.project.JavaModels +import com.itsaky.androidide.projects.api.JavaModule +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.util.BootClasspathProvider +import com.itsaky.androidide.utils.Environment +import io.mockk.every +import io.mockk.mockk +import openjdk.source.util.TreePath +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child +import org.junit.After +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.nio.file.Path +import java.nio.file.Paths + +@RunWith(JUnit4::class) +class ImportCompletionProviderTest { + companion object { + @JvmStatic + @BeforeClass + fun setUpEnvironment() { + // JavaCompilerService's static initializer reads it; only a device sets it. + if (Environment.ANDROID_JAR == null) { + Environment.ANDROID_JAR = File("android.jar") + } + } + } + + private val fixture = JavacFixture("import java.util.List;\nclass Fixture {}") + + @After + fun tearDown() = fixture.close() + + private fun module(): ModuleProject = + JavaModule( + GradleModels.GradleProject + .newBuilder() + .setName("app") + .setPath(":app") + .setJavaProject(JavaModels.JavaProject.getDefaultInstance()) + .build(), + ) + + private fun complete( + importPath: String, + module: ModuleProject?, + classpath: ClasspathClassNames?, + bootClasses: Set = emptySet(), + ): List { + val file: Path = Paths.get("Fixture.java") + val task = mockk { every { root(any()) } returns fixture.root } + val importTree = TreePath(TreePath(fixture.root), fixture.root.imports.single()) + val compiler = compilerWithClasspath(module, classpath, bootClasses) + val provider = ImportCompletionProvider(file, 0, compiler, DefaultServerSettings()) + provider.importPath = importPath + return provider.complete(task, importTree, "", false).items + } + + private fun List.labels() = map { it.ideLabel } + + @Test + fun `a package only the class index holds is completed`() { + val module = module() + module.compileClasspathClasses.append("com.trie.Widget") + + val items = complete("com.", module, FakeClasspathPackages(listOf("com.indexed.Widget"))) + + assertThat(items.labels()).contains("indexed") + } + + @Test + fun `a package only the classpath trie holds is not completed`() { + val module = module() + module.compileClasspathClasses.append("com.trie.Widget") + + val items = complete("com.", module, FakeClasspathPackages(listOf("com.indexed.Widget"))) + + assertThat(items.labels()).doesNotContain("trie") + } + + @Test + fun `classpath packages and classes are itemised as packages and classes`() { + val items = + complete("com.lib.", module(), FakeClasspathPackages(listOf("com.lib.Widget", "com.lib.impl.Engine"))) + + assertThat(items.associate { it.ideLabel to it.completionKind }) + .containsExactly("impl", CompletionItemKind.MODULE, "Widget", CompletionItemKind.CLASS) + } + + @Test + fun `without a module the boot classpath is still completed`() { + // Any real JAR stands in for android.jar; kotlin-stdlib is on the test classpath. + val bootJar = + File( + Unit::class.java.protectionDomain.codeSource.location + .toURI(), + ).path + BootClasspathProvider.update(listOf(bootJar)) + try { + val items = complete("kotlin.La", module = null, classpath = null) + + assertThat(items.labels()).contains("Lazy") + } finally { + BootClasspathProvider.drop(bootJar) + } + } + + @Test + fun `one completion request queries the classpath children at most once`() { + val classpath = + CountingChildrenClasspathPackages(FakeClasspathPackages(listOf("com.lib.Widget", "com.lib.impl.Engine"))) + + complete("com.lib.", module(), classpath) + + assertThat(classpath.childrenCalls).isAtMost(1) + } +} + +/** Counts calls to [children] so a test can assert the classpath is queried at most once per request. */ +private class CountingChildrenClasspathPackages( + private val delegate: ClasspathClassNames, +) : ClasspathClassNames by delegate { + var childrenCalls = 0 + private set + + override fun children(packageName: String): List { + childrenCalls++ + return delegate.children(packageName) + } +} diff --git a/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/ImportPathChildrenTest.kt b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/ImportPathChildrenTest.kt new file mode 100644 index 0000000000..10ecdc3bf7 --- /dev/null +++ b/lsp/java/src/test/java/com/itsaky/androidide/lsp/java/providers/completion/ImportPathChildrenTest.kt @@ -0,0 +1,162 @@ +package com.itsaky.androidide.lsp.java.providers.completion + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.lsp.java.compiler.ClasspathPackages +import com.itsaky.androidide.lsp.java.providers.completion.ImportCompletionProvider.RequireMemberCompletionException +import com.itsaky.androidide.utils.ClassTrie +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child +import org.junit.Assert.assertThrows +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class ImportPathChildrenTest { + private fun trie(vararg classes: String) = ClassTrie().apply { classes.forEach(::append) } + + private fun children( + sources: ClassTrie = trie(), + classpath: ClasspathPackages? = null, + boot: List = emptyList(), + ) = ImportPathChildren(sources, classpath, boot) + + private fun Sequence.names() = map { it.name }.toList() + + @Test + fun `a classpath class before the last segment of a package walk requires member completion`() { + val children = children(classpath = FakeClasspathPackages(listOf("com.lib.Outer"))) + + assertThrows(RequireMemberCompletionException::class.java) { + children.ofPackage("com.lib.Outer.Inner") + } + } + + @Test + fun `a classpath class as the last segment of a package walk does not require member completion`() { + val children = children(classpath = FakeClasspathPackages(listOf("com.lib.Outer"))) + + assertThat(children.ofPackage("com.lib.Outer").names()).isEmpty() + } + + @Test + fun `a classpath class named by the whole path requires member completion`() { + val children = children(classpath = FakeClasspathPackages(listOf("com.lib.Outer"))) + + assertThrows(RequireMemberCompletionException::class.java) { + children.of("com.lib.Outer").toList() + } + } + + @Test + fun `a source class named by the whole path requires member completion`() { + val children = children(sources = trie("com.app.Main")) + + assertThrows(RequireMemberCompletionException::class.java) { + children.of("com.app.Main").toList() + } + } + + @Test + fun `packages and classes of the classpath are told apart`() { + val children = children(classpath = FakeClasspathPackages(listOf("com.lib.Widget", "com.lib.impl.Engine"))) + + assertThat(children.ofPackage("com.lib").toList()).containsExactly( + Child("impl", "com.lib.impl", isClass = false), + Child("Widget", "com.lib.Widget", isClass = true), + ) + } + + @Test + fun `children come from the sources, then the classpath, then the boot classpath`() { + val children = + children( + sources = trie("com.app.Main"), + classpath = FakeClasspathPackages(listOf("com.lib.Widget")), + boot = listOf(trie("com.boot.Thing")), + ) + + assertThat(children.of("com").names()).containsExactly("app", "lib", "boot").inOrder() + } + + @Test + fun `a name the sources offered is not offered again by the classpath or the boot classpath`() { + val children = + children( + sources = trie("com.shared.A"), + classpath = FakeClasspathPackages(listOf("com.shared.B")), + boot = listOf(trie("com.shared.C")), + ) + + assertThat(children.of("com").toList()).containsExactly(Child("shared", "com.shared", isClass = false)) + } + + @Test + fun `a name the package walk offered is not offered again by the lookup`() { + val children = children(classpath = FakeClasspathPackages(listOf("com.lib.Widget"))) + + assertThat(children.ofPackage("com.lib").names()).containsExactly("Widget") + assertThat(children.of("com.lib").names()).isEmpty() + } + + @Test + fun `a package only the classpath holds is offered`() { + val children = children(sources = trie("com.app.Main"), classpath = FakeClasspathPackages(listOf("com.indexed.Widget"))) + + assertThat(children.ofPackage("com").names()).contains("indexed") + } + + @Test + fun `boot classpath packages are offered`() { + val children = children(boot = listOf(trie("java.util.List"), trie("android.view.View"))) + + assertThat(children.of("").names()).containsExactly("java", "android") + } + + @Test + fun `without a classpath the sources and the boot classpath still answer`() { + val children = children(sources = trie("com.app.Main"), classpath = null, boot = listOf(trie("com.boot.Thing"))) + + assertThat(children.ofPackage("com").names()).containsExactly("app", "boot") + } + + @Test + fun `requireNotClass throws for a class in the sources`() { + val children = children(sources = trie("com.app.Main")) + + assertThrows(RequireMemberCompletionException::class.java) { + children.requireNotClass("com.app.Main") + } + } + + @Test + fun `requireNotClass throws for a class in the classpath`() { + val children = children(classpath = FakeClasspathPackages(listOf("com.lib.Widget"))) + + assertThrows(RequireMemberCompletionException::class.java) { + children.requireNotClass("com.lib.Widget") + } + } + + @Test + fun `requireNotClass throws for a class in the boot classpath`() { + val children = children(boot = listOf(trie("java.util.List"))) + + assertThrows(RequireMemberCompletionException::class.java) { + children.requireNotClass("java.util.List") + } + } + + @Test + fun `requireNotClass does not throw for a package`() { + val children = + children( + sources = trie("com.app.Main"), + classpath = FakeClasspathPackages(listOf("com.lib.Widget")), + boot = listOf(trie("java.util.List")), + ) + + children.requireNotClass("com.app") + children.requireNotClass("com.lib") + children.requireNotClass("java.util") + } +} diff --git a/lsp/jvm-symbol-index/build.gradle.kts b/lsp/jvm-symbol-index/build.gradle.kts index a26497c9c2..12ffbecef1 100644 --- a/lsp/jvm-symbol-index/build.gradle.kts +++ b/lsp/jvm-symbol-index/build.gradle.kts @@ -22,6 +22,8 @@ dependencies { api(projects.subprojects.kotlinAnalysisApi) api(projects.subprojects.projects) + implementation(projects.shared) + testImplementation(projects.testing.unit) testImplementation(libs.tests.kotlinx.coroutines) } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt index 84542cd92c..1934f8941b 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt @@ -1,5 +1,6 @@ package org.appdevforall.codeonthego.indexing.jvm +import org.appdevforall.codeonthego.indexing.util.CloseableSequence import org.jetbrains.kotlin.analysis.api.KaImplementationDetail import org.jetbrains.kotlin.analysis.api.impl.base.util.LibraryUtils import org.jetbrains.kotlin.com.intellij.openapi.vfs.VfsUtilCore @@ -11,7 +12,9 @@ import org.jetbrains.org.objectweb.asm.Opcodes import org.slf4j.LoggerFactory import java.io.ByteArrayOutputStream import java.nio.file.Path +import java.util.concurrent.atomic.AtomicReference import java.util.jar.JarFile +import kotlin.io.path.notExists import kotlin.io.path.pathString /** @@ -46,47 +49,70 @@ object CombinedJarScanner { } } + /** + * Scans [jarPath], routing each class to [KotlinMetadataScanner] or [JarSymbolScanner]. + * + * @throws UnreadableJarException if [jarPath] cannot be opened as a JAR (a truncated download, + * corrupt file, etc.). A per-entry parse failure is logged and skipped instead, since the JAR as + * a whole is still readable. A JAR that no longer exists (deleted after the caller listed it) + * rethrows the open failure unwrapped instead, since the file is gone rather than unreadable. + * + * The JAR is opened on first iteration and closed when iteration ends. A consumer that may stop + * part-way must [close][CloseableSequence.close] the result, or the JAR stays open until it is + * garbage collected. + */ fun scan( jarPath: Path, sourceId: String = jarPath.pathString, - ): Sequence = - sequence { - val jar = - try { - JarFile(jarPath.toFile()) - } catch (e: Exception) { - log.warn("Failed to open JAR: {}", jarPath, e) - return@sequence - } + ): CloseableSequence { + val opened = AtomicReference() + val symbols = + sequence { + val jar = + try { + JarFile(jarPath.toFile()) + } catch (e: Exception) { + if (jarPath.notExists()) throw e + throw UnreadableJarException(jarPath, e) + } + opened.set(jar) + scanEntries(jar, sourceId) + } + return CloseableSequence(symbols) { opened.getAndSet(null)?.close() } + } - jar.use { - val entries = jar.entries() - while (entries.hasMoreElements()) { - val entry = entries.nextElement() - if (!isIndexableClassEntry(entry.name)) continue + private suspend fun SequenceScope.scanEntries( + jar: JarFile, + sourceId: String, + ) { + jar.use { + val entries = jar.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + if (!isIndexableClassEntry(entry.name)) continue - try { - val bytes = - jar.getInputStream(entry).use { input -> - val buf = ByteArrayOutputStream(entry.size.toInt().coerceAtLeast(1024)) - input.copyTo(buf) - buf.toByteArray() - } + try { + val bytes = + jar.getInputStream(entry).use { input -> + val buf = ByteArrayOutputStream(entry.size.toInt().coerceAtLeast(1024)) + input.copyTo(buf) + buf.toByteArray() + } - val symbols = - if (hasKotlinMetadata(bytes)) { - KotlinMetadataScanner.parseKotlinClass(bytes.inputStream(), sourceId) - } else { - JarSymbolScanner.parseClassFile(bytes.inputStream(), sourceId) - } + val symbols = + if (hasKotlinMetadata(bytes)) { + KotlinMetadataScanner.parseKotlinClass(bytes.inputStream(), sourceId) + } else { + JarSymbolScanner.parseClassFile(bytes.inputStream(), sourceId) + } - symbols?.forEach { yield(it) } - } catch (e: Exception) { - log.debug("Failed to parse {}: {}", entry.name, e.message) - } + symbols?.forEach { yield(it) } + } catch (e: Exception) { + log.debug("Failed to parse {}: {}", entry.name, e.message) } } } + } /* * Everything under META-INF is skipped. A multi-release JAR's META-INF/versions// copy of a @@ -120,3 +146,9 @@ object CombinedJarScanner { return found } } + +/** Thrown by [CombinedJarScanner.scan] when [path] cannot be opened as a JAR. */ +class UnreadableJarException( + val path: Path, + cause: Throwable, +) : Exception("Failed to open JAR: $path", cause) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingService.kt new file mode 100644 index 0000000000..bfe7764f6f --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingService.kt @@ -0,0 +1,230 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import com.itsaky.androidide.projects.ProjectManagerImpl +import com.itsaky.androidide.projects.api.Workspace +import com.itsaky.androidide.projects.reportUnreadableClasspathJars +import com.itsaky.androidide.tasks.cancelIfActive +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.appdevforall.codeonthego.indexing.service.IndexKey +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.appdevforall.codeonthego.indexing.service.IndexingService +import org.appdevforall.codeonthego.indexing.util.CloseableSequence +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.file.Paths +import java.util.Collections + +/** + * Base [IndexingService] for a persistent [JvmSymbolIndex] over a set of JARs derived from the + * project model. + * + * Owns the index instance, the single-submitter mutex that serializes [refresh] passes, and the + * fingerprint-driven reindex loop: only a JAR whose [jarFingerprint] changed since it was last + * indexed is re-scanned, and [JvmSymbolIndex.optimizeAfter] runs once the whole pass finishes. A + * subclass supplies [indexKey], [dbName], [indexName] and [jarsToIndex]. + * + * Every pass reports the JARs it submits to [progressTracker] and closes its tracker pass in a + * `finally`, so a pass that throws or is cancelled cannot leave the indexing state stuck. + * + * A JAR whose scan throws [UnreadableJarException] (see [CombinedJarScanner.scan]) is recorded and + * rethrown, so [BackgroundIndexer][org.appdevforall.codeonthego.indexing.util.BackgroundIndexer] + * logs the failure and writes no fingerprint for it, which makes the next pass retry it. Once the + * pass's jobs have all finished, every recorded JAR is passed through [unreadableJarFilter] - which + * a caller shares across every service built against the same project session (typically + * [IndexingServiceManager.filterNewlyUnreadableJars][org.appdevforall.codeonthego.indexing.service.IndexingServiceManager.filterNewlyUnreadableJars]) + * so a JAR already reported this session is not reported again - and [unreadableJarReporter] is + * called once with the names of whatever [unreadableJarFilter] returns, or not at all if that is + * empty. The JAR is still retried every pass either way, since retry depends only on the missing + * fingerprint. A JAR deleted between [jarsToIndex] and its scan also gets no fingerprint, but is + * neither recorded nor reported. + * + * The scope carries a [SupervisorJob] and a [CoroutineExceptionHandler]: a [refresh] whose + * [jarsToIndex] throws logs the failure instead of cancelling the scope, so a later [refresh] + * still runs its pass. + */ +abstract class JarIndexingService( + protected val context: Context, + private val progressTracker: IndexingProgressTracker, + private val workspaceSupplier: () -> Workspace? = { ProjectManagerImpl.getInstance().workspace }, + private val unreadableJarReporter: (List) -> Unit = ::reportUnreadableClasspathJars, + private val unreadableJarFilter: (Collection) -> List = { it.toList() }, +) : IndexingService { + companion object { + private val log = LoggerFactory.getLogger(JarIndexingService::class.java) + } + + /** Key this service's index is registered under. */ + protected abstract val indexKey: IndexKey + + /** Database file backing this service's index. */ + protected abstract val dbName: String + + /** Index name within [dbName]. */ + protected abstract val indexName: String + + override val providedKeys: List> get() = listOf(indexKey) + + /** Every JAR that should be indexed and visible for the current [workspace]. */ + protected abstract fun jarsToIndex(workspace: Workspace): Set + + private var jarIndex: JvmSymbolIndex? = null + private val indexingMutex = Mutex() + private val coroutineScope = + CoroutineScope( + SupervisorJob() + + Dispatchers.Default + + CoroutineExceptionHandler { _, throwable -> + log.error("[{}] Unhandled exception while indexing", id, throwable) + }, + ) + + override suspend fun initialize(registry: IndexRegistry) { + val jvmIndex = + JvmSymbolIndex.createSqliteIndex( + context = context, + dbName = dbName, + indexName = indexName, + ) + + this.jarIndex = jvmIndex + registry.register(indexKey, jvmIndex) + log.info("[{}] index initialized", id) + } + + /** + * Submits a reindex pass under the single-submitter mutex, then optimizes once it completes. + * + * Returns the pass's job, which completes once every JAR it submitted is indexed. + */ + fun refresh(): Job = + coroutineScope.launch { + progressTracker.openPass().use { pass -> + val unreadableJars = Collections.synchronizedSet(linkedSetOf()) + val jobs = indexingMutex.withLock { reindex(pass, unreadableJars) } + completePass(jobs, pass.newlyCounted) + if (unreadableJars.isNotEmpty()) { + val newlyUnreadable = unreadableJarFilter(unreadableJars) + if (newlyUnreadable.isNotEmpty()) { + unreadableJarReporter(newlyUnreadable.map { File(it).name }.distinct()) + } + } + } + } + + /** + * Waits for [jobs], every JAR one pass submitted, then optimizes the index; runs outside the mutex. + * + * [newlyCounted] is how many of [jobs] the pass added to the progress total rather than folded + * into another pass's still-running jobs. + */ + protected open suspend fun completePass( + jobs: List, + newlyCounted: Int, + ) { + jarIndex?.optimizeAfter(jobs) + } + + /** + * Submits every JAR whose fingerprint changed since it was last indexed, tracking each on [pass], + * and returns the submitted jobs. A JAR whose scan cannot open it has its path added to + * [unreadableJars]. + */ + private suspend fun reindex( + pass: IndexingProgressTracker.Pass, + unreadableJars: MutableSet, + ): List { + val index = + this.jarIndex ?: run { + log.warn("[{}] Not indexing. Index not initialized.", id) + return emptyList() + } + + val workspace = + workspaceSupplier() ?: run { + log.warn("[{}] Not indexing. Workspace model not available.", id) + return emptyList() + } + + val jars = jarsToIndex(workspace) + log.info("[{}] {} JARs found", id, jars.size) + + /* + * Step 1: Set the active set - this is instant. JARs not in the set become invisible to + * queries; JARs in the set that are already cached become visible immediately. + */ + index.setActiveSources(jars) + + /* + * Step 2: Index any JAR that is not cached, or was cached with a different fingerprint: + * a JAR rebuilt at the same path (a snapshot, a local file dependency) is re-scanned. + * Newly cached JARs are automatically visible because they're already in the active set. + */ + val jobs = mutableListOf() + for (jarPath in jars) { + val fingerprint = jarFingerprint(File(jarPath)) + if (index.sourceFingerprint(jarPath) != fingerprint) { + val job = + index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> + scanTrackingUnreadable(jarPath, sourceId, unreadableJars) + } + pass.track(jarPath, job) + jobs += job + } + } + + if (jobs.isNotEmpty()) { + log.info("[{}] {} new or changed JARs submitted for background indexing", id, jobs.size) + } else { + log.info("[{}] All JARs already cached, nothing to index", id) + } + return jobs + } + + override fun close() { + coroutineScope.cancelIfActive("$id indexing service closed") + jarIndex?.close() + jarIndex = null + } + + /** + * Scans [jarPath], adding it to [unreadableJars] and rethrowing if the scan throws + * [UnreadableJarException]. The caller's consumption of the returned sequence, not this call + * itself, is what actually runs the scan, and closing it closes the JAR. + */ + private fun scanTrackingUnreadable( + jarPath: String, + sourceId: String, + unreadableJars: MutableSet, + ): CloseableSequence { + val scan = CombinedJarScanner.scan(Paths.get(jarPath), sourceId) + val tracked = + sequence { + try { + yieldAll(scan) + } catch (e: UnreadableJarException) { + unreadableJars += jarPath + throw e + } + } + return CloseableSequence(tracked, scan::close) + } +} + +/** + * Identifies the content of [jar] by its size and last-modified time, which a rewrite of the file + * changes without the cost of reading it. Stats the file, so never call it on the main thread. + * + * Accepted limitation: a rewrite that keeps the exact same size and lands within the + * filesystem's modification-time granularity (a second on some filesystems) produces the same + * fingerprint as the original, so that rewrite goes undetected. + */ +internal fun jarFingerprint(jar: File): String = "${jar.length()}:${jar.lastModified()}" diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt index 937b4d8666..604de074f9 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt @@ -1,21 +1,11 @@ package org.appdevforall.codeonthego.indexing.jvm import android.content.Context -import com.itsaky.androidide.projects.ProjectManagerImpl import com.itsaky.androidide.projects.api.ModuleProject -import com.itsaky.androidide.tasks.cancelIfActive -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.launch -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock +import com.itsaky.androidide.projects.api.Workspace import org.appdevforall.codeonthego.indexing.service.IndexKey import org.appdevforall.codeonthego.indexing.service.IndexRegistry -import org.appdevforall.codeonthego.indexing.service.IndexingService -import org.slf4j.LoggerFactory -import java.io.File -import java.nio.file.Paths +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker import kotlin.io.path.extension /** @@ -28,7 +18,7 @@ import kotlin.io.path.extension val JVM_GENERATED_SYMBOL_INDEX = IndexKey("jvm-generated-symbols") /** - * [IndexingService] that scans build-generated JARs (R.jar, etc.) and + * [JarIndexingService] that scans build-generated JARs (R.jar, etc.) and * maintains a dedicated [JvmSymbolIndex] for them. * * A build completion re-scans every generated JAR's size and modification time (see @@ -38,100 +28,38 @@ val JVM_GENERATED_SYMBOL_INDEX = IndexKey("jvm-generated-symbols * would otherwise miss until some later build happened to touch it again. */ class JvmGeneratedIndexingService( - private val context: Context, -) : IndexingService { + context: Context, + progressTracker: IndexingProgressTracker, + unreadableJarFilter: (Collection) -> List = { it.toList() }, +) : JarIndexingService(context, progressTracker, unreadableJarFilter = unreadableJarFilter) { companion object { const val ID = "jvm-generated-indexing-service" private const val DB_NAME = "jvm_generated_symbol_index.db" private const val INDEX_NAME = "jvm-generated-cache" - private val log = LoggerFactory.getLogger(JvmGeneratedIndexingService::class.java) } override val id = ID - - override val providedKeys = listOf(JVM_GENERATED_SYMBOL_INDEX) - - private var generatedIndex: JvmSymbolIndex? = null - private val indexingMutex = Mutex() - private val coroutineScope = CoroutineScope(Dispatchers.Default) + override val indexKey = JVM_GENERATED_SYMBOL_INDEX + override val dbName = DB_NAME + override val indexName = INDEX_NAME override suspend fun initialize(registry: IndexRegistry) { - val index = - JvmSymbolIndex.createSqliteIndex( - context = context, - dbName = DB_NAME, - indexName = INDEX_NAME, - ) - - this.generatedIndex = index - registry.register(JVM_GENERATED_SYMBOL_INDEX, index) - log.info("JVM generated symbol index initialized") - + super.initialize(registry) // Kick off an initial index pass for any already-built JARs. - coroutineScope.launch { - val jobs = indexingMutex.withLock { reindexGeneratedJars() } - generatedIndex?.optimizeAfter(jobs) - } + refresh() } override suspend fun onBuildCompleted() { - coroutineScope.launch { - val jobs = indexingMutex.withLock { reindexGeneratedJars() } - generatedIndex?.optimizeAfter(jobs) - } + refresh() } - /** Submits every generated JAR whose fingerprint changed and returns the submitted jobs. */ - private suspend fun reindexGeneratedJars(): List { - val index = - this.generatedIndex ?: run { - log.warn("Not indexing generated JARs - index not initialized.") - return emptyList() - } - - val workspace = - ProjectManagerImpl.getInstance().workspace ?: run { - log.warn("Not indexing generated JARs - workspace model not available.") - return emptyList() - } - - val generatedJars = - workspace.subProjects - .asSequence() - .filterIsInstance() - .filter { it.path != workspace.rootProject.path } - .flatMap { project -> project.getIntermediateClasspaths() } - .filter { jar -> jar.exists() && jar.toPath().extension.lowercase() == "jar" } - .map { jar -> jar.absolutePath } - .toSet() - - log.info("{} generated JARs found", generatedJars.size) - - // Make exactly these JARs visible; remove stale ones from scope. - index.setActiveSources(generatedJars) - - val jobs = mutableListOf() - for (jarPath in generatedJars) { - val fingerprint = jarFingerprint(File(jarPath)) - if (index.sourceFingerprint(jarPath) != fingerprint) { - jobs += - index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> - CombinedJarScanner.scan(Paths.get(jarPath), sourceId) - } - } - } - - if (jobs.isNotEmpty()) { - log.info("{} generated JARs submitted for background indexing", jobs.size) - } else { - log.info("All generated JARs already cached, nothing to index") - } - return jobs - } - - override fun close() { - coroutineScope.cancelIfActive("generated indexing service closed") - generatedIndex?.close() - generatedIndex = null - } + override fun jarsToIndex(workspace: Workspace): Set = + workspace.subProjects + .asSequence() + .filterIsInstance() + .filter { it.path != workspace.rootProject.path } + .flatMap { project -> project.getIntermediateClasspaths() } + .filter { jar -> jar.exists() && jar.toPath().extension.lowercase() == "jar" } + .map { jar -> jar.absolutePath } + .toSet() } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt index 0a4b1d1be8..97159c8203 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt @@ -1,154 +1,97 @@ package org.appdevforall.codeonthego.indexing.jvm import android.content.Context +import com.itsaky.androidide.memprof.Memprof import com.itsaky.androidide.projects.ProjectManagerImpl import com.itsaky.androidide.projects.api.AndroidModule import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.api.Workspace import com.itsaky.androidide.projects.models.bootClassPaths -import com.itsaky.androidide.tasks.cancelIfActive -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job -import kotlinx.coroutines.launch -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock import org.appdevforall.codeonthego.indexing.service.IndexKey import org.appdevforall.codeonthego.indexing.service.IndexRegistry -import org.appdevforall.codeonthego.indexing.service.IndexingService -import org.greenrobot.eventbus.Subscribe -import org.greenrobot.eventbus.ThreadMode -import org.slf4j.LoggerFactory -import java.io.File +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker import java.nio.file.Path -import java.nio.file.Paths import kotlin.io.path.extension /** * Well-known key for the JVM library symbol index. * * Both the Kotlin and Java LSPs use this key to retrieve the - * shared index from the [IndexRegistry]. + * shared index from the [IndexRegistry][org.appdevforall.codeonthego.indexing.service.IndexRegistry]. */ val JVM_LIBRARY_SYMBOL_INDEX = IndexKey("jvm-library-symbols") /** - * [IndexingService] that scans classpath JARs/AARs and builds - * a [JvmSymbolIndex]. + * [JarIndexingService] that scans classpath JARs/AARs and builds a [JvmSymbolIndex]. + * + * A pass runs on initialization and whenever a language server calls [refresh] at project setup. The + * initialization pass is what indexes the libraries when those calls came before initialization, + * since a [refresh] before then finds no index and does nothing. Every pass sets the active source + * set and stats each JAR; one already indexed costs only that stat and fingerprint read and opens no + * "Index libraries" phase, but a JAR the earlier pass failed to index still has no fingerprint and is + * rescanned, opening a phase of its own. + * + * A pass that submits at least one JAR not already being indexed is profiled as the [Memprof] phase + * "Index libraries", which ends, emitting `library_index_complete`, once the pass's JARs are indexed + * and the index optimized. * * Thread safety: all methods are called from the * [IndexingServiceManager][org.appdevforall.codeonthego.indexing.service.IndexingServiceManager]'s * coroutine scope. The [JvmSymbolIndex] handles its own internal thread safety. */ class JvmLibraryIndexingService( - private val context: Context, -) : IndexingService { + context: Context, + progressTracker: IndexingProgressTracker, + workspaceSupplier: () -> Workspace? = { ProjectManagerImpl.getInstance().workspace }, + unreadableJarFilter: (Collection) -> List = { it.toList() }, +) : JarIndexingService(context, progressTracker, workspaceSupplier, unreadableJarFilter = unreadableJarFilter) { companion object { const val ID = "jvm-indexing-service" - private val log = LoggerFactory.getLogger(JvmLibraryIndexingService::class.java) } override val id = ID - - override val providedKeys = listOf(JVM_LIBRARY_SYMBOL_INDEX) - - private var libraryIndex: JvmSymbolIndex? = null - private var indexingMutex = Mutex() - private val coroutineScope = CoroutineScope(Dispatchers.Default) + override val indexKey = JVM_LIBRARY_SYMBOL_INDEX + override val dbName = JvmSymbolIndex.DB_NAME_DEFAULT + override val indexName = JvmSymbolIndex.INDEX_NAME_LIBRARY override suspend fun initialize(registry: IndexRegistry) { - val jvmIndex = - JvmSymbolIndex.createSqliteIndex( - context = context, - dbName = JvmSymbolIndex.DB_NAME_DEFAULT, - indexName = JvmSymbolIndex.INDEX_NAME_LIBRARY, - ) - - this.libraryIndex = jvmIndex - registry.register(JVM_LIBRARY_SYMBOL_INDEX, jvmIndex) - log.info("JVM symbol index initialized") - } - - @Subscribe(threadMode = ThreadMode.ASYNC) - @Suppress("UNUSED") - fun onProjectSynced() { + super.initialize(registry) refresh() } - fun refresh() { - coroutineScope.launch { - val jobs = indexingMutex.withLock { reindexLibraries() } - libraryIndex?.optimizeAfter(jobs) - } - } - - /** Submits every library JAR that needs indexing and returns the submitted jobs. */ - private suspend fun reindexLibraries(): List { - val index = - this.libraryIndex ?: run { - log.warn("Not indexing libraries. Index not initialized.") - return emptyList() - } - - val workspace = - ProjectManagerImpl.getInstance().workspace ?: run { - log.warn("Not indexing libraries. Workspace model not available.") - return emptyList() - } - - val currentJars = - workspace.subProjects - .asSequence() - .filterIsInstance() - .filter { it.path != workspace.rootProject.path } - .flatMap { project -> - buildList { - if (project is AndroidModule) { - addAll(project.bootClassPaths) - } - - addAll(project.getCompileClasspaths(excludeSourceGeneratedClassPath = true)) - } - }.filter { jar -> jar.exists() && isIndexableJar(jar.toPath()) } - .map { jar -> jar.absolutePath } - .toSet() - - log.info("{} JARs on classpath", currentJars.size) - + override suspend fun completePass( + jobs: List, + newlyCounted: Int, + ) { /* - * Step 1: Set the active set - this is instant. JARs not in the set become invisible to - * queries; JARs in the set that are already cached become visible immediately. + * Both LSPs refresh this index at project open. A pass that only folded into the other + * pass's running jobs must not open a second phase, or the report gets two rows and two + * markers for one indexing run. */ - index.setActiveSources(currentJars) - - /* - * Step 2: Index any JAR that is not cached, or was cached with a different fingerprint: - * a JAR rebuilt at the same path (a snapshot, a local file dependency) is re-scanned. - * Newly cached JARs are automatically visible because they're already in the active set. - */ - val jobs = mutableListOf() - for (jarPath in currentJars) { - val fingerprint = jarFingerprint(File(jarPath)) - if (index.sourceFingerprint(jarPath) != fingerprint) { - jobs += - index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> - CombinedJarScanner.scan(Paths.get(jarPath), sourceId) - } - } + if (newlyCounted == 0) { + return super.completePass(jobs, newlyCounted) } - if (jobs.isNotEmpty()) { - log.info("{} new or changed JARs submitted for background indexing", jobs.size) - } else { - log.info("All JARs already cached, nothing to index") - } - return jobs + indexLibrariesPhase(jobs.size) { super.completePass(jobs, newlyCounted) } } - override fun close() { - coroutineScope.cancelIfActive("indexing service closed") - libraryIndex?.close() - libraryIndex = null - } + override fun jarsToIndex(workspace: Workspace): Set = + workspace.subProjects + .asSequence() + .filterIsInstance() + .filter { it.path != workspace.rootProject.path } + .flatMap { project -> + buildList { + if (project is AndroidModule) { + addAll(project.bootClassPaths) + } + + addAll(project.getCompileClasspaths(excludeSourceGeneratedClassPath = true)) + } + }.filter { jar -> jar.exists() && isIndexableJar(jar.toPath()) } + .map { jar -> jar.absolutePath } + .toSet() private fun isIndexableJar(path: Path): Boolean { val ext = path.extension.lowercase() @@ -157,11 +100,18 @@ class JvmLibraryIndexingService( } /** - * Identifies the content of [jar] by its size and last-modified time, which a rewrite of the file - * changes without the cost of reading it. Stats the file, so never call it on the main thread. + * Runs [action], a library pass over [jarCount] submitted JARs, as the "Index libraries" phase. * - * Accepted limitation: a rewrite that keeps the exact same size and lands within the - * filesystem's modification-time granularity (a second on some filesystems) produces the same - * fingerprint as the original, so that rewrite goes undetected. + * The phase begins once the pass has submitted its JARs, so it leaves out the submit loop's own time + * (a stat and a fingerprint read per JAR) although the submitted scans are already running by then. */ -internal fun jarFingerprint(jar: File): String = "${jar.length()}:${jar.lastModified()}" +internal inline fun indexLibrariesPhase( + jarCount: Int, + action: () -> R, +): R = + Memprof.phase("Index libraries", "library_index_complete") { span -> + if (span.isRecording) { + span.put("jars", jarCount.toLong()) + } + action() + } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmModuleOutputIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmModuleOutputIndexingService.kt new file mode 100644 index 0000000000..c499bd2722 --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmModuleOutputIndexingService.kt @@ -0,0 +1,80 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import com.itsaky.androidide.projects.api.AndroidModule +import com.itsaky.androidide.projects.api.JavaModule +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.api.Workspace +import org.appdevforall.codeonthego.indexing.service.IndexKey +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import kotlin.io.path.extension + +/** + * Well-known key for the index of the project modules' own compiled output JARs. + * + * A scope of its own rather than part of [JVM_LIBRARY_SYMBOL_INDEX] or + * [JVM_GENERATED_SYMBOL_INDEX]: the Kotlin LSP resolves project classes from its source index and + * also reads those two, so folding these JARs into either would show it every project class twice. + * The Java and XML layout-completion LSPs, which resolve project classes from the compile + * classpath, read this one through [ModuleClasspathLookup]. + */ +val JVM_MODULE_OUTPUT_SYMBOL_INDEX = IndexKey("jvm-module-output-symbols") + +/** + * [JarIndexingService] over each project module's own compiled output JAR. + * + * These are the entries `getCompileClasspaths()` holds and + * `getCompileClasspaths(excludeSourceGeneratedClassPath = true)`, the library index's set, leaves + * out: an Android module's [generated JAR][AndroidModule.getGeneratedJar] and a Java module's + * [classes JAR][JavaModule.getClassesJar]. Every module is covered, so the JARs of the project + * modules a module compiles against are too. `R.jar` is not, since the library and generated + * indexes already hold it. + * + * A pass runs on initialization, after every build and on every project sync. Each JAR is + * re-indexed only when its [fingerprint][jarFingerprint] changed, so a build that failed without + * rewriting a JAR costs one file stat per module. + */ +class JvmModuleOutputIndexingService( + context: Context, + progressTracker: IndexingProgressTracker, + unreadableJarFilter: (Collection) -> List = { it.toList() }, +) : JarIndexingService(context, progressTracker, unreadableJarFilter = unreadableJarFilter) { + companion object { + const val ID = "jvm-module-output-indexing-service" + private const val DB_NAME = "jvm_module_output_symbol_index.db" + private const val INDEX_NAME = "jvm-module-output-cache" + } + + override val id = ID + override val indexKey = JVM_MODULE_OUTPUT_SYMBOL_INDEX + override val dbName = DB_NAME + override val indexName = INDEX_NAME + + override suspend fun initialize(registry: IndexRegistry) { + super.initialize(registry) + refresh() + } + + override suspend fun onBuildCompleted() { + refresh() + } + + override fun jarsToIndex(workspace: Workspace): Set = moduleOutputJars(workspace) +} + +/** Absolute paths of the built output JARs of every module in [workspace]. Stats each JAR. */ +internal fun moduleOutputJars(workspace: Workspace): Set = + workspace.subProjects + .asSequence() + .filterIsInstance() + .filter { it.path != workspace.rootProject.path } + .mapNotNull { module -> + when (module) { + is AndroidModule -> module.getGeneratedJar() + is JavaModule -> module.getClassesJar() + else -> null + } + }.filter { jar -> jar.exists() && jar.toPath().extension.lowercase() == "jar" } + .map { jar -> jar.absolutePath } + .toSet() diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt index 0988a03a9f..61b87c0c7b 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt @@ -16,6 +16,8 @@ import org.appdevforall.codeonthego.indexing.jvm.proto.JvmSymbolProtos.JvmSymbol * - `containingClass`: exact, for member lookup * - `language` : exact, for Java-only or Kotlin-only queries * + * Its [packageOf] makes a package exist for each package holding a top-level class file. + * * Blob serialization uses Protobuf with `oneof` for type-specific data. */ object JvmSymbolDescriptor : IndexDescriptor { @@ -48,6 +50,14 @@ object JvmSymbolDescriptor : IndexDescriptor { KEY_LANGUAGE to entry.language.name, ) + /** + * Returns the package of a top-level class file, `null` for anything else. + * + * A package exists only where a top-level class file is, as on the classpath: members, nested + * classes and type aliases name a package but never put a class file in it. + */ + override fun packageOf(entry: JvmSymbol): String? = entry.packageName.takeIf { entry.kind.isJvmClass && entry.isTopLevel } + override fun serialize(entry: JvmSymbol): ByteArray = toProto(entry).toByteArray() override fun deserialize(bytes: ByteArray): JvmSymbol = fromProto(JvmSymbolData.parseFrom(bytes)) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index 34d1759411..9537b17126 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -7,6 +7,7 @@ import kotlinx.coroutines.joinAll import org.appdevforall.codeonthego.indexing.FilteredIndex import org.appdevforall.codeonthego.indexing.SQLiteIndex import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.api.IndexQueryBuilder import org.appdevforall.codeonthego.indexing.api.WritableIndex import org.appdevforall.codeonthego.indexing.api.indexQuery import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_CONTAINING_CLASS @@ -21,6 +22,10 @@ import java.io.Closeable /** * An index of symbols from JVM source and binary files. + * + * Its [package tree][subpackages] holds the packages of the active sources' top-level class files. + * Its queries and package lookups block on disk I/O over a persistent backing index, so never call + * them on the main thread. */ open class JvmSymbolIndex( private val backing: Index, @@ -32,6 +37,7 @@ open class JvmSymbolIndex( /** Kind names as the descriptor stores them, for set-membership predicates. */ private val CLASSIFIER_KIND_NAMES = JvmSymbolKind.CLASSIFIER_KINDS.map { it.name } private val CALLABLE_KIND_NAMES = JvmSymbolKind.CALLABLE_KINDS.map { it.name } + private val JVM_CLASS_KIND_NAMES = JvmSymbolKind.JVM_CLASS_KINDS.map { it.name } const val DB_NAME_DEFAULT = "jvm_symbol_index.db" const val INDEX_NAME_LIBRARY = "jvm-library-cache" @@ -45,10 +51,16 @@ open class JvmSymbolIndex( * source file's symbols are re-indexed only when its metadata row says so; dropping the * symbols alone would leave files recorded as indexed with no symbols. * - * Version 3 keys extension callables by their receiver, stores the key in the payload, and - * stops treating a top-level class with '$' in its name as nested. + * Version 3 added the package table: a version 2 file holds symbols with no packages, and an + * unchanged source is never re-indexed, so it would answer every package lookup with nothing. + * + * Version 4 added the `_gen` column that lets a source be re-indexed in place. + * + * Version 5 keys extension callables by their receiver and stores the key in the payload: a + * version 4 source kept only the last of its overloads that differ by receiver alone. It also + * stops indexing a top-level class with '$' in its name, such as `Foo$$ViewBinder`, as nested. */ - const val FORMAT_VERSION = 3 + const val FORMAT_VERSION = 5 /** * Create (or get) a JVM symbol index backed by SQLite. @@ -214,6 +226,77 @@ open class JvmSymbolIndex( }, ) + /** + * Top-level classes named exactly [simpleName] in any of [sourceIds], up to [limit]. + * + * A top-level class is one with a class file of its own (a [file facade][JvmSymbolKind.FILE_FACADE] + * included, a type alias not) and no containing class, of any visibility. `sourceIds` follows + * [IndexQuery.sourceIds][org.appdevforall.codeonthego.indexing.api.IndexQuery.sourceIds] and is + * further narrowed to the active sources. The default [limit] of 0 is unbounded, which is safe + * here because the result is at most the number of classes sharing one simple name. + */ + fun findTopLevelClassesNamed( + simpleName: String, + sourceIds: Collection?, + limit: Int = 0, + ): Sequence = findTopLevelClasses(sourceIds, limit) { eq(KEY_NAME, simpleName) } + + /** + * Top-level classes whose simple name starts with [prefix], ignoring case, in any of [sourceIds]. + * + * Top-level and [sourceIds] are as in [findTopLevelClassesNamed]. There is no default [limit]: a + * short prefix matches a large share of the classpath, so the caller must choose one. + */ + fun findTopLevelClassesByPrefix( + prefix: String, + sourceIds: Collection?, + limit: Int, + ): Sequence = findTopLevelClasses(sourceIds, limit) { prefix(KEY_NAME, prefix) } + + /** + * Top-level classes declared directly in [packageName] in any of [sourceIds], up to [limit]. + * + * Top-level and [sourceIds] are as in [findTopLevelClassesNamed]. The default [limit] of 0 is + * unbounded, which is safe here because the result is at most the number of classes in one + * package. + */ + fun findTopLevelClassesInPackage( + packageName: String, + sourceIds: Collection?, + limit: Int = 0, + ): Sequence = findTopLevelClasses(sourceIds, limit) { eq(KEY_PACKAGE, packageName) } + + /** + * Returns whether any of [sourceIds] has a top-level class named [simpleName] declared directly + * in [packageName] (`""` for the default package). + * + * Top-level and [sourceIds] are as in [findTopLevelClassesNamed]. + */ + fun containsTopLevelClass( + packageName: String, + simpleName: String, + sourceIds: Collection?, + ): Boolean = + findTopLevelClasses(sourceIds, limit = 1) { + eq(KEY_PACKAGE, packageName) + eq(KEY_NAME, simpleName) + }.any() + + private inline fun findTopLevelClasses( + sourceIds: Collection?, + limit: Int, + crossinline match: IndexQueryBuilder.() -> Unit, + ): Sequence = + query( + indexQuery { + match() + anyOf(KEY_KIND, JVM_CLASS_KIND_NAMES) + notExists(KEY_CONTAINING_CLASS) + this.sourceIds = sourceIds + this.limit = limit + }, + ) + suspend fun findByKey(key: String): JvmSymbol? = get(key) fun allPackages(): Sequence = distinctValues(KEY_PACKAGE) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookup.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookup.kt new file mode 100644 index 0000000000..6466025c82 --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookup.kt @@ -0,0 +1,198 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.itsaky.androidide.projects.api.ModuleProject +import org.appdevforall.codeonthego.indexing.service.IndexRegistry + +/** + * Class and package lookups over one module's compile classpath, answered from the JVM symbol indexes. + * + * It offers what the module's classpath trie held: the top-level classes, of any visibility, of every + * entry of the module's compile classpath, and the packages they are in. Boot classpath classes are + * not included. Each index is queried with the module's share of it: the + * [library][JVM_LIBRARY_SYMBOL_INDEX] and [generated][JVM_GENERATED_SYMBOL_INDEX] indexes with the + * whole compile classpath, and the [module-output][JVM_MODULE_OUTPUT_SYMBOL_INDEX] index, which + * holds the output JARs of every module, with the output JARs of this module and of the project + * modules it compiles against only, so a module is never offered classes of a module it does not + * depend on. A `null` index contributes nothing. + * + * Results are deduplicated by qualified name and never cached: an index being built answers with what + * it holds so far. + * + * Construction does no I/O. Every lookup blocks on disk I/O, and the first one also resolves the + * module's classpath, which stats files, so never call them on the main thread. + * + * Build one per request, with [of]. A lookup captures the index instances and the module's classpath + * as they were: a re-registered index closes the one it replaces, a re-sync replaces the module + * model, and a build can create a Java module's classes JAR after the classpath was resolved. A + * lookup held across any of these answers from a stale classpath or a closed index, which yields + * nothing. + */ +class ModuleClasspathLookup internal constructor( + libraryIndex: JvmSymbolIndex?, + generatedIndex: JvmSymbolIndex?, + moduleOutputIndex: JvmSymbolIndex?, + scope: () -> ClasspathScope, +) { + /** Creates the lookup for [module] over the given indexes, any of which may be unregistered. */ + constructor( + module: ModuleProject, + libraryIndex: JvmSymbolIndex?, + generatedIndex: JvmSymbolIndex?, + moduleOutputIndex: JvmSymbolIndex?, + ) : this(libraryIndex, generatedIndex, moduleOutputIndex, { ClasspathScope.of(module) }) + + /** A direct child of a package: a subpackage, or a top-level class when [isClass]. */ + data class Child( + val name: String, + val qualifiedName: String, + val isClass: Boolean, + ) + + private class ScopedIndex( + val index: JvmSymbolIndex, + val sourceIds: Collection, + ) + + private val scopedIndexes by lazy { + val scope = scope() + listOfNotNull( + libraryIndex?.let { ScopedIndex(it, scope.classpath) }, + generatedIndex?.let { ScopedIndex(it, scope.classpath) }, + moduleOutputIndex?.let { ScopedIndex(it, scope.moduleOutputs) }, + ) + } + + /** Returns the qualified names of the top-level classes whose simple name is exactly [simpleName]. */ + fun qualifiedNamesOf(simpleName: String): List = classesNamed(simpleName).map { it.fqName } + + /** + * Returns the top-level classes whose simple name is exactly [simpleName], one per qualified name. + * + * Of a class held by more than one source, the one returned is the first found. + */ + fun classesNamed(simpleName: String): List = + scopedIndexes + .asSequence() + .flatMap { it.index.findTopLevelClassesNamed(simpleName, it.sourceIds) } + .distinctBy { it.fqName } + .toList() + + /** + * Returns the qualified names of top-level classes whose simple name starts with [prefix], + * ignoring case: the exact simple-name matches first, then up to [limit] more from each index. + * + * Exact matches come first so that a short prefix, which matches far more classes than [limit], + * cannot crowd out the class named exactly [prefix]. Each index has its own limit so that the + * library index, which holds far more classes, cannot crowd out the module's own output. + */ + fun qualifiedNamesByPrefix( + prefix: String, + limit: Int, + ): List { + require(limit > 0) { "limit must be positive, was $limit" } + + val names = LinkedHashSet() + qualifiedNamesOf(prefix).take(limit).forEach(names::add) + for (scoped in scopedIndexes) { + val target = (names.size.toLong() + limit).coerceAtMost(Int.MAX_VALUE.toLong()).toInt() + addPrefixMatches(scoped, prefix, names, target) + } + return names.toList() + } + + /** + * Adds [scoped]'s prefix matches to [names] until it holds [target] names or the index has no more. + * + * One fetch of the missing count is not enough: a fetched row can repeat a name already taken, and + * one index can hold the same class from several JARs in scope (the Kotlin stdlib split into + * `-jdk7`/`-jdk8`, a class duplicated across AARs). So the fetch doubles until it fills or a + * fetch comes back short, which means the index is exhausted. + */ + private fun addPrefixMatches( + scoped: ScopedIndex, + prefix: String, + names: MutableSet, + target: Int, + ) { + var fetch = target - names.size + while (names.size < target) { + val rows = scoped.index.findTopLevelClassesByPrefix(prefix, scoped.sourceIds, fetch).toList() + rows + .asSequence() + .map { it.fqName } + .filterNot { it in names } + .take(target - names.size) + .forEach(names::add) + if (rows.size < fetch || fetch == Int.MAX_VALUE) return + fetch = (fetch.toLong() * 2).coerceAtMost(Int.MAX_VALUE.toLong()).toInt() + } + } + + /** + * Returns the direct subpackages of [packageName], then its top-level classes. + * + * [packageName] `""` is the default package: its children are the root packages and the classes + * declared in no package. + */ + fun children(packageName: String): List { + val children = LinkedHashSet() + for (scoped in scopedIndexes) { + scoped.index.subpackages(packageName, scoped.sourceIds).mapTo(children) { subpackage -> + Child(subpackage.substringAfterLast('.'), subpackage, isClass = false) + } + } + for (scoped in scopedIndexes) { + scoped.index.findTopLevelClassesInPackage(packageName, scoped.sourceIds).mapTo(children) { symbol -> + Child(symbol.shortName, symbol.fqName, isClass = true) + } + } + return children.toList() + } + + /** Returns whether a top-level class of the classpath is in the package [name] or one of its subpackages. */ + fun isPackage(name: String): Boolean = scopedIndexes.any { it.index.containsPackage(name, it.sourceIds) } + + /** Returns whether [qualifiedName] names a top-level class of the classpath. */ + fun isClass(qualifiedName: String): Boolean { + val packageName = qualifiedName.substringBeforeLast('.', missingDelimiterValue = "") + val simpleName = qualifiedName.substringAfterLast('.') + return scopedIndexes.any { it.index.containsTopLevelClass(packageName, simpleName, it.sourceIds) } + } + + companion object { + /** Creates the lookup for [module] over the indexes currently registered in [registry]. */ + fun of( + module: ModuleProject, + registry: IndexRegistry, + ) = ModuleClasspathLookup( + module, + registry.get(JVM_LIBRARY_SYMBOL_INDEX), + registry.get(JVM_GENERATED_SYMBOL_INDEX), + registry.get(JVM_MODULE_OUTPUT_SYMBOL_INDEX), + ) + } +} + +/** + * The source ids a module's lookups are scoped to, as absolute JAR paths. + * + * @property classpath The module's whole compile classpath. + * @property moduleOutputs The part of [classpath] that is the output of project modules: this module's + * own output JAR and those of the project modules it compiles against. + */ +internal class ClasspathScope( + val classpath: List, + val moduleOutputs: List, +) { + companion object { + /** Resolves [module]'s scope from its project model. Stats files, so never on the main thread. */ + fun of(module: ModuleProject): ClasspathScope { + val classpath = module.getCompileClasspaths(excludeSourceGeneratedClassPath = false) + val external = module.getCompileClasspaths(excludeSourceGeneratedClassPath = true) + return ClasspathScope( + classpath = classpath.map { it.absolutePath }, + moduleOutputs = (classpath - external).map { it.absolutePath }, + ) + } + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt index a92676508f..7c003c8b0a 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt @@ -1,7 +1,6 @@ package org.appdevforall.codeonthego.indexing.jvm import com.google.common.truth.Truth.assertThat -import com.google.common.truth.Truth.assertWithMessage import com.itsaky.androidide.projects.classpath.JarFsClasspathReader import org.junit.Assume.assumeFalse import org.junit.Test @@ -36,17 +35,7 @@ class ClasspathTrieDifferentialTest( ) } - private val jar: File by lazy { - val location = - File( - anchor.protectionDomain.codeSource.location - .toURI(), - ) - assertWithMessage("$corpus must resolve to a JAR on the test classpath, got $location") - .that(location.isFile && location.name.endsWith(".jar")) - .isTrue() - location - } + private val jar: File by lazy { testClasspathJar(anchor) } private val isKotlinCorpus get() = anchor == Unit::class.java @@ -68,14 +57,6 @@ class ClasspathTrieDifferentialTest( /** What the index offers for the same JAR. */ private fun indexNames(): Set = indexClasses().map { it.fqName }.toSet() - /** - * A fragment of a `@JvmMultifileClass` facade, e.g. `LazyKt__LazyJVMKt`. - * - * The only thing the index deliberately withholds: the declarations belong to the facade, which - * is indexed, and neither Java nor Kotlin names the part. - */ - private fun isMultiFilePart(qualifiedName: String) = qualifiedName.substringAfterLast('.').contains("__") - @Test fun `every class the trie held is offered by the index, bar multi-file parts`() { val missing = trieNames() - indexNames() diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerCloseTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerCloseTest.kt new file mode 100644 index 0000000000..24addf4fd1 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerCloseTest.kt @@ -0,0 +1,47 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.util.jar.JarEntry +import java.util.jar.JarOutputStream + +/** Closing a JAR scan part-way closes the JAR, which an abandoned `sequence {}` never does itself. */ +@RunWith(JUnit4::class) +class CombinedJarScannerCloseTest { + @get:Rule + val temp = TemporaryFolder() + + private fun emptyClass(internalName: String): ByteArray = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, internalName, null, "java/lang/Object", null) + visitEnd() + }.toByteArray() + + @Test + fun `closing a scan part-way closes its jar`() { + val jar = temp.newFile("two.jar") + JarOutputStream(jar.outputStream()).use { out -> + for (name in listOf("com/example/A", "com/example/B")) { + out.putNextEntry(JarEntry("$name.class")) + out.write(emptyClass(name)) + out.closeEntry() + } + } + val scan = CombinedJarScanner.scan(jar.toPath(), "two") + val symbols = scan.iterator() + symbols.next() + + scan.close() + + val afterClose = runCatching { while (symbols.hasNext()) symbols.next() }.exceptionOrNull() + assertThat(afterClose).isInstanceOf(IllegalStateException::class.java) + assertThat(afterClose).hasMessageThat().contains("closed") + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerUnreadableJarTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerUnreadableJarTest.kt new file mode 100644 index 0000000000..2f5deed4e5 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerUnreadableJarTest.kt @@ -0,0 +1,41 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.junit.Assert.assertThrows +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * An open failure must surface as [UnreadableJarException] rather than being swallowed: silently + * yielding nothing records the JAR as indexed with no symbols, and it is never retried. + */ +@RunWith(JUnit4::class) +class CombinedJarScannerUnreadableJarTest { + @get:Rule + val temp = TemporaryFolder() + + @Test + fun `scanning a jar that cannot be opened throws UnreadableJarException`() { + val garbage = temp.newFile("garbage.jar").apply { writeBytes(byteArrayOf(1, 2, 3, 4)) } + + val exception = + assertThrows(UnreadableJarException::class.java) { + CombinedJarScanner.scan(garbage.toPath()).toList() + } + + assertThat(exception.path).isEqualTo(garbage.toPath()) + } + + @Test + fun `scanning a jar that no longer exists does not throw UnreadableJarException`() { + val missing = temp.root.toPath().resolve("deleted.jar") + + val failure = runCatching { CombinedJarScanner.scan(missing).toList() }.exceptionOrNull() + + assertThat(failure).isNotNull() + assertThat(failure).isNotInstanceOf(UnreadableJarException::class.java) + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceProgressTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceProgressTest.kt new file mode 100644 index 0000000000..cb44d6513d --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceProgressTest.kt @@ -0,0 +1,141 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.memprof.Memprof +import com.itsaky.androidide.projects.api.Workspace +import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Job +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.service.IndexKey +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.appdevforall.codeonthego.indexing.service.IndexingState +import org.junit.After +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File + +/** A [JarIndexingService.refresh] pass reports its submitted JARs to the tracker, and always closes. */ +@RunWith(RobolectricTestRunner::class) +class JarIndexingServiceProgressTest { + @get:Rule + val temp = TemporaryFolder() + + private val context = ApplicationProvider.getApplicationContext() + private val tracker = IndexingProgressTracker() + + @After + fun tearDown() { + Memprof.sink = null + context.deleteDatabase(DB_NAME) + } + + @Test + fun `a refresh counts each JAR it submits while its pass is open`() { + val service = TestService(jars = setOf(jar("a.jar"), jar("b.jar"))) + + service.refreshAndAwait() + + assertThat((service.stateWhileCompleting as? IndexingState.Indexing)?.total).isEqualTo(2) + } + + @Test + fun `the state is Idle once a refresh finishes`() { + val service = TestService(jars = setOf(jar("a.jar"))) + + service.refreshAndAwait() + + assertThat(tracker.state.value).isEqualTo(IndexingState.Idle) + } + + @Test + fun `a refresh that throws after submitting still returns the state to Idle`() { + val service = TestService(jars = setOf(jar("a.jar")), onComplete = { error("optimize failed") }) + + service.refreshAndAwait() + + assertThat(tracker.state.value).isEqualTo(IndexingState.Idle) + } + + @Test + fun `a refresh cancelled after submitting still returns the state to Idle`() { + val completing = CompletableDeferred() + val service = + TestService(jars = setOf(jar("a.jar")), onComplete = { + completing.complete(Unit) + awaitCancellation() + }) + + service.use { + runBlocking { + it.initialize(IndexRegistry()) + val pass = it.refresh() + completing.await() + pass.cancel() + pass.join() + } + } + + assertThat(tracker.state.value).isEqualTo(IndexingState.Idle) + } + + @Test + fun `a pass of a service other than the library one records no Memprof phase`() { + val sink = RecordingMemprofSink() + Memprof.sink = sink + val service = TestService(jars = setOf(jar("a.jar"))) + + service.refreshAndAwait() + + assertThat(sink.phases).isEmpty() + } + + private fun TestService.refreshAndAwait() { + use { + runBlocking { + it.initialize(IndexRegistry()) + it.refresh().join() + } + } + } + + private fun jar(name: String): String = + File(temp.root, name) + .apply { writeBytes(ByteArray(1)) } + .absolutePath + + private inner class TestService( + private val jars: Set, + private val onComplete: suspend () -> Unit = {}, + ) : JarIndexingService(context, tracker, workspaceSupplier = { mockk() }) { + var stateWhileCompleting: IndexingState? = null + + override val id = "jar-indexing-service-progress-test" + override val indexKey = IndexKey("jar-indexing-service-progress-test") + override val dbName = DB_NAME + override val indexName = INDEX_NAME + + override fun jarsToIndex(workspace: Workspace): Set = jars + + override suspend fun completePass( + jobs: List, + newlyCounted: Int, + ) { + stateWhileCompleting = tracker.state.value + onComplete() + super.completePass(jobs, newlyCounted) + } + } + + private companion object { + const val DB_NAME = "jar_indexing_service_progress_test.db" + const val INDEX_NAME = "jar-indexing-service-progress-test-index" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceSupervisorTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceSupervisorTest.kt new file mode 100644 index 0000000000..d32def5a88 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceSupervisorTest.kt @@ -0,0 +1,79 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.projects.api.Workspace +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.service.IndexKey +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +/** + * A [refresh] whose [JarIndexingService.jarsToIndex] throws must not take down the service's + * coroutine scope: a later [refresh] still has to run its pass. + */ +@RunWith(RobolectricTestRunner::class) +class JarIndexingServiceSupervisorTest { + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + @Test + fun `a refresh whose jarsToIndex throws does not stop a later refresh from running`() { + val callCount = AtomicInteger(0) + val firstCallStarted = CountDownLatch(1) + val secondCallRan = CountDownLatch(1) + + val service = + object : JarIndexingService( + context = context, + progressTracker = IndexingProgressTracker(), + workspaceSupplier = { mockk() }, + ) { + override val id = "jar-indexing-service-supervisor-test" + override val indexKey = IndexKey("jar-indexing-service-supervisor-test") + override val dbName = DB_NAME + override val indexName = INDEX_NAME + + override fun jarsToIndex(workspace: Workspace): Set { + if (callCount.getAndIncrement() == 0) { + firstCallStarted.countDown() + throw IllegalStateException("jarsToIndex failed") + } + secondCallRan.countDown() + return emptySet() + } + } + + try { + runBlocking { service.initialize(IndexRegistry()) } + + service.refresh() + assertThat(firstCallStarted.await(5, TimeUnit.SECONDS)).isTrue() + + service.refresh() + assertThat(secondCallRan.await(5, TimeUnit.SECONDS)).isTrue() + + assertThat(callCount.get()).isAtLeast(2) + } finally { + service.close() + } + } + + private companion object { + const val DB_NAME = "jar_indexing_service_supervisor_test.db" + const val INDEX_NAME = "jar-indexing-service-supervisor-test-index" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceUnreadableJarDedupTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceUnreadableJarDedupTest.kt new file mode 100644 index 0000000000..88009ce64a --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceUnreadableJarDedupTest.kt @@ -0,0 +1,106 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.projects.api.Workspace +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.service.IndexKey +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.appdevforall.codeonthego.indexing.service.IndexingServiceManager +import org.junit.After +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File + +/** + * A JAR that stays unreadable is retried every pass but reported to the user at most once per + * project session: [JarIndexingService.refresh] runs every recorded JAR through the shared + * `unreadableJarFilter`, typically [IndexingServiceManager.filterNewlyUnreadableJars]. + */ +@RunWith(RobolectricTestRunner::class) +class JarIndexingServiceUnreadableJarDedupTest { + @get:Rule + val temp = TemporaryFolder() + + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + @Test + fun `two passes finding the same unreadable jar report it only once`() { + val garbage = garbageJar("garbage.jar") + val manager = IndexingServiceManager() + val reported = mutableListOf>() + val service = testService(jars = setOf(garbage), reporter = { reported += it }, filter = manager::filterNewlyUnreadableJars) + + try { + runBlocking { + service.initialize(IndexRegistry()) + service.refresh().join() + service.refresh().join() + } + + assertThat(reported).containsExactly(listOf("garbage.jar")) + } finally { + service.close() + manager.close() + } + } + + @Test + fun `a new session reports the same unreadable jar again`() { + val garbage = garbageJar("garbage.jar") + val manager = IndexingServiceManager() + val reported = mutableListOf>() + val service = testService(jars = setOf(garbage), reporter = { reported += it }, filter = manager::filterNewlyUnreadableJars) + + try { + runBlocking { + service.initialize(IndexRegistry()) + service.refresh().join() + } + manager.close() + runBlocking { service.refresh().join() } + + assertThat(reported).containsExactly(listOf("garbage.jar"), listOf("garbage.jar")) + } finally { + service.close() + } + } + + private fun garbageJar(name: String): String = File(temp.root, name).apply { writeBytes(byteArrayOf(1, 2, 3, 4)) }.absolutePath + + private fun testService( + jars: Set, + reporter: (List) -> Unit, + filter: (Collection) -> List, + ): JarIndexingService = + object : JarIndexingService( + context = context, + progressTracker = IndexingProgressTracker(), + workspaceSupplier = { mockk() }, + unreadableJarReporter = reporter, + unreadableJarFilter = filter, + ) { + override val id = "jar-indexing-service-unreadable-jar-dedup-test" + override val indexKey = IndexKey("jar-indexing-service-unreadable-jar-dedup-test") + override val dbName = DB_NAME + override val indexName = INDEX_NAME + + override fun jarsToIndex(workspace: Workspace): Set = jars + } + + private companion object { + const val DB_NAME = "jar_indexing_service_unreadable_jar_dedup_test.db" + const val INDEX_NAME = "jar-indexing-service-unreadable-jar-dedup-test-index" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceUnreadableJarTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceUnreadableJarTest.kt new file mode 100644 index 0000000000..59c45370bd --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarIndexingServiceUnreadableJarTest.kt @@ -0,0 +1,146 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.projects.api.Workspace +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.service.IndexKey +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.After +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** + * A pass over a JAR whose scan cannot open it must report that JAR once through the injected + * reporter, and must leave no fingerprint recorded for it so a later pass retries it. + */ +@RunWith(RobolectricTestRunner::class) +class JarIndexingServiceUnreadableJarTest { + @get:Rule + val temp = TemporaryFolder() + + private val context = ApplicationProvider.getApplicationContext() + private val indexKey = IndexKey("jar-indexing-service-unreadable-jar-test") + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + @Test + fun `a pass reports exactly the unreadable jar once and does not fingerprint it`() { + val good = validJar("good.jar") + val garbage = garbageJar("garbage.jar") + val reported = mutableListOf>() + val service = TestService(jars = setOf(good, garbage), reporter = { reported += it }) + + try { + val registry = IndexRegistry() + runBlocking { + service.initialize(registry) + service.refresh().join() + } + val index = registry.require(indexKey) + + assertThat(reported).containsExactly(listOf("garbage.jar")) + runBlocking { + assertThat(index.sourceFingerprint(good)).isNotNull() + assertThat(index.sourceFingerprint(garbage)).isNull() + } + } finally { + service.close() + } + } + + @Test + fun `a pass over only readable jars never calls the reporter`() { + val good = validJar("good.jar") + val reported = mutableListOf>() + val service = TestService(jars = setOf(good), reporter = { reported += it }) + + try { + runBlocking { + service.initialize(IndexRegistry()) + service.refresh().join() + } + + assertThat(reported).isEmpty() + } finally { + service.close() + } + } + + @Test + fun `a jar deleted before its scan is not reported and not fingerprinted`() { + val deleted = File(temp.root, "deleted.jar").absolutePath + val reported = mutableListOf>() + val service = TestService(jars = setOf(deleted), reporter = { reported += it }) + + try { + val registry = IndexRegistry() + runBlocking { + service.initialize(registry) + service.refresh().join() + } + + assertThat(reported).isEmpty() + runBlocking { assertThat(registry.require(indexKey).sourceFingerprint(deleted)).isNull() } + } finally { + service.close() + } + } + + /** A JAR holding one public class, which scans and fingerprints cleanly. */ + private fun validJar(name: String): String { + val internalName = "p/${name.substringBefore('.').replaceFirstChar(Char::uppercase)}" + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, internalName, null, "java/lang/Object", null) + visitEnd() + }.toByteArray() + val jar = File(temp.root, name) + ZipOutputStream(jar.outputStream()).use { out -> + out.putNextEntry(ZipEntry("$internalName.class")) + out.write(bytes) + out.closeEntry() + } + return jar.absolutePath + } + + /** Not a valid zip, so opening it as a [java.util.jar.JarFile] fails. */ + private fun garbageJar(name: String): String = File(temp.root, name).apply { writeBytes(byteArrayOf(1, 2, 3, 4)) }.absolutePath + + private inner class TestService( + private val jars: Set, + reporter: (List) -> Unit, + ) : JarIndexingService( + context = context, + progressTracker = IndexingProgressTracker(), + workspaceSupplier = { mockk() }, + unreadableJarReporter = reporter, + ) { + override val id = "jar-indexing-service-unreadable-jar-test" + override val indexKey = this@JarIndexingServiceUnreadableJarTest.indexKey + override val dbName = DB_NAME + override val indexName = INDEX_NAME + + override fun jarsToIndex(workspace: Workspace): Set = jars + } + + private companion object { + const val DB_NAME = "jar_indexing_service_unreadable_jar_test.db" + const val INDEX_NAME = "jar-indexing-service-unreadable-jar-test-index" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingServiceInitialRefreshTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingServiceInitialRefreshTest.kt new file mode 100644 index 0000000000..82f64f87c3 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingServiceInitialRefreshTest.kt @@ -0,0 +1,100 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.projects.api.GradleProject +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.api.Workspace +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.After +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** + * The language servers can refresh the library index before the manager has initialized it, and + * that refresh is dropped, so initialization must run a pass of its own. + */ +@RunWith(RobolectricTestRunner::class) +class JvmLibraryIndexingServiceInitialRefreshTest { + @get:Rule + val temp = TemporaryFolder() + + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(JvmSymbolIndex.DB_NAME_DEFAULT) + } + + @Test + fun `a refresh before initialization still leaves the index populated once initialized`() { + val jar = classJar("lib.jar", "p/Lib") + val workspace = workspaceOf(jar) + val service = JvmLibraryIndexingService(context, IndexingProgressTracker(), workspaceSupplier = { workspace }) + val registry = IndexRegistry() + + val indexed = + service.use { + runBlocking { + it.refresh().join() + it.initialize(registry) + val index = registry.require(JVM_LIBRARY_SYMBOL_INDEX) + withTimeoutOrNull(10_000) { + while (index.findTopLevelClassesNamed("Lib", listOf(jar)).none()) { + delay(20) + } + true + } ?: false + } + } + + assertThat(indexed).isTrue() + } + + private fun workspaceOf(jar: String): Workspace { + val module = + mockk { + every { path } returns ":app" + every { getCompileClasspaths(excludeSourceGeneratedClassPath = true) } returns setOf(File(jar)) + } + val root = mockk { every { path } returns ":" } + return mockk { + every { rootProject } returns root + every { subProjects } returns listOf(module) + } + } + + private fun classJar( + name: String, + internalName: String, + ): String { + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, internalName, null, "java/lang/Object", null) + visitEnd() + }.toByteArray() + val jar = File(temp.root, name) + ZipOutputStream(jar.outputStream()).use { out -> + out.putNextEntry(ZipEntry("$internalName.class")) + out.write(bytes) + out.closeEntry() + } + return jar.absolutePath + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingServicePhaseTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingServicePhaseTest.kt new file mode 100644 index 0000000000..d25245d0a6 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingServicePhaseTest.kt @@ -0,0 +1,191 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.memprof.Memprof +import com.itsaky.androidide.projects.api.GradleProject +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.api.Workspace +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.appdevforall.codeonthego.indexing.service.IndexingState +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.After +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File +import java.io.FileOutputStream +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** A library pass that submits JARs is profiled as the "Index libraries" phase. */ +@RunWith(RobolectricTestRunner::class) +class JvmLibraryIndexingServicePhaseTest { + @get:Rule + val temp = TemporaryFolder() + + private val context = ApplicationProvider.getApplicationContext() + private val sink = RecordingMemprofSink() + private val tracker = IndexingProgressTracker() + + @Before + fun setUp() { + Memprof.sink = sink + } + + @After + fun tearDown() { + Memprof.sink = null + context.deleteDatabase(JvmSymbolIndex.DB_NAME_DEFAULT) + } + + @Test + fun `a library pass that submits JARs ends the Index libraries phase with its JAR count`() { + val service = libraryService(jar("a.jar"), jar("b.jar")) + + service.use { runBlocking { refreshAndAwait(it) } } + + assertThat(sink.phases.map { listOf(it.title, it.marker, it.details, it.ended) }) + .containsExactly(listOf("Index libraries", "library_index_complete", mapOf("jars" to 2L), true)) + } + + @Test + fun `a library pass with nothing new to index records no phase`() { + val service = libraryService(jar("a.jar")) + + service.use { + runBlocking { + refreshAndAwait(it) + it.refresh().join() + } + } + + assertThat(sink.phases).hasSize(1) + } + + @Test + fun `a library pass that only folds into another pass's running jobs records no phase`() { + val fifo = File(temp.root, "blocking.jar") + assumeTrue(ProcessBuilder("mkfifo", fifo.path).start().waitFor() == 0) + + /* + * The first pass is the one initialization starts. Opening the FIFO blocks its scan until a + * writer opens it, so its job is still running when the second pass submits the same JAR. The + * third workspace read happens only once the second pass has released the mutex, so it marks + * that pass as fully submitted. + */ + val reads = AtomicInteger(0) + val firstPassSubmitting = CountDownLatch(1) + val secondPassSubmitted = CountDownLatch(1) + val workspace = workspaceOf(fifo.path) + val service = + JvmLibraryIndexingService(context, tracker, workspaceSupplier = { + when (reads.incrementAndGet()) { + 1 -> workspace.also { firstPassSubmitting.countDown() } + 2 -> workspace + else -> null.also { secondPassSubmitted.countDown() } + } + }) + + service.use { + runBlocking { + it.initialize(IndexRegistry()) + assertThat(firstPassSubmitting.await(5, TimeUnit.SECONDS)).isTrue() + val others = listOf(it.refresh(), it.refresh()) + assertThat(secondPassSubmitted.await(5, TimeUnit.SECONDS)).isTrue() + + FileOutputStream(fifo).close() + withTimeout(10_000) { + others.joinAll() + awaitIdle() + } + } + } + + assertThat(sink.phases).hasSize(1) + } + + @Test + fun `the Index libraries phase is abandoned when its pass fails`() { + runCatching { indexLibrariesPhase(jarCount = 1) { error("optimize failed") } } + + assertThat(sink.phases.map { it.abandoned }).containsExactly(true) + } + + /** + * Initializes [service] and waits for the pass that starts on its own to finish. + * + * Subscribing to [tracker] before calling [initialize][JvmLibraryIndexingService.initialize] + * means the wait observes that pass actually start, so a tracker already back to + * [IndexingState.Idle] cannot be mistaken for one that never ran a pass at all. + */ + private suspend fun refreshAndAwait(service: JvmLibraryIndexingService) = + coroutineScope { + val initPass = + async(start = CoroutineStart.UNDISPATCHED) { + tracker.state.first { it is IndexingState.Indexing } + awaitIdle() + } + service.initialize(IndexRegistry()) + withTimeout(10_000) { initPass.await() } + } + + private suspend fun awaitIdle() { + tracker.state.first { it is IndexingState.Idle } + } + + private fun libraryService(vararg jars: String): JvmLibraryIndexingService { + val workspace = workspaceOf(*jars) + return JvmLibraryIndexingService(context, tracker, workspaceSupplier = { workspace }) + } + + private fun workspaceOf(vararg jars: String): Workspace { + val module = + mockk { + every { path } returns ":app" + every { getCompileClasspaths(excludeSourceGeneratedClassPath = true) } returns jars.map(::File).toSet() + } + val root = mockk { every { path } returns ":" } + return mockk { + every { rootProject } returns root + every { subProjects } returns listOf(module) + } + } + + /** Writes a JAR holding one public class, which indexes cleanly and so records its fingerprint. */ + private fun jar(name: String): String { + val internalName = "p/${name.substringBefore('.').uppercase()}" + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, internalName, null, "java/lang/Object", null) + visitEnd() + }.toByteArray() + val jar = File(temp.root, name) + ZipOutputStream(jar.outputStream()).use { out -> + out.putNextEntry(ZipEntry("$internalName.class")) + out.write(bytes) + out.closeEntry() + } + return jar.absolutePath + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmModuleOutputIndexingServiceTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmModuleOutputIndexingServiceTest.kt new file mode 100644 index 0000000000..e8ceaff46d --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmModuleOutputIndexingServiceTest.kt @@ -0,0 +1,158 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.projects.ProjectManagerImpl +import com.itsaky.androidide.projects.api.ModuleProject +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingProgressTracker +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.After +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File +import java.util.zip.CRC32 +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +@RunWith(RobolectricTestRunner::class) +class JvmModuleOutputIndexingServiceTest { + @get:Rule + val temp = TemporaryFolder() + + private val context = ApplicationProvider.getApplicationContext() + + private val fixtures = ModuleFixtures(temp) + + @After + fun tearDown() { + runCatching { ProjectManagerImpl.getInstance().workspace = null } + context.deleteDatabase("jvm_module_output_symbol_index.db") + } + + @Test + fun `the jar set is the union of the modules' classpath deltas`() { + val app = fixtures.androidModule(":app", moduleDeps = listOf(":lib"), externalJars = listOf(fixtures.builtFile("ext/guava.jar"))) + val lib = fixtures.androidModule(":lib") + val unbuilt = fixtures.androidModule(":unbuilt", built = false) + val jlib = fixtures.javaModule(":jlib") + val workspace = fixtures.installWorkspace(app, lib, unbuilt, jlib) + + val deltas = + workspace.subProjects + .filterIsInstance() + .flatMap { it.getCompileClasspaths(false) - it.getCompileClasspaths(true) } + .filter { it.exists() } + .map { it.absolutePath } + .toSet() + + assertThat(moduleOutputJars(workspace)).isEqualTo(deltas) + assertThat(deltas.map { File(it).name }).doesNotContain("R.jar") + } + + @Test + fun `a jar whose fingerprint is unchanged is not re-indexed after a build`() { + val lib = fixtures.androidModule(":lib") + fixtures.installWorkspace(lib) + val jar = lib.getGeneratedJar() + writeClassJar(jar, "p/Aa") + val service = JvmModuleOutputIndexingService(context, IndexingProgressTracker()) + try { + val index = initializeAndAwait(service) + + writeClassJar(jar, "p/Bb") + /* + * The build-completed pass and this join()'d pass race for the mutex; either order + * leaves the jar unindexed here, since its fingerprint has not changed. + */ + runBlocking { + service.onBuildCompleted() + service.refresh().join() + } + + assertThat(index.topLevelClassNames(jar)).containsExactly("Aa") + } finally { + service.close() + } + } + + @Test + fun `a jar whose fingerprint changed is re-indexed after a build`() { + val lib = fixtures.androidModule(":lib") + fixtures.installWorkspace(lib) + val jar = lib.getGeneratedJar() + writeClassJar(jar, "p/Aa") + val service = JvmModuleOutputIndexingService(context, IndexingProgressTracker()) + try { + val index = initializeAndAwait(service) + + writeClassJar(jar, "p/Bb", modifiedAt = MODIFIED_AT + 60_000) + runBlocking { + service.onBuildCompleted() + service.refresh().join() + } + + assertThat(index.topLevelClassNames(jar)).containsExactly("Bb") + } finally { + service.close() + } + } + + private fun initializeAndAwait(service: JvmModuleOutputIndexingService): JvmSymbolIndex { + val registry = IndexRegistry() + runBlocking { + service.initialize(registry) + service.refresh().join() + } + return registry.require(JVM_MODULE_OUTPUT_SYMBOL_INDEX) + } + + private fun JvmSymbolIndex.topLevelClassNames(jar: File): List = + listOf("Aa", "Bb").filter { name -> + findTopLevelClassesNamed(name, listOf(jar.absolutePath)).any() + } + + /** + * Writes a JAR holding one public class, with stored entries and fixed timestamps, so two JARs + * whose class names have the same length have the same size and, with [modifiedAt], the same + * fingerprint. + */ + private fun writeClassJar( + jar: File, + internalName: String, + modifiedAt: Long = MODIFIED_AT, + ) { + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, internalName, null, "java/lang/Object", null) + visitEnd() + }.toByteArray() + val entry = + ZipEntry("$internalName.class").apply { + method = ZipEntry.STORED + size = bytes.size.toLong() + compressedSize = bytes.size.toLong() + crc = CRC32().apply { update(bytes) }.value + time = MODIFIED_AT + } + + jar.parentFile.mkdirs() + ZipOutputStream(jar.outputStream()).use { out -> + out.putNextEntry(entry) + out.write(bytes) + out.closeEntry() + } + jar.setLastModified(modifiedAt) + } + + private companion object { + const val MODIFIED_AT = 1_700_000_000_000L + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexFormatVersionTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexFormatVersionTest.kt new file mode 100644 index 0000000000..1546af4597 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexFormatVersionTest.kt @@ -0,0 +1,79 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.SQLiteIndex +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** A database written before the index stored package rows is rebuilt rather than reused. */ +@RunWith(RobolectricTestRunner::class) +class JvmSymbolIndexFormatVersionTest { + private val context = ApplicationProvider.getApplicationContext() + + /** The descriptor as it was at the previous format: symbol rows only, no package rows. */ + private val descriptorWithoutPackages = + object : IndexDescriptor by JvmSymbolDescriptor { + override fun packageOf(entry: JvmSymbol): String? = null + } + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + @Test + fun `an index stored before package rows existed is dropped on open`() = + runTest { + SQLiteIndex( + descriptor = descriptorWithoutPackages, + context = context, + dbName = DB_NAME, + formatVersion = FORMAT_WITHOUT_PACKAGES, + name = INDEX_NAME, + ).use { it.insert(widget()) } + + JvmSymbolIndex.createSqliteIndex(context, DB_NAME, INDEX_NAME).use { + it.setActiveSources(setOf(JAR)) + + assertThat(it.isCached(JAR)).isFalse() + } + } + + @Test + fun `a rebuilt index stores the package rows of what it indexes`() = + runTest { + JvmSymbolIndex.createSqliteIndex(context, DB_NAME, INDEX_NAME).use { + it.insert(widget()) + it.setActiveSources(setOf(JAR)) + + assertThat(it.containsPackage("com.example", sourceIds = null)).isTrue() + } + } + + private fun widget() = + JvmSymbol( + key = "com/example/Widget", + sourceId = JAR, + name = "com/example/Widget", + shortName = "Widget", + packageName = "com.example", + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(internalName = "com/example/Widget"), + ) + + private companion object { + const val DB_NAME = "jvm_format_version_test.db" + const val INDEX_NAME = "jvm-format-version-test" + const val JAR = "widget.jar" + + /** The last format version whose rows carry no package tree. */ + const val FORMAT_WITHOUT_PACKAGES = 2 + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexTopLevelClassTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexTopLevelClassTest.kt new file mode 100644 index 0000000000..e4428fc9d6 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexTopLevelClassTest.kt @@ -0,0 +1,251 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * The top-level class lookups and the package tree hold what the classpath trie held: every + * top-level class file of any visibility, and the packages those class files are in. + */ +@RunWith(JUnit4::class) +class JvmSymbolIndexTopLevelClassTest { + private fun unfilteredIndex(): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { + override fun visibleSourceIds(): Collection? = null + } + } + + private fun filteredIndex(vararg activeSources: String): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return JvmSymbolIndex(backing, BackgroundIndexer(backing)).apply { setActiveSources(activeSources.toSet()) } + } + + private fun symbol( + shortName: String, + kind: JvmSymbolKind, + pkg: String = PKG, + sourceId: String = JAR_A, + visibility: JvmVisibility = JvmVisibility.PUBLIC, + data: JvmSymbolInfo = JvmClassInfo(internalName = "${pkg.replace('.', '/')}/$shortName"), + ) = JvmSymbol( + key = "${pkg.replace('.', '/')}/$shortName", + sourceId = sourceId, + name = "${pkg.replace('.', '/')}/$shortName", + shortName = shortName, + packageName = pkg, + kind = kind, + language = JvmSourceLanguage.KOTLIN, + visibility = visibility, + data = data, + ) + + private fun topLevelClass( + shortName: String, + pkg: String = PKG, + sourceId: String = JAR_A, + kind: JvmSymbolKind = JvmSymbolKind.CLASS, + visibility: JvmVisibility = JvmVisibility.PUBLIC, + ) = symbol(shortName, kind, pkg, sourceId, visibility) + + private fun nestedClass( + shortName: String, + outer: String, + pkg: String = PKG, + sourceId: String = JAR_A, + ) = symbol( + shortName = "$outer\$$shortName", + kind = JvmSymbolKind.CLASS, + pkg = pkg, + sourceId = sourceId, + data = JvmClassInfo(containingClassName = "${pkg.replace('.', '/')}/$outer"), + ).copy(shortName = shortName) + + private fun member( + shortName: String, + owner: String, + pkg: String = PKG, + ) = symbol( + shortName = "$owner#$shortName", + kind = JvmSymbolKind.FUNCTION, + pkg = pkg, + data = JvmFunctionInfo(containingClassName = "${pkg.replace('.', '/')}/$owner"), + ).copy(shortName = shortName) + + private fun topLevelFunction( + shortName: String, + pkg: String = PKG, + ) = symbol("#$shortName", JvmSymbolKind.FUNCTION, pkg, data = JvmFunctionInfo()).copy(shortName = shortName) + + private fun typeAlias( + shortName: String, + pkg: String = PKG, + ) = symbol(shortName, JvmSymbolKind.TYPE_ALIAS, pkg, data = JvmTypeAliasInfo()) + + private suspend fun JvmSymbolIndex.insertEveryShapeNamed(name: String) { + insert(topLevelClass(name)) + insert(nestedClass(name, outer = "Outer")) + insert(member(name, owner = "Outer")) + insert(topLevelFunction(name)) + insert(typeAlias(name, pkg = "$PKG.aliases")) + } + + @Test + fun `an exact-name lookup returns only the top-level class`() = + runTest { + val index = unfilteredIndex() + index.insertEveryShapeNamed("Widget") + + val found = index.findTopLevelClassesNamed("Widget", sourceIds = null).toList() + + assertThat(found.map { it.key }).containsExactly("com/example/Widget") + } + + @Test + fun `a prefix lookup returns only top-level classes`() = + runTest { + val index = unfilteredIndex() + index.insertEveryShapeNamed("Widget") + + val found = index.findTopLevelClassesByPrefix("Widg", sourceIds = null, limit = 10).toList() + + assertThat(found.map { it.key }).containsExactly("com/example/Widget") + } + + @Test + fun `a prefix lookup ignores case`() = + runTest { + val index = unfilteredIndex() + index.insert(topLevelClass("Widget")) + + val found = index.findTopLevelClassesByPrefix("wIDG", sourceIds = null, limit = 10).toList() + + assertThat(found.map { it.shortName }).containsExactly("Widget") + } + + @Test + fun `a prefix lookup spends its limit only on top-level classes`() = + runTest { + val index = unfilteredIndex() + repeat(20) { index.insert(nestedClass("Widget$it", outer = "Outer")) } + index.insert(topLevelClass("WidgetZ")) + + val found = index.findTopLevelClassesByPrefix("Widget", sourceIds = null, limit = 5).toList() + + assertThat(found.map { it.shortName }).containsExactly("WidgetZ") + } + + @Test + fun `an in-package lookup returns only the package's top-level classes`() = + runTest { + val index = unfilteredIndex() + index.insertEveryShapeNamed("Widget") + index.insert(typeAlias("Alias")) + index.insert(topLevelClass("Elsewhere", pkg = "com.other")) + + val found = index.findTopLevelClassesInPackage(PKG, sourceIds = null).toList() + + assertThat(found.map { it.key }).containsExactly("com/example/Widget") + } + + @Test + fun `a file facade and a package-private class are top-level classes`() = + runTest { + val index = unfilteredIndex() + index.insert(topLevelClass("WidgetKt", kind = JvmSymbolKind.FILE_FACADE)) + index.insert(topLevelClass("Hidden", visibility = JvmVisibility.PACKAGE_PRIVATE)) + + assertThat(index.findTopLevelClassesNamed("WidgetKt", sourceIds = null).toList()).hasSize(1) + assertThat(index.findTopLevelClassesNamed("Hidden", sourceIds = null).toList()).hasSize(1) + assertThat(index.findTopLevelClassesByPrefix("", sourceIds = null, limit = 10).map { it.shortName }.toList()) + .containsExactly("WidgetKt", "Hidden") + assertThat(index.findTopLevelClassesInPackage(PKG, sourceIds = null).map { it.shortName }.toList()) + .containsExactly("WidgetKt", "Hidden") + } + + @Test + fun `every class query is scoped to the requested sources`() = + runTest { + val index = unfilteredIndex() + index.insert(topLevelClass("Widget", sourceId = JAR_A)) + index.insert(topLevelClass("Widget", sourceId = JAR_B)) + val scope = listOf(JAR_B) + + assertThat(index.findTopLevelClassesNamed("Widget", scope).map { it.sourceId }.toList()).containsExactly(JAR_B) + assertThat(index.findTopLevelClassesByPrefix("Wid", scope, limit = 10).map { it.sourceId }.toList()) + .containsExactly(JAR_B) + assertThat(index.findTopLevelClassesInPackage(PKG, scope).map { it.sourceId }.toList()).containsExactly(JAR_B) + } + + @Test + fun `every class query drops a requested source that is not active`() = + runTest { + val index = filteredIndex(JAR_A) + index.insert(topLevelClass("Widget", sourceId = JAR_A)) + index.insert(topLevelClass("Widget", sourceId = JAR_B)) + val scope = listOf(JAR_A, JAR_B) + + assertThat(index.findTopLevelClassesNamed("Widget", scope).map { it.sourceId }.toList()).containsExactly(JAR_A) + assertThat(index.findTopLevelClassesByPrefix("Wid", scope, limit = 10).map { it.sourceId }.toList()) + .containsExactly(JAR_A) + assertThat(index.findTopLevelClassesInPackage(PKG, scope).map { it.sourceId }.toList()).containsExactly(JAR_A) + } + + @Test + fun `a package exists when it holds a top-level class`() = + runTest { + val index = unfilteredIndex() + index.insert(topLevelClass("Widget", pkg = "com.example.ui")) + + assertThat(index.containsPackage("com.example.ui", sourceIds = null)).isTrue() + assertThat(index.subpackages("com.example", sourceIds = null)).containsExactly("com.example.ui") + assertThat(index.subpackages("", sourceIds = null)).containsExactly("com") + } + + @Test + fun `a package holding only a file facade exists`() = + runTest { + val index = unfilteredIndex() + index.insert(topLevelClass("UtilsKt", pkg = "com.example.util", kind = JvmSymbolKind.FILE_FACADE)) + + assertThat(index.containsPackage("com.example.util", sourceIds = null)).isTrue() + } + + @Test + fun `a package holding no top-level class file does not exist`() = + runTest { + val index = unfilteredIndex() + index.insert(nestedClass("Inner", outer = "Outer", pkg = "com.nested")) + index.insert(member("helper", owner = "Outer", pkg = "com.members")) + index.insert(topLevelFunction("launch", pkg = "com.functions")) + index.insert(typeAlias("Alias", pkg = "com.aliases")) + + assertThat(index.subpackages("", sourceIds = null)).isEmpty() + assertThat(index.containsPackage("com.nested", sourceIds = null)).isFalse() + } + + @Test + fun `package lookups are scoped to the requested active sources`() = + runTest { + val index = filteredIndex(JAR_A, JAR_B) + index.insert(topLevelClass("A", pkg = "com.fromA", sourceId = JAR_A)) + index.insert(topLevelClass("B", pkg = "com.fromB", sourceId = JAR_B)) + index.insert(topLevelClass("C", pkg = "com.fromC", sourceId = JAR_C)) + + assertThat(index.subpackages("com", sourceIds = listOf(JAR_B, JAR_C))).containsExactly("com.fromB") + assertThat(index.containsPackage("com.fromA", sourceIds = listOf(JAR_B))).isFalse() + assertThat(index.containsPackage("com.fromC", sourceIds = listOf(JAR_C))).isFalse() + } + + private companion object { + const val PKG = "com.example" + const val JAR_A = "a.jar" + const val JAR_B = "b.jar" + const val JAR_C = "c.jar" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupDifferentialTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupDifferentialTest.kt new file mode 100644 index 0000000000..7785f628f8 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupDifferentialTest.kt @@ -0,0 +1,370 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import com.itsaky.androidide.projects.classpath.JarFsClasspathReader +import com.itsaky.androidide.utils.ClassTrie +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.SQLiteIndex +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File +import java.nio.file.Paths + +/** + * A module's [ModuleClasspathLookup] answers every query its consumers asked of the module's + * classpath trie the way the trie did. + * + * The oracle is built the way `ModuleProject.indexClasspaths` fills the trie: `JarFsClasspathReader` + * filtered to top-level classes, appended to a [ClassTrie]. The lookup reads real SQLite-backed + * indexes filled by [CombinedJarScanner], so the scanner, the package table and the query layer are + * all under test. kotlin-stdlib sits in the library index and, like `R.jar`, in the generated index + * too, so every answer must be deduplicated; JUnit sits in the module-output index, standing in for + * the output JAR of a project module the module compiles against. + * + * It runs against real JARs because the shapes that matter do not occur in a hand-built fixture: + * Kotlin file facades, multi-file class parts and deep package trees in kotlin-stdlib, and + * package-private and nested Java classes in JUnit. + * + * Two differences are intended, and each has a test proving it is non-empty and of its kind: + * multi-file class parts are withheld, and a prefix query no longer adds the fuzzy matches + * (`FuzzySearch.ratio > 59`) the trie's consumers added on top of prefix matches. Any other + * difference fails. + */ +@RunWith(RobolectricTestRunner::class) +class ModuleClasspathLookupDifferentialTest { + private class Corpus( + anchor: Class<*>, + ) { + val jar: File by lazy { testClasspathJar(anchor) } + val sourceId: String get() = jar.absolutePath + val symbols: List by lazy { CombinedJarScanner.scan(Paths.get(sourceId), sourceId).toList() } + } + + private companion object { + val stdlib = Corpus(Unit::class.java) + val junit = Corpus(Test::class.java) + + /** Typos the fuzzy tier matched to a class in the corpora, and no class name starts with. */ + val TYPOS = listOf("Asert", "Colections") + } + + private val context = ApplicationProvider.getApplicationContext() + private val indexes = mutableListOf() + + private val libraryIndex by lazy { sqliteIndex(stdlib) } + private val generatedIndex by lazy { sqliteIndex(stdlib) } + private val moduleOutputIndex by lazy { sqliteIndex(junit) } + + @After + fun tearDown() { + indexes.forEach(JvmSymbolIndex::close) + } + + private fun sqliteIndex(corpus: Corpus): JvmSymbolIndex { + val backing = + SQLiteIndex( + descriptor = JvmSymbolDescriptor, + context = context, + dbName = null, + formatVersion = JvmSymbolIndex.FORMAT_VERSION, + ) + val index = JvmSymbolIndex(backing, BackgroundIndexer(backing)).also(indexes::add) + index.setActiveSources(setOf(corpus.sourceId)) + index.indexSource(corpus.sourceId) { corpus.symbols.asSequence() } + runBlocking { index.awaitIndexing() } + return index + } + + /** The lookup of a module whose compile classpath is [classpath]. */ + private fun lookup(vararg classpath: Corpus): ModuleClasspathLookup = + ModuleClasspathLookup(libraryIndex, generatedIndex, moduleOutputIndex) { + ClasspathScope( + classpath = classpath.map { it.sourceId }, + moduleOutputs = classpath.filter { it === junit }.map { it.sourceId }, + ) + } + + /** What the module's classpath trie held for [classpath]. */ + private class Oracle( + vararg classpath: Corpus, + ) { + val trie = ClassTrie() + + init { + JarFsClasspathReader() + .listClasses(classpath.map { it.jar }) + .filter { it.isTopLevel } + .forEach { trie.append(it.name) } + } + + val classes: Set = trie.allClassNames() + + /** The trie's classes the index offers too: all but the multi-file parts. */ + val offered: Set = classes.filterNot(::isMultiFilePart).toSet() + + val packages: List = packageNodes(trie.root) + + private fun packageNodes(node: ClassTrie.Node): List = + listOf(node) + + node.children.values + .filter { it.children.isNotEmpty() } + .flatMap(::packageNodes) + + fun subpackagesOf(node: ClassTrie.Node): Set = + node.children.values + .filter { it.children.isNotEmpty() } + .map { it.qualifiedName } + .toSet() + + fun offeredClassesIn(node: ClassTrie.Node): Set = + node.children.values + .filter { it.isClass && it.qualifiedName in offered } + .map { it.qualifiedName } + .toSet() + + /** Whether the trie matches [prefix] against [qualifiedName]'s simple name, ignoring case. */ + fun matchesPrefix( + qualifiedName: String, + prefix: String, + ) = simpleName(qualifiedName).lowercase().startsWith(prefix.lowercase()) + } + + /** Where the import completion's walk over the segments of a name ends. */ + private sealed interface Walk { + /** A segment before the last is a class: its members must be completed instead. */ + data object ClassMember : Walk + + data class Found( + val isClass: Boolean, + val isPackage: Boolean, + ) : Walk + + data object Absent : Walk + } + + /** The trie walk of the import completion: it stops at the first segment that is a class. */ + private fun Oracle.walk(name: String): Walk { + var node: ClassTrie.Node? = trie.root + for (segment in trie.segments(name)) { + if (node == null) break + if (node.isClass) return Walk.ClassMember + node = node.children[segment] + } + return node?.let { Walk.Found(it.isClass, it.children.isNotEmpty()) } ?: Walk.Absent + } + + /** The same walk as point lookups: a class check on every proper prefix, then the name itself. */ + private fun ModuleClasspathLookup.walk(name: String): Walk { + val segments = name.split('.') + for (end in 1 until segments.size) { + if (isClass(segments.subList(0, end).joinToString("."))) return Walk.ClassMember + } + val isClass = isClass(name) + val isPackage = isPackage(name) + return if (isClass || isPackage) Walk.Found(isClass, isPackage) else Walk.Absent + } + + private fun ModuleClasspathLookup.subpackagesOf(packageName: String) = + children(packageName).filterNot { it.isClass }.map { it.qualifiedName } + + private fun ModuleClasspathLookup.classesIn(packageName: String) = children(packageName).filter { it.isClass }.map { it.qualifiedName } + + @Test + fun `the root children are the trie's root children`() { + val oracle = Oracle(stdlib, junit) + val expected = + oracle.trie.root.children.values.flatMap { node -> + listOfNotNull( + Child(node.name, node.qualifiedName, isClass = false).takeIf { node.children.isNotEmpty() }, + Child(node.name, node.qualifiedName, isClass = true).takeIf { node.qualifiedName in oracle.offered }, + ) + } + + val children = lookup(stdlib, junit).children("") + + assertThat(children).containsExactlyElementsIn(expected) + assertThat(children.map { it.name }).containsAtLeast("kotlin", "org", "junit") + } + + @Test + fun `every package's subpackages are the trie node's package children`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + + for (node in oracle.packages) { + assertWithMessage("subpackages of '${node.qualifiedName}'") + .that(lookup.subpackagesOf(node.qualifiedName)) + .containsExactlyElementsIn(oracle.subpackagesOf(node)) + } + assertThat(oracle.packages.size).isGreaterThan(20) + } + + @Test + fun `every package's classes are the trie node's class children`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + + for (node in oracle.packages) { + assertWithMessage("classes in '${node.qualifiedName}'") + .that(lookup.classesIn(node.qualifiedName)) + .containsExactlyElementsIn(oracle.offeredClassesIn(node)) + } + } + + @Test + fun `every trie class and package ends the segment walk where the trie's walk ended`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + val names = oracle.offered + oracle.packages.map { it.qualifiedName }.filter { it.isNotEmpty() } + + for (name in names) { + assertWithMessage("walk of '$name'").that(lookup.walk(name)).isEqualTo(oracle.walk(name)) + } + } + + @Test + fun `a name below a trie class asks for member completion, as the trie's walk did`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + + for (name in oracle.offered.map { "$it.x" }) { + assertWithMessage("walk of '$name'").that(oracle.walk(name)).isEqualTo(Walk.ClassMember) + assertWithMessage("walk of '$name'").that(lookup.walk(name)).isEqualTo(Walk.ClassMember) + } + } + + @Test + fun `every simple name resolves to the trie classes of that name`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + val bySimpleName = oracle.offered.groupBy(::simpleName) + + for (simpleName in oracle.classes.map(::simpleName).toSet()) { + val found = lookup.qualifiedNamesOf(simpleName) + + assertWithMessage("classes named '$simpleName'") + .that(found) + .containsExactlyElementsIn(bySimpleName[simpleName].orEmpty()) + assertWithMessage("classes named '$simpleName'").that(found).containsNoDuplicates() + } + } + + @Test + fun `every one and two character prefix matches the trie classes it matched, ignoring case`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + val present = + oracle.offered + .map(::simpleName) + .flatMap { listOf(it.take(1), it.take(2)) } + .toSet() + val prefixes = present + present.map(::swapCase) + + for (prefix in prefixes) { + val found = lookup.qualifiedNamesByPrefix(prefix, limit = Int.MAX_VALUE) + + assertWithMessage("classes by prefix '$prefix'") + .that(found) + .containsExactlyElementsIn(oracle.offered.filter { oracle.matchesPrefix(it, prefix) }) + assertWithMessage("classes by prefix '$prefix'").that(found).containsNoDuplicates() + } + assertThat(prefixes.size).isGreaterThan(200) + } + + @Test + fun `a module scoped to one JAR is offered nothing only the other JAR holds`() { + for ((kept, other) in listOf(stdlib to junit, junit to stdlib)) { + val keptOracle = Oracle(kept) + val otherOracle = Oracle(other) + val onlyOther = otherOracle.offered - keptOracle.classes + val otherPackages = otherOracle.packages.map { it.qualifiedName } - keptOracle.packages.map { it.qualifiedName }.toSet() + val lookup = lookup(kept) + + assertThat(onlyOther).isNotEmpty() + assertThat(otherPackages).isNotEmpty() + for (name in onlyOther) { + assertWithMessage("class '$name'").that(lookup.isClass(name)).isFalse() + assertWithMessage("classes named '${simpleName(name)}'") + .that(lookup.qualifiedNamesOf(simpleName(name))) + .doesNotContain(name) + } + for (name in otherPackages) { + assertWithMessage("package '$name'").that(lookup.isPackage(name)).isFalse() + } + assertThat(lookup.children("").map { it.qualifiedName }.toSet()) + .containsExactlyElementsIn(keptOracle.trie.root.children.keys) + } + } + + @Test + fun `the classes the lookup withholds are multi-file parts, and there are some`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + + val withheld = oracle.classes.filterNot(lookup::isClass) + + assertThat(withheld).isNotEmpty() + assertThat(withheld.filterNot(::isMultiFilePart)).isEmpty() + assertThat(withheld.flatMap { lookup.qualifiedNamesOf(simpleName(it)) }).containsNoneIn(withheld) + } + + @Test + fun `a prefix query leaves out exactly the fuzzy matches the trie's consumers added, and there are some`() { + val oracle = Oracle(stdlib, junit) + val lookup = lookup(stdlib, junit) + + for (typo in TYPOS) { + val fuzzyOnly = + oracle.offered.filter { !oracle.matchesPrefix(it, typo) && fuzzyRatio(simpleName(it), typo) > FUZZY_MIN_RATIO } + val consumerMatches = oracle.offered.filter { oracle.matchesPrefix(it, typo) } + fuzzyOnly + + val found = lookup.qualifiedNamesByPrefix(typo, limit = Int.MAX_VALUE) + + assertWithMessage("fuzzy-only matches of '$typo'").that(fuzzyOnly).isNotEmpty() + assertWithMessage("matches of '$typo' the lookup leaves out") + .that(consumerMatches - found.toSet()) + .containsExactlyElementsIn(fuzzyOnly) + assertWithMessage("matches of '$typo'").that(found).containsNoneIn(fuzzyOnly) + } + } +} + +/** The minimum ratio above which the trie's completion consumers took a fuzzy match. */ +private const val FUZZY_MIN_RATIO = 59 + +/** + * `FuzzySearch.ratio`, which is not on this module's classpath: the similarity of [a] and [b] in + * percent, from their edit distance with insertions and deletions costing 1 and a substitution 2. + * + * That distance is `a.length + b.length - 2 * lcs`, so the ratio is `2 * lcs` over the total length. + */ +private fun fuzzyRatio( + a: String, + b: String, +): Int = Math.round(200.0 * longestCommonSubsequence(a, b) / (a.length + b.length)).toInt() + +private fun longestCommonSubsequence( + a: String, + b: String, +): Int { + var previous = IntArray(b.length + 1) + for (i in a.indices) { + val current = IntArray(b.length + 1) + for (j in b.indices) { + current[j + 1] = if (a[i] == b[j]) previous[j] + 1 else maxOf(previous[j + 1], current[j]) + } + previous = current + } + return previous[b.length] +} + +private fun simpleName(qualifiedName: String) = qualifiedName.substringAfterLast('.') + +private fun swapCase(text: String) = text.map { if (it.isUpperCase()) it.lowercaseChar() else it.uppercaseChar() }.joinToString("") diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupModuleScopeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupModuleScopeTest.kt new file mode 100644 index 0000000000..e48418d6c4 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupModuleScopeTest.kt @@ -0,0 +1,94 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.projects.ProjectManagerImpl +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.After +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * A module's lookup is scoped to its own compile classpath, although the indexes it reads hold the + * JARs of every module. + */ +@RunWith(JUnit4::class) +class ModuleClasspathLookupModuleScopeTest { + @get:Rule + val temp = TemporaryFolder() + + private val fixtures = ModuleFixtures(temp) + + @After + fun tearDown() { + runCatching { ProjectManagerImpl.getInstance().workspace = null } + } + + /** An index whose active set is [activeSources], as a registered indexing service leaves it. */ + private fun index(activeSources: Set): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return JvmSymbolIndex(backing, BackgroundIndexer(backing)).apply { setActiveSources(activeSources) } + } + + private fun topLevelClass( + qualifiedName: String, + sourceId: String, + ): JvmSymbol { + val internalName = qualifiedName.replace('.', '/') + return JvmSymbol( + key = internalName, + sourceId = sourceId, + name = internalName, + shortName = qualifiedName.substringAfterLast('.'), + packageName = qualifiedName.substringBeforeLast('.'), + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.KOTLIN, + data = JvmClassInfo(internalName = internalName), + ) + } + + @Test + fun `a module does not see the output classes of a module it does not depend on`() = + runTest { + val app = fixtures.androidModule(":app", moduleDeps = listOf(":lib")) + val lib = fixtures.androidModule(":lib") + fixtures.installWorkspace(app, lib) + val appJar = app.getGeneratedJar().absolutePath + val libJar = lib.getGeneratedJar().absolutePath + val moduleOutput = index(setOf(appJar, libJar)) + moduleOutput.insert(topLevelClass("com.app.AppScreen", appJar)) + moduleOutput.insert(topLevelClass("com.lib.LibWidget", libJar)) + + val libLookup = ModuleClasspathLookup(lib, null, null, moduleOutput) + val appLookup = ModuleClasspathLookup(app, null, null, moduleOutput) + + assertThat(libLookup.qualifiedNamesByPrefix("", limit = 10)).containsExactly("com.lib.LibWidget") + assertThat(libLookup.isPackage("com.app")).isFalse() + assertThat(appLookup.qualifiedNamesByPrefix("", limit = 10)) + .containsExactly("com.app.AppScreen", "com.lib.LibWidget") + } + + @Test + fun `a module sees the library jars of the modules it depends on`() = + runTest { + val guava = fixtures.builtFile("ext/guava.jar") + val okio = fixtures.builtFile("ext/okio.jar") + val app = fixtures.androidModule(":app", moduleDeps = listOf(":lib"), externalJars = listOf(okio)) + val lib = fixtures.androidModule(":lib", externalJars = listOf(guava)) + fixtures.installWorkspace(app, lib) + val library = index(setOf(guava, okio)) + library.insert(topLevelClass("com.google.common.Lists", guava)) + library.insert(topLevelClass("okio.Buffer", okio)) + + val libLookup = ModuleClasspathLookup(lib, library, null, null) + val appLookup = ModuleClasspathLookup(app, library, null, null) + + assertThat(libLookup.qualifiedNamesByPrefix("", limit = 10)).containsExactly("com.google.common.Lists") + assertThat(appLookup.qualifiedNamesByPrefix("", limit = 10)) + .containsExactly("com.google.common.Lists", "okio.Buffer") + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupTest.kt new file mode 100644 index 0000000000..e8bdc67f90 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleClasspathLookupTest.kt @@ -0,0 +1,275 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** A module's class lookups answer what its classpath trie did, from the three JVM symbol indexes. */ +@RunWith(JUnit4::class) +class ModuleClasspathLookupTest { + private val library = index() + private val generated = index() + private val moduleOutput = index() + + private fun index(): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { + override fun visibleSourceIds(): Collection? = null + } + } + + /** + * An index returning its matches in simple-name order, as SQLite does when it walks the name + * index, so which rows a limited query returns does not depend on hash order. + */ + private fun nameOrderedIndex(): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { + override fun visibleSourceIds(): Collection? = null + + override fun query(query: IndexQuery): Sequence { + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + return super.query(query.copy(limit = 0)).sortedBy { it.shortName }.take(limit) + } + } + } + + private fun lookup( + library: JvmSymbolIndex? = this.library, + generated: JvmSymbolIndex? = this.generated, + moduleOutput: JvmSymbolIndex? = this.moduleOutput, + classpath: List = listOf(LIB_JAR, R_JAR, OWN_JAR), + moduleOutputs: List = listOf(OWN_JAR), + ) = ModuleClasspathLookup(library, generated, moduleOutput) { ClasspathScope(classpath, moduleOutputs) } + + private fun topLevelClass( + qualifiedName: String, + sourceId: String = LIB_JAR, + visibility: JvmVisibility = JvmVisibility.PUBLIC, + ): JvmSymbol { + val internalName = qualifiedName.replace('.', '/') + return JvmSymbol( + key = internalName, + sourceId = sourceId, + name = internalName, + shortName = qualifiedName.substringAfterLast('.'), + packageName = qualifiedName.substringBeforeLast('.', missingDelimiterValue = ""), + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + visibility = visibility, + data = JvmClassInfo(internalName = internalName), + ) + } + + private fun nestedClass( + outer: String, + simpleName: String, + ): JvmSymbol { + val outerInternal = outer.replace('.', '/') + return topLevelClass("$outer\$$simpleName").copy( + shortName = simpleName, + packageName = outer.substringBeforeLast('.'), + data = JvmClassInfo(internalName = "$outerInternal\$$simpleName", containingClassName = outerInternal), + ) + } + + @Test + fun `a class only on a jar outside the module's classpath is not offered`() = + runTest { + library.insert(topLevelClass("com.other.Widget", sourceId = OTHER_JAR)) + + val lookup = lookup() + + assertThat(lookup.qualifiedNamesOf("Widget")).isEmpty() + assertThat(lookup.qualifiedNamesByPrefix("Wid", limit = 10)).isEmpty() + assertThat(lookup.children("com")).isEmpty() + assertThat(lookup.isPackage("com.other")).isFalse() + assertThat(lookup.isClass("com.other.Widget")).isFalse() + } + + @Test + fun `each index is scoped to its own share of the module's classpath`() = + runTest { + library.insert(topLevelClass("com.lib.Widget", sourceId = LIB_JAR)) + generated.insert(topLevelClass("com.app.R", sourceId = R_JAR)) + moduleOutput.insert(topLevelClass("com.app.Screen", sourceId = OWN_JAR)) + moduleOutput.insert(topLevelClass("com.unrelated.Screen", sourceId = UNRELATED_MODULE_JAR)) + + val lookup = lookup() + + assertThat(lookup.qualifiedNamesOf("Widget")).containsExactly("com.lib.Widget") + assertThat(lookup.qualifiedNamesOf("R")).containsExactly("com.app.R") + assertThat(lookup.qualifiedNamesOf("Screen")).containsExactly("com.app.Screen") + } + + @Test + fun `a class held by two indexes is offered once`() = + runTest { + library.insert(topLevelClass("com.example.Widget", sourceId = LIB_JAR)) + generated.insert(topLevelClass("com.example.Widget", sourceId = R_JAR)) + library.insert(topLevelClass("com.example.sub.Gadget", sourceId = LIB_JAR)) + generated.insert(topLevelClass("com.example.sub.Gadget", sourceId = R_JAR)) + + val lookup = lookup() + + assertThat(lookup.qualifiedNamesOf("Widget")).containsExactly("com.example.Widget") + assertThat(lookup.classesNamed("Widget")).hasSize(1) + assertThat(lookup.qualifiedNamesByPrefix("Wid", limit = 10)).containsExactly("com.example.Widget") + assertThat(lookup.children("com.example")) + .containsExactly( + Child("sub", "com.example.sub", isClass = false), + Child("Widget", "com.example.Widget", isClass = true), + ) + } + + @Test + fun `a prefix lookup puts exact simple-name matches first under a small limit`() = + runTest { + repeat(5) { library.insert(topLevelClass("com.example.Widget$it")) } + generated.insert(topLevelClass("com.example.Widget", sourceId = R_JAR)) + + val found = lookup().qualifiedNamesByPrefix("Widget", limit = 2) + + assertThat(found.first()).isEqualTo("com.example.Widget") + } + + @Test + fun `a prefix lookup matches prefixes ignoring case`() = + runTest { + library.insert(topLevelClass("com.example.WidgetA")) + generated.insert(topLevelClass("com.example.widgetB", sourceId = R_JAR)) + moduleOutput.insert(topLevelClass("com.example.WIDGETC", sourceId = OWN_JAR)) + + val found = lookup().qualifiedNamesByPrefix("widget", limit = 2) + + assertThat(found).containsExactly("com.example.WidgetA", "com.example.widgetB", "com.example.WIDGETC") + } + + @Test + fun `a prefix lookup gives each index its own limit`() = + runTest { + repeat(5) { library.insert(topLevelClass("com.example.Widget$it")) } + moduleOutput.insert(topLevelClass("com.example.WidgetOwn", sourceId = OWN_JAR)) + + val found = lookup().qualifiedNamesByPrefix("Wid", limit = 3) + + assertThat(found).contains("com.example.WidgetOwn") + } + + @Test + fun `a prefix lookup fills its limit when one index holds a class from two jars`() = + runTest { + val library = nameOrderedIndex() + val jars = listOf("/libs/kotlin-stdlib.jar", "/libs/kotlin-stdlib-jdk8.jar") + jars.forEach { library.insert(topLevelClass("com.example.Widget", sourceId = it)) } + library.insert(topLevelClass("com.example.WidgetB", sourceId = jars.first())) + + val found = lookup(library = library, classpath = jars).qualifiedNamesByPrefix("Wid", limit = 2) + + assertThat(found).containsExactly("com.example.Widget", "com.example.WidgetB") + } + + @Test + fun `children mixes a package's direct subpackages and top-level classes`() = + runTest { + library.insert(topLevelClass("com.example.Widget")) + library.insert(topLevelClass("com.example.ui.Button")) + library.insert(topLevelClass("com.example.ui.deep.Deep")) + library.insert(nestedClass("com.example.Widget", "Inner")) + + val children = lookup().children("com.example") + + assertThat(children) + .containsExactly( + Child("ui", "com.example.ui", isClass = false), + Child("Widget", "com.example.Widget", isClass = true), + ) + } + + @Test + fun `children of the default package are the root packages and default-package classes`() = + runTest { + library.insert(topLevelClass("com.example.Widget")) + library.insert(topLevelClass("Loose")) + + val lookup = lookup() + + assertThat(lookup.children("")) + .containsExactly( + Child("com", "com", isClass = false), + Child("Loose", "Loose", isClass = true), + ) + assertThat(lookup.isClass("Loose")).isTrue() + } + + @Test + fun `a package or class check answers for top-level classes only`() = + runTest { + library.insert(topLevelClass("com.example.Widget")) + library.insert(nestedClass("com.example.Widget", "Inner")) + library.insert(topLevelClass("com.other.Gadget")) + + val lookup = lookup() + + assertThat(lookup.isPackage("com.example")).isTrue() + assertThat(lookup.isPackage("com")).isTrue() + assertThat(lookup.isPackage("com.example.Widget")).isFalse() + assertThat(lookup.isClass("com.example.Widget")).isTrue() + assertThat(lookup.isClass("com.example.Widget.Inner")).isFalse() + assertThat(lookup.isClass("com.example")).isFalse() + assertThat(lookup.isClass("com.example.Gadget")).isFalse() + } + + @Test + fun `classes named carry the package and visibility a visibility filter needs`() = + runTest { + library.insert(topLevelClass("com.example.Widget", visibility = JvmVisibility.PACKAGE_PRIVATE)) + + val found = lookup().classesNamed("Widget").single() + + assertThat(found.fqName).isEqualTo("com.example.Widget") + assertThat(found.packageName).isEqualTo("com.example") + assertThat(found.visibility).isEqualTo(JvmVisibility.PACKAGE_PRIVATE) + } + + @Test + fun `an unregistered index contributes nothing`() = + runTest { + library.insert(topLevelClass("com.example.Widget")) + + val lookup = lookup(generated = null, moduleOutput = null) + + assertThat(lookup.qualifiedNamesOf("Widget")).containsExactly("com.example.Widget") + assertThat(lookup.qualifiedNamesByPrefix("W", limit = 5)).containsExactly("com.example.Widget") + assertThat(lookup.children("com")).containsExactly(Child("example", "com.example", isClass = false)) + assertThat(lookup.isPackage("com.example")).isTrue() + assertThat(lookup.isClass("com.example.Widget")).isTrue() + assertThat(lookup(library = null, generated = null, moduleOutput = null).qualifiedNamesOf("Widget")).isEmpty() + } + + @Test + fun `constructing a lookup does not resolve the module's classpath`() { + var resolved = false + + ModuleClasspathLookup(library, generated, moduleOutput) { + resolved = true + ClasspathScope(emptyList(), emptyList()) + } + + assertThat(resolved).isFalse() + } + + private companion object { + const val LIB_JAR = "/libs/lib.jar" + const val R_JAR = "/app/build/R.jar" + const val OWN_JAR = "/app/build/classes.jar" + const val OTHER_JAR = "/libs/other.jar" + const val UNRELATED_MODULE_JAR = "/unrelated/build/classes.jar" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleFixtures.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleFixtures.kt new file mode 100644 index 0000000000..ca666e48ad --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ModuleFixtures.kt @@ -0,0 +1,160 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.itsaky.androidide.project.AndroidModels +import com.itsaky.androidide.project.GradleModels +import com.itsaky.androidide.project.JavaModels +import com.itsaky.androidide.projects.ProjectManagerImpl +import com.itsaky.androidide.projects.api.AndroidModule +import com.itsaky.androidide.projects.api.GradleProject +import com.itsaky.androidide.projects.api.JavaModule +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.api.Workspace +import io.mockk.every +import io.mockk.spyk +import org.junit.rules.TemporaryFolder +import java.io.File + +/** Builds project modules laid out under [temp] and installs them as the project manager's workspace. */ +internal class ModuleFixtures( + private val temp: TemporaryFolder, +) { + /** Creates a one-byte file at [relativePath] under [temp] and returns its absolute path. */ + fun builtFile(relativePath: String): String = + File(temp.root, relativePath) + .apply { + parentFile.mkdirs() + writeBytes(ByteArray(1)) + }.absolutePath + + private fun moduleDir(path: String) = File(temp.root, path.replace(':', '/')) + + /** + * Builds an [AndroidModule] whose compile graph lists [moduleDeps] as project libraries and + * [externalJars] as external Java libraries, with an `R.jar` on its selected variant. Its + * generated JAR and `R.jar` exist on disk when [built]. + */ + fun androidModule( + path: String, + moduleDeps: List = emptyList(), + externalJars: List = emptyList(), + built: Boolean = true, + ): AndroidModule { + val dir = moduleDir(path) + val generatedJar = File(dir, "build/classes.jar") + val rJar = File(dir, "build/R.jar") + if (built) { + builtFile(generatedJar.relativeTo(temp.root).path) + builtFile(rJar.relativeTo(temp.root).path) + } + + val graph = AndroidModels.DependencyGraph.newBuilder() + val variantDeps = AndroidModels.VariantDependencies.newBuilder().setName(VARIANT) + + fun addRoot(library: AndroidModels.Library) { + val keyId = graph.keyCount + graph.addKey(library.key) + graph.addRoot(graph.nodeCount) + graph.addNode( + AndroidModels.GraphNode + .newBuilder() + .setKeyId(keyId) + .build(), + ) + variantDeps.putLibraries(library.key, library) + } + + for (depPath in moduleDeps) { + addRoot( + AndroidModels.Library + .newBuilder() + .setKey("project$depPath") + .setType(AndroidModels.LibraryType.Project) + .setProjectInfo( + AndroidModels.ProjectInfo + .newBuilder() + .setBuildId(":") + .setProjectPath(depPath) + .build(), + ).build(), + ) + } + + for (jar in externalJars) { + addRoot( + AndroidModels.Library + .newBuilder() + .setKey("external$jar") + .setType(AndroidModels.LibraryType.ExternalJavaLibrary) + .setArtifactPath(jar) + .build(), + ) + } + + variantDeps.setMainArtifact( + AndroidModels.ArtifactDependencies + .newBuilder() + .setCompileGraph(graph.build()) + .build(), + ) + + val module = + AndroidModule( + gradleProject(path) + .setAndroidProject( + AndroidModels.AndroidProject + .newBuilder() + .setProjectType(AndroidModels.ProjectType.LibraryProject) + .setVariantDependencies(variantDeps.build()) + .setClassesJarPath(generatedJar.absolutePath) + .build(), + ).build(), + ) + + // The selected variant comes from the project manager's sync state, which a test cannot set. + val variant = + AndroidModels.AndroidVariant + .newBuilder() + .setName(VARIANT) + .setMainArtifact( + AndroidModels.AndroidArtifact + .newBuilder() + .setName(VARIANT) + .addClassJarPaths(rJar.absolutePath) + .build(), + ).build() + return spyk(module).also { every { it.getSelectedVariant() } returns variant } + } + + /** Builds a [JavaModule] whose `build/libs/.jar` exists on disk. */ + fun javaModule(path: String): JavaModule { + builtFile("${path.replace(':', '/')}/build/libs/${path.trimStart(':')}.jar") + return JavaModule( + gradleProject(path) + .setJavaProject(JavaModels.JavaProject.getDefaultInstance()) + .build(), + ) + } + + private fun gradleProject(path: String): GradleModels.GradleProject.Builder { + val dir = moduleDir(path) + return GradleModels.GradleProject + .newBuilder() + .setName(path.trimStart(':')) + .setPath(path) + .setProjectDirPath(dir.path) + .setBuildDirPath(File(dir, "build").path) + .setBuildScriptPath(File(dir, "build.gradle").path) + } + + /** Installs [modules] under a root project as the project manager's workspace. */ + fun installWorkspace(vararg modules: ModuleProject): Workspace { + val root = GradleProject(gradleProject(":").setName("root").build()) + val workspace = Workspace(rootProject = root, subProjects = modules.toList(), syncIssues = emptyList()) + ProjectManagerImpl.getInstance().workspace = workspace + return workspace + } + + private companion object { + const val VARIANT = "debug" + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/RecordingMemprofSink.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/RecordingMemprofSink.kt new file mode 100644 index 0000000000..4ddaa18f5f --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/RecordingMemprofSink.kt @@ -0,0 +1,49 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.itsaky.androidide.memprof.MemprofSink +import com.itsaky.androidide.memprof.MemprofSpan +import java.util.concurrent.CopyOnWriteArrayList + +/** A [MemprofSink] that keeps every phase it was asked to begin. */ +class RecordingMemprofSink : MemprofSink { + /** A phase begun on this sink, with the details put on it and how it finished. */ + class Phase( + val title: String, + val marker: String, + ) : MemprofSpan { + val details = mutableMapOf() + var ended = false + var abandoned = false + + override val isRecording = true + + override fun put( + key: String, + value: Long, + ) { + details[key] = value + } + + override fun end() { + if (!abandoned) ended = true + } + + override fun abandon() { + if (!ended) abandoned = true + } + } + + val phases = CopyOnWriteArrayList() + + override fun beginPhase( + title: String, + marker: String, + ): MemprofSpan = Phase(title, marker).also { phases += it } + + override fun beginSection( + name: String, + detail: String?, + ): MemprofSpan = MemprofSpan.None + + override fun mark(marker: String) = Unit +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/TrieCorpora.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/TrieCorpora.kt new file mode 100644 index 0000000000..9f111d4a8c --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/TrieCorpora.kt @@ -0,0 +1,29 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertWithMessage +import java.io.File + +/** + * Returns the JAR on the test classpath that [anchor] was loaded from, failing the test when it + * does not come from a JAR. + */ +fun testClasspathJar(anchor: Class<*>): File { + val location = + File( + anchor.protectionDomain.codeSource.location + .toURI(), + ) + assertWithMessage("${anchor.name} must be loaded from a JAR on the test classpath, got $location") + .that(location.isFile && location.name.endsWith(".jar")) + .isTrue() + return location +} + +/** + * Returns whether [qualifiedName] is a fragment of a `@JvmMultifileClass` facade, e.g. + * `LazyKt__LazyJVMKt`. + * + * The trie held these and the index deliberately withholds them: the declarations belong to the + * facade, which is indexed, and neither Java nor Kotlin names the part. + */ +fun isMultiFilePart(qualifiedName: String) = qualifiedName.substringAfterLast('.').contains("__") diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt index a4b3193c3f..edcc759a86 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt @@ -195,9 +195,11 @@ class CollectUnimportedSymbolMatchesTest { }, ) - // The same row (same source and key) is a match in both the exact and prefix stages; once - // `accept` has turned it down, asking again wastes the visibility check and item-building - // work it would otherwise repeat. + /* + * The same row (same source and key) is a match in both the exact and prefix stages; once + * `accept` has turned it down, asking again wastes the visibility check and item-building + * work it would otherwise repeat. + */ assertThat(acceptCalls).isEqualTo(1) } diff --git a/lsp/xml/build.gradle.kts b/lsp/xml/build.gradle.kts index b7d944882e..11d6375667 100644 --- a/lsp/xml/build.gradle.kts +++ b/lsp/xml/build.gradle.kts @@ -44,6 +44,7 @@ dependencies { implementation(projects.actions) implementation(projects.lsp.api) + implementation(projects.lsp.jvmSymbolIndex) implementation(projects.lexers) implementation(projects.subprojects.xmlDom) implementation(projects.subprojects.xmlUtils) diff --git a/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/ClasspathLookup.kt b/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/ClasspathLookup.kt new file mode 100644 index 0000000000..bd781d6e3c --- /dev/null +++ b/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/ClasspathLookup.kt @@ -0,0 +1,52 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.xml.providers.completion.layout + +import com.itsaky.androidide.projects.ProjectManagerImpl +import com.itsaky.androidide.projects.api.ModuleProject +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup + +/** + * The package and class children a layout tag completer reads from a module's compile classpath. + * + * Narrows [ModuleClasspathLookup] to what the completers need, so a test can supply a fake without + * a real module or a registered index. + */ +internal interface ClasspathLookup { + /** Returns the direct subpackages of [packageName], then its top-level classes. */ + fun children(packageName: String): List + + /** Returns whether a top-level class of the classpath is in the package [name] or a subpackage. */ + fun isPackage(name: String): Boolean + + /** Returns whether [qualifiedName] names a top-level class of the classpath. */ + fun isClass(qualifiedName: String): Boolean +} + +/** Builds the classpath lookup for [module] from the indexes currently registered for its project. */ +internal fun classpathLookupOf(module: ModuleProject): ClasspathLookup { + val lookup = + ModuleClasspathLookup.of(module, ProjectManagerImpl.getInstance().indexingServiceManager.registry) + return object : ClasspathLookup { + override fun children(packageName: String) = lookup.children(packageName) + + override fun isPackage(name: String) = lookup.isPackage(name) + + override fun isClass(qualifiedName: String) = lookup.isClass(qualifiedName) + } +} diff --git a/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/QualifiedTagCompleter.kt b/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/QualifiedTagCompleter.kt index b6a9c82197..186cffdcdd 100644 --- a/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/QualifiedTagCompleter.kt +++ b/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/QualifiedTagCompleter.kt @@ -37,58 +37,79 @@ import org.eclipse.lemminx.dom.DOMDocument * * @author Akash Yadav */ -class QualifiedTagCompleter(provider: ICompletionProvider) : LayoutTagCompletionProvider(provider) { +class QualifiedTagCompleter + internal constructor( + provider: ICompletionProvider, + private val lookupFactory: (ModuleProject) -> ClasspathLookup = ::classpathLookupOf, + ) : LayoutTagCompletionProvider(provider) { + override fun doComplete( + params: CompletionParams, + pathData: ResourcePathData, + document: DOMDocument, + type: NodeType, + prefix: String, + ): CompletionResult { + val result = mutableListOf() + val (widgets, module) = doLookup() + var fqn = prefix + if (prefix.endsWith('.')) { + fqn = fqn.substringBeforeLast('.') + } - override fun doComplete( - params: CompletionParams, - pathData: ResourcePathData, - document: DOMDocument, - type: NodeType, - prefix: String - ): CompletionResult { - val result = mutableListOf() - val (widgets, module) = doLookup() - var fqn = prefix - if (prefix.endsWith('.')) { - fqn = fqn.substringBeforeLast('.') - } + widgets.getNode(name = fqn, createIfNotPresent = false)?.children?.values?.forEach { + val qualifiedName = "$fqn.${it.name}" + val match = match(it.name, qualifiedName, prefix) + result.add(createTagCompletionItem(it.name, qualifiedName, match)) + } - widgets.getNode(name = fqn, createIfNotPresent = false)?.children?.values?.forEach { - val qualifiedName = "$fqn.${it.name}" - val match = match(it.name, qualifiedName, prefix) - result.add(createTagCompletionItem(it.name, qualifiedName, match)) - } + addFromLookup(lookupFactory(module), fqn, prefix, result) + addFromTrie(module.compileJavaSourceClasses, fqn, prefix, result) - addFromTrie(module.compileClasspathClasses, fqn, prefix, result) - addFromTrie(module.compileJavaSourceClasses, fqn, prefix, result) + return CompletionResult(result) + } - return CompletionResult(result) - } + private fun addFromLookup( + lookup: ClasspathLookup, + fqn: String, + prefix: String, + result: MutableList, + ) { + val packageName = if (lookup.isPackage(fqn) || lookup.isClass(fqn)) fqn else fqn.substringBeforeLast('.') - private fun addFromTrie( - trie: ClassTrie, - fqn: String, - prefix: String, - result: MutableList - ) { - val node = trie.findNode(fqn) ?: trie.findNode(fqn.substringBeforeLast('.')) ?: return - node.children.values.forEach { - val match = match(it.name, it.qualifiedName, prefix) - if (match == NO_MATCH) { - return@forEach - } - - result.add(createTagCompletionItem(it.name, it.qualifiedName, match)) - } - } + lookup.children(packageName).forEach { + val match = match(it.name, it.qualifiedName, prefix) + if (match == NO_MATCH) { + return@forEach + } - private fun doLookup(): Pair { - val widgets = - Lookup.getDefault().lookup(WidgetTable.COMPLETION_LOOKUP_KEY) - ?: throw IllegalStateException("No widget table provided") - val module = - Lookup.getDefault().lookup(ModuleProject.COMPLETION_MODULE_KEY) - ?: throw IllegalStateException("No module project provided") - return widgets as DefaultWidgetTable to module - } -} + result.add(createTagCompletionItem(it.name, it.qualifiedName, match)) + } + } + + private fun addFromTrie( + trie: ClassTrie, + fqn: String, + prefix: String, + result: MutableList, + ) { + val node = trie.findNode(fqn) ?: trie.findNode(fqn.substringBeforeLast('.')) ?: return + node.children.values.forEach { + val match = match(it.name, it.qualifiedName, prefix) + if (match == NO_MATCH) { + return@forEach + } + + result.add(createTagCompletionItem(it.name, it.qualifiedName, match)) + } + } + + private fun doLookup(): Pair { + val widgets = + Lookup.getDefault().lookup(WidgetTable.COMPLETION_LOOKUP_KEY) + ?: throw IllegalStateException("No widget table provided") + val module = + Lookup.getDefault().lookup(ModuleProject.COMPLETION_MODULE_KEY) + ?: throw IllegalStateException("No module project provided") + return widgets as DefaultWidgetTable to module + } + } diff --git a/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/SimpleTagCompleter.kt b/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/SimpleTagCompleter.kt index 8b03d222a2..eedb8a22f1 100644 --- a/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/SimpleTagCompleter.kt +++ b/lsp/xml/src/main/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/SimpleTagCompleter.kt @@ -27,6 +27,7 @@ import com.itsaky.androidide.lsp.xml.utils.XmlUtils.NodeType import com.itsaky.androidide.projects.api.ModuleProject import com.itsaky.androidide.utils.ClassTrie import com.itsaky.androidide.xml.widgets.WidgetTable +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup import org.eclipse.lemminx.dom.DOMDocument /** @@ -34,41 +35,59 @@ import org.eclipse.lemminx.dom.DOMDocument * * @author Akash Yadav */ -class SimpleTagCompleter(provider: ICompletionProvider) : LayoutTagCompletionProvider(provider) { +class SimpleTagCompleter + internal constructor( + provider: ICompletionProvider, + private val lookupFactory: (ModuleProject) -> ClasspathLookup = ::classpathLookupOf, + ) : LayoutTagCompletionProvider(provider) { + override fun doComplete( + params: CompletionParams, + pathData: ResourcePathData, + document: DOMDocument, + type: NodeType, + prefix: String, + ): CompletionResult { + val widgets = + Lookup.getDefault().lookup(WidgetTable.COMPLETION_LOOKUP_KEY)?.getAllWidgets() + ?: return CompletionResult.EMPTY + val result = mutableListOf() - override fun doComplete( - params: CompletionParams, - pathData: ResourcePathData, - document: DOMDocument, - type: NodeType, - prefix: String - ): CompletionResult { - val widgets = - Lookup.getDefault().lookup(WidgetTable.COMPLETION_LOOKUP_KEY)?.getAllWidgets() - ?: return CompletionResult.EMPTY - val result = mutableListOf() + // Complete all tags which do not require fully qualified name + for (widget in widgets) { + val match = matchLevel(widget.simpleName, prefix) + result.add(createTagCompletionItem(widget.simpleName, widget.qualifiedName, match, true)) + } - // Complete all tags which do not require fully qualified name - for (widget in widgets) { - val match = matchLevel(widget.simpleName, prefix) - result.add(createTagCompletionItem(widget.simpleName, widget.qualifiedName, match, true)) - } + // Complete the root package names if possible + val module = + Lookup.getDefault().lookup(ModuleProject.COMPLETION_MODULE_KEY) ?: return CompletionResult(result) - // Complete the root package names if possible - val module = - Lookup.getDefault().lookup(ModuleProject.COMPLETION_MODULE_KEY) ?: return CompletionResult(result) + // Add root packages from the compile classpath and source paths + addFromLookup(lookupFactory(module).children(""), prefix, result) + addFromTrie(module.compileJavaSourceClasses, prefix, result) - // Add root packages from the compile classpath and source paths - addFromTrie(module.compileClasspathClasses, prefix, result) - addFromTrie(module.compileJavaSourceClasses, prefix, result) + return CompletionResult(result) + } - return CompletionResult(result) - } + private fun addFromLookup( + children: List, + prefix: String, + result: MutableList, + ) { + children.forEach { + val match = matchLevel(it.name, prefix) + result.add(createTagCompletionItem(it.name, it.qualifiedName, match)) + } + } - private fun addFromTrie(trie: ClassTrie, prefix: String, result: MutableList) { - trie.root.children.values.forEach { - val match = matchLevel(it.name, prefix) - result.add(createTagCompletionItem(it.name, it.qualifiedName, match)) - } - } -} + private fun addFromTrie( + trie: ClassTrie, + prefix: String, + result: MutableList, + ) { + trie.root.children.values.forEach { + val match = matchLevel(it.name, prefix) + result.add(createTagCompletionItem(it.name, it.qualifiedName, match)) + } + } + } diff --git a/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/FakeClasspathLookup.kt b/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/FakeClasspathLookup.kt new file mode 100644 index 0000000000..ca6d7b112e --- /dev/null +++ b/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/FakeClasspathLookup.kt @@ -0,0 +1,56 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.xml.providers.completion.layout + +import com.itsaky.androidide.project.GradleModels +import com.itsaky.androidide.project.JavaModels +import com.itsaky.androidide.projects.api.JavaModule +import com.itsaky.androidide.projects.api.ModuleProject +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup + +/** A hand-built [ClasspathLookup] a test can shape without a real module or index. */ +class FakeClasspathLookup( + private val childrenByPackage: Map> = emptyMap(), + private val packages: Set = emptySet(), + private val classes: Set = emptySet(), +) : ClasspathLookup { + override fun children(packageName: String): List = childrenByPackage[packageName] ?: emptyList() + + override fun isPackage(name: String): Boolean = name in packages + + override fun isClass(qualifiedName: String): Boolean = qualifiedName in classes +} + +/** + * Builds a bare [ModuleProject] a test can register in [com.itsaky.androidide.lookup.Lookup], + * with no real Gradle sync or disk I/O. Its own classpath is never read: a test that needs one + * supplies a [ClasspathLookup] fake instead. + */ +fun fakeModuleProject(path: String = ":app"): ModuleProject { + val delegate = + GradleModels.GradleProject + .newBuilder() + .setName(path.trimStart(':')) + .setPath(path) + .setProjectDirPath("/fake$path") + .setBuildDirPath("/fake$path/build") + .setBuildScriptPath("/fake$path/build.gradle") + .setJavaProject(JavaModels.JavaProject.getDefaultInstance()) + .build() + return JavaModule(delegate) +} diff --git a/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/QualifiedTagCompleterTest.kt b/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/QualifiedTagCompleterTest.kt new file mode 100644 index 0000000000..6904e7d01d --- /dev/null +++ b/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/QualifiedTagCompleterTest.kt @@ -0,0 +1,150 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.xml.providers.completion.layout + +import com.android.aaptcompiler.extractPathData +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.lookup.Lookup +import com.itsaky.androidide.lsp.api.ICompletionProvider +import com.itsaky.androidide.lsp.models.CompletionItem +import com.itsaky.androidide.lsp.models.CompletionParams +import com.itsaky.androidide.lsp.models.CompletionResult +import com.itsaky.androidide.lsp.xml.utils.XmlUtils.NodeType.TAG +import com.itsaky.androidide.models.Position +import com.itsaky.androidide.progress.ICancelChecker +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.xml.internal.widgets.DefaultWidgetTable +import com.itsaky.androidide.xml.widgets.WidgetTable +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child +import org.eclipse.lemminx.dom.DOMParser +import org.eclipse.lemminx.uriresolver.URIResolverExtensionManager +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File + +/** + * [QualifiedTagCompleter] reads a package's children from the injected [ClasspathLookup]: the + * package or class named by the prefix if it is one, otherwise its parent. + */ +@RunWith(RobolectricTestRunner::class) +class QualifiedTagCompleterTest { + @get:Rule + val temp = TemporaryFolder() + + @Before + fun setup() { + Lookup.getDefault().update(WidgetTable.COMPLETION_LOOKUP_KEY, DefaultWidgetTable()) + } + + @Test + fun `a package fqn offers its subpackages and classes`() { + val fake = + FakeClasspathLookup( + packages = setOf("android.widget"), + childrenByPackage = + mapOf( + "android.widget" to + listOf( + Child("inner", "android.widget.inner", isClass = false), + Child("Button", "android.widget.Button", isClass = true), + ), + ), + ) + + val items = complete(fake, prefix = "android.widget.") + + assertThat(items.map { it.detail }) + .containsAtLeast("android.widget.inner", "android.widget.Button") + } + + @Test + fun `a class fqn offers nothing from the classpath`() { + // The parent package has children of its own: a wrong fall-back to it (instead of + // recognizing "android.widget.Button" as a class) would leak "android.widget.Other" here. + val fake = + FakeClasspathLookup( + classes = setOf("android.widget.Button"), + childrenByPackage = + mapOf( + "android.widget" to listOf(Child("Other", "android.widget.Other", isClass = true)), + ), + ) + + val items = complete(fake, prefix = "android.widget.Button") + + assertThat(items).isEmpty() + } + + @Test + fun `an fqn that is neither a package nor a class falls back to its parent`() { + val fake = + FakeClasspathLookup( + packages = setOf("android.widget"), + childrenByPackage = + mapOf( + "android.widget" to listOf(Child("Button", "android.widget.Button", isClass = true)), + ), + ) + + // "android.widget.But" is a partial simple name, so it is neither a package nor a class + // itself: the completer must fall back to the parent package's children. + val items = complete(fake, prefix = "android.widget.But") + + assertThat(items.map { it.detail }).contains("android.widget.Button") + } + + @Test + fun `a package only the index holds is offered`() { + // Regression: a trie built only from the module's classpath JARs never held this name. + val fake = + FakeClasspathLookup( + packages = setOf("only"), + childrenByPackage = + mapOf("only" to listOf(Child("inindex", "only.inindex", isClass = false))), + ) + + val items = complete(fake, prefix = "only.") + + assertThat(items.map { it.detail }).contains("only.inindex") + } + + private fun complete( + lookup: ClasspathLookup, + prefix: String, + ): List { + Lookup.getDefault().update(ModuleProject.COMPLETION_MODULE_KEY, fakeModuleProject()) + + val provider = + object : ICompletionProvider { + override fun complete(params: CompletionParams): CompletionResult = CompletionResult.EMPTY + } + val completer = QualifiedTagCompleter(provider) { lookup } + + val layoutDir = temp.newFolder("layout") + val file = File(layoutDir, "test.xml").apply { writeText("") } + val params = CompletionParams(Position(0, 0, 1), file.toPath(), ICancelChecker.NOOP) + val pathData = extractPathData(file) + val document = DOMParser.getInstance().parse("", "test.xml", URIResolverExtensionManager()) + + return completer.complete(params, pathData, document, TAG, prefix).items + } +} diff --git a/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/SimpleTagCompleterTest.kt b/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/SimpleTagCompleterTest.kt new file mode 100644 index 0000000000..ce116014e6 --- /dev/null +++ b/lsp/xml/src/test/java/com/itsaky/androidide/lsp/xml/providers/completion/layout/SimpleTagCompleterTest.kt @@ -0,0 +1,100 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.lsp.xml.providers.completion.layout + +import com.android.aaptcompiler.extractPathData +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.lookup.Lookup +import com.itsaky.androidide.lsp.api.ICompletionProvider +import com.itsaky.androidide.lsp.models.CompletionItem +import com.itsaky.androidide.lsp.models.CompletionParams +import com.itsaky.androidide.lsp.models.CompletionResult +import com.itsaky.androidide.lsp.xml.utils.XmlUtils.NodeType.TAG +import com.itsaky.androidide.models.Position +import com.itsaky.androidide.progress.ICancelChecker +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.xml.internal.widgets.DefaultWidgetTable +import com.itsaky.androidide.xml.widgets.WidgetTable +import org.appdevforall.codeonthego.indexing.jvm.ModuleClasspathLookup.Child +import org.eclipse.lemminx.dom.DOMParser +import org.eclipse.lemminx.uriresolver.URIResolverExtensionManager +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.io.File + +/** [SimpleTagCompleter] reads root package/class names from the injected [ClasspathLookup]. */ +@RunWith(RobolectricTestRunner::class) +class SimpleTagCompleterTest { + @get:Rule + val temp = TemporaryFolder() + + @Before + fun setup() { + Lookup.getDefault().update(WidgetTable.COMPLETION_LOOKUP_KEY, DefaultWidgetTable()) + } + + @Test + fun `root package names from the lookup are offered`() { + val fake = + FakeClasspathLookup( + childrenByPackage = mapOf("" to listOf(Child("com", "com", isClass = false))), + ) + + val items = complete(fake, prefix = "") + + assertThat(items.map { it.detail }).contains("com") + } + + @Test + fun `a package only the index holds is offered`() { + // Regression: a trie built only from the module's classpath JARs never held this name. + val fake = + FakeClasspathLookup( + childrenByPackage = mapOf("" to listOf(Child("onlyinindex", "onlyinindex", isClass = false))), + ) + + val items = complete(fake, prefix = "") + + assertThat(items.map { it.detail }).contains("onlyinindex") + } + + private fun complete( + lookup: ClasspathLookup, + prefix: String, + ): List { + Lookup.getDefault().update(ModuleProject.COMPLETION_MODULE_KEY, fakeModuleProject()) + + val provider = + object : ICompletionProvider { + override fun complete(params: CompletionParams): CompletionResult = CompletionResult.EMPTY + } + val completer = SimpleTagCompleter(provider) { lookup } + + val layoutDir = temp.newFolder("layout") + val file = File(layoutDir, "test.xml").apply { writeText("") } + val params = CompletionParams(Position(0, 0, 1), file.toPath(), ICancelChecker.NOOP) + val pathData = extractPathData(file) + val document = DOMParser.getInstance().parse("", "test.xml", URIResolverExtensionManager()) + + return completer.complete(params, pathData, document, TAG, prefix).items + } +} diff --git a/resources/src/main/res/values/strings.xml b/resources/src/main/res/values/strings.xml index 05ed0eafe8..ca6abca880 100644 --- a/resources/src/main/res/values/strings.xml +++ b/resources/src/main/res/values/strings.xml @@ -98,6 +98,10 @@ No references found No imports found The file changed. Try importing again. + + No importable class named \'%1$s\'. + + No importable class named \'%1$s\'. Libraries are still being indexed. Diagnostics Installation failed Failed to install assets @@ -1275,6 +1279,8 @@ Quick Run failed. Building… Installing plugin… + + Indexing classes (%1$d/%2$d) Install Plugin Plugin Already Installed \'%1$s\' v%2$s is already installed. Replace it with v%3$s? diff --git a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt index 389d9b928e..fa43658911 100644 --- a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt +++ b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt @@ -79,18 +79,23 @@ import kotlin.io.path.pathString class ProjectManagerImpl : IProjectManager, EventReceiver { + private val indexingServiceManagerLock = Any() private var _indexingServiceManager: IndexingServiceManager? = null lateinit var projectPath: String + /** + * The indexing service manager of the current project session, created on first access after + * construction or [destroy]. + * + * Read from language-server worker threads as well as the main thread, so creation is guarded by + * a lock: two racing first reads must not each create, and leak, a manager. + */ val indexingServiceManager: IndexingServiceManager - get() { - if (_indexingServiceManager == null) { - _indexingServiceManager = IndexingServiceManager() + get() = + synchronized(indexingServiceManagerLock) { + _indexingServiceManager ?: IndexingServiceManager().also { _indexingServiceManager = it } } - return _indexingServiceManager!! - } - @Volatile internal var pluginProjectCached: Boolean? = null @@ -190,37 +195,6 @@ class ProjectManagerImpl : jobs.toList().awaitAll() } } - - reportUnreadableClasspathJars(workspace) - } - - /** - * Surface any classpath JARs that were corrupt/unreadable during indexing (e.g. a truncated - * download or incomplete offline provisioning) to the user — naming the offending dependency and - * offering a recovery path (re-sync) — instead of silently dropping its code-completion symbols. - */ - private fun reportUnreadableClasspathJars(workspace: Workspace) { - val names = - workspace.subProjects - .filterIsInstance() - .flatMap { it.unreadableClasspathJars } - .map { it.name } - .distinct() - if (names.isEmpty()) { - return - } - - log.warn("Skipped {} unreadable classpath JAR(s) during indexing: {}", names.size, names) - - val context = BaseApplication.baseInstance - val shown = names.take(3).joinToString(", ") - val list = - if (names.size > 3) { - context.getString(R.string.msg_unreadable_classpath_jars_overflow, shown, names.size - 3) - } else { - shown - } - flashError(context.getString(R.string.msg_unreadable_classpath_jars, list)) } override fun getAndroidModules(): List { @@ -291,8 +265,13 @@ class ProjectManagerImpl : this.workspace = null pluginProjectCached = null - _indexingServiceManager?.close() - _indexingServiceManager = null + // Closed outside the lock: close() blocks until every service has shut down, and a worker + // thread reading the manager meanwhile must not wait on that. + val indexingServiceManager = + synchronized(indexingServiceManagerLock) { + _indexingServiceManager.also { _indexingServiceManager = null } + } + indexingServiceManager?.close() (this.androidBuildVariants as? MutableMap?)?.clear() } @@ -518,3 +497,30 @@ class ProjectManagerImpl : } } } + +private val unreadableClasspathJarsLog = LoggerFactory.getLogger("UnreadableClasspathJars") + +/** + * Surfaces unreadable classpath JARs [names] to the user - naming the offending dependency and + * offering a recovery path (re-sync) - instead of silently dropping its code-completion symbols. + * + * Called once per indexing pass with the names of every JAR that pass could not open; a no-op for + * an empty list. + */ +fun reportUnreadableClasspathJars(names: List) { + if (names.isEmpty()) { + return + } + + unreadableClasspathJarsLog.warn("Skipped {} unreadable classpath JAR(s) during indexing: {}", names.size, names) + + val context = BaseApplication.baseInstance + val shown = names.take(3).joinToString(", ") + val list = + if (names.size > 3) { + context.getString(R.string.msg_unreadable_classpath_jars_overflow, shown, names.size - 3) + } else { + shown + } + flashError(context.getString(R.string.msg_unreadable_classpath_jars, list)) +} diff --git a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/JavaModule.kt b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/JavaModule.kt index 556696ddf9..9732a9a3c2 100644 --- a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/JavaModule.kt +++ b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/JavaModule.kt @@ -46,18 +46,24 @@ class JavaModule( } } - private val classesJar by lazy { - var jar = File(delegate.buildDir, "libs/${delegate.name}.jar") - if (jar.exists()) { - return@lazy jar + /** + * Returns this module's built JAR under `build/libs`, or a non-existent sentinel file when it + * has not been built. + * + * Resolved on every call, since a build can create the JAR after the first lookup. Stats the + * filesystem, so never call it on the main thread. + */ + fun getClassesJar(): File { + val libs = File(delegate.buildDir, "libs") + val named = File(libs, "${delegate.name}.jar") + if (named.exists()) { + return named } - jar = File(delegate.buildDir, "libs") + return libs .listFiles() - ?.first { delegate.name?.let(it.name::startsWith) ?: false } + ?.firstOrNull { delegate.name?.let(it.name::startsWith) ?: false } ?: File("module-jar-does-not-exist.jar") - - return@lazy jar } override fun isInitialized(): Boolean = super.isInitialized() @@ -84,7 +90,7 @@ class JavaModule( return dirs } - override fun getModuleClasspaths(): Set = mutableSetOf(classesJar) + override fun getModuleClasspaths(): Set = mutableSetOf(getClassesJar()) override fun getCompileClasspaths( excludeSourceGeneratedClassPath: Boolean, diff --git a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/ModuleProject.kt b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/ModuleProject.kt index 775db620c0..f54eda8814 100644 --- a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/ModuleProject.kt +++ b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/ModuleProject.kt @@ -225,16 +225,6 @@ abstract class ModuleProject( indexClasspaths() } - /** - * Classpath JARs skipped during the most recent [indexClasspaths] because they were - * corrupt/unreadable (e.g. a truncated download / incomplete offline provisioning). Aggregated by - * the project setup after indexing so the user can be told which dependency failed and offered a - * recovery path (re-sync). Empty when everything indexed cleanly. - */ - @Volatile - var unreadableClasspathJars: List = emptyList() - private set - @RestrictTo(RestrictTo.Scope.LIBRARY_GROUP_PREFIX) fun indexClasspaths() { this.compileClasspathClasses.clear() @@ -251,7 +241,6 @@ abstract class ModuleProject( val reader = JarFsClasspathReader() val topLevelClasses = reader.listClasses(paths).filter { it.isTopLevel } topLevelClasses.forEach { this.compileClasspathClasses.append(it.name) } - unreadableClasspathJars = reader.unreadableJars.toList() watch.log() log.debug("Found {} classpaths.", topLevelClasses.size) diff --git a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/Workspace.kt b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/Workspace.kt index cdd9c6611d..fbd063dfa4 100644 --- a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/Workspace.kt +++ b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/api/Workspace.kt @@ -89,7 +89,7 @@ data class Workspace( file: File, checkExistence: Boolean = false, ): ModuleProject? { - if (!file.exists() && checkExistence) { + if (checkExistence && !file.exists()) { return null } diff --git a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/classpath/JarFsClasspathReader.kt b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/classpath/JarFsClasspathReader.kt index f203c2d97d..3156eda1ba 100644 --- a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/classpath/JarFsClasspathReader.kt +++ b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/classpath/JarFsClasspathReader.kt @@ -35,93 +35,93 @@ import kotlin.io.path.pathString /** @author Akash Yadav */ class JarFsClasspathReader : IClasspathReader { - - companion object { - private val log = LoggerFactory.getLogger(JarFsClasspathReader::class.java) - } - - private val _unreadableJars = mutableListOf() - - /** - * JARs skipped during the most recent [listClasses] call because they were corrupt or could not - * be read (e.g. a truncated download / incomplete offline provisioning). Exposed so a caller can - * surface them to the user and offer a recovery path, instead of silently dropping that library's - * symbols. Reset at the start of every [listClasses] call. - */ - val unreadableJars: List - get() = _unreadableJars.toList() - - /** - * Lists the classes contained in the given JAR files. Any JAR that is corrupt or cannot be read is - * skipped (rather than aborting the whole scan) and recorded in [unreadableJars] for reporting. - * [unreadableJars] is reset at the start of each call. - */ - override fun listClasses(files: Collection): ImmutableSet { - _unreadableJars.clear() - val builder = ImmutableSet.builder() - for (path in files.map(File::toPath)) { - if (!Files.exists(path)) { - continue - } - - try { - val fs = CachingJarFileSystemProvider.newFileSystem(path) as CachedJarFileSystem - for (rootDirectory in fs.rootDirectories) { - Files.walkFileTree( - rootDirectory, - emptySet(), - Int.MAX_VALUE, - object : SimpleFileVisitor() { - - override fun preVisitDirectory( - dir: Path?, - attrs: BasicFileAttributes? - ): FileVisitResult { - return if (fs.storeJARPackageDir(dir)) { - CONTINUE - } else { - SKIP_SUBTREE - } - } - - override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { - var name = file.pathString - if (name.endsWith("/package-info.class") || !name.endsWith(".class")) { - return CONTINUE - } - - name = name.substringBeforeLast(".class") - - if (name.isBlank()) { - return CONTINUE - } - - if (name.startsWith('/')) { - name = name.substring(1) - } - - if (name.contains('/')) { - name = name.replace('/', '.') - } - - ClassInfo.create(name)?.also { - builder.add(it) - } - - return super.visitFile(file, attrs) - } - } - ) - } - } catch (e: ZipException) { - // A corrupt or truncated JAR must not abort indexing of the remaining classpath entries. - log.warn("Skipping corrupt/unreadable JAR while indexing classpath: {}", path, e) - _unreadableJars.add(path.toFile()) - } catch (e: IOException) { - log.warn("Skipping JAR that could not be read while indexing classpath: {}", path, e) - _unreadableJars.add(path.toFile()) - } - } - return builder.build() - } + companion object { + private val log = LoggerFactory.getLogger(JarFsClasspathReader::class.java) + } + + private val _unreadableJars = mutableListOf() + + /** + * JARs skipped during the most recent [listClasses] call because they were corrupt or could not + * be read (e.g. a truncated download / incomplete offline provisioning). Not reported to the + * user: the JAR indexing pass in `:lsp:jvm-symbol-index` is what surfaces an unreadable JAR now, + * from its own scan of the same JARs. Reset at the start of every [listClasses] call. + */ + val unreadableJars: List + get() = _unreadableJars.toList() + + /** + * Lists the classes contained in the given JAR files. Any JAR that is corrupt or cannot be read is + * skipped (rather than aborting the whole scan) and recorded in [unreadableJars]. [unreadableJars] + * is reset at the start of each call. + */ + override fun listClasses(files: Collection): ImmutableSet { + _unreadableJars.clear() + val builder = ImmutableSet.builder() + for (path in files.map(File::toPath)) { + if (!Files.exists(path)) { + continue + } + + try { + val fs = CachingJarFileSystemProvider.newFileSystem(path) as CachedJarFileSystem + for (rootDirectory in fs.rootDirectories) { + Files.walkFileTree( + rootDirectory, + emptySet(), + Int.MAX_VALUE, + object : SimpleFileVisitor() { + override fun preVisitDirectory( + dir: Path?, + attrs: BasicFileAttributes?, + ): FileVisitResult = + if (fs.storeJARPackageDir(dir)) { + CONTINUE + } else { + SKIP_SUBTREE + } + + override fun visitFile( + file: Path, + attrs: BasicFileAttributes, + ): FileVisitResult { + var name = file.pathString + if (name.endsWith("/package-info.class") || !name.endsWith(".class")) { + return CONTINUE + } + + name = name.substringBeforeLast(".class") + + if (name.isBlank()) { + return CONTINUE + } + + if (name.startsWith('/')) { + name = name.substring(1) + } + + if (name.contains('/')) { + name = name.replace('/', '.') + } + + ClassInfo.create(name)?.also { + builder.add(it) + } + + return super.visitFile(file, attrs) + } + }, + ) + } + } catch (e: ZipException) { + // A corrupt or truncated JAR must not abort indexing of the remaining classpath entries. + log.warn("Skipping corrupt/unreadable JAR while indexing classpath: {}", path, e) + _unreadableJars.add(path.toFile()) + } catch (e: IOException) { + log.warn("Skipping JAR that could not be read while indexing classpath: {}", path, e) + _unreadableJars.add(path.toFile()) + } + } + return builder.build() + } } diff --git a/subprojects/projects/src/test/java/com/itsaky/androidide/projects/ProjectManagerImplIndexingServiceManagerTest.kt b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/ProjectManagerImplIndexingServiceManagerTest.kt new file mode 100644 index 0000000000..ff7cd1ad41 --- /dev/null +++ b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/ProjectManagerImplIndexingServiceManagerTest.kt @@ -0,0 +1,58 @@ +package com.itsaky.androidide.projects + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.service.IndexingServiceManager +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import java.util.Collections +import java.util.IdentityHashMap +import java.util.concurrent.CyclicBarrier +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** The indexing service manager is read from worker threads, so racing first reads share one instance. */ +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE) +class ProjectManagerImplIndexingServiceManagerTest { + @Test + fun `racing first reads all get the same indexing service manager`() { + val seen = Collections.newSetFromMap(IdentityHashMap()) + repeat(ROUNDS) { + val manager = ProjectManagerImpl() + val barrier = CyclicBarrier(THREADS) + val pool = Executors.newFixedThreadPool(THREADS) + val reads = + (1..THREADS).map { + pool.submit { + barrier.await() + manager.indexingServiceManager + } + } + val managers = reads.map { it.get(10, TimeUnit.SECONDS) } + pool.shutdown() + + seen.clear() + seen.addAll(managers) + assertThat(seen).hasSize(1) + manager.destroy() + } + } + + @Test + fun `a read after destroy gets a new indexing service manager`() { + val manager = ProjectManagerImpl() + val first = manager.indexingServiceManager + + manager.destroy() + + assertThat(manager.indexingServiceManager).isNotSameInstanceAs(first) + manager.destroy() + } + + private companion object { + const val ROUNDS = 50 + const val THREADS = 16 + } +} diff --git a/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/JavaModuleClassesJarTest.kt b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/JavaModuleClassesJarTest.kt new file mode 100644 index 0000000000..9f85d2a6bb --- /dev/null +++ b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/JavaModuleClassesJarTest.kt @@ -0,0 +1,53 @@ +package com.itsaky.androidide.projects.api + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.project.GradleModels +import com.itsaky.androidide.project.JavaModels +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File + +@RunWith(JUnit4::class) +class JavaModuleClassesJarTest { + @get:Rule + val temp = TemporaryFolder() + + private fun javaModule(buildDir: File): JavaModule = + JavaModule( + GradleModels.GradleProject + .newBuilder() + .setName("lib") + .setPath(":lib") + .setProjectDirPath(buildDir.parentFile.absolutePath) + .setBuildDirPath(buildDir.absolutePath) + .setBuildScriptPath(File(buildDir.parentFile, "build.gradle").absolutePath) + .setJavaProject(JavaModels.JavaProject.getDefaultInstance()) + .build(), + ) + + @Test + fun `a libs directory without a matching jar resolves to the missing-jar sentinel`() { + val buildDir = temp.newFolder("lib", "build") + File(buildDir, "libs").mkdirs() + File(buildDir, "libs/other.jar").writeBytes(ByteArray(1)) + + val jar = javaModule(buildDir).getClassesJar() + + assertThat(jar.exists()).isFalse() + } + + @Test + fun `a jar built after the first lookup is found by the next one`() { + val buildDir = temp.newFolder("lib", "build") + val module = javaModule(buildDir) + module.getClassesJar() + + val built = File(buildDir, "libs/lib-1.0.jar").apply { parentFile.mkdirs() } + built.writeBytes(ByteArray(1)) + + assertThat(module.getClassesJar()).isEqualTo(built) + } +} diff --git a/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/ModuleOutputClasspathDeltaTest.kt b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/ModuleOutputClasspathDeltaTest.kt new file mode 100644 index 0000000000..330761ba94 --- /dev/null +++ b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/ModuleOutputClasspathDeltaTest.kt @@ -0,0 +1,204 @@ +package com.itsaky.androidide.projects.api + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.project.AndroidModels +import com.itsaky.androidide.project.GradleModels +import com.itsaky.androidide.projects.ProjectManagerImpl +import io.mockk.every +import io.mockk.spyk +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import java.io.File + +/** + * Pins the classpath delta the module-output symbol index has to cover. + * + * Per-module class lookup used to cover `getCompileClasspaths()`, while the library index covers + * `getCompileClasspaths(excludeSourceGeneratedClassPath = true)`. The difference must be exactly + * the generated JARs of the module and of every project module it compiles against, and never + * `R.jar`, which both variants already contain. These characterise existing behaviour. + */ +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE) +class ModuleOutputClasspathDeltaTest { + @After + fun tearDown() { + runCatching { ProjectManagerImpl.getInstance().workspace = null } + } + + @Test + fun `the delta of a standalone module is its own generated jar`() { + val a = androidModule(":a") + installWorkspace(a) + + assertThat(a.classpathDelta()).containsExactly(generatedJarOf(":a")) + } + + @Test + fun `the delta includes the generated jar of a project dependency`() { + val a = androidModule(":a", moduleDeps = listOf(":b")) + val b = androidModule(":b") + installWorkspace(a, b) + + assertThat(a.classpathDelta()).containsExactly(generatedJarOf(":a"), generatedJarOf(":b")) + } + + @Test + fun `the delta reaches transitively through project dependencies`() { + val a = androidModule(":a", moduleDeps = listOf(":b")) + val b = androidModule(":b", moduleDeps = listOf(":c")) + val c = androidModule(":c") + installWorkspace(a, b, c) + + assertThat(a.classpathDelta()) + .containsExactly(generatedJarOf(":a"), generatedJarOf(":b"), generatedJarOf(":c")) + } + + @Test + fun `an external jar is not part of the delta`() { + val a = androidModule(":a", externalJars = listOf("$ROOT/ext/guava.jar")) + installWorkspace(a) + + assertThat(a.classpathDelta()).containsExactly(generatedJarOf(":a")) + } + + @Test + fun `the delta is the generated jars of the compile module closure and holds no R jar`() { + val a = androidModule(":a", moduleDeps = listOf(":b"), externalJars = listOf("$ROOT/ext/guava.jar")) + val b = androidModule(":b", moduleDeps = listOf(":c")) + val c = androidModule(":c") + installWorkspace(a, b, c) + + val closure = listOf(a) + a.getCompileModuleProjects().filterIsInstance() + val delta = a.classpathDelta() + + assertThat(delta).isEqualTo(closure.map { it.getGeneratedJar() }.toSet()) + assertThat(a.getCompileClasspaths(false)).containsAtLeast(rJarOf(":a"), rJarOf(":b"), rJarOf(":c")) + assertThat(delta.map { it.name }).doesNotContain("R.jar") + } + + private fun ModuleProject.classpathDelta(): Set = getCompileClasspaths(false) - getCompileClasspaths(true) + + private fun generatedJarOf(path: String) = File(moduleDir(path), "build/classes.jar") + + private fun rJarOf(path: String) = File(moduleDir(path), "build/R.jar") + + private fun moduleDir(path: String) = File("$ROOT${path.replace(':', '/')}") + + /** + * Builds an [AndroidModule] whose compile graph lists [moduleDeps] as project libraries and + * [externalJars] as external Java libraries, with an `R.jar` on its selected variant. + */ + private fun androidModule( + path: String, + moduleDeps: List = emptyList(), + externalJars: List = emptyList(), + ): AndroidModule { + val graph = AndroidModels.DependencyGraph.newBuilder() + val variantDeps = AndroidModels.VariantDependencies.newBuilder().setName(VARIANT) + + fun addRoot(library: AndroidModels.Library) { + val keyId = graph.keyCount + graph.addKey(library.key) + graph.addRoot(graph.nodeCount) + graph.addNode( + AndroidModels.GraphNode + .newBuilder() + .setKeyId(keyId) + .build(), + ) + variantDeps.putLibraries(library.key, library) + } + + for (depPath in moduleDeps) { + addRoot( + AndroidModels.Library + .newBuilder() + .setKey("project$depPath") + .setType(AndroidModels.LibraryType.Project) + .setProjectInfo( + AndroidModels.ProjectInfo + .newBuilder() + .setBuildId(":") + .setProjectPath(depPath) + .build(), + ).build(), + ) + } + + for (jar in externalJars) { + addRoot( + AndroidModels.Library + .newBuilder() + .setKey("external$jar") + .setType(AndroidModels.LibraryType.ExternalJavaLibrary) + .setArtifactPath(jar) + .build(), + ) + } + + variantDeps.setMainArtifact( + AndroidModels.ArtifactDependencies + .newBuilder() + .setCompileGraph(graph.build()) + .build(), + ) + + val androidProject = + AndroidModels.AndroidProject + .newBuilder() + .setProjectType(AndroidModels.ProjectType.LibraryProject) + .setVariantDependencies(variantDeps.build()) + .setClassesJarPath(generatedJarOf(path).path) + .build() + + val dir = moduleDir(path) + val module = + AndroidModule( + GradleModels.GradleProject + .newBuilder() + .setName(path.trimStart(':')) + .setPath(path) + .setProjectDirPath(dir.path) + .setBuildDirPath(File(dir, "build").path) + .setBuildScriptPath(File(dir, "build.gradle").path) + .setAndroidProject(androidProject) + .build(), + ) + + // The selected variant comes from the project manager's sync state, which a test cannot set. + val variant = + AndroidModels.AndroidVariant + .newBuilder() + .setName(VARIANT) + .setMainArtifact( + AndroidModels.AndroidArtifact + .newBuilder() + .setName(VARIANT) + .addClassJarPaths(rJarOf(path).path) + .build(), + ).build() + return spyk(module).also { every { it.getSelectedVariant() } returns variant } + } + + private fun installWorkspace(vararg modules: AndroidModule) { + val root = + GradleProject( + GradleModels.GradleProject + .newBuilder() + .setName("root") + .setPath(":") + .build(), + ) + ProjectManagerImpl.getInstance().workspace = + Workspace(rootProject = root, subProjects = modules.toList(), syncIssues = emptyList()) + } + + private companion object { + const val ROOT = "/tmp/module-output-delta-test" + const val VARIANT = "debug" + } +} diff --git a/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/WorkspaceFindModuleForFileTest.kt b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/WorkspaceFindModuleForFileTest.kt new file mode 100644 index 0000000000..1713ee83ec --- /dev/null +++ b/subprojects/projects/src/test/java/com/itsaky/androidide/projects/api/WorkspaceFindModuleForFileTest.kt @@ -0,0 +1,50 @@ +package com.itsaky.androidide.projects.api + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.project.GradleModels +import org.junit.Test +import java.io.File + +class WorkspaceFindModuleForFileTest { + private val workspace = + Workspace( + rootProject = + GradleProject( + GradleModels.GradleProject + .newBuilder() + .setName("root") + .setPath(":") + .build(), + ), + subProjects = emptyList(), + syncIssues = emptyList(), + ) + + @Test + fun `does not stat the file when existence is not checked`() { + val file = StatCountingFile("/project/app/src/Main.java") + + workspace.findModuleForFile(file, checkExistence = false) + + assertThat(file.existsCalls).isEqualTo(0) + } + + @Test + fun `returns null for a missing file when existence is checked`() { + val file = StatCountingFile("/project/app/src/Main.java") + + assertThat(workspace.findModuleForFile(file, checkExistence = true)).isNull() + assertThat(file.existsCalls).isEqualTo(1) + } + + private class StatCountingFile( + path: String, + ) : File(path) { + var existsCalls = 0 + + override fun exists(): Boolean { + existsCalls++ + return false + } + } +} diff --git a/testing/resources/test-project/app/src/main/resources/actions/AddImportActionNone_template.java b/testing/resources/test-project/app/src/main/resources/actions/AddImportActionNone_template.java new file mode 100644 index 0000000000..c9580e8db9 --- /dev/null +++ b/testing/resources/test-project/app/src/main/resources/actions/AddImportActionNone_template.java @@ -0,0 +1,7 @@ +package com.itsaky.androidide.test; + +public class AddImportActionNone_template { + public static void main(String[] args) { + NoSuchClassAnywhere nope = null; + } +}