diff --git a/.github/workflows/build-db-container.yml b/.github/workflows/build-db-container.yml index 643f44ce30..c6b61cd4cb 100644 --- a/.github/workflows/build-db-container.yml +++ b/.github/workflows/build-db-container.yml @@ -55,7 +55,7 @@ jobs: $containers = cat src/ServiceControl.RavenDB/containers.json | ConvertFrom-Json $containers | ForEach-Object -Process { $FULLTAG="${{ env.TAG_NAME }}-$($_.tag)" - $BASETAG="$($RavenVersion)-ubuntu.22.04-$($_.tag)" + $BASETAG="$($RavenVersion)-ubuntu.24.04-$($_.tag)" Write-Output "::group::Building $FULLTAG for architecture $($_.arch) from $BASETAG" docker build -t ghcr.io/particular/servicecontrol-ravendb:$FULLTAG --file src/ServiceControl.RavenDB/Dockerfile --build-arg VERSION=${{ env.MinVerVersion }} --build-arg BASETAG=$BASETAG --platform linux/$($_.arch) . Write-Output "::endgroup::" diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 1a712572da..aa6f636013 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -79,7 +79,7 @@ - + diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs index 787a06d9c1..451bc7701b 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -109,7 +109,7 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin var logPath = GetLogPath(settings); - var logsMode = "Operations"; + var logsMode = RavenDbLogLevelToLogsModeMapper.WarnMinLevel; if (settings.PersisterSpecificSettings.TryGetValue(RavenDbLogLevelKey, out var ravenDbLogLevel)) { diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs index c3bf3fb38e..53cda30b39 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs @@ -34,7 +34,7 @@ public static async Task GetInstance(CancellationToken cancell var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "AuditData"); var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "Audit"); - var logsMode = "Operations"; + var logsMode = RavenDbLogLevelToLogsModeMapper.WarnMinLevel; var serverUrl = $"http://localhost:{PortUtility.GetAssignedOrAvailablePort(33334)}"; var databaseConfiguration = new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode), TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(30)); diff --git a/src/ServiceControl.Infrastructure/RavenDbLogLevelToLogsModeMapper.cs b/src/ServiceControl.Infrastructure/RavenDbLogLevelToLogsModeMapper.cs index d3316ba830..166560dae7 100644 --- a/src/ServiceControl.Infrastructure/RavenDbLogLevelToLogsModeMapper.cs +++ b/src/ServiceControl.Infrastructure/RavenDbLogLevelToLogsModeMapper.cs @@ -2,28 +2,37 @@ { using Microsoft.Extensions.Logging; + // Maps the user supplied 'RavenDBLogLevel' setting onto the value of the RavenDB server's 'Logs.MinLevel' setting. + // HINT: RavenDB 7 removed the 'Logs.Mode' setting (and its None/Operations/Information values). The server now + // exposes 'Logs.MinLevel', typed as Sparrow.Logging.LogLevel, which only accepts Trace, Debug, Info, Warn, + // Error, Fatal or Off. The three verbosity tiers used before the upgrade are preserved: + // None -> Off, Operations (quiet/production) -> Warn, Information (verbose) -> Info public class RavenDbLogLevelToLogsModeMapper { + public const string OffMinLevel = "Off"; + public const string WarnMinLevel = "Warn"; + public const string InfoMinLevel = "Info"; + public static string Map(string ravenDbLogLevel, ILogger logger) { switch (ravenDbLogLevel.ToLower()) { case "off": // Backwards compatibility with 4.x case "none": - return "None"; + return OffMinLevel; case "trace": // Backwards compatibility with 4.x case "debug": // Backwards compatibility with 4.x case "info": // Backwards compatibility with 4.x case "information": - return "Information"; + return InfoMinLevel; case "error": // Backwards compatibility with 4.x case "warn": // Backwards compatibility with 4.x case "fatal": // Backwards compatibility with 4.x case "operations": - return "Operations"; + return WarnMinLevel; default: - logger.LogWarning("Unknown log level '{RavenDbLogLevel}', mapped to 'Operations'", ravenDbLogLevel); - return "Operations"; + logger.LogWarning("Unknown log level '{RavenDbLogLevel}', mapped to '{FallbackLogLevel}'", ravenDbLogLevel, WarnMinLevel); + return WarnMinLevel; } } } diff --git a/src/ServiceControl.Persistence.RavenDB/Editing/EditFailedMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Editing/EditFailedMessagesDataStore.cs index 55cd75225d..5c5b078e65 100644 --- a/src/ServiceControl.Persistence.RavenDB/Editing/EditFailedMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Editing/EditFailedMessagesDataStore.cs @@ -2,6 +2,7 @@ namespace ServiceControl.Persistence.RavenDB.Editing { using System.Threading; using System.Threading.Tasks; + using Raven.Client.Documents.Session; using Raven.Client.Exceptions; using ServiceControl.MessageFailures; using ServiceControl.Persistence.Recoverability.Editing; @@ -18,7 +19,7 @@ public async Task GetCurrentEditingRequestId(string failedMessageId, Can public async Task TryBeginEdit(string failedMessageId, string editingMessageId, CancellationToken cancellationToken = default) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); - session.Advanced.UseOptimisticConcurrency = true; + session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes; var failedMessage = await session.LoadAsync(FailedMessageIdGenerator.MakeDocumentId(failedMessageId), cancellationToken); if (failedMessage is null) diff --git a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs index 52bb6f3b5e..970a74f7d5 100644 --- a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs @@ -358,7 +358,7 @@ public async Task MarkAsResolved(string failedMessageId, CancellationToken var documentId = FailedMessageIdGenerator.MakeDocumentId(failedMessageId); using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); - session.Advanced.UseOptimisticConcurrency = true; + session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes; var failedMessage = await session.LoadAsync(documentId, cancellationToken); @@ -452,7 +452,7 @@ public async Task UnArchiveMessages(IEnumerable failedMessageI Dictionary failedMessages; using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); - session.Advanced.UseOptimisticConcurrency = true; + session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes; var documentIds = failedMessageIds.Select(FailedMessageIdGenerator.MakeDocumentId); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersisterSettings.cs index 0f59fd9554..456750f8de 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersisterSettings.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersisterSettings.cs @@ -36,6 +36,8 @@ class RavenPersisterSettings : PersistenceSettings, IRavenClientCertificateInfo public const string DatabaseNameDefault = "primary"; public const int DatabaseMaintenancePortDefault = 33334; public const int ExpirationProcessTimerInSecondsDefault = 600; - public const string LogsModeDefault = "Operations"; + // Value of the RavenDB 'Logs.MinLevel' setting. Must be a Sparrow.Logging.LogLevel name (see + // RavenDbLogLevelToLogsModeMapper) because RavenDB 7 removed the 'Logs.Mode' setting. + public const string LogsModeDefault = ServiceControl.RavenDbLogLevelToLogsModeMapper.WarnMinLevel; public const int ExternalIntegrationsDispatchingBatchSizeDefault = 100; } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDB/Recoverability/Archiving/MessageArchiver.cs b/src/ServiceControl.Persistence.RavenDB/Recoverability/Archiving/MessageArchiver.cs index 9171a7df6f..2294c0347a 100644 --- a/src/ServiceControl.Persistence.RavenDB/Recoverability/Archiving/MessageArchiver.cs +++ b/src/ServiceControl.Persistence.RavenDB/Recoverability/Archiving/MessageArchiver.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; + using Raven.Client.Documents.Session; using RavenDB; using ServiceControl.Infrastructure.Auth; using ServiceControl.Infrastructure.DomainEvents; @@ -45,7 +46,7 @@ public async Task ArchiveAllInGroup(string groupId, AuditUser? initiatedBy = nul using (var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken)) { - session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once + session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes; // Ensure 2 messages don't split the same operation into batches at once archiveOperation = await archiveDocumentManager.LoadArchiveOperation(session, groupId, ArchiveType.FailureGroup, cancellationToken); @@ -145,7 +146,7 @@ public async Task UnarchiveAllInGroup(string groupId, AuditUser? initiatedBy = n using (var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken)) { - session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once + session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes; // Ensure 2 messages don't split the same operation into batches at once unarchiveOperation = await unarchiveDocumentManager.LoadUnarchiveOperation(session, groupId, ArchiveType.FailureGroup, cancellationToken); diff --git a/src/ServiceControl.Persistence.RavenDB/RetryDocumentDataStore.cs b/src/ServiceControl.Persistence.RavenDB/RetryDocumentDataStore.cs index 3b67232127..bc0a8bc583 100644 --- a/src/ServiceControl.Persistence.RavenDB/RetryDocumentDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/RetryDocumentDataStore.cs @@ -29,7 +29,7 @@ public async Task AssignMessagesToBatch(string batchId, string[] messageIds, Can using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); - var batch = new SingleNodeBatchCommand(documentStore.Conventions, session.Advanced.Context, commands); + var batch = new SingleNodeBatchCommand(documentStore.Conventions, commands); await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context, token: cancellationToken); } diff --git a/src/ServiceControl.Persistence.RavenDB/RetryStagingStore.cs b/src/ServiceControl.Persistence.RavenDB/RetryStagingStore.cs index 1c64c3bc67..c2dec5ddf7 100644 --- a/src/ServiceControl.Persistence.RavenDB/RetryStagingStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/RetryStagingStore.cs @@ -158,7 +158,7 @@ public async Task RecordStagingFailure(IReadOnlyCollection uniqueMessage using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken); - var batch = new SingleNodeBatchCommand(documentStore.Conventions, session.Advanced.Context, commands); + var batch = new SingleNodeBatchCommand(documentStore.Conventions, commands); await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context, token: cancellationToken); } catch (ConcurrencyException) diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs index cd7c23e4dc..333c17fc0a 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs @@ -27,7 +27,9 @@ public async Task Should_report_defaults_when_nothing_is_configured() using (Assert.EnterMultipleScope()) { - Assert.That(data["Storage.LogLevel"], Is.EqualTo("Operations")); + // RavenDB 7 replaced 'Logs.Mode' (None/Operations/Information) with 'Logs.MinLevel', which + // takes a Sparrow.Logging.LogLevel name, so the quiet default now reports as "Warn". + Assert.That(data["Storage.LogLevel"], Is.EqualTo("Warn")); Assert.That(data["Storage.QueryTimeoutSeconds"], Is.EqualTo("Default")); Assert.That(data["Storage.FreeSpaceThresholdPercent"], Is.EqualTo("Default")); Assert.That(data["Storage.MinimumFreeSpaceForIngestionPercent"], Is.EqualTo("Default")); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs b/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs index 49b3cddb56..ff5148d5e1 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs @@ -34,7 +34,7 @@ public static async Task GetInstance(CancellationToken cancell var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "PrimaryData"); var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "Primary"); - var logsMode = "Operations"; + var logsMode = RavenDbLogLevelToLogsModeMapper.WarnMinLevel; var settings = new RavenPersisterSettings { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepCustomCheckTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepCustomCheckTests.cs index b73d460c3b..dd11006bdb 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepCustomCheckTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepCustomCheckTests.cs @@ -11,6 +11,7 @@ namespace ServiceControl.Persistence.Tests; using ServiceControl.CustomChecks; using ServiceControl.Infrastructure.DomainEvents; using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.EFCore.Infrastructure.Metrics; @@ -67,6 +68,12 @@ public async Task Successful_sweep_resets_the_consecutive_failure_count() [Test] public async Task Check_does_not_expire_with_time() { + // The hourly sweeper shares the fake clock, so advancing it by two hours lets sweeps run in + // the background. Subtracting this retention period from the clock cannot be represented, so + // those sweeps fail as well and keep the failure alive: the alert has to survive the passage + // of time and clear only on a fully successful sweep. + ((EFPersisterSettings)PersistenceSettings).ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000); + State.ReportError(RetentionEntity.FailedMessages, "db timeout"); CompleteThreeSweeps(); AdvanceClock(TimeSpan.FromHours(2)); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index 3a73a8350d..fe95902905 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -612,15 +612,19 @@ public async Task Scheduled_sweep_with_a_failing_pass_reports_failed() [Test] public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first() { + // The failed messages pass overflows the cutoff and throws before any database call, so its + // error is already recorded by the time the cancellation lands in the event log pass. EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000); + EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14); + await Store([.. Enumerable.Range(0, 1500).Select(i => EventLogRow($"expired-{i}", Now.AddDays(-15)))]); using var cancellation = new CancellationTokenSource(); - var sweeper = GetSweeper(); + // The cancellation is raised from inside the sweep's own batch loop rather than from the test + // thread, so it cannot arrive after the sweep has finished and been reported as Failed. + cancelAfterBatchDelete.Cancellation = cancellation; - // The failed messages pass throws before any database call, so by the time the start returns - // the event log pass is waiting on the database and the cancellation lands there. + var sweeper = GetSweeper(); sweeper.TryStartManualSweep(null, null, cancellation.Token); - await cancellation.CancelAsync(); await WaitForManualSweepToFinish(); diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index efdf66b06f..01170c4610 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -15,7 +15,6 @@ namespace ServiceControl.RavenDB using Raven.Client.Documents.Conventions; using Raven.Embedded; using ServiceControl.Infrastructure; - using Sparrow.Logging; public sealed class EmbeddedDatabase : IDisposable { @@ -53,18 +52,16 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi var nugetPackagesPath = Path.Combine(databaseConfiguration.DbPath, "Packages", "NuGet"); - var logMode = Enum.Parse(databaseConfiguration.LogsMode); - - if (logMode == LogMode.Information) // Most verbose + // HINT: RavenDB 7 removed the client-side Sparrow logging configuration (LoggingSource/LogMode) and the + // 'Logs.Mode' server setting. The embedded server runs in its own process, so its logging is driven by + // the 'Logs.MinLevel' server setting passed below together with ServerOptions.LogsPath. + // + // The server only creates the logs folder when it actually writes an entry, so with the quiet 'Warn' + // minimum level the configured folder would never appear. Create it up front so the configured LogPath + // always exists and warnings always have somewhere to land. + if (!string.IsNullOrWhiteSpace(databaseConfiguration.LogPath)) { - LoggingSource.Instance.EnableConsoleLogging(); - LoggingSource.Instance.SetupLogMode( - logMode, - Path.Combine(databaseConfiguration.LogPath, "Raven.Embedded"), - retentionTime: TimeSpan.FromDays(14), - retentionSize: 1024 * 1024 * 10, - compress: false - ); + Directory.CreateDirectory(databaseConfiguration.LogPath); } logger.LogInformation("Loading RavenDB license from {LicenseFileName}", licenseFileNameAndServerDirectory.LicenseFileName); @@ -79,7 +76,7 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi { CommandLineArgs = [ - $"--Logs.Mode={databaseConfiguration.LogsMode}", + $"--Logs.MinLevel={databaseConfiguration.LogsMode}", // HINT: If this is not set, then Raven will pick a default location relative to the server binaries // See https://github.com/ravendb/ravendb/issues/15694 $"--Indexing.NuGetPackagesPath=\"{nugetPackagesPath}\"",