Ravendb upgrade

.github/workflows/build-db-container.yml

@@ -55,7 +55,7 @@ jobs:
           $containers = cat src/ServiceControl.RavenDB/containers.json | ConvertFrom-Json
           $containers | ForEach-Object -Process {
             $FULLTAG="${{ env.TAG_NAME }}-$($_.tag)"
-            $BASETAG="$($RavenVersion)-ubuntu.22.04-$($_.tag)"
+            $BASETAG="$($RavenVersion)-ubuntu.24.04-$($_.tag)"
             Write-Output "::group::Building $FULLTAG for architecture $($_.arch) from $BASETAG"
             docker build -t ghcr.io/particular/servicecontrol-ravendb:$FULLTAG --file src/ServiceControl.RavenDB/Dockerfile --build-arg VERSION=${{ env.MinVerVersion }} --build-arg BASETAG=$BASETAG --platform linux/$($_.arch) .
             Write-Output "::endgroup::"

src/Directory.Packages.props

@@ -79,7 +79,7 @@
     <PackageVersion Include="PropertyChanged.Fody" Version="4.1.0" />
     <PackageVersion Include="PropertyChanging.Fody" Version="1.31.0" />
     <PackageVersion Include="PublicApiGenerator" Version="11.5.4" />
-    <PackageVersion Include="RavenDB.Embedded" Version="6.2.18" />
+    <PackageVersion Include="RavenDB.Embedded" Version="7.2.6" />
     <PackageVersion Include="ReactiveUI.WPF" Version="22.3.1" />
     <PackageVersion Include="Seq.Extensions.Logging" Version="9.0.0" />
     <PackageVersion Include="ServiceControl.Contracts" Version="5.1.0" />

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs

@@ -109,7 +109,7 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin
 
                 var logPath = GetLogPath(settings);
 
-                var logsMode = "Operations";
+                var logsMode = RavenDbLogLevelToLogsModeMapper.WarnMinLevel;
 
                 if (settings.PersisterSpecificSettings.TryGetValue(RavenDbLogLevelKey, out var ravenDbLogLevel))
                 {

src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs

@@ -34,7 +34,7 @@ public static async Task<EmbeddedDatabase> GetInstance(CancellationToken cancell
 
                 var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "AuditData");
                 var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "Audit");
-                var logsMode = "Operations";
+                var logsMode = RavenDbLogLevelToLogsModeMapper.WarnMinLevel;
                 var serverUrl = $"http://localhost:{PortUtility.GetAssignedOrAvailablePort(33334)}";
 
                 var databaseConfiguration = new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode), TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(30));

src/ServiceControl.Infrastructure/RavenDbLogLevelToLogsModeMapper.cs

@@ -2,28 +2,37 @@
 {
     using Microsoft.Extensions.Logging;
 
+    // Maps the user supplied 'RavenDBLogLevel' setting onto the value of the RavenDB server's 'Logs.MinLevel' setting.
+    // HINT: RavenDB 7 removed the 'Logs.Mode' setting (and its None/Operations/Information values). The server now
+    // exposes 'Logs.MinLevel', typed as Sparrow.Logging.LogLevel, which only accepts Trace, Debug, Info, Warn,
+    // Error, Fatal or Off. The three verbosity tiers used before the upgrade are preserved:
+    //   None -> Off, Operations (quiet/production) -> Warn, Information (verbose) -> Info
     public class RavenDbLogLevelToLogsModeMapper
     {
+        public const string OffMinLevel = "Off";
+        public const string WarnMinLevel = "Warn";
+        public const string InfoMinLevel = "Info";
+
         public static string Map(string ravenDbLogLevel, ILogger logger)
         {
             switch (ravenDbLogLevel.ToLower())
             {
                 case "off": // Backwards compatibility with 4.x
                 case "none":
-                    return "None";
+                    return OffMinLevel;
                 case "trace": // Backwards compatibility with 4.x
                 case "debug": // Backwards compatibility with 4.x
                 case "info": // Backwards compatibility with 4.x
                 case "information":
-                    return "Information";
+                    return InfoMinLevel;
                 case "error": // Backwards compatibility with 4.x
                 case "warn": // Backwards compatibility with 4.x
                 case "fatal": // Backwards compatibility with 4.x
                 case "operations":
-                    return "Operations";
+                    return WarnMinLevel;
                 default:
-                    logger.LogWarning("Unknown log level '{RavenDbLogLevel}', mapped to 'Operations'", ravenDbLogLevel);
-                    return "Operations";
+                    logger.LogWarning("Unknown log level '{RavenDbLogLevel}', mapped to '{FallbackLogLevel}'", ravenDbLogLevel, WarnMinLevel);
+                    return WarnMinLevel;
             }
         }
     }

src/ServiceControl.Persistence.RavenDB/Editing/EditFailedMessagesDataStore.cs

@@ -2,6 +2,7 @@ namespace ServiceControl.Persistence.RavenDB.Editing
 {
     using System.Threading;
     using System.Threading.Tasks;
+    using Raven.Client.Documents.Session;
     using Raven.Client.Exceptions;
     using ServiceControl.MessageFailures;
     using ServiceControl.Persistence.Recoverability.Editing;
@@ -18,7 +19,7 @@ public async Task<string> GetCurrentEditingRequestId(string failedMessageId, Can
         public async Task<BeginEditResult> TryBeginEdit(string failedMessageId, string editingMessageId, CancellationToken cancellationToken = default)
         {
             using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken);
-            session.Advanced.UseOptimisticConcurrency = true;
+            session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes;
 
             var failedMessage = await session.LoadAsync<FailedMessage>(FailedMessageIdGenerator.MakeDocumentId(failedMessageId), cancellationToken);
             if (failedMessage is null)

src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs

@@ -358,7 +358,7 @@ public async Task<bool> MarkAsResolved(string failedMessageId, CancellationToken
             var documentId = FailedMessageIdGenerator.MakeDocumentId(failedMessageId);
 
             using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken);
-            session.Advanced.UseOptimisticConcurrency = true;
+            session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes;
 
             var failedMessage = await session.LoadAsync<FailedMessage>(documentId, cancellationToken);
 
@@ -452,7 +452,7 @@ public async Task<string[]> UnArchiveMessages(IEnumerable<string> failedMessageI
             Dictionary<string, FailedMessage> failedMessages;
 
             using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken);
-            session.Advanced.UseOptimisticConcurrency = true;
+            session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes;
 
             var documentIds = failedMessageIds.Select(FailedMessageIdGenerator.MakeDocumentId);
 

src/ServiceControl.Persistence.RavenDB/RavenPersisterSettings.cs

@@ -36,6 +36,8 @@ class RavenPersisterSettings : PersistenceSettings, IRavenClientCertificateInfo
     public const string DatabaseNameDefault = "primary";
     public const int DatabaseMaintenancePortDefault = 33334;
     public const int ExpirationProcessTimerInSecondsDefault = 600;
-    public const string LogsModeDefault = "Operations";
+    // Value of the RavenDB 'Logs.MinLevel' setting. Must be a Sparrow.Logging.LogLevel name (see
+    // RavenDbLogLevelToLogsModeMapper) because RavenDB 7 removed the 'Logs.Mode' setting.
+    public const string LogsModeDefault = ServiceControl.RavenDbLogLevelToLogsModeMapper.WarnMinLevel;
     public const int ExternalIntegrationsDispatchingBatchSizeDefault = 100;
 }

src/ServiceControl.Persistence.RavenDB/Recoverability/Archiving/MessageArchiver.cs

@@ -6,6 +6,7 @@
     using System.Threading;
     using System.Threading.Tasks;
     using Microsoft.Extensions.Logging;
+    using Raven.Client.Documents.Session;
     using RavenDB;
     using ServiceControl.Infrastructure.Auth;
     using ServiceControl.Infrastructure.DomainEvents;
@@ -45,7 +46,7 @@ public async Task ArchiveAllInGroup(string groupId, AuditUser? initiatedBy = nul
 
             using (var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken))
             {
-                session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once
+                session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes; // Ensure 2 messages don't split the same operation into batches at once
 
                 archiveOperation = await archiveDocumentManager.LoadArchiveOperation(session, groupId, ArchiveType.FailureGroup, cancellationToken);
 
@@ -145,7 +146,7 @@ public async Task UnarchiveAllInGroup(string groupId, AuditUser? initiatedBy = n
 
             using (var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken))
             {
-                session.Advanced.UseOptimisticConcurrency = true; // Ensure 2 messages don't split the same operation into batches at once
+                session.Advanced.OptimisticConcurrencyMode = OptimisticConcurrencyMode.Writes; // Ensure 2 messages don't split the same operation into batches at once
 
                 unarchiveOperation = await unarchiveDocumentManager.LoadUnarchiveOperation(session, groupId, ArchiveType.FailureGroup, cancellationToken);
 

src/ServiceControl.Persistence.RavenDB/RetryDocumentDataStore.cs

@@ -29,7 +29,7 @@ public async Task AssignMessagesToBatch(string batchId, string[] messageIds, Can
 
             using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken);
             var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken);
-            var batch = new SingleNodeBatchCommand(documentStore.Conventions, session.Advanced.Context, commands);
+            var batch = new SingleNodeBatchCommand(documentStore.Conventions, commands);
             await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context, token: cancellationToken);
         }
 

src/ServiceControl.Persistence.RavenDB/RetryStagingStore.cs

@@ -158,7 +158,7 @@ public async Task RecordStagingFailure(IReadOnlyCollection<string> uniqueMessage
                 using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken);
                 var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken);
 
-                var batch = new SingleNodeBatchCommand(documentStore.Conventions, session.Advanced.Context, commands);
+                var batch = new SingleNodeBatchCommand(documentStore.Conventions, commands);
                 await session.Advanced.RequestExecutor.ExecuteAsync(batch, session.Advanced.Context, token: cancellationToken);
             }
             catch (ConcurrencyException)

src/ServiceControl.Persistence.Tests.RavenDB/EnvironmentDataConfigurationTests.cs

@@ -27,7 +27,9 @@ public async Task Should_report_defaults_when_nothing_is_configured()
 
         using (Assert.EnterMultipleScope())
         {
-            Assert.That(data["Storage.LogLevel"], Is.EqualTo("Operations"));
+            // RavenDB 7 replaced 'Logs.Mode' (None/Operations/Information) with 'Logs.MinLevel', which
+            // takes a Sparrow.Logging.LogLevel name, so the quiet default now reports as "Warn".
+            Assert.That(data["Storage.LogLevel"], Is.EqualTo("Warn"));
             Assert.That(data["Storage.QueryTimeoutSeconds"], Is.EqualTo("Default"));
             Assert.That(data["Storage.FreeSpaceThresholdPercent"], Is.EqualTo("Default"));
             Assert.That(data["Storage.MinimumFreeSpaceForIngestionPercent"], Is.EqualTo("Default"));

src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs

@@ -34,7 +34,7 @@ public static async Task<EmbeddedDatabase> GetInstance(CancellationToken cancell
 
                 var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "PrimaryData");
                 var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "Primary");
-                var logsMode = "Operations";
+                var logsMode = RavenDbLogLevelToLogsModeMapper.WarnMinLevel;
 
                 var settings = new RavenPersisterSettings
                 {

src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepCustomCheckTests.cs

@@ -11,6 +11,7 @@ namespace ServiceControl.Persistence.Tests;
 using ServiceControl.CustomChecks;
 using ServiceControl.Infrastructure.DomainEvents;
 using ServiceControl.Operations;
+using ServiceControl.Persistence.EFCore.Abstractions;
 using ServiceControl.Persistence.EFCore.Infrastructure;
 using ServiceControl.Persistence.EFCore.Infrastructure.Metrics;
 
@@ -67,6 +68,12 @@ public async Task Successful_sweep_resets_the_consecutive_failure_count()
     [Test]
     public async Task Check_does_not_expire_with_time()
     {
+        // The hourly sweeper shares the fake clock, so advancing it by two hours lets sweeps run in
+        // the background. Subtracting this retention period from the clock cannot be represented, so
+        // those sweeps fail as well and keep the failure alive: the alert has to survive the passage
+        // of time and clear only on a fully successful sweep.
+        ((EFPersisterSettings)PersistenceSettings).ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000);
+
         State.ReportError(RetentionEntity.FailedMessages, "db timeout");
         CompleteThreeSweeps();
         AdvanceClock(TimeSpan.FromHours(2));

src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs

@@ -612,15 +612,19 @@ public async Task Scheduled_sweep_with_a_failing_pass_reports_failed()
     [Test]
     public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first()
     {
+        // The failed messages pass overflows the cutoff and throws before any database call, so its
+        // error is already recorded by the time the cancellation lands in the event log pass.
         EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000);
+        EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14);
+        await Store([.. Enumerable.Range(0, 1500).Select(i => EventLogRow($"expired-{i}", Now.AddDays(-15)))]);
 
         using var cancellation = new CancellationTokenSource();
-        var sweeper = GetSweeper();
+        // The cancellation is raised from inside the sweep's own batch loop rather than from the test
+        // thread, so it cannot arrive after the sweep has finished and been reported as Failed.
+        cancelAfterBatchDelete.Cancellation = cancellation;
 
-        // The failed messages pass throws before any database call, so by the time the start returns
-        // the event log pass is waiting on the database and the cancellation lands there.
+        var sweeper = GetSweeper();
         sweeper.TryStartManualSweep(null, null, cancellation.Token);
-        await cancellation.CancelAsync();
 
         await WaitForManualSweepToFinish();
 

src/ServiceControl.RavenDB/EmbeddedDatabase.cs

@@ -15,7 +15,6 @@ namespace ServiceControl.RavenDB
     using Raven.Client.Documents.Conventions;
     using Raven.Embedded;
     using ServiceControl.Infrastructure;
-    using Sparrow.Logging;
 
     public sealed class EmbeddedDatabase : IDisposable
     {
@@ -53,18 +52,16 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi
 
             var nugetPackagesPath = Path.Combine(databaseConfiguration.DbPath, "Packages", "NuGet");
 
-            var logMode = Enum.Parse<LogMode>(databaseConfiguration.LogsMode);
-
-            if (logMode == LogMode.Information) // Most verbose
+            // HINT: RavenDB 7 removed the client-side Sparrow logging configuration (LoggingSource/LogMode) and the
+            // 'Logs.Mode' server setting. The embedded server runs in its own process, so its logging is driven by
+            // the 'Logs.MinLevel' server setting passed below together with ServerOptions.LogsPath.
+            //
+            // The server only creates the logs folder when it actually writes an entry, so with the quiet 'Warn'
+            // minimum level the configured folder would never appear. Create it up front so the configured LogPath
+            // always exists and warnings always have somewhere to land.
+            if (!string.IsNullOrWhiteSpace(databaseConfiguration.LogPath))
             {
-                LoggingSource.Instance.EnableConsoleLogging();
-                LoggingSource.Instance.SetupLogMode(
-                    logMode,
-                    Path.Combine(databaseConfiguration.LogPath, "Raven.Embedded"),
-                    retentionTime: TimeSpan.FromDays(14),
-                    retentionSize: 1024 * 1024 * 10,
-                    compress: false
-                );
+                Directory.CreateDirectory(databaseConfiguration.LogPath);
             }
 
             logger.LogInformation("Loading RavenDB license from {LicenseFileName}", licenseFileNameAndServerDirectory.LicenseFileName);
@@ -79,7 +76,7 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi
             {
                 CommandLineArgs =
                 [
-                    $"--Logs.Mode={databaseConfiguration.LogsMode}",
+                    $"--Logs.MinLevel={databaseConfiguration.LogsMode}",
                     // HINT: If this is not set, then Raven will pick a default location relative to the server binaries
                     // See https://github.com/ravendb/ravendb/issues/15694
                     $"--Indexing.NuGetPackagesPath=\"{nugetPackagesPath}\"",