Update RavenDB to 7.2.7

.github/workflows/build-db-container.yml

@@ -55,7 +55,7 @@ jobs:
           $containers = cat src/ServiceControl.RavenDB/containers.json | ConvertFrom-Json
           $containers | ForEach-Object -Process {
             $FULLTAG="${{ env.TAG_NAME }}-$($_.tag)"
-            $BASETAG="$($RavenVersion)-ubuntu.22.04-$($_.tag)"
+            $BASETAG="$($RavenVersion)-ubuntu.24.04-$($_.tag)"
             Write-Output "::group::Building $FULLTAG for architecture $($_.arch) from $BASETAG"
             docker build -t ghcr.io/particular/servicecontrol-ravendb:$FULLTAG --file src/ServiceControl.RavenDB/Dockerfile --build-arg VERSION=${{ env.MinVerVersion }} --build-arg BASETAG=$BASETAG --platform linux/$($_.arch) .
             Write-Output "::endgroup::"

docs/usage-report.md

@@ -89,7 +89,7 @@ Releases up to 6.21.2 report `SqlVersion` as the full text of `@@VERSION` on SQL
 | `Storage.BodyStorage.Type` | `RavenAttachments`, `FileSystem`, `AzureBlob`, `S3` | the persister | 6.20.0 |
 | `Limits.MaxBodySizeToStore` | bytes | `MaxBodySizeToStore`, SQL Server and PostgreSQL only | 6.20.0 |
 | `Storage.Schema` | `Default`, `Custom` | `Database/Schema`, SQL Server and PostgreSQL only | Unreleased |
-| `Storage.LogLevel` | `None`, `Information`, `Operations` | `RavenDBLogLevel`, RavenDB only | Unreleased |
+| `Storage.LogLevel` | `Trace`, `Debug`, `Info`, `Warn`, `Error`, `Fatal`, `Off` | `RavenDBLogLevel`, RavenDB only | Unreleased |
 | `Storage.CommandTimeoutSeconds` | `Default` or number | `Database/CommandTimeout`, SQL Server and PostgreSQL only | Unreleased |
 | `Storage.QueryTimeoutSeconds` | `Default` or number | `QueryTimeoutInSeconds` | Unreleased |
 | `Storage.SubscriptionCacheSeconds` | `Default` or number | `SubscriptionCacheDuration`, SQL Server and PostgreSQL only | Unreleased |

src/Directory.Packages.props

@@ -79,7 +79,7 @@
     <PackageVersion Include="PropertyChanged.Fody" Version="4.1.0" />
     <PackageVersion Include="PropertyChanging.Fody" Version="1.31.0" />
     <PackageVersion Include="PublicApiGenerator" Version="11.5.4" />
-    <PackageVersion Include="RavenDB.Embedded" Version="6.2.18" />
+    <PackageVersion Include="RavenDB.Embedded" Version="7.2.7" />
     <PackageVersion Include="ReactiveUI.WPF" Version="22.3.1" />
     <PackageVersion Include="Seq.Extensions.Logging" Version="9.0.0" />
     <PackageVersion Include="ServiceControl.Contracts" Version="5.1.0" />

src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs

@@ -39,7 +39,7 @@ public async Task Initialize(CancellationToken cancellationToken = default)
 
                 var serverConfig = databaseConfiguration.ServerConfiguration;
 
-                var embeddedConfig = new EmbeddedDatabaseConfiguration(serverConfig.ServerUrl, databaseConfiguration.Name, serverConfig.DbPath, serverConfig.LogPath, serverConfig.LogsMode);
+                var embeddedConfig = new EmbeddedDatabaseConfiguration(serverConfig.ServerUrl, databaseConfiguration.Name, serverConfig.DbPath, serverConfig.LogPath, serverConfig.LogsMinLevel);
 
                 database = EmbeddedDatabase.Start(embeddedConfig, lifetime);
 

src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs

@@ -5,11 +5,12 @@ namespace ServiceControl.Audit.Persistence.RavenDB
     using System;
     using System.Threading;
     using System.Threading.Tasks;
+    using Microsoft.Extensions.Logging;
     using Raven.Client.Documents;
     using Raven.Client.Documents.Conventions;
     using ServiceControl.RavenDB;
 
-    sealed class RavenExternalPersistenceLifecycle(DatabaseConfiguration configuration) : IRavenPersistenceLifecycle, IRavenDocumentStoreProvider, IDisposable
+    sealed class RavenExternalPersistenceLifecycle(DatabaseConfiguration configuration, ILogger<RavenExternalPersistenceLifecycle> logger) : IRavenPersistenceLifecycle, IRavenDocumentStoreProvider, IDisposable
     {
         public async ValueTask<IDocumentStore> GetDocumentStore(CancellationToken cancellationToken = default)
         {
@@ -42,7 +43,8 @@ public async Task Initialize(CancellationToken cancellationToken = default)
                     Certificate = RavenClientCertificate.FindClientCertificate(configuration.ServerConfiguration),
                     Conventions = new DocumentConventions
                     {
-                        SaveEnumsAsIntegers = true
+                        SaveEnumsAsIntegers = true,
+                        SaveEnumsAsIntegersForPatching = true
                     }
                 };
 
@@ -51,7 +53,7 @@ public async Task Initialize(CancellationToken cancellationToken = default)
                 await StartupChecks.EnsureServerVersion(store, cancellationToken);
 
                 var databaseSetup = new DatabaseSetup(configuration);
-                await databaseSetup.Execute(store, cancellationToken);
+                await DatabaseLoadRetry.Execute(token => databaseSetup.Execute(store, token), logger, cancellationToken);
             }
             finally
             {

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs

@@ -109,14 +109,14 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin
 
                 var logPath = GetLogPath(settings);
 
-                var logsMode = "Operations";
+                var logsMinLevel = "Warn";
 
                 if (settings.PersisterSpecificSettings.TryGetValue(RavenDbLogLevelKey, out var ravenDbLogLevel))
                 {
-                    logsMode = RavenDbLogLevelToLogsModeMapper.Map(ravenDbLogLevel, Logger);
+                    logsMinLevel = RavenDbLogLevelToLogsMinLevelMapper.Map(ravenDbLogLevel, Logger);
                 }
 
-                serverConfiguration = new ServerConfiguration(dbPath, serverUrl, logPath, logsMode);
+                serverConfiguration = new ServerConfiguration(dbPath, serverUrl, logPath, logsMinLevel);
             }
 
             var dataSpaceRemainingThreshold = CheckFreeDiskSpace.Parse(settings.PersisterSpecificSettings, Logger);

src/ServiceControl.Audit.Persistence.RavenDB/ServerConfiguration.cs

@@ -10,13 +10,13 @@ public ServerConfiguration(string connectionString)
             ConnectionString = connectionString;
         }
 
-        public ServerConfiguration(string dbPath, string serverUrl, string logPath, string logsMode)
+        public ServerConfiguration(string dbPath, string serverUrl, string logPath, string logsMinLevel)
         {
             UseEmbeddedServer = true;
             DbPath = dbPath;
             ServerUrl = serverUrl;
             LogPath = logPath;
-            LogsMode = logsMode;
+            LogsMinLevel = logsMinLevel;
         }
 
         public string ConnectionString { get; }
@@ -27,6 +27,6 @@ public ServerConfiguration(string dbPath, string serverUrl, string logPath, stri
         public string DbPath { get; internal set; } //Setter for ATT only
         public string ServerUrl { get; }
         public string LogPath { get; }
-        public string LogsMode { get; set; }
+        public string LogsMinLevel { get; set; }
     }
 }

src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionTests.VerifyGeneratedDefinitions.approved.txt

@@ -0,0 +1,121 @@
+Index: FailedAuditImportIndex
+Type: Map
+SourceType: Documents
+Maps:
+  docs.FailedAuditImports.Select(cc => new {
+    Id = Id(cc)
+})
+Reduce: (none)
+Fields:
+Configuration:
+  Indexing.Static.SearchEngineType=Lucene
+AdditionalSources:
+AdditionalAssemblies: 0
+OutputReduceToCollection: (none)
+
+Index: MessagesViewIndex
+Type: Map
+SourceType: Documents
+Maps:
+  docs.ProcessedMessages.Select(message => new {
+    MessageId = ((string) message.MessageMetadata["MessageId"]),
+    MessageType = ((string) message.MessageMetadata["MessageType"]),
+    IsSystemMessage = ((bool) message.MessageMetadata["IsSystemMessage"]),
+    Status = ((bool) message.MessageMetadata["IsRetried"]) ? 4 : 3,
+    TimeSent = ((DateTime) message.MessageMetadata["TimeSent"]),
+    ProcessedAt = message.ProcessedAt,
+    ReceivingEndpointName = ((message.MessageMetadata["ReceivingEndpoint"])).Name,
+    SendingEndpointName = ((message.MessageMetadata["SendingEndpoint"])).Name,
+    CriticalTime = ((TimeSpan ? ) message.MessageMetadata["CriticalTime"]),
+    ProcessingTime = ((TimeSpan ? ) message.MessageMetadata["ProcessingTime"]),
+    DeliveryTime = ((TimeSpan ? ) message.MessageMetadata["DeliveryTime"]),
+    Query = Enumerable.ToArray(DynamicEnumerable.Union(message.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] {
+        String.Join(" ", message.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value))
+    })),
+    ConversationId = ((string) message.MessageMetadata["ConversationId"])
+})
+Reduce: (none)
+Fields:
+  Query: Storage=, Indexing=Search, TermVector=, Suggestions=, Analyzer=Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181, HasSpatial=False
+Configuration:
+  Indexing.Static.SearchEngineType=Lucene
+AdditionalSources:
+AdditionalAssemblies: 0
+OutputReduceToCollection: (none)
+
+Index: MessagesViewIndexWithFullTextSearch
+Type: Map
+SourceType: Documents
+Maps:
+  docs.ProcessedMessages.Select(message => new {
+    MessageId = ((string) message.MessageMetadata["MessageId"]),
+    MessageType = ((string) message.MessageMetadata["MessageType"]),
+    IsSystemMessage = ((bool) message.MessageMetadata["IsSystemMessage"]),
+    Status = ((bool) message.MessageMetadata["IsRetried"]) ? 4 : 3,
+    TimeSent = ((DateTime) message.MessageMetadata["TimeSent"]),
+    ProcessedAt = message.ProcessedAt,
+    ReceivingEndpointName = ((message.MessageMetadata["ReceivingEndpoint"])).Name,
+    SendingEndpointName = ((message.MessageMetadata["SendingEndpoint"])).Name,
+    CriticalTime = ((TimeSpan ? ) message.MessageMetadata["CriticalTime"]),
+    ProcessingTime = ((TimeSpan ? ) message.MessageMetadata["ProcessingTime"]),
+    DeliveryTime = ((TimeSpan ? ) message.MessageMetadata["DeliveryTime"]),
+    Query = Enumerable.ToArray(DynamicEnumerable.Union(message.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] {
+        String.Join(" ", message.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value)),
+        this.LoadAttachment(message, "body").GetContentAsString()
+    })),
+    ConversationId = ((string) message.MessageMetadata["ConversationId"])
+})
+Reduce: (none)
+Fields:
+  Query: Storage=, Indexing=Search, TermVector=, Suggestions=, Analyzer=Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181, HasSpatial=False
+Configuration:
+  Indexing.Static.SearchEngineType=Lucene
+AdditionalSources:
+AdditionalAssemblies: 0
+OutputReduceToCollection: (none)
+
+Index: SagaDetailsIndex
+Type: MapReduce
+SourceType: Documents
+Maps:
+  docs.SagaHistories.Select(doc => new {
+    SagaId = doc.SagaId,
+    Id = doc.SagaId,
+    SagaType = doc.SagaType,
+    Changes = doc.Changes
+})
+  docs.SagaSnapshots.Select(doc => new {
+    SagaId = doc.SagaId,
+    Id = doc.SagaId,
+    SagaType = doc.SagaType,
+    Changes = new[] {
+        new {
+            Endpoint = doc.Endpoint,
+            FinishTime = doc.FinishTime,
+            InitiatingMessage = doc.InitiatingMessage,
+            OutgoingMessages = doc.OutgoingMessages,
+            StartTime = doc.StartTime,
+            StateAfterChange = doc.StateAfterChange,
+            Status = doc.Status
+        }
+    }
+})
+Reduce: results.GroupBy(result => result.SagaId).Select(g => new {
+    g = g,
+    first = DynamicEnumerable.FirstOrDefault(g)
+}).Select(this0 => new {
+    Id = this0.first.SagaId,
+    SagaId = this0.first.SagaId,
+    SagaType = this0.first.SagaType,
+    Changes = Enumerable.ToList(DynamicEnumerable.OrderByDescending(this0.g.SelectMany(x => x.Changes), x0 => x0.FinishTime).Take(50000))
+})
+Fields:
+  Changes: Storage=, Indexing=No, TermVector=, Suggestions=, Analyzer=(none), HasSpatial=False
+  SagaId: Storage=, Indexing=Exact, TermVector=, Suggestions=, Analyzer=(none), HasSpatial=False
+  SagaType: Storage=, Indexing=No, TermVector=, Suggestions=, Analyzer=(none), HasSpatial=False
+Configuration:
+  Indexing.Static.SearchEngineType=Lucene
+AdditionalSources:
+AdditionalAssemblies: 0
+OutputReduceToCollection: (none)
+

src/ServiceControl.Audit.Persistence.Tests.RavenDB/AuditTestData.cs

@@ -0,0 +1,57 @@
+namespace ServiceControl.Audit.Persistence.Tests;
+
+using System;
+using System.Collections.Generic;
+using Auditing;
+using Monitoring;
+using NServiceBus;
+
+static class AuditTestData
+{
+    // Carries every metadata entry that the audit indexes read, as the audit ingestion does.
+    public static ProcessedMessage CreateProcessedMessage(
+        string messageId = null,
+        string messageType = "MyMessageType",
+        string conversationId = null,
+        string receivingEndpoint = "ReceivingEndpoint",
+        DateTime? timeSent = null,
+        TimeSpan? criticalTime = null,
+        TimeSpan? processingTime = null,
+        TimeSpan? deliveryTime = null,
+        DateTimeOffset? processedAt = null)
+    {
+        messageId ??= Guid.NewGuid().ToString();
+        conversationId ??= Guid.NewGuid().ToString();
+        timeSent ??= DateTime.UtcNow;
+        processedAt ??= DateTimeOffset.UtcNow;
+
+        var metadata = new Dictionary<string, object>
+        {
+            { "MessageId", messageId },
+            { "MessageIntent", MessageIntent.Send },
+            { "TimeSent", timeSent.Value },
+            { "CriticalTime", criticalTime ?? TimeSpan.FromSeconds(5) },
+            { "ProcessingTime", processingTime ?? TimeSpan.FromSeconds(1) },
+            { "DeliveryTime", deliveryTime ?? TimeSpan.FromSeconds(4) },
+            { "IsSystemMessage", false },
+            { "MessageType", messageType },
+            { "IsRetried", false },
+            { "ConversationId", conversationId },
+            { "SendingEndpoint", new EndpointDetails { Name = "SendingEndpoint", HostId = Guid.Empty, Host = "host" } },
+            { "ReceivingEndpoint", new EndpointDetails { Name = receivingEndpoint, HostId = Guid.Empty, Host = "host" } }
+        };
+
+        var headers = new Dictionary<string, string>
+        {
+            { Headers.MessageId, messageId },
+            { Headers.ProcessingEndpoint, receivingEndpoint },
+            { Headers.MessageIntent, MessageIntent.Send.ToString() },
+            { Headers.ConversationId, conversationId },
+            { Headers.ProcessingStarted, DateTimeOffsetHelper.ToWireFormattedString(processedAt.Value) },
+            { Headers.ProcessingEnded, DateTimeOffsetHelper.ToWireFormattedString(processedAt.Value) },
+            { Headers.EnclosedMessageTypes, messageType }
+        };
+
+        return new ProcessedMessage(headers, metadata);
+    }
+}

src/ServiceControl.Audit.Persistence.Tests.RavenDB/CleanDatabaseFixture.cs

@@ -0,0 +1,55 @@
+namespace ServiceControl.Audit.Persistence.Tests;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using Raven.Client.Documents;
+using Raven.Client.Documents.Conventions;
+using Raven.Client.ServerWide;
+using Raven.Client.ServerWide.Operations;
+
+// A new database on the shared embedded server with no settings and no indexes, as opposed to the one DatabaseSetup prepares.
+abstract class CleanDatabaseFixture
+{
+    protected IDocumentStore Store { get; private set; }
+
+    protected CancellationToken TestTimeoutCancellationToken => testCancellationTokenSource.Token;
+
+    [SetUp]
+    public async Task CreateCleanDatabase()
+    {
+        testCancellationTokenSource = new CancellationTokenSource(TestTimeout);
+
+        var embeddedDatabase = await SharedEmbeddedServer.GetInstance(TestContext.CurrentContext.CancellationToken);
+        databaseName = Guid.NewGuid().ToString();
+
+        Store = new DocumentStore
+        {
+            Urls = [embeddedDatabase.ServerUrl],
+            Database = databaseName,
+            Conventions = new DocumentConventions { SaveEnumsAsIntegers = true, SaveEnumsAsIntegersForPatching = true }
+        };
+        Store.Initialize();
+
+        await Store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(databaseName)), TestTimeoutCancellationToken);
+    }
+
+    [TearDown]
+    public async Task DeleteCleanDatabase()
+    {
+        if (Store != null)
+        {
+            await Store.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(
+                new DeleteDatabasesOperation.Parameters { DatabaseNames = [databaseName], HardDelete = true }));
+            Store.Dispose();
+        }
+
+        testCancellationTokenSource?.Dispose();
+    }
+
+    string databaseName;
+    CancellationTokenSource testCancellationTokenSource;
+
+    static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(60);
+}