From 2e8f6ed7cad391fec5ee7a8bcef5a4fc58933c78 Mon Sep 17 00:00:00 2001 From: march <106459595+marchc1@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:16:39 -0700 Subject: [PATCH 1/4] Use a dictionary for finding sounds via soundentry The LinkedList Find ate ~25% of startup... --- Source.SoundEmitterSystem/SoundEmitterSystemBase.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs b/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs index 994f31c7b..a79f36bd0 100644 --- a/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs +++ b/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs @@ -75,12 +75,14 @@ public class SoundEmitterSystemBase : ISoundEmitterSystemBase readonly Dictionary m_ActorGenders = new(SymbolStringComparer_OrdinalIgnoreCase.Instance); readonly LinkedList Sounds = new(); + readonly Dictionary> SoundEntryLookup = new(); readonly Dictionary> HandleToSound = []; readonly Dictionary, int> SoundToHandle = []; int CurrentHandle; int Sounds_AllocHandle(LinkedListNode node) { Sounds.AddLast(node); + SoundEntryLookup[node.Value] = node; int handle = Interlocked.Increment(ref CurrentHandle); HandleToSound[handle] = node; SoundToHandle[node] = handle; @@ -143,8 +145,7 @@ public bool AddSound(ReadOnlySpan soundname, ReadOnlySpan scriptfile newEntry.ScriptFileIndex = i; newEntry.SoundParams.CopyFrom(parms); - LinkedListNode? tryFindingDuplicate = Sounds.Find(newEntry); - if (tryFindingDuplicate != null) + if (SoundEntryLookup.TryGetValue(newEntry, out LinkedListNode? tryFindingDuplicate)) Sounds_ReplaceKey(SoundToHandle[tryFindingDuplicate], entryNode); else Sounds_AllocHandle(entryNode); @@ -495,8 +496,8 @@ public int GetSoundIndex(ReadOnlySpan name) { SoundEntry search = default; search.Name = name; - var node = Sounds.Find(search); - if (node == null) return -1; + if (!SoundEntryLookup.TryGetValue(search, out var node)) + return -1; return SoundToHandle[node]; } @@ -728,8 +729,7 @@ public void AddSoundsFromFile(ReadOnlySpan filename, bool preload, bool is bool isDuplicate; int lookup = -1; { - var lookupEntry = Sounds.Find(entry); - if (lookupEntry != null) { + if (SoundEntryLookup.TryGetValue(entry, out var lookupEntry)) { isDuplicate = true; lookup = SoundToHandle[lookupEntry]; } From 5163784405b6bf3958f0dd2bc43859721b05eed4 Mon Sep 17 00:00:00 2001 From: march <106459595+marchc1@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:22:57 -0700 Subject: [PATCH 2/4] Slight VPK rework for performance --- Source.Filesystem/PackStoreSearchPath.cs | 107 ++++++++++++----- Source.VPK/VpkArchive.cs | 141 +++++++++++++---------- Source.VPK/VpkEntry.cs | 3 +- Source.VPK/VpkReaderBase.cs | 96 +++++++++------ 4 files changed, 219 insertions(+), 128 deletions(-) diff --git a/Source.Filesystem/PackStoreSearchPath.cs b/Source.Filesystem/PackStoreSearchPath.cs index 4a8f7cbeb..0878d586a 100644 --- a/Source.Filesystem/PackStoreSearchPath.cs +++ b/Source.Filesystem/PackStoreSearchPath.cs @@ -28,15 +28,86 @@ public void Dispose() { public class PackStoreSearchPath : BaseSearchPath { + sealed class PackStore + { + public readonly string Key; + public readonly VpkArchive Vpk = new(); + public readonly Dictionary EntryLookups = []; + public readonly Dictionary DirectoryLookups = []; + public int References; + + public PackStore(string key, string vpkPath) { + Key = key; + Vpk.Load(vpkPath); + + int entryCount = 0; + foreach (var dir in Vpk.Directories) + entryCount += dir.Entries.Count; + + DirectoryLookups.EnsureCapacity(Vpk.Directories.Count); + EntryLookups.EnsureCapacity(entryCount); + + Span buildPath = stackalloc char[260]; + foreach (var dir in Vpk.Directories) { + DirectoryLookups[dir.Path.Hash()] = dir; + + foreach (var entry in dir.Entries) { + var path = entry.Path.Replace('\\', '/'); + var filename = entry.Filename; + var ext = entry.Extension; + + int strlen = 0; + if (path.Length > 0 && path[0] != ' ') { + path.CopyTo(buildPath[strlen..]); strlen += path.Length; + buildPath[strlen] = '/'; strlen += 1; + } + filename.CopyTo(buildPath[strlen..]); strlen += filename.Length; + buildPath[strlen] = '.'; strlen += 1; + ext.CopyTo(buildPath[strlen..]); strlen += ext.Length; + + ReadOnlySpan finalSpan = buildPath[..strlen]; + EntryLookups[finalSpan.Hash()] = entry; + } + } + } + } + + static readonly Dictionary packStores = new(OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + + static PackStore AcquirePackStore(string vpkPath) { + string key = Path.GetFullPath(vpkPath); + lock (packStores) { + if (!packStores.TryGetValue(key, out PackStore? store)) { + store = new(key, vpkPath); + packStores[key] = store; + } + + store.References++; + return store; + } + } + + static void ReleasePackStore(PackStore store) { + lock (packStores) { + if (--store.References > 0) + return; + + packStores.Remove(store.Key); + } + + store.Vpk.Dispose(); + } + private readonly IFileSystem parent; - private readonly VpkArchive vpk; + private readonly PackStore? store; private readonly string VpkPath; - private Dictionary vpkEntryLookups = []; - private Dictionary vpkDirectoryLookups = []; + private readonly Dictionary vpkEntryLookups; + private readonly Dictionary vpkDirectoryLookups; ~PackStoreSearchPath(){ - vpk.Dispose(); + if (store != null) + ReleasePackStore(store); } public PackStoreSearchPath(IFileSystem filesystem, string absPath) { @@ -46,31 +117,9 @@ public PackStoreSearchPath(IFileSystem filesystem, string absPath) { absPath = $"{absPath}_dir.vpk"; VpkPath = absPath; parent = filesystem; - vpk = new VpkArchive(); - vpk.Load(absPath); - - Span buildPath = stackalloc char[260]; - foreach (var dir in vpk.Directories) { - vpkDirectoryLookups[dir.Path.Hash()] = dir; - - foreach (var entry in dir.Entries) { - var path = entry.Path.Replace('\\', '/'); - var filename = entry.Filename; - var ext = entry.Extension; - - int strlen = 0; - if (path.Length > 0 && path[0] != ' ') { - path.CopyTo(buildPath[strlen..]); strlen += path.Length; - buildPath[strlen] = '/'; strlen += 1; - } - filename.CopyTo(buildPath[strlen..]); strlen += filename.Length; - buildPath[strlen] = '.'; strlen += 1; - ext.CopyTo(buildPath[strlen..]); strlen += ext.Length; - - ReadOnlySpan finalSpan = buildPath[..strlen]; - vpkEntryLookups[finalSpan.Hash()] = entry; - } - } + store = AcquirePackStore(absPath); + vpkEntryLookups = store.EntryLookups; + vpkDirectoryLookups = store.DirectoryLookups; if (!Path.IsPathFullyQualified(absPath)) absPath = Path.GetFullPath(absPath); diff --git a/Source.VPK/VpkArchive.cs b/Source.VPK/VpkArchive.cs index 99826b1e7..0438b8697 100644 --- a/Source.VPK/VpkArchive.cs +++ b/Source.VPK/VpkArchive.cs @@ -21,9 +21,10 @@ public VpkArchive() { } public void Dispose() { - foreach (var part in Parts) - part.Dispose(); - FileHandle.Dispose(); + if (Parts != null) + foreach (var part in Parts) + part.Dispose(); + FileHandle?.Dispose(); } public void Load(string filename, VpkVersions.Versions version = VpkVersions.Versions.Any) { @@ -32,75 +33,89 @@ public void Load(string filename, VpkVersions.Versions version = VpkVersions.Ver if (IsMultiPart) LoadParts(filename); - switch (version) { - case VpkVersions.Versions.Any: - // Try V2 first - _reader = new V2.VpkReaderV2(filename); - var hdr_anyv2 = _reader.ReadArchiveHeader(); - if (!hdr_anyv2.Verify()) { - // Try V1 + IVpkArchiveHeader header = null!; + try { + switch (version) { + case VpkVersions.Versions.Any: + // Try V2 first + _reader = new V2.VpkReaderV2(filename); + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) { + // Try V1 + _reader.Dispose(); + _reader = new VpkReaderV1(filename); + + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) + throw new ArchiveParsingException("Invalid archive header (tried V2, then V1)"); + } + break; + case VpkVersions.Versions.V1: _reader = new VpkReaderV1(filename); - var hdr_anyv1 = _reader.ReadArchiveHeader(); - if (!hdr_anyv1.Verify()) - throw new ArchiveParsingException("Invalid archive header (tried V2, then V1)"); - } - break; - case VpkVersions.Versions.V1: - _reader = new VpkReaderV1(filename); - - var hdr_v1 = _reader.ReadArchiveHeader(); - if (!hdr_v1.Verify()) - throw new ArchiveParsingException("Invalid V1 archive header"); - break; - case VpkVersions.Versions.V2: - _reader = new V2.VpkReaderV2(filename); - - var hdr_v2 = _reader.ReadArchiveHeader(); - if (!hdr_v2.Verify()) - throw new ArchiveParsingException("Invalid V2 archive header"); - break; - } + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) + throw new ArchiveParsingException("Invalid V1 archive header"); + break; + case VpkVersions.Versions.V2: + _reader = new V2.VpkReaderV2(filename); + + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) + throw new ArchiveParsingException("Invalid V2 archive header"); + break; + } - var reader = _reader.ReadDirectories(this); - while (reader.MoveNext()) - Directories.Add(reader.Current); + var reader = _reader.ReadDirectories(this, header.TreeLength); + while (reader.MoveNext()) + Directories.Add(reader.Current); + } + finally { + _reader?.Dispose(); + } } public void Load(byte[] file, VpkVersions.Versions version = VpkVersions.Versions.V1) { - switch (version) { - case VpkVersions.Versions.Any: - // Try V2 first - _reader = new V2.VpkReaderV2(file); - var hdr_anyv2 = _reader.ReadArchiveHeader(); - if (!hdr_anyv2.Verify()) { - // Try V1 + IVpkArchiveHeader header = null!; + try { + switch (version) { + case VpkVersions.Versions.Any: + // Try V2 first + _reader = new V2.VpkReaderV2(file); + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) { + // Try V1 + _reader.Dispose(); + _reader = new VpkReaderV1(file); + + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) + throw new ArchiveParsingException("Invalid archive header (tried V2, then V1)"); + } + break; + case VpkVersions.Versions.V1: _reader = new VpkReaderV1(file); - var hdr_anyv1 = _reader.ReadArchiveHeader(); - if (!hdr_anyv1.Verify()) - throw new ArchiveParsingException("Invalid archive header (tried V2, then V1)"); - } - break; - case VpkVersions.Versions.V1: - _reader = new VpkReaderV1(file); - - var hdr_v1 = _reader.ReadArchiveHeader(); - if (!hdr_v1.Verify()) - throw new ArchiveParsingException("Invalid V1 archive header"); - break; - case VpkVersions.Versions.V2: - _reader = new V2.VpkReaderV2(file); - - var hdr_v2 = _reader.ReadArchiveHeader(); - if (!hdr_v2.Verify()) - throw new ArchiveParsingException("Invalid V2 archive header"); - break; - } + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) + throw new ArchiveParsingException("Invalid V1 archive header"); + break; + case VpkVersions.Versions.V2: + _reader = new V2.VpkReaderV2(file); + + header = _reader.ReadArchiveHeader(); + if (!header.Verify()) + throw new ArchiveParsingException("Invalid V2 archive header"); + break; + } - var reader = _reader.ReadDirectories(this); - while (reader.MoveNext()) - Directories.Add(reader.Current); + var reader = _reader.ReadDirectories(this, header.TreeLength); + while (reader.MoveNext()) + Directories.Add(reader.Current); + } + finally { + _reader?.Dispose(); + } } private void LoadParts(string filePath) { diff --git a/Source.VPK/VpkEntry.cs b/Source.VPK/VpkEntry.cs index c6aa98bac..69fcc5172 100644 --- a/Source.VPK/VpkEntry.cs +++ b/Source.VPK/VpkEntry.cs @@ -5,7 +5,7 @@ public class VpkEntry public string Extension { get; set; } public string Path { get; set; } public string Filename { get; set; } - public string FilenameAndExtension { get; set; } + public string FilenameAndExtension { get => field ??= Filename + "." + Extension; set; } public byte[] PreloadData { get { return ReadPreloadData(); } } public byte[] Data { get { return ReadData(); } } public bool HasPreloadData { get; set; } @@ -32,7 +32,6 @@ internal VpkEntry(VpkArchive parentArchive, uint crc, ushort preloadBytes, nuint Extension = extension; Path = path; Filename = filename; - FilenameAndExtension = filename + "." + Extension; HasPreloadData = preloadBytes > 0; } diff --git a/Source.VPK/VpkReaderBase.cs b/Source.VPK/VpkReaderBase.cs index 5f34d66a9..b0f185611 100644 --- a/Source.VPK/VpkReaderBase.cs +++ b/Source.VPK/VpkReaderBase.cs @@ -1,31 +1,54 @@ -using Microsoft.Win32.SafeHandles; - -using System.IO.MemoryMappedFiles; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; namespace Source.VPK { - internal abstract class VpkReaderBase + internal abstract class VpkReaderBase : IDisposable { - readonly BinaryReader reader; + const int EntryDataSize = sizeof(uint) + sizeof(ushort) + sizeof(ushort) + sizeof(uint) + sizeof(uint) + sizeof(ushort); + + readonly Stream stream; + readonly List entryBuffer = []; + byte[] tree = []; + int treePosition; + long treeOffset; - protected VpkReaderBase(string filename) => reader = new(File.OpenRead(filename)); - protected VpkReaderBase(byte[] file) =>reader = new(new MemoryStream(file)); + protected VpkReaderBase(string filename) => stream = File.OpenRead(filename); + protected VpkReaderBase(byte[] file) => stream = new MemoryStream(file); + + public void Dispose() => stream.Dispose(); public abstract IVpkArchiveHeader ReadArchiveHeader(); - readonly List StringBuffer = []; - public string ReadNullTerminatedString() { - StringBuffer.Clear(); - do { - int ic = reader.ReadByte(); - if (ic == -1 || ic == 0) - break; - StringBuffer.Add((char)ic); - } while (reader.BaseStream.Position < reader.BaseStream.Length); - return new(CollectionsMarshal.AsSpan(StringBuffer)[..StringBuffer.Count]); + void LoadTree(uint treeLength) { + treeOffset = stream.Position; + long remaining = Math.Max(stream.Length - treeOffset, 0); + tree = new byte[Math.Min(treeLength, remaining)]; + int read = stream.ReadAtLeast(tree, tree.Length, false); + if (read != tree.Length) + Array.Resize(ref tree, read); + treePosition = 0; + } + + string ReadNullTerminatedString() { + ReadOnlySpan remaining = tree.AsSpan(treePosition); + int length = remaining.IndexOf((byte)0); + if (length == -1) { + length = remaining.Length; + treePosition += length; + } + else + treePosition += length + 1; + + return length == 0 ? string.Empty : Encoding.Latin1.GetString(remaining[..length]); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + T ReadTree() where T : unmanaged { + T value = MemoryMarshal.Read(tree.AsSpan(treePosition)); + treePosition += Unsafe.SizeOf(); + return value; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -63,7 +86,10 @@ public bool MoveNext() { } #region default - public DirectoryReader ReadDirectories(VpkArchive parentArchive) => new(this, parentArchive); + public DirectoryReader ReadDirectories(VpkArchive parentArchive, uint treeLength) { + LoadTree(treeLength); + return new(this, parentArchive); + } public struct EntryReader(VpkReaderBase reader, VpkArchive parentArchive, string ext, string path) { @@ -76,42 +102,45 @@ public bool MoveNext() { if (string.IsNullOrEmpty(fileName)) return false; - var crc = Reader.Read(); - var preloadBytes = Reader.Read(); - var archiveIdx = Reader.Read(); - var entryOffset = Reader.Read(); - var entryLen = Reader.Read(); + if (Reader.tree.Length - Reader.treePosition < EntryDataSize) + return false; + + var crc = Reader.ReadTree(); + var preloadBytes = Reader.ReadTree(); + var archiveIdx = Reader.ReadTree(); + var entryOffset = Reader.ReadTree(); + var entryLen = Reader.ReadTree(); // skip terminator - Reader.Read(); - nuint preloadDataOffset = (nuint)Reader.reader.BaseStream.Position; - if (preloadBytes > 0) - Reader.reader.Read(stackalloc byte[preloadBytes]); + Reader.ReadTree(); + nuint preloadDataOffset = (nuint)(Reader.treeOffset + Reader.treePosition); + Reader.treePosition = Math.Min(Reader.treePosition + preloadBytes, Reader.tree.Length); Current = new VpkEntry(ParentArchive, crc, preloadBytes, preloadDataOffset, archiveIdx, entryOffset, entryLen, Ext, Path, fileName.ToLowerInvariant()); return true; } public List ToList() { - List entries = []; + List buffer = Reader.entryBuffer; + buffer.Clear(); while (MoveNext()) - entries.Add(Current); - return entries; + buffer.Add(Current); + return new(buffer); } public VpkEntry Current = null!; } - public EntryReader ReadEntries(VpkArchive parentArchive, string ext, string path) => new(this, parentArchive, ext.ToLowerInvariant(), path.ToLowerInvariant()); + public EntryReader ReadEntries(VpkArchive parentArchive, string ext, string path) => new(this, parentArchive, ext, path.ToLowerInvariant()); [MethodImpl(MethodImplOptions.AggressiveInlining)] protected T Read() where T : unmanaged { int sizeofT = Unsafe.SizeOf(); Span data = stackalloc byte[sizeofT]; - reader.Read(data); + stream.ReadAtLeast(data, sizeofT, false); return MemoryMarshal.Cast(data)[0]; } protected ReadOnlySpan ReadBytes(Span data) { - reader.Read(data); + stream.ReadAtLeast(data, data.Length, false); return data; } @@ -120,4 +149,3 @@ protected ReadOnlySpan ReadBytes(Span data) { public abstract uint CalculateEntryOffset(uint offset); } } - From 178dd5c1bb0456e179aa513a85adac2e52cb68fa Mon Sep 17 00:00:00 2001 From: march <106459595+marchc1@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:24:45 -0700 Subject: [PATCH 3/4] Fix nettables excludeprop checking --- Source.Common/DtCommon.cs | 1 + Source.Common/DtRecv.cs | 2 +- Source.Common/DtSend.cs | 7 +++---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source.Common/DtCommon.cs b/Source.Common/DtCommon.cs index d314be804..766737b39 100644 --- a/Source.Common/DtCommon.cs +++ b/Source.Common/DtCommon.cs @@ -101,6 +101,7 @@ public struct ExcludeProp public struct BuildHierarchyStruct { public ExcludeProp[]? ExcludeProps; + public int NumExcludeProps; public InlineArrayMaxTotalSendTableProps DataTableProps; public int NumDataTableProps; diff --git a/Source.Common/DtRecv.cs b/Source.Common/DtRecv.cs index 1f72b7bdc..54ff7abc5 100644 --- a/Source.Common/DtRecv.cs +++ b/Source.Common/DtRecv.cs @@ -526,7 +526,7 @@ public bool SetupFlatPropertyArray() { // Now build the hierarchy. BuildHierarchyStruct bhs = default; bhs.ExcludeProps = excludeProps; - bhs.ExcludeProps = excludeProps; + bhs.NumExcludeProps = numExcludeProps; bhs.NumProps = bhs.NumDataTableProps = 0; bhs.PropProxies = 0; SendTable.BuildHierarchy(GetRootNode(), table, ref bhs); diff --git a/Source.Common/DtSend.cs b/Source.Common/DtSend.cs index 06786b541..12913443b 100644 --- a/Source.Common/DtSend.cs +++ b/Source.Common/DtSend.cs @@ -807,9 +807,8 @@ private static void BuildHierarchy_IterateProps(SendNode node, SendTable? table, for (i = 0; i < table?.Props?.Length; i++) { SendProp prop = table.Props![i]; - if (prop.IsExcludeProp() || prop.IsInsideArray() || FindExcludeProp(table.GetName(), prop.GetName(), bhs.ExcludeProps!)) { + if (prop.IsExcludeProp() || prop.IsInsideArray() || FindExcludeProp(table.GetName(), prop.GetName(), bhs.ExcludeProps!.AsSpan()[..bhs.NumExcludeProps])) continue; - } if (prop.GetPropType() == SendPropType.DataTable) { if ((prop.GetFlags() & PropFlags.Collapsible) != 0) { @@ -844,8 +843,8 @@ private static void BuildHierarchy_IterateProps(SendNode node, SendTable? table, } } - private static bool FindExcludeProp(ReadOnlySpan tableName, ReadOnlySpan propName, ExcludeProp[]? excludeProps) { - for (int i = 0; i < excludeProps?.Length; i++) { + private static bool FindExcludeProp(ReadOnlySpan tableName, ReadOnlySpan propName, Span excludeProps) { + for (int i = 0, c = excludeProps.Length; i < c; i++) { if (tableName.Equals(excludeProps[i].TableName!, StringComparison.OrdinalIgnoreCase) && propName.Equals(excludeProps[i].PropName!, StringComparison.OrdinalIgnoreCase)) { return true; } From d21e0c14561b6eba4df395fd8f4a1fec1fc1cb67 Mon Sep 17 00:00:00 2001 From: march <106459595+marchc1@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:29:00 -0700 Subject: [PATCH 4/4] Forgot to handle any removals that may happen here --- Source.SoundEmitterSystem/SoundEmitterSystemBase.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs b/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs index a79f36bd0..0f2dfedd3 100644 --- a/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs +++ b/Source.SoundEmitterSystem/SoundEmitterSystemBase.cs @@ -96,6 +96,8 @@ void Sounds_ReplaceKey(int handle, LinkedListNode newNode) { Sounds.AddAfter(oldNode, newNode); Sounds.Remove(oldNode); SoundToHandle.Remove(oldNode); + + SoundEntryLookup[newNode.Value] = newNode; HandleToSound[handle] = newNode; SoundToHandle[newNode] = handle; } @@ -870,6 +872,7 @@ private void InternalModShutdown() { SoundKeyValues.Clear(); Sounds.Clear(); + SoundEntryLookup.Clear(); HandleToSound.Clear(); CurrentHandle = 0; @@ -962,6 +965,7 @@ public void RenameSound(ReadOnlySpan soundname, ReadOnlySpan newname entry.Name = newname; // Re-insert in new spot Sounds.AddLast(entryNode); + SoundEntryLookup[entry] = entryNode; // Mark associated script as dirty SoundKeyValues.AsSpan()[entry.ScriptFileIndex].Dirty = true;