Skip to content

Commit 6e0392d

Browse files
marchc1marchc1
authored
Merge pull request #198 from ZehMatt/vpk-optimization
Store VPK entries as structs indexing the tree buffer
2 parents c888905 + 33f9f54 commit 6e0392d

5 files changed

Lines changed: 206 additions & 249 deletions

File tree

‎Source.Filesystem/PackStoreSearchPath.cs‎

Lines changed: 48 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
using Source.FileSystem;
33
using Source.VPK;
44

5+
using System.Runtime.CompilerServices;
6+
using System.Runtime.InteropServices;
7+
using System.Text;
58

69
namespace Source.Filesystem;
710

@@ -40,34 +43,32 @@ public PackStore(string key, string vpkPath) {
4043
Key = key;
4144
Vpk.Load(vpkPath);
4245

43-
int entryCount = 0;
44-
foreach (var dir in Vpk.Directories)
45-
entryCount += dir.Entries.Count;
46-
4746
DirectoryLookups.EnsureCapacity(Vpk.Directories.Count);
48-
EntryLookups.EnsureCapacity(entryCount);
47+
EntryLookups.EnsureCapacity(Vpk.EntryCount);
4948

5049
Span<char> buildPath = stackalloc char[260];
51-
foreach (var dir in Vpk.Directories) {
52-
DirectoryLookups[dir.Path.Hash()] = dir;
53-
54-
foreach (var entry in dir.Entries) {
55-
var path = entry.Path.Replace('\\', '/');
56-
var filename = entry.Filename;
57-
var ext = entry.Extension;
58-
59-
int strlen = 0;
60-
if (path.Length > 0 && path[0] != ' ') {
61-
path.CopyTo(buildPath[strlen..]); strlen += path.Length;
62-
buildPath[strlen] = '/'; strlen += 1;
63-
}
64-
filename.CopyTo(buildPath[strlen..]); strlen += filename.Length;
65-
buildPath[strlen] = '.'; strlen += 1;
66-
ext.CopyTo(buildPath[strlen..]); strlen += ext.Length;
67-
68-
ReadOnlySpan<char> finalSpan = buildPath[..strlen];
69-
EntryLookups[finalSpan.Hash()] = entry;
50+
foreach (VpkDirectory dir in Vpk.Directories) {
51+
int pathLength = Encoding.Latin1.GetChars(Vpk.GetPath(in dir), buildPath);
52+
DirectoryLookups[((ReadOnlySpan<char>)buildPath[..pathLength]).Hash()] = dir;
53+
}
54+
55+
foreach (VpkEntry entry in Vpk.EnumerateEntries()) {
56+
ReadOnlySpan<byte> path = Vpk.GetPath(in entry);
57+
ReadOnlySpan<byte> filename = Vpk.GetFilename(in entry);
58+
ReadOnlySpan<byte> ext = Vpk.GetExtension(in entry);
59+
60+
int strlen = 0;
61+
if (path.Length > 0 && path[0] != ' ') {
62+
strlen += Encoding.Latin1.GetChars(path, buildPath[strlen..]);
63+
buildPath[..strlen].Replace('\\', '/');
64+
buildPath[strlen] = '/'; strlen += 1;
7065
}
66+
strlen += Encoding.Latin1.GetChars(filename, buildPath[strlen..]);
67+
buildPath[strlen] = '.'; strlen += 1;
68+
strlen += Encoding.Latin1.GetChars(ext, buildPath[strlen..]);
69+
70+
ReadOnlySpan<char> finalSpan = buildPath[..strlen];
71+
EntryLookups[finalSpan.Hash()] = entry;
7172
}
7273
}
7374
}
@@ -141,8 +142,9 @@ public override bool IsFileWritable(ReadOnlySpan<char> path) {
141142
}
142143

143144
public override IFileHandle? Open(ReadOnlySpan<char> path, FileOpenOptions options) {
144-
if (vpkEntryLookups.TryGetValue(path.Hash(), out VpkEntry? entry))
145-
return new VpkFileHandle(parent, parent.FindOrAddFileName(path), new MemoryStream(entry.Data));
145+
ref VpkEntry entry = ref CollectionsMarshal.GetValueRefOrNullRef(vpkEntryLookups, path.Hash());
146+
if (!Unsafe.IsNullRef(ref entry))
147+
return new VpkFileHandle(parent, parent.FindOrAddFileName(path), new MemoryStream(store!.Vpk.ReadData(in entry)));
146148

147149
return null;
148150
}
@@ -153,7 +155,8 @@ public override bool IsFileWritable(ReadOnlySpan<char> path) {
153155
public override bool SetFileWritable(ReadOnlySpan<char> path, bool writable) => false;
154156

155157
public override long Size(ReadOnlySpan<char> path) {
156-
if (vpkEntryLookups.TryGetValue(path.Hash(), out VpkEntry? entry))
158+
ref VpkEntry entry = ref CollectionsMarshal.GetValueRefOrNullRef(vpkEntryLookups, path.Hash());
159+
if (!Unsafe.IsNullRef(ref entry))
157160
return entry.EntryLength;
158161
return -1;
159162
}
@@ -184,9 +187,14 @@ public override void PrepareFinds(List<string> files, List<string> dirs, string?
184187
int wildcardAsteriskAtFile = wildcardFile.IndexOf('*');
185188
int wildcardAsteriskAtExt = wildcardExt.IndexOf('*');
186189

190+
VpkArchive vpk = store!.Vpk;
191+
Span<char> pathBuffer = stackalloc char[260];
192+
Span<char> filenameBuffer = stackalloc char[260];
193+
Span<char> extBuffer = stackalloc char[260];
194+
187195
foreach (var directory in vpkDirectoryLookups) {
188196
VpkDirectory dir = directory.Value;
189-
string path = dir.Path;
197+
ReadOnlySpan<char> path = pathBuffer[..Encoding.Latin1.GetChars(vpk.GetPath(in dir), pathBuffer)];
190198
path.FileInfo(null, out var baseDirectory, out var baseName, out _);
191199

192200
if (wildcardAsteriskAtDir == -1) {
@@ -206,35 +214,40 @@ public override void PrepareFinds(List<string> files, List<string> dirs, string?
206214

207215
foreach (var entryKVP in vpkEntryLookups) {
208216
VpkEntry entry = entryKVP.Value;
217+
ReadOnlySpan<char> entryPath = pathBuffer[..Encoding.Latin1.GetChars(vpk.GetPath(in entry), pathBuffer)];
209218

210219
if (wildcardAsteriskAtDir == -1) {
211-
if (!entry.Path.PathEquals(wildcardDir))
220+
if (!entryPath.PathEquals(wildcardDir))
212221
continue;
213222
}
214223
else {
215-
if (wildcardDir != "*" && !entry.Path.PathStartsWith(wildcardDir))
224+
if (wildcardDir != "*" && !entryPath.PathStartsWith(wildcardDir))
216225
continue;
217226
}
218227

228+
ReadOnlySpan<char> entryFilename = filenameBuffer[..Encoding.Latin1.GetChars(vpk.GetFilename(in entry), filenameBuffer)];
229+
219230
if (wildcardAsteriskAtFile == -1) {
220-
if (!entry.Filename.PathEquals(wildcardFile))
231+
if (!entryFilename.PathEquals(wildcardFile))
221232
continue;
222233
}
223234
else {
224-
if (wildcardFile[0] != '*' && !entry.Filename.PathStartsWith(wildcardFile))
235+
if (wildcardFile[0] != '*' && !entryFilename.PathStartsWith(wildcardFile))
225236
continue;
226237
}
227238

239+
ReadOnlySpan<char> entryExt = extBuffer[..Encoding.Latin1.GetChars(vpk.GetExtension(in entry), extBuffer)];
240+
228241
if (wildcardAsteriskAtExt == -1) {
229-
if (!entry.Extension.PathEquals(wildcardExt))
242+
if (!entryExt.PathEquals(wildcardExt))
230243
continue;
231244
}
232245
else {
233-
if (!wildcardExt.IsEmpty && wildcardExt[0] != '*' && !entry.Extension.PathStartsWith(wildcardExt))
246+
if (!wildcardExt.IsEmpty && wildcardExt[0] != '*' && !entryExt.PathStartsWith(wildcardExt))
234247
continue;
235248
}
236249

237-
files.Add(entry.FilenameAndExtension);
250+
files.Add(string.Concat(entryFilename, ".", entryExt));
238251
}
239252
}
240253
}

‎Source.VPK/VpkArchive.cs‎

Lines changed: 126 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
using Microsoft.Win32.SafeHandles;
22

3+
using System.Runtime.CompilerServices;
4+
using System.Runtime.InteropServices;
5+
36
using Source.VPK.Exceptions;
47
using Source.VPK.V1;
58

@@ -13,6 +16,125 @@ public class VpkArchive : IDisposable
1316
internal List<ArchivePart> Parts { get; set; }
1417
internal string ArchivePath { get; set; }
1518
public readonly SafeFileHandle FileHandle;
19+
public byte[] Tree { get; private set; } = [];
20+
public int EntryCount { get; private set; }
21+
22+
const int EntryDataSize = sizeof(uint) + sizeof(ushort) + sizeof(ushort) + sizeof(uint) + sizeof(uint) + sizeof(ushort);
23+
24+
struct TreeReader(byte[] tree)
25+
{
26+
readonly byte[] Tree = tree;
27+
int Position;
28+
29+
public bool ReadString(out int offset, out ushort length) {
30+
ReadOnlySpan<byte> remaining = Tree.AsSpan(Position);
31+
int end = remaining.IndexOf((byte)0);
32+
offset = Position;
33+
if (end == -1) {
34+
end = remaining.Length;
35+
Position += end;
36+
}
37+
else
38+
Position += end + 1;
39+
40+
length = (ushort)end;
41+
return end != 0;
42+
}
43+
44+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
45+
T Read<T>() where T : unmanaged {
46+
T value = MemoryMarshal.Read<T>(Tree.AsSpan(Position));
47+
Position += Unsafe.SizeOf<T>();
48+
return value;
49+
}
50+
51+
public bool ReadEntry(int pathOffset, ushort pathLength, int extensionOffset, ushort extensionLength, out VpkEntry entry) {
52+
entry = default;
53+
if (!ReadString(out int filenameOffset, out ushort filenameLength))
54+
return false;
55+
56+
if (Tree.Length - Position < EntryDataSize)
57+
return false;
58+
59+
uint crc = Read<uint>();
60+
ushort preloadBytes = Read<ushort>();
61+
ushort archiveIndex = Read<ushort>();
62+
uint entryOffset = Read<uint>();
63+
uint entryLength = Read<uint>();
64+
// skip terminator
65+
Read<ushort>();
66+
int preloadTreeOffset = Position;
67+
Position = Math.Min(Position + preloadBytes, Tree.Length);
68+
69+
entry = new VpkEntry(pathOffset, pathLength, filenameOffset, filenameLength, extensionOffset, extensionLength,
70+
crc, preloadBytes, preloadTreeOffset, archiveIndex, entryOffset, entryLength);
71+
return true;
72+
}
73+
}
74+
75+
static void LowerAscii(Span<byte> text) {
76+
for (int i = 0; i < text.Length; i++) {
77+
if ((uint)(text[i] - 'A') <= 'Z' - 'A')
78+
text[i] |= 0x20;
79+
}
80+
}
81+
82+
void ParseTree() {
83+
TreeReader reader = new(Tree);
84+
while (reader.ReadString(out int extensionOffset, out ushort extensionLength)) {
85+
LowerAscii(Tree.AsSpan(extensionOffset, extensionLength)); // once per extension block
86+
while (reader.ReadString(out int pathOffset, out ushort pathLength)) {
87+
LowerAscii(Tree.AsSpan(pathOffset, pathLength));
88+
Directories.Add(new VpkDirectory(pathOffset, pathLength));
89+
while (reader.ReadEntry(pathOffset, pathLength, extensionOffset, extensionLength, out VpkEntry entry)) {
90+
LowerAscii(Tree.AsSpan(entry.FilenameOffset, entry.FilenameLength));
91+
EntryCount++;
92+
}
93+
}
94+
}
95+
}
96+
97+
public IEnumerable<VpkEntry> EnumerateEntries() {
98+
TreeReader reader = new(Tree);
99+
while (reader.ReadString(out int extensionOffset, out ushort extensionLength)) {
100+
while (reader.ReadString(out int pathOffset, out ushort pathLength)) {
101+
while (reader.ReadEntry(pathOffset, pathLength, extensionOffset, extensionLength, out VpkEntry entry))
102+
yield return entry;
103+
}
104+
}
105+
}
106+
107+
public ReadOnlySpan<byte> GetPath(in VpkDirectory directory) => Tree.AsSpan(directory.PathOffset, directory.PathLength);
108+
public ReadOnlySpan<byte> GetPath(in VpkEntry entry) => Tree.AsSpan(entry.PathOffset, entry.PathLength);
109+
public ReadOnlySpan<byte> GetFilename(in VpkEntry entry) => Tree.AsSpan(entry.FilenameOffset, entry.FilenameLength);
110+
public ReadOnlySpan<byte> GetExtension(in VpkEntry entry) => Tree.AsSpan(entry.ExtensionOffset, entry.ExtensionLength);
111+
public ReadOnlySpan<byte> GetPreloadData(in VpkEntry entry) => Tree.AsSpan(entry.PreloadTreeOffset, Math.Min(entry.PreloadBytes, Tree.Length - entry.PreloadTreeOffset));
112+
113+
public byte[]? ReadPreloadData(in VpkEntry entry) => entry.HasPreloadData ? GetPreloadData(in entry).ToArray() : null;
114+
public byte[]? ReadAnyData(in VpkEntry entry) => entry.HasPreloadData ? ReadPreloadData(in entry) : ReadData(in entry);
115+
116+
ArchivePart? FindPart(ushort archiveIndex) {
117+
if (Parts == null)
118+
return null;
119+
120+
for (int i = 0, c = Parts.Count; i < c; i++) {
121+
ArchivePart part = Parts[i];
122+
if (part.Index == archiveIndex)
123+
return part;
124+
}
125+
return null;
126+
}
127+
128+
public byte[] ReadData(in VpkEntry entry) {
129+
ArchivePart? part = FindPart(entry.ArchiveIndex);
130+
if (part == null)
131+
return [];
132+
133+
byte[] data = new byte[entry.PreloadBytes + entry.EntryLength];
134+
GetPreloadData(in entry).CopyTo(data);
135+
RandomAccess.Read(part.FileHandle, data.AsSpan(entry.PreloadBytes), entry.EntryOffset);
136+
return data;
137+
}
16138

17139
public override string ToString() => $"VpkArchive '{ArchivePath}' [{Directories.Count} directories, {Parts.Count} parts]";
18140

@@ -66,9 +188,8 @@ public void Load(string filename, VpkVersions.Versions version = VpkVersions.Ver
66188
break;
67189
}
68190

69-
var reader = _reader.ReadDirectories(this, header.TreeLength);
70-
while (reader.MoveNext())
71-
Directories.Add(reader.Current);
191+
Tree = _reader.ReadTree(header.TreeLength);
192+
ParseTree();
72193
}
73194
finally {
74195
_reader?.Dispose();
@@ -109,9 +230,8 @@ public void Load(byte[] file, VpkVersions.Versions version = VpkVersions.Version
109230
break;
110231
}
111232

112-
var reader = _reader.ReadDirectories(this, header.TreeLength);
113-
while (reader.MoveNext())
114-
Directories.Add(reader.Current);
233+
Tree = _reader.ReadTree(header.TreeLength);
234+
ParseTree();
115235
}
116236
finally {
117237
_reader?.Dispose();

‎Source.VPK/VpkDirectory.cs‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,10 @@
11
namespace Source.VPK
22
{
3-
public class VpkDirectory
4-
{
5-
public List<VpkEntry> Entries { get; set; }
6-
public string Path { get; set; }
7-
internal VpkArchive ParentArchive { get; set; }
3+
public readonly struct VpkDirectory(int pathOffset, ushort pathLength)
4+
{
5+
public readonly int PathOffset = pathOffset;
6+
public readonly ushort PathLength = pathLength;
87

9-
public override string ToString() => $"VpkDirectory '{Path}' [{Entries.Count} entries]";
10-
11-
internal VpkDirectory(VpkArchive parentArchive, string path, List<VpkEntry> entries)
12-
{
13-
ParentArchive = parentArchive;
14-
Path = path.ToLower();
15-
Entries = entries;
16-
}
17-
}
8+
public override string ToString() => $"VpkDirectory [path<{PathOffset}-{PathLength}>]";
9+
}
1810
}

0 commit comments

Comments
 (0)