From f4416dce9b2ad5bfc8bee0b303666f36ade384ba Mon Sep 17 00:00:00 2001 From: Diogo Trindade Date: Fri, 27 Dec 2024 15:44:37 +0000 Subject: [PATCH] dont inline file records --- .../Properties/launchSettings.json | 2 +- src/StarBreaker.DataCore/DataCoreBinary.cs | 60 ++++++++++++------- src/StarBreaker.DataCore/DataCoreDatabase.cs | 9 +++ src/StarBreaker.DataCore/DataForge.cs | 2 +- src/StarBreaker.Sandbox/TagDatabase.cs | 6 +- 5 files changed, 52 insertions(+), 27 deletions(-) diff --git a/src/StarBreaker.Cli/Properties/launchSettings.json b/src/StarBreaker.Cli/Properties/launchSettings.json index 8a68fb8..d032df2 100644 --- a/src/StarBreaker.Cli/Properties/launchSettings.json +++ b/src/StarBreaker.Cli/Properties/launchSettings.json @@ -14,7 +14,7 @@ }, "p4k-extract-datacore": { "commandName": "Project", - "commandLineArgs": "p4k-extract --p4k \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" --output \"D:\\StarCitizen\\p4keptu\" --filter \"*system.socpak*\"" + "commandLineArgs": "p4k-extract --p4k \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" --output \"D:\\StarCitizen\\p4keptu\" --filter \"*.dcb\"" }, "chf-download": { "commandName": "Project", diff --git a/src/StarBreaker.DataCore/DataCoreBinary.cs b/src/StarBreaker.DataCore/DataCoreBinary.cs index 32de04e..9bf3357 100644 --- a/src/StarBreaker.DataCore/DataCoreBinary.cs +++ b/src/StarBreaker.DataCore/DataCoreBinary.cs @@ -4,13 +4,6 @@ namespace StarBreaker.DataCore; -public sealed class VisitedTracker -{ - private readonly HashSet<(int, int)> _visited = []; - - public bool TryVisit(int structIndex, int instanceIndex) => _visited.Add((structIndex, instanceIndex)); -} - public sealed class DataCoreBinary { public DataCoreDatabase Database { get; } @@ -20,7 +13,7 @@ public DataCoreBinary(Stream fs) Database = new DataCoreDatabase(fs); } - private XElement GetFromStruct(int structIndex, ref SpanReader reader, VisitedTracker tracker) + private XElement GetFromStruct(int structIndex, ref SpanReader reader, Stack<(int, int)> tracker) { var node = new XElement(Database.StructDefinitions[structIndex].GetName(Database)); @@ -40,7 +33,7 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, VisitedTr return node; } - private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader, VisitedTracker tracker) + private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader, Stack<(int, int)> tracker) { var count = reader.ReadInt32(); var firstIndex = reader.ReadInt32(); @@ -52,7 +45,7 @@ private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader arrayNode.Add(prop.DataType switch { - DataType.Reference => GetReference(Database.ReferenceValues[instanceIndex], tracker), + DataType.Reference => GetFromReference(Database.ReferenceValues[instanceIndex], tracker), DataType.WeakPointer => GetFromPointer(Database.WeakValues[instanceIndex], tracker), DataType.StrongPointer => GetFromPointer(Database.StrongValues[instanceIndex], tracker), DataType.Class => GetFromInstance(prop.StructIndex, instanceIndex, tracker), @@ -76,17 +69,14 @@ private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader }); } - if (count == 0) - arrayNode.Add(new XAttribute("Empty", "true")); - return arrayNode; } - private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader reader, VisitedTracker tracker) + private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader reader, Stack<(int, int)> tracker) { return prop.DataType switch { - DataType.Reference => GetReference(reader.Read(), tracker), + DataType.Reference => GetFromReference(reader.Read(), tracker), DataType.WeakPointer => GetFromPointer(reader.Read(), tracker), DataType.StrongPointer => GetFromPointer(reader.Read(), tracker), DataType.Class => GetFromStruct(prop.StructIndex, ref reader, tracker), @@ -110,21 +100,31 @@ private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader rea }; } - private XElement GetReference(DataCoreReference reference, VisitedTracker tracker) + private XElement GetFromReference(DataCoreReference reference, Stack<(int, int)> tracker) { if (reference.IsInvalid) { var invalidNode = new XElement("InvalidReference"); invalidNode.Add(new XAttribute("__guid", reference.RecordId.ToString())); + invalidNode.Add(new XAttribute("__instanceIndex", reference.InstanceIndex.ToString(CultureInfo.InvariantCulture))); return invalidNode; } var record = Database.GetRecord(reference.RecordId); + if (IsReferenceForFile(reference)) + { + //if we're referencing a full on file, just add a small mention to it + var fileReferenceNode = new XElement("FileReference"); + fileReferenceNode.Add(new XAttribute("__guid", reference.RecordId.ToString())); + fileReferenceNode.Add(new XAttribute("__fileName", record.GetFileName(Database))); + return fileReferenceNode; + } + return GetFromInstance(record.StructIndex, record.InstanceIndex, tracker); } - private XElement GetFromPointer(DataCorePointer pointer, VisitedTracker tracker) + private XElement GetFromPointer(DataCorePointer pointer, Stack<(int, int)> tracker) { if (pointer.IsInvalid) { @@ -139,21 +139,35 @@ private XElement GetFromPointer(DataCorePointer pointer, VisitedTracker tracker) public XElement GetFromRecord(DataCoreRecord record) { - return GetFromInstance(record.StructIndex, record.InstanceIndex, new VisitedTracker()); + return GetFromInstance(record.StructIndex, record.InstanceIndex, new Stack<(int, int)>()); } - private XElement GetFromInstance(int structIndex, int instanceIndex, VisitedTracker tracker) + private XElement GetFromInstance(int structIndex, int instanceIndex, Stack<(int, int)> tracker) { - if (!tracker.TryVisit(structIndex, instanceIndex)) + if (tracker.Contains((structIndex, instanceIndex))) { var circularNode = new XElement("CircularReference"); + circularNode.Add(new XAttribute("__structName", Database.StructDefinitions[structIndex].GetName(Database))); circularNode.Add(new XAttribute("__structIndex", structIndex.ToString(CultureInfo.InvariantCulture))); circularNode.Add(new XAttribute("__instanceIndex", instanceIndex.ToString(CultureInfo.InvariantCulture))); - circularNode.Add(new XAttribute("__structName", Database.StructDefinitions[structIndex].GetName(Database))); + //TODO: add more info here? return circularNode; } - var reader = Database.GetReader(Database.Offsets[structIndex][instanceIndex]); - return GetFromStruct(structIndex, ref reader, tracker); + tracker.Push((structIndex, instanceIndex)); + try + { + var reader = Database.GetReader(Database.Offsets[structIndex][instanceIndex]); + return GetFromStruct(structIndex, ref reader, tracker); + } + finally + { + tracker.Pop(); + } + } + + public bool IsReferenceForFile(DataCoreReference reference) + { + return Database.MainRecords.Contains(reference.RecordId); } } \ No newline at end of file diff --git a/src/StarBreaker.DataCore/DataCoreDatabase.cs b/src/StarBreaker.DataCore/DataCoreDatabase.cs index 12eaef1..f119328 100644 --- a/src/StarBreaker.DataCore/DataCoreDatabase.cs +++ b/src/StarBreaker.DataCore/DataCoreDatabase.cs @@ -48,6 +48,7 @@ public class DataCoreDatabase public readonly FrozenDictionary CachedStrings; public readonly FrozenDictionary CachedStrings2; public readonly FrozenDictionary RecordMap; + public readonly FrozenSet MainRecords; public DataCoreDatabase(Stream fs) { @@ -130,6 +131,14 @@ public DataCoreDatabase(Stream fs) RecordMap = RecordDefinitions.ToDictionary(x => x.Id).ToFrozenDictionary(); + var mainRecords = new Dictionary(); + foreach (var record in RecordDefinitions) + { + mainRecords[record.GetFileName(this)] = record; + } + + MainRecords = mainRecords.Values.Select(x => x.Id).ToFrozenSet(); + #if DEBUG DebugGlobal.Database = this; #endif diff --git a/src/StarBreaker.DataCore/DataForge.cs b/src/StarBreaker.DataCore/DataForge.cs index 68b970a..aa919b2 100644 --- a/src/StarBreaker.DataCore/DataForge.cs +++ b/src/StarBreaker.DataCore/DataForge.cs @@ -85,7 +85,7 @@ public void ExtractAllParallel(string outputFolder, string? fileNameFilter = nul var recordsByFileName = GetRecordsByFileName(fileNameFilter); var total = recordsByFileName.Count; - Parallel.ForEach(recordsByFileName, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, kvp => + Parallel.ForEach(recordsByFileName, kvp => { var (fileName, record) = kvp; var filePath = Path.Combine(outputFolder, fileName); diff --git a/src/StarBreaker.Sandbox/TagDatabase.cs b/src/StarBreaker.Sandbox/TagDatabase.cs index e132b62..82595bb 100644 --- a/src/StarBreaker.Sandbox/TagDatabase.cs +++ b/src/StarBreaker.Sandbox/TagDatabase.cs @@ -28,12 +28,14 @@ public static void Run() var megaMap = dcb.GetRecordsByFileName("*megamap.pu*").Values.Single(); var tagDatabase = dcb.GetRecordsByFileName("*TagDatabase*").Values.Single(); var broker = dcb.GetRecordsByFileName("*missionbroker.pu*").Values.Single(); + var unittest = dcb.GetRecordsByFileName("*unittesta*").Values.Single(); var someActorThing = dcb.DataCore.Database.GetRecord(new CigGuid("41d8fb15-72bb-446e-81df-eaecbc01e195")); - dcb.GetFromRecord(someActorThing).Save(@"D:\someActorThing.xml"); + dcb.GetFromRecord(broker).Save(@"D:\broker.xml"); + dcb.GetFromRecord(unittest).Save(@"D:\unittesta.xml"); + dcb.GetFromRecord(someActorThing).Save(@"D:\someActorThing.xml"); dcb.GetFromRecord(tagDatabase).Save(@"D:\tagdatabase.xml"); dcb.GetFromRecord(megaMap).Save(@"D:\megamap.xml"); - dcb.GetFromRecord(broker).Save(@"D:\broker.xml"); //dcb.ExtractAll(@"D:\DataCore2"); }