Skip to content

Commit

Permalink
dont inline file records
Browse files Browse the repository at this point in the history
  • Loading branch information
@diogotr7
diogotr7 committed Dec 27, 2024
1 parent 211ddda commit f4416dc
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 27 deletions.
2 changes: 1 addition & 1 deletion src/StarBreaker.Cli/Properties/launchSettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
},
"p4k-extract-datacore": {
"commandName": "Project",
"commandLineArgs": "p4k-extract --p4k \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" --output \"D:\\StarCitizen\\p4keptu\" --filter \"*system.socpak*\""
"commandLineArgs": "p4k-extract --p4k \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\4.0_PREVIEW\\Data.p4k\" --output \"D:\\StarCitizen\\p4keptu\" --filter \"*.dcb\""
},
"chf-download": {
"commandName": "Project",
Expand Down
60 changes: 37 additions & 23 deletions src/StarBreaker.DataCore/DataCoreBinary.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,6 @@

namespace StarBreaker.DataCore;

public sealed class VisitedTracker
{
private readonly HashSet<(int, int)> _visited = [];

public bool TryVisit(int structIndex, int instanceIndex) => _visited.Add((structIndex, instanceIndex));
}

public sealed class DataCoreBinary
{
public DataCoreDatabase Database { get; }
Expand All @@ -20,7 +13,7 @@ public DataCoreBinary(Stream fs)
Database = new DataCoreDatabase(fs);
}

private XElement GetFromStruct(int structIndex, ref SpanReader reader, VisitedTracker tracker)
private XElement GetFromStruct(int structIndex, ref SpanReader reader, Stack<(int, int)> tracker)
{
var node = new XElement(Database.StructDefinitions[structIndex].GetName(Database));

Expand All @@ -40,7 +33,7 @@ private XElement GetFromStruct(int structIndex, ref SpanReader reader, VisitedTr
return node;
}

private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader, VisitedTracker tracker)
private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader, Stack<(int, int)> tracker)
{
var count = reader.ReadInt32();
var firstIndex = reader.ReadInt32();
Expand All @@ -52,7 +45,7 @@ private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader

arrayNode.Add(prop.DataType switch
{
DataType.Reference => GetReference(Database.ReferenceValues[instanceIndex], tracker),
DataType.Reference => GetFromReference(Database.ReferenceValues[instanceIndex], tracker),
DataType.WeakPointer => GetFromPointer(Database.WeakValues[instanceIndex], tracker),
DataType.StrongPointer => GetFromPointer(Database.StrongValues[instanceIndex], tracker),
DataType.Class => GetFromInstance(prop.StructIndex, instanceIndex, tracker),
Expand All @@ -76,17 +69,14 @@ private XElement GetArray(DataCorePropertyDefinition prop, ref SpanReader reader
});
}

if (count == 0)
arrayNode.Add(new XAttribute("Empty", "true"));

return arrayNode;
}

private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader reader, VisitedTracker tracker)
private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader reader, Stack<(int, int)> tracker)
{
return prop.DataType switch
{
DataType.Reference => GetReference(reader.Read<DataCoreReference>(), tracker),
DataType.Reference => GetFromReference(reader.Read<DataCoreReference>(), tracker),
DataType.WeakPointer => GetFromPointer(reader.Read<DataCorePointer>(), tracker),
DataType.StrongPointer => GetFromPointer(reader.Read<DataCorePointer>(), tracker),
DataType.Class => GetFromStruct(prop.StructIndex, ref reader, tracker),
Expand All @@ -110,21 +100,31 @@ private XObject GetAttribute(DataCorePropertyDefinition prop, ref SpanReader rea
};
}

private XElement GetReference(DataCoreReference reference, VisitedTracker tracker)
private XElement GetFromReference(DataCoreReference reference, Stack<(int, int)> tracker)
{
if (reference.IsInvalid)
{
var invalidNode = new XElement("InvalidReference");
invalidNode.Add(new XAttribute("__guid", reference.RecordId.ToString()));
invalidNode.Add(new XAttribute("__instanceIndex", reference.InstanceIndex.ToString(CultureInfo.InvariantCulture)));
return invalidNode;
}

var record = Database.GetRecord(reference.RecordId);

if (IsReferenceForFile(reference))
{
//if we're referencing a full on file, just add a small mention to it
var fileReferenceNode = new XElement("FileReference");
fileReferenceNode.Add(new XAttribute("__guid", reference.RecordId.ToString()));
fileReferenceNode.Add(new XAttribute("__fileName", record.GetFileName(Database)));
return fileReferenceNode;
}

return GetFromInstance(record.StructIndex, record.InstanceIndex, tracker);
}

private XElement GetFromPointer(DataCorePointer pointer, VisitedTracker tracker)
private XElement GetFromPointer(DataCorePointer pointer, Stack<(int, int)> tracker)
{
if (pointer.IsInvalid)
{
Expand All @@ -139,21 +139,35 @@ private XElement GetFromPointer(DataCorePointer pointer, VisitedTracker tracker)

public XElement GetFromRecord(DataCoreRecord record)
{
return GetFromInstance(record.StructIndex, record.InstanceIndex, new VisitedTracker());
return GetFromInstance(record.StructIndex, record.InstanceIndex, new Stack<(int, int)>());
}

private XElement GetFromInstance(int structIndex, int instanceIndex, VisitedTracker tracker)
private XElement GetFromInstance(int structIndex, int instanceIndex, Stack<(int, int)> tracker)
{
if (!tracker.TryVisit(structIndex, instanceIndex))
if (tracker.Contains((structIndex, instanceIndex)))
{
var circularNode = new XElement("CircularReference");
circularNode.Add(new XAttribute("__structName", Database.StructDefinitions[structIndex].GetName(Database)));
circularNode.Add(new XAttribute("__structIndex", structIndex.ToString(CultureInfo.InvariantCulture)));
circularNode.Add(new XAttribute("__instanceIndex", instanceIndex.ToString(CultureInfo.InvariantCulture)));
circularNode.Add(new XAttribute("__structName", Database.StructDefinitions[structIndex].GetName(Database)));
//TODO: add more info here?
return circularNode;
}

var reader = Database.GetReader(Database.Offsets[structIndex][instanceIndex]);
return GetFromStruct(structIndex, ref reader, tracker);
tracker.Push((structIndex, instanceIndex));
try
{
var reader = Database.GetReader(Database.Offsets[structIndex][instanceIndex]);
return GetFromStruct(structIndex, ref reader, tracker);
}
finally
{
tracker.Pop();
}
}

public bool IsReferenceForFile(DataCoreReference reference)
{
return Database.MainRecords.Contains(reference.RecordId);
}
}
9 changes: 9 additions & 0 deletions src/StarBreaker.DataCore/DataCoreDatabase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public class DataCoreDatabase
public readonly FrozenDictionary<int, string> CachedStrings;
public readonly FrozenDictionary<int, string> CachedStrings2;
public readonly FrozenDictionary<CigGuid, DataCoreRecord> RecordMap;
public readonly FrozenSet<CigGuid> MainRecords;

public DataCoreDatabase(Stream fs)
{
Expand Down Expand Up @@ -130,6 +131,14 @@ public DataCoreDatabase(Stream fs)

RecordMap = RecordDefinitions.ToDictionary(x => x.Id).ToFrozenDictionary();

var mainRecords = new Dictionary<string, DataCoreRecord>();
foreach (var record in RecordDefinitions)
{
mainRecords[record.GetFileName(this)] = record;
}

MainRecords = mainRecords.Values.Select(x => x.Id).ToFrozenSet();

#if DEBUG
DebugGlobal.Database = this;
#endif
Expand Down
2 changes: 1 addition & 1 deletion src/StarBreaker.DataCore/DataForge.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public void ExtractAllParallel(string outputFolder, string? fileNameFilter = nul
var recordsByFileName = GetRecordsByFileName(fileNameFilter);
var total = recordsByFileName.Count;

Parallel.ForEach(recordsByFileName, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, kvp =>
Parallel.ForEach(recordsByFileName, kvp =>
{
var (fileName, record) = kvp;
var filePath = Path.Combine(outputFolder, fileName);
Expand Down
6 changes: 4 additions & 2 deletions src/StarBreaker.Sandbox/TagDatabase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@ public static void Run()
var megaMap = dcb.GetRecordsByFileName("*megamap.pu*").Values.Single();
var tagDatabase = dcb.GetRecordsByFileName("*TagDatabase*").Values.Single();
var broker = dcb.GetRecordsByFileName("*missionbroker.pu*").Values.Single();
var unittest = dcb.GetRecordsByFileName("*unittesta*").Values.Single();
var someActorThing = dcb.DataCore.Database.GetRecord(new CigGuid("41d8fb15-72bb-446e-81df-eaecbc01e195"));
dcb.GetFromRecord(someActorThing).Save(@"D:\someActorThing.xml");

dcb.GetFromRecord(broker).Save(@"D:\broker.xml");
dcb.GetFromRecord(unittest).Save(@"D:\unittesta.xml");
dcb.GetFromRecord(someActorThing).Save(@"D:\someActorThing.xml");
dcb.GetFromRecord(tagDatabase).Save(@"D:\tagdatabase.xml");
dcb.GetFromRecord(megaMap).Save(@"D:\megamap.xml");
dcb.GetFromRecord(broker).Save(@"D:\broker.xml");

//dcb.ExtractAll(@"D:\DataCore2");
}
Expand Down

0 comments on commit f4416dc

Please sign in to comment.