From b171b65aa580ae0854297004e5305c37e524066c Mon Sep 17 00:00:00 2001 From: Samuele Lorefice Date: Thu, 20 Feb 2025 19:58:33 +0100 Subject: [PATCH] Added handling of pointers, huge memory improvements --- BlendFile/FileInfoExt.cs | 7 ++ BlendFile/Reader.cs | 107 ++++++++++++++++++------------ BlendFileTester/Program.cs | 14 ++-- BlenderSharp.sln.DotSettings.user | 6 +- 4 files changed, 85 insertions(+), 49 deletions(-) create mode 100644 BlendFile/FileInfoExt.cs diff --git a/BlendFile/FileInfoExt.cs b/BlendFile/FileInfoExt.cs new file mode 100644 index 0000000..b8a1276 --- /dev/null +++ b/BlendFile/FileInfoExt.cs @@ -0,0 +1,7 @@ +using System.Reflection; + +namespace BlendFile; + +public static class FileInfoExt { + static bool IsFieldPointer(this FieldInfo field) => field.GetCustomAttribute()!.IsPointer; +} \ No newline at end of file diff --git a/BlendFile/Reader.cs b/BlendFile/Reader.cs index 3932cac..064343c 100644 --- a/BlendFile/Reader.cs +++ b/BlendFile/Reader.cs @@ -1,4 +1,5 @@ using System.Reflection; +using System.Runtime.CompilerServices; using System.Text; using Kaitai; @@ -8,9 +9,19 @@ namespace BlendFile; public class Reader { string _path; private readonly Dictionary dnaTypes = new(); + private readonly Dictionary dnaTypesDb = new(); private Dictionary<(long, Type), object> objects = new(); public Dictionary<(long, Type), object> Objects => objects; + + /// + /// A dictionary that contains pointers to pointers + /// + /// + /// Key: Memory address of the pointer + /// Value: Memory address of the object we are pointing to + /// + private Dictionary pointers = new(); public List GetObjects() => objects.Values.ToList(); public List GetObjects() => objects.Values.OfType().ToList(); @@ -42,6 +53,7 @@ public class Reader { var attrib = type.GetCustomAttribute(); if (attrib == null) continue; dnaTypes.Add(attrib.OriginalIndex, type); + dnaTypesDb.Add(attrib.OriginalName, type); } } @@ -49,52 +61,74 @@ public class Reader { _path = path; Read(); } - + public void Read() { var file = new KaitaiStream(_path); var blend = new Kaitai.BlendFile(file); - + bool isLe = blend.Hdr.Endian == Kaitai.BlendFile.Endian.Le; //TODO: two blocks somehow have the same mem address... this sounds wrong. blend.Blocks.ForEach(block => memBlocks.TryAdd(block.MemAddr.ToMemAddr(isLe), block)); - - foreach (var block in blend.Blocks) - { - //We need to read all blocks of data regardeless of the type + + foreach (var block in blend.Blocks) { + //We need to read all blocks of data regardless of the type //if (block.Code != "DATA") continue; //Checks if the block has a known SDNA type, meaning that we know how it is structured - if(!dnaTypes.ContainsKey((int)block.SdnaIndex)) continue; + if (!dnaTypes.ContainsKey((int)block.SdnaIndex)) continue; + + //Get the type of the block + dnaTypesDb.TryGetValue(block.SdnaStruct.Type, out Type? t); + if (t == null) continue; //should never happen - Type t = dnaTypes.Values.First( - x => x.GetCustomAttribute()!.OriginalName == block.SdnaStruct.Type); // How many objects are in the block var count = block.Count; //offset for the next object in the block var blockOffset = 0; //for each expected object in the block - for(var i=0; i()!.Size; } } + objects.AsParallel().ForAll(x => + { + FieldInfo[] fieldInfo = x.Value.GetType().GetFields(); + fieldInfo.Where(fldInfo => fldInfo.GetCustomAttribute()!.IsPointer).ToList().ForEach(f => + { + var addr = GetBlockFieldDataOffset(x.Key.Item1, f.GetCustomAttribute()!.OriginalIndex, fieldInfo); + var obj = objects.GetValueOrDefault((addr, f.FieldType)); + if (obj != null) f.SetValue(x.Value, obj); + }); + }); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private long GetBlockFieldDataOffset(long blockAddress, int fieldIndex, FieldInfo[] fieldMetadata) => + blockAddress + GetFieldDataOffset(fieldIndex, fieldMetadata); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private long GetFieldDataOffset(int fieldIndex, FieldInfo[] fieldMetadata) => + fieldMetadata.Where(f => f.GetCustomAttribute()!.OriginalIndex < fieldIndex) + .Sum(f => f.GetCustomAttribute()!.Size); + + private object? ActivateInstance(string type) => dnaTypesDb.TryGetValue(type, out Type? t) ? Activator.CreateInstance(t) : null; + + /// - /// + /// Filles a given object with the data from a block, starting to read it from the specified offset /// - /// - /// - /// - /// - /// - private void FillObject(Kaitai.BlendFile.FileBlock block, ref object? obj, FieldInfo[] fieldMetadata, int startOffset = 0) { + /// data block from where to read the values + /// object of same struct type as the one that is being decoded + /// Array of s containing attributes + /// offset in bytes from where structure starts in the block + private void FillObject(Kaitai.BlendFile.FileBlock block, ref object? obj, FieldInfo[] fieldMetadata, long startOffset = 0) { if(block.Code == "ENDB") return; foreach (var field in fieldMetadata) { //Get the DNAFieldAttribute of the current field @@ -103,8 +137,7 @@ public class Reader { //Calculate the offset from where the data of the field starts. //Because the order of the fields is not guaranteed we need to compute it each time - int offset = fieldMetadata.Where(f => f.GetCustomAttribute()!.OriginalIndex < attrib.OriginalIndex) - .Sum(f => f.GetCustomAttribute()!.Size) + startOffset; + long offset = GetFieldDataOffset(attrib.OriginalIndex, fieldMetadata); int size = attrib.Size; var data = new byte[size]; @@ -114,39 +147,27 @@ public class Reader { if(value == null){ //if the data could not be converted //Check if the field is a pointer to another DNA structure - if (dnaTypes.Values.FirstOrDefault(x => x.GetCustomAttribute()!.OriginalName == attrib.OriginalType) != null) { + //if (dnaTypes.Values.FirstOrDefault(x => x.GetCustomAttribute()!.OriginalName == attrib.OriginalType) != null) { + if (dnaTypesDb.ContainsKey(attrib.OriginalType)) { //Create a new instance of the DNA structure type - object? newObj = Activator.CreateInstance(dnaTypes.Values.First(x => - x.GetCustomAttribute()!.OriginalName == attrib.OriginalType)); + object? newObj = ActivateInstance(attrib.OriginalType); if(newObj == null) continue; //should never happen... type is missing? //Get the information of the fields of the new object var fieldInfo = newObj.GetType().GetFields(); //If the field is a pointer, we need to dereference it - if (attrib.IsPointer) { - //Get the memory address of the data - long addr = data.ToMemAddr(); - //Get the closest block that contains the data - var derefBlock = GetBlock(addr); - //Calculate the relative address of the data in the block - long relAddr = addr - derefBlock!.MemAddr.ToMemAddr(); - - if(objects.TryGetValue((addr, newObj.GetType()), out object? o)){ //If the object is already created, we can just assign it - field.SetValue(obj, o); - continue; - } - - //Fill the object with the data from the block (this is recursive) - FillObject(derefBlock, ref newObj, fieldInfo, (int)relAddr); - } else { + if (!attrib.IsPointer) { long relAddr = block.MemAddr.ToMemAddr() + offset; - if(objects.TryGetValue((relAddr, newObj.GetType()), out object? o)){ //If the object is already created, we can just assign it + if (objects.TryGetValue((relAddr, newObj.GetType()), out object? o)) { + //If the object is already created, we can just assign it field.SetValue(obj, o); continue; } //Fill the object with the data from the block (this is recursive) FillObject(block, ref newObj, fieldInfo, offset); + } else { // if is a pointer, make a pointer to the pointer + pointers.TryAdd(block.MemAddr.ToMemAddr() + offset, data.ToMemAddr()); } } continue; //should never happen, but means the data could not be converted diff --git a/BlendFileTester/Program.cs b/BlendFileTester/Program.cs index 55c8abe..ba0dd8e 100644 --- a/BlendFileTester/Program.cs +++ b/BlendFileTester/Program.cs @@ -1,11 +1,15 @@ using System; +using System.Linq; using BlendFile; var reader = new Reader("cube.blend"); reader.Read(); -foreach (var obj in reader.GetObjects()) -{ - Console.WriteLine(obj.GetType()); - Console.WriteLine(obj.ToString()); -} \ No newline at end of file +var counts = reader.Objects.GroupBy(x => x.Key.Item2).ToList(); + +foreach (var count in counts) Console.WriteLine($"{count.Key}: {count.Count()}"); + +var Meshes = reader.GetObjects(); +var Objects = reader.GetObjects(); +Console.WriteLine($"Meshes: {Meshes.Count}"); +Console.WriteLine($"Objects: {Objects.Count}"); diff --git a/BlenderSharp.sln.DotSettings.user b/BlenderSharp.sln.DotSettings.user index 8a0f6e6..df4eaff 100644 --- a/BlenderSharp.sln.DotSettings.user +++ b/BlenderSharp.sln.DotSettings.user @@ -9,4 +9,8 @@ ForceIncluded ForceIncluded ForceIncluded - ForceIncluded \ No newline at end of file + ForceIncluded + True + 26214400 + 26214400 + 52428800 \ No newline at end of file