REDCODE/BlenderSharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added handling of pointers, huge memory improvements

Browse Source
This commit is contained in:
Samuele Lorefice
2025-02-20 19:58:33 +01:00
parent 6d565377a4
commit b171b65aa5
4 changed files with 85 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
BlendFile/FileInfoExt.cs Normal file
View File

@@ -0,0 +1,7 @@
using System.Reflection;
namespace BlendFile;
public static class FileInfoExt {
static bool IsFieldPointer(this FieldInfo field) => field.GetCustomAttribute<DNAFieldAttribute>()!.IsPointer;
}

107
BlendFile/Reader.cs
View File

@@ -1,4 +1,5 @@
using System.Reflection; using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text; using System.Text;
using Kaitai; using Kaitai;
@@ -8,9 +9,19 @@ namespace BlendFile;
public class Reader { public class Reader {
string _path; string _path;
private readonly Dictionary<int, Type> dnaTypes = new(); private readonly Dictionary<int, Type> dnaTypes = new();
private readonly Dictionary<string, Type> dnaTypesDb = new();
private Dictionary<(long, Type), object> objects = new(); private Dictionary<(long, Type), object> objects = new();
public Dictionary<(long, Type), object> Objects => objects; public Dictionary<(long, Type), object> Objects => objects;
/// <summary>
/// A dictionary that contains pointers to pointers
/// </summary>
/// <remarks>
/// Key: Memory address of the pointer
/// Value: Memory address of the object we are pointing to
/// </remarks>
private Dictionary<long, long> pointers = new();
public List<object> GetObjects() => objects.Values.ToList(); public List<object> GetObjects() => objects.Values.ToList();
public List<T> GetObjects<T>() => objects.Values.OfType<T>().ToList(); public List<T> GetObjects<T>() => objects.Values.OfType<T>().ToList();
@@ -42,6 +53,7 @@ public class Reader {
var attrib = type.GetCustomAttribute<DNAClassAttribute>(); var attrib = type.GetCustomAttribute<DNAClassAttribute>();
if (attrib == null) continue; if (attrib == null) continue;
dnaTypes.Add(attrib.OriginalIndex, type); dnaTypes.Add(attrib.OriginalIndex, type);
dnaTypesDb.Add(attrib.OriginalName, type);
} }
} }
@@ -49,52 +61,74 @@ public class Reader {
_path = path; _path = path;
Read(); Read();
} }
public void Read() { public void Read() {
var file = new KaitaiStream(_path); var file = new KaitaiStream(_path);
var blend = new Kaitai.BlendFile(file); var blend = new Kaitai.BlendFile(file);
bool isLe = blend.Hdr.Endian == Kaitai.BlendFile.Endian.Le; bool isLe = blend.Hdr.Endian == Kaitai.BlendFile.Endian.Le;
//TODO: two blocks somehow have the same mem address... this sounds wrong. //TODO: two blocks somehow have the same mem address... this sounds wrong.
blend.Blocks.ForEach(block => memBlocks.TryAdd(block.MemAddr.ToMemAddr(isLe), block)); blend.Blocks.ForEach(block => memBlocks.TryAdd(block.MemAddr.ToMemAddr(isLe), block));
foreach (var block in blend.Blocks) foreach (var block in blend.Blocks) {
{ //We need to read all blocks of data regardless of the type
//We need to read all blocks of data regardeless of the type
//if (block.Code != "DATA") continue; //if (block.Code != "DATA") continue;
//Checks if the block has a known SDNA type, meaning that we know how it is structured //Checks if the block has a known SDNA type, meaning that we know how it is structured
if(!dnaTypes.ContainsKey((int)block.SdnaIndex)) continue; if (!dnaTypes.ContainsKey((int)block.SdnaIndex)) continue;
//Get the type of the block
dnaTypesDb.TryGetValue(block.SdnaStruct.Type, out Type? t);
if (t == null) continue; //should never happen
Type t = dnaTypes.Values.First(
x => x.GetCustomAttribute<DNAClassAttribute>()!.OriginalName == block.SdnaStruct.Type);
// How many objects are in the block // How many objects are in the block
var count = block.Count; var count = block.Count;
//offset for the next object in the block //offset for the next object in the block
var blockOffset = 0; var blockOffset = 0;
//for each expected object in the block //for each expected object in the block
for(var i=0; i<count; i++) { for (var i = 0; i < count; i++) {
//create an instance of type "t" (the dna type we inferred before) //create an instance of type "t" (the dna type we inferred before)
var obj = Activator.CreateInstance(t); var obj = ActivateInstance(block.SdnaStruct.Type);
if(obj == null) continue; //should never happen if (obj == null) continue; //should never happen
//fill the object with the data from the block //fill the object with the data from the block
FillObject(block, ref obj, t.GetFields(), blockOffset); FillObject(block, ref obj, t.GetFields(), blockOffset);
//move the offset to the next object in the block //move the offset to the next object in the block
blockOffset += t.GetCustomAttribute<DNAClassAttribute>()!.Size; blockOffset += t.GetCustomAttribute<DNAClassAttribute>()!.Size;
} }
} }
objects.AsParallel().ForAll(x =>
{
FieldInfo[] fieldInfo = x.Value.GetType().GetFields();
fieldInfo.Where(fldInfo => fldInfo.GetCustomAttribute<DNAFieldAttribute>()!.IsPointer).ToList().ForEach(f =>
{
var addr = GetBlockFieldDataOffset(x.Key.Item1, f.GetCustomAttribute<DNAFieldAttribute>()!.OriginalIndex, fieldInfo);
var obj = objects.GetValueOrDefault((addr, f.FieldType));
if (obj != null) f.SetValue(x.Value, obj);
});
});
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private long GetBlockFieldDataOffset(long blockAddress, int fieldIndex, FieldInfo[] fieldMetadata) =>
blockAddress + GetFieldDataOffset(fieldIndex, fieldMetadata);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private long GetFieldDataOffset(int fieldIndex, FieldInfo[] fieldMetadata) =>
fieldMetadata.Where(f => f.GetCustomAttribute<DNAFieldAttribute>()!.OriginalIndex < fieldIndex)
.Sum(f => f.GetCustomAttribute<DNAFieldAttribute>()!.Size);
private object? ActivateInstance(string type) => dnaTypesDb.TryGetValue(type, out Type? t) ? Activator.CreateInstance(t) : null;
/// <summary> /// <summary>
/// /// Filles a given object with the data from a block, starting to read it from the specified offset
/// </summary> /// </summary>
/// <param name="block"></param> /// <param name="block">data block from where to read the values</param>
/// <param name="obj"></param> /// <param name="obj">object of same struct type as the one that is being decoded</param>
/// <param name="fieldMetadata"></param> /// <param name="fieldMetadata">Array of <see cref="FieldInfo"/>s containing <see cref="DNAFieldAttribute"/> attributes</param>
/// <param name="startOffset"></param> /// <param name="startOffset">offset in bytes from where structure starts in the block</param>
/// <returns></returns> private void FillObject(Kaitai.BlendFile.FileBlock block, ref object? obj, FieldInfo[] fieldMetadata, long startOffset = 0) {
private void FillObject(Kaitai.BlendFile.FileBlock block, ref object? obj, FieldInfo[] fieldMetadata, int startOffset = 0) {
if(block.Code == "ENDB") return; if(block.Code == "ENDB") return;
foreach (var field in fieldMetadata) { foreach (var field in fieldMetadata) {
//Get the DNAFieldAttribute of the current field //Get the DNAFieldAttribute of the current field
@@ -103,8 +137,7 @@ public class Reader {
//Calculate the offset from where the data of the field starts. //Calculate the offset from where the data of the field starts.
//Because the order of the fields is not guaranteed we need to compute it each time //Because the order of the fields is not guaranteed we need to compute it each time
int offset = fieldMetadata.Where(f => f.GetCustomAttribute<DNAFieldAttribute>()!.OriginalIndex < attrib.OriginalIndex) long offset = GetFieldDataOffset(attrib.OriginalIndex, fieldMetadata);
.Sum(f => f.GetCustomAttribute<DNAFieldAttribute>()!.Size) + startOffset;
int size = attrib.Size; int size = attrib.Size;
var data = new byte[size]; var data = new byte[size];
@@ -114,39 +147,27 @@ public class Reader {
if(value == null){ //if the data could not be converted if(value == null){ //if the data could not be converted
//Check if the field is a pointer to another DNA structure //Check if the field is a pointer to another DNA structure
if (dnaTypes.Values.FirstOrDefault(x => x.GetCustomAttribute<DNAClassAttribute>()!.OriginalName == attrib.OriginalType) != null) { //if (dnaTypes.Values.FirstOrDefault(x => x.GetCustomAttribute<DNAClassAttribute>()!.OriginalName == attrib.OriginalType) != null) {
if (dnaTypesDb.ContainsKey(attrib.OriginalType)) {
//Create a new instance of the DNA structure type //Create a new instance of the DNA structure type
object? newObj = Activator.CreateInstance(dnaTypes.Values.First(x => object? newObj = ActivateInstance(attrib.OriginalType);
x.GetCustomAttribute<DNAClassAttribute>()!.OriginalName == attrib.OriginalType));
if(newObj == null) continue; //should never happen... type is missing? if(newObj == null) continue; //should never happen... type is missing?
//Get the information of the fields of the new object //Get the information of the fields of the new object
var fieldInfo = newObj.GetType().GetFields(); var fieldInfo = newObj.GetType().GetFields();
//If the field is a pointer, we need to dereference it //If the field is a pointer, we need to dereference it
if (attrib.IsPointer) { if (!attrib.IsPointer) {
//Get the memory address of the data
long addr = data.ToMemAddr();
//Get the closest block that contains the data
var derefBlock = GetBlock(addr);
//Calculate the relative address of the data in the block
long relAddr = addr - derefBlock!.MemAddr.ToMemAddr();
if(objects.TryGetValue((addr, newObj.GetType()), out object? o)){ //If the object is already created, we can just assign it
field.SetValue(obj, o);
continue;
}
//Fill the object with the data from the block (this is recursive)
FillObject(derefBlock, ref newObj, fieldInfo, (int)relAddr);
} else {
long relAddr = block.MemAddr.ToMemAddr() + offset; long relAddr = block.MemAddr.ToMemAddr() + offset;
if(objects.TryGetValue((relAddr, newObj.GetType()), out object? o)){ //If the object is already created, we can just assign it if (objects.TryGetValue((relAddr, newObj.GetType()), out object? o)) {
//If the object is already created, we can just assign it
field.SetValue(obj, o); field.SetValue(obj, o);
continue; continue;
} }
//Fill the object with the data from the block (this is recursive) //Fill the object with the data from the block (this is recursive)
FillObject(block, ref newObj, fieldInfo, offset); FillObject(block, ref newObj, fieldInfo, offset);
} else { // if is a pointer, make a pointer to the pointer
pointers.TryAdd(block.MemAddr.ToMemAddr() + offset, data.ToMemAddr());
} }
} }
continue; //should never happen, but means the data could not be converted continue; //should never happen, but means the data could not be converted

14
BlendFileTester/Program.cs
View File

@@ -1,11 +1,15 @@
using System; using System;
using System.Linq;
using BlendFile; using BlendFile;
var reader = new Reader("cube.blend"); var reader = new Reader("cube.blend");
reader.Read(); reader.Read();
foreach (var obj in reader.GetObjects()) var counts = reader.Objects.GroupBy(x => x.Key.Item2).ToList();
{
Console.WriteLine(obj.GetType()); foreach (var count in counts) Console.WriteLine($"{count.Key}: {count.Count()}");
Console.WriteLine(obj.ToString());
} var Meshes = reader.GetObjects<BlendFile.DNA.Mesh>();
var Objects = reader.GetObjects<BlendFile.DNA.Object>();
Console.WriteLine($"Meshes: {Meshes.Count}");
Console.WriteLine($"Objects: {Objects.Count}");

6
BlenderSharp.sln.DotSettings.user
View File

@@ -9,4 +9,8 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASortedDictionary_002Ecs_002Fl_003AC_0021_003FUsers_003Fairon_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Febdb3cec3b3875204585daa9fc42159a24cae33b2087ff4dc114d0e6a5a3e9_003FSortedDictionary_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASortedDictionary_002Ecs_002Fl_003AC_0021_003FUsers_003Fairon_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Febdb3cec3b3875204585daa9fc42159a24cae33b2087ff4dc114d0e6a5a3e9_003FSortedDictionary_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003AC_0021_003FUsers_003Fairon_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F2c8e7ca976f350cba9836d5565dac56b11e0b56656fa786460eb1395857a6fa_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003AC_0021_003FUsers_003Fairon_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F2c8e7ca976f350cba9836d5565dac56b11e0b56656fa786460eb1395857a6fa_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0beb96d31db641cf82014cb1a758a330b2dc00_003F3e_003F433607bb_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThrowHelper_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0beb96d31db641cf82014cb1a758a330b2dc00_003F3e_003F433607bb_003FThrowHelper_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AType_002Ecs_002Fl_003AC_0021_003FUsers_003Fairon_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fe7dbd6fe331ff9a3c4b24dd470ec1f19a71b7c5acf258b81ae7f761cd2b319b_003FType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AType_002Ecs_002Fl_003AC_0021_003FUsers_003Fairon_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fe7dbd6fe331ff9a3c4b24dd470ec1f19a71b7c5acf258b81ae7f761cd2b319b_003FType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:Boolean x:Key="/Default/Dpa/IsEnabledInDebug/@EntryValue">True</s:Boolean>
<s:Int64 x:Key="/Default/Dpa/Thresholds/=AllocationClosure/@EntryIndexedValue">26214400</s:Int64>
<s:Int64 x:Key="/Default/Dpa/Thresholds/=AllocationLoh/@EntryIndexedValue">26214400</s:Int64>
<s:Int64 x:Key="/Default/Dpa/Thresholds/=AllocationTopSoh/@EntryIndexedValue">52428800</s:Int64></wpf:ResourceDictionary>