using System; using System.CodeDom; using System.CodeDom.Compiler; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text; using System.Threading; using Kaitai; using Microsoft.CSharp; using static Kaitai.BlendFile; // ReSharper disable BitwiseOperatorOnEnumWithoutFlags namespace CodeGenerator { using CParamDeclExp = CodeParameterDeclarationExpression; using CArgRefExp = CodeArgumentReferenceExpression; using CThisRefExp = CodeThisReferenceExpression; using CFieldRefExp = CodeFieldReferenceExpression; public static class Program { private static BlendFile _blendfile; private static readonly StringBuilder Sb = new(); private const string OutPath = @"GeneratedOutput"; private const string Namespace = "BlendFile"; private static HashSet _customTypes; private static readonly string[] ListMarkerStr = { "list", "array" }; private static readonly string[] ListLenghtStr = { "num", "len", "size" }; private static ConcurrentQueue> _logQueue = new(); enum LogType { Info, Warning, Error } private static void LogNow(string message, LogType type = LogType.Info) { string msg = $"{DateTime.Now:yy-MM-dd HH:mm:ss}|{type.ToString()}|{message}"; lock (Sb) { Sb.AppendLine(msg); } Console.WriteLine(msg); } private static void Log(string message, LogType type = LogType.Info) { _logQueue.Enqueue(new(message, type)); } public static void Main(string[] args) { LogNow("Setting up logger..."); Stopwatch sw = new(); sw.Start(); long start = 0; bool loggerExit = false; Thread logger = new(() => { start = sw.ElapsedTicks; LogNow($"Logger started! In {sw.ElapsedMilliseconds}ms"); // ReSharper disable once AccessToModifiedClosure LoopVariableIsNeverChangedInsideLoop while (!loggerExit || !_logQueue.IsEmpty) if (_logQueue.TryDequeue(out var msg)) LogNow(msg.Item1, msg.Item2); LogNow("Logger exiting..."); }); logger.Start(); var initTime = sw.ElapsedTicks; Log("Reading blend file"); sw.Restart(); ReadBlendFile(); var readTime = sw.ElapsedTicks; Log("Generating C# code..."); Log("Pass 1: Generating types"); sw.Restart(); CodeNamespace ns = GenerateTypes(out var rootNs); var genTime = sw.ElapsedTicks; sw.Restart(); Log("Pass 2: Writing out code"); OutputCodeFiles(ns); OutputCodeFiles(rootNs, false); var writeTime = sw.ElapsedTicks; sw.Stop(); Log("Finished generating C# code!"); Log($""" Timings: Initialization: {(decimal)initTime / TimeSpan.TicksPerMillisecond,10:N4} ms Logger Startup: {(decimal)start / TimeSpan.TicksPerMillisecond,10:N4} ms Reading: {(decimal)readTime / TimeSpan.TicksPerMillisecond,10:N4} ms Generating: {(decimal)genTime / TimeSpan.TicksPerMillisecond,10:N4} ms Writing: {(decimal)writeTime / TimeSpan.TicksPerMillisecond,10:N4} ms ---------------------------- Total: {(decimal)(initTime + readTime + genTime + writeTime) / TimeSpan.TicksPerMillisecond,10:N4} ms """); loggerExit = true; while (logger.IsAlive) { } Thread.Sleep(1000); lock (Sb) { File.AppendAllText("Log.txt", Sb.ToString()); } } private static void ReadBlendFile() { Log("Reading empty.blend file"); _blendfile = FromFile("empty.blend"); Log($"Header: Blender v{_blendfile.Hdr.Version} {_blendfile.Hdr.Endian}\n" + $"DataBlocks: {_blendfile.Blocks.Count}\n" + $"DNA1: {_blendfile.SdnaStructs.Count} structures\n"); } private static int AddNormalField(DnaField field, ref CodeTypeDeclaration ctd, int index, int totalSize) { CodeMemberField cmf; string name = field.Name; if (name.Contains("()")) return 0; Log($"Generating field {field.Name}"); cmf = CreateMemberField(field); var attributes = GenerateDnaFieldAttribute(index, field, field.M_Parent.M_Parent, totalSize, out int size); cmf.CustomAttributes.Add(attributes); ctd.Members.Add(cmf); return size; } private static int AddArrayField(ref CodeTypeDeclaration ctd, DnaField field, int index, int totalSize) { Log($"Generating Array field {field.Name}"); var cmf = CreateArrayMemberField(field); var attribute = GenerateDnaArrayAttribute(index, field, field.M_Parent.M_Parent, totalSize, out int size); cmf.CustomAttributes.Add(attribute); ctd.Members.Add(cmf); return size; } private static int AddListField(ref CodeTypeDeclaration ctd, int totalSize, int index, int listLenghtOffset, DnaField listPointer, DnaField listLength, int sizeIndex) { var cmf = CreateListMemberField(listPointer, listLength); var attribute = GenerateDnaListAttribute(index, listPointer, sizeIndex, listLength, totalSize, listLenghtOffset, out int size); cmf.CustomAttributes.Add(attribute); ctd.Members.Add(cmf); return size; } private static CodeNamespace GenerateTypes(out CodeNamespace additionalNs) { //Initialize the namespaces CodeNamespace rootNs = new CodeNamespace(Namespace); CodeNamespace ns = new CodeNamespace(Namespace + ".DNA"); //Fill the attribute types then add them to the namespaces rootNs.Types.AddRange(GenerateTypeDeclarations()); ns.Imports.Add(new(rootNs.Name)); _customTypes = new(); foreach (var type in _blendfile.SdnaStructs) { //Add the type to the custom types list _customTypes.Add(type.Type); //Create a new type declaration var ctd = new CodeTypeDeclaration(type.Type); ctd.CustomAttributes.Add(new("DNAClassAttribute", new CodeAttributeArgument(new CodePrimitiveExpression(type.IdxType)), new CodeAttributeArgument(new CodePrimitiveExpression(type.Type)) )); Log($"Generating type from struct {type.Type}"); if (IsClass(type)) { Log($"Marking {type.Type} as class"); ctd.IsClass = true; } else { Log($"Marking {type.Type} as struct"); ctd.IsStruct = !ctd.IsClass; } //Add the class to the namespace ns.Types.Add(ctd); //TODO: when encountering a list, run trough the fields to find a count/lenght or similar data. // ReSharper disable InlineOutVariableDeclaration List normalFields; //Fields that are not lists nor lengths of lists List<(DnaField, DnaField)> listFields; //Fields that are lists, and their corresponding length fields // ReSharper restore InlineOutVariableDeclaration //filter the fields we want to include in the class minus the lists FilterFields(type.Fields, out normalFields, out listFields); var totalSize = 0; Dictionary listCountOffsets = new(); //Add the fields to the class Log($"Fields: {type.Fields.Count}"); for (var index = 0; index < type.Fields.Count; index++) { var field = type.Fields[index]; //Check if the field is a normal field or a list field if (normalFields.Contains(field) && !listFields.Select(f => f.Item2).Contains(field)) { //check if the field is an array if (field.Name.Contains('[')) totalSize += AddArrayField(ref ctd, field, index, totalSize); else totalSize += AddNormalField(field, ref ctd, index, totalSize); } else if (listFields.Select(f => f.Item1).Contains(field)) { //Retrieve the list pointer and the list length fields var (listPointer, listLength) = listFields.FirstOrDefault(x => x.Item1 == field); Log($"Generating list field {listPointer.Name}"); //retrieve the offset of the list length field if exists listCountOffsets.TryGetValue(listLength.Name.ParseFName(), out int listLenghtOffset); //Retrieve the index of the list length field int sizeIndex = type.Fields.IndexOf(listLength); totalSize += AddListField(ref ctd, totalSize, index, listLenghtOffset, listPointer, listLength, sizeIndex); } else if (listFields.Select(f => f.Item2).Contains(field)) { //update the size of the list attribute string fName = field.Name.ParseFName(); //retrieve the name of the list pointer string listPointerName = listFields.First(f => f.Item2.Name.ParseFName() == fName).Item1.Name.ParseFName(); //Try seeing if the list attribute is already present var x = ctd.Members.OfType() .FirstOrDefault(member => member.Name.ParseFName() == listPointerName); if (x != null) //Update the existing list attribute x.CustomAttributes[0].Arguments[9] = new(new CodePrimitiveExpression(totalSize)); else //Store the data for when the list attribute is made listCountOffsets.Add(fName, totalSize); totalSize += field.Type.ParseFSize(); } else { Log($"Field {field.Name} is of unknown or unsupported type"); } } ctd.CustomAttributes[0].Arguments.Add(new(new CodePrimitiveExpression(totalSize))); Log("Generating Parameterless constructor"); if (ctd.Members.Count > 0) ctd.Members.Add(GenerateParameterlessConstructor(type, ctd)); Log("Generating Default Constructor"); ctd.Members.Add(GenerateConstructor(type, ctd)); Log("Finished generating struct"); } additionalNs = rootNs; return ns; } /// /// Determines if the type has to be serialized as a class or a struct /// /// istance to analyze /// if there is any pointer or self reference, otherwise private static bool IsClass(DnaStruct type) { foreach (var field in type.Fields) { if (field.Name.Contains("*")) { Log($"Pointer detected. {field.Type} {field.Name}"); return true; } if (field.Type.Contains(type.Type)) { Log($"Self reference detected. {field.Type} {field.Name}"); return true; } } Log($"No pointer or self reference detected in {type.Type}"); return false; } /// /// Filters the fields into normal fields and list fields pairs /// /// of s from all parameters /// of containing all fields not part of a List /// of (, ) collection where Item1 is the ListPointer and Item2 is the list lenght private static void FilterFields(IEnumerable fields, out List normalFields, out List<(DnaField, DnaField)> listFields) { normalFields = new(); //Fields that are not lists nor lengths of lists listFields = new(); //Fields that are lists, and their corresponding length fields //Cast to array the fields to avoid multiple enumerations var dnaFields = fields as DnaField[] ?? fields.ToArray(); foreach (var field in dnaFields) { if (ListMarkerStr.Any(s => field.Name.Contains(s)) && !ListLenghtStr.Any(s2 => field.Name.Contains(s2))) { Log($"Found list field {field.Name}"); Log("Searching for list length field"); var listLengthField = dnaFields.FirstOrDefault(f => f.Name.Contains(field.Name.ParseFName()) && ListLenghtStr.Any(s2 => f.Name.Contains(s2))); if (listLengthField == null) Log($"No list length field found for {field.Name}"); else { Log($"Found list length field {listLengthField.Name}"); listFields.Add((field, listLengthField)); //Remove the list length field from the normal fields (if present) if (normalFields.Remove(listLengthField)) Log($"Removed list length field {listLengthField.Name}"); } continue; } //Skip fields that are recognized as listLengths if (listFields.Select(f => f.Item2).Contains(field)) { Log($"Skipping known list length field {field.Name}"); continue; } Log($"Adding normal field {field.Name}"); normalFields.Add(field); } } /// /// Generates the following attribute types DNAAttribute, DNAFieldAttribute, DNAClassAttribute, DNAListAttribute /// /// array containing the generated attribute types /// This internally uses a single instance of and sequentially generates the various attributes. private static CodeTypeDeclaration[] GenerateTypeDeclarations() { var attributeBuilder = new AttributeBuilder(); var typeDeclarations = new[] { attributeBuilder.New().SetName("DNAAttribute") .SetAttributeUsage(new(new CodeSnippetExpression("AttributeTargets.All"))) .DeriveFromClass() .AddAutoProperty(typeof(int), "OriginalIndex") .AddAutoProperty(typeof(string), "OriginalName") .AddPropertiesConstructor() .Build(), attributeBuilder.New().SetName("DNAFieldAttribute") .SetAttributeUsage(new(new CodeSnippetExpression("AttributeTargets.Field"))) .DeriveFromClass($"{Namespace}.DNAAttribute") .AddAutoProperty(typeof(int), "Size") .AddAutoProperty(typeof(string), "OriginalType") .AddAutoProperty(typeof(int), "OriginalIndex") .AddAutoProperty(typeof(string), "OriginalName") .AddAutoProperty(typeof(string), "UnderlyingType") .AddAutoProperty(typeof(bool), "IsPointer") .AddAutoProperty(typeof(int), "MemoryOffset") .AddPropertiesConstructor() .AddBaseConstructorParams(["OriginalIndex", "OriginalName"]) .Build(), attributeBuilder.New().SetName("DNAArrayAttribute") .SetAttributeUsage(new(new CodeSnippetExpression("AttributeTargets.Field"))) .DeriveFromClass($"{Namespace}.DNAAttribute") .AddAutoProperty(typeof(int), "Size") .AddAutoProperty(typeof(string), "OriginalType") .AddAutoProperty(typeof(int), "OriginalIndex") .AddAutoProperty(typeof(string), "OriginalName") .AddAutoProperty(typeof(string), "UnderlyingType") .AddAutoProperty(typeof(int), "ArrayLenght") .AddAutoProperty(typeof(int), "MemoryOffset") .AddPropertiesConstructor() .AddBaseConstructorParams(["OriginalIndex", "OriginalName"]) .Build(), attributeBuilder.New().SetName("DNAClassAttribute") .SetAttributeUsage(new(new CodeSnippetExpression("AttributeTargets.Class | AttributeTargets.Struct"))) .DeriveFromClass($"{Namespace}.DNAAttribute") .AddAutoProperty(typeof(int), "OriginalIndex") .AddAutoProperty(typeof(string), "OriginalName") .AddAutoProperty(typeof(int), "Size") .AddPropertiesConstructor() .AddBaseConstructorParams(["OriginalIndex", "OriginalName"]) .Build(), attributeBuilder.New().SetName("DNAListAttribute") .SetAttributeUsage(new(new CodeSnippetExpression("AttributeTargets.Property | AttributeTargets.Field"))) .DeriveFromClass($"{Namespace}.DNAAttribute") .AddAutoProperty(typeof(int), "Size") .AddAutoProperty(typeof(string), "OriginalType") .AddAutoProperty(typeof(string), "OriginalName") .AddAutoProperty(typeof(int), "OriginalIndex") .AddAutoProperty(typeof(string), "UnderlyingType") .AddAutoProperty(typeof(string), "CountFieldType") .AddAutoProperty(typeof(string), "CountFieldName") .AddAutoProperty(typeof(int), "CountFieldIndex") .AddAutoProperty(typeof(int), "PtrMemoryOffset") .AddAutoProperty(typeof(int), "CountMemoryOffset") .AddPropertiesConstructor() .AddBaseConstructorParams(["OriginalIndex", "OriginalName"]) .Build() }; return typeDeclarations; } //TODO: use AttributeBuilder inside here private static CodeAttributeDeclaration GenerateDnaFieldAttribute(int index, DnaField field, Dna1Body body, int offset, out int size) { var isPointer = false; size = body.Lengths[field.IdxType]; string t = field.Type; if (field.Name.StartsWith('*')) { size = 8; isPointer = true; } CodeAttributeDeclaration cad = new("DNAFieldAttribute"); cad.Arguments.AddRange(new CodeAttributeArgumentCollection { new(new CodePrimitiveExpression(size)), new(new CodePrimitiveExpression(field.Type)), new(new CodePrimitiveExpression(index)), new(new CodePrimitiveExpression(field.Name)), new(new CodePrimitiveExpression(t)), new(new CodePrimitiveExpression(isPointer)), new(new CodePrimitiveExpression(offset)) }); return cad; } private static CodeAttributeDeclaration GenerateDnaArrayAttribute(int index, DnaField field, Dna1Body body, int offset, out int size) { //Grab the lenght of the single item in the array size = body.Lengths[field.IdxType]; //Generate the array declaration again... to grab the base type CodeMemberField amf = CreateArrayMemberField(field); //Generate the type string var sb = new StringBuilder(); sb.Append(amf.Type.BaseType); sb.Append('['); for (int i = 1; i < amf.Type.ArrayRank; i++) { sb.Append(','); } sb.Append(']'); var t = sb.ToString(); var dimensions = GetArrayDimensions(field.Name); int length = 0; foreach (int dim in dimensions) { length += dim; size *= dim; } CodeAttributeDeclaration cad = new("DNAArrayAttribute"); cad.Arguments.AddRange(new CodeAttributeArgumentCollection { new(new CodePrimitiveExpression(size)), new(new CodePrimitiveExpression(field.Type)), new(new CodePrimitiveExpression(index)), new(new CodePrimitiveExpression(field.Name)), new(new CodePrimitiveExpression(t)), new(new CodePrimitiveExpression(length)), new(new CodePrimitiveExpression(offset)) }); return cad; } private static CodeAttributeDeclaration GenerateDnaListAttribute(int listIndex, DnaField listField, int lenghtIndex, DnaField lenghtField, int ptrOffset, int countOffset, out int size) { size = 8; var cad = new CodeAttributeDeclaration("DNAListAttribute"); cad.Arguments.AddRange(new CodeAttributeArgumentCollection { new(new CodeSnippetExpression("8")), //pointer size new(new CodePrimitiveExpression(listField.Type)), new(new CodePrimitiveExpression(listField.Name)), new(new CodePrimitiveExpression(listIndex)), new(new CodePrimitiveExpression(listField.Type)), //TODO: double check this new(new CodePrimitiveExpression(lenghtField.Type)), new(new CodePrimitiveExpression(lenghtField.Name)), new(new CodePrimitiveExpression(lenghtIndex)), new(new CodePrimitiveExpression(ptrOffset)), new(new CodePrimitiveExpression(countOffset)) }); return cad; } private static CodeMemberField CreateMemberField(DnaField field) { Type t = Type.GetType(field.Type.ParseFType()); CodeMemberField cmf; //Check if the type is a built-in type or a custom type if (t != null) cmf = new(t, field.Name.ParseFName()); //Built-in type else { cmf = new(new CodeTypeReference(field.Type), field.Name.ParseFName()); //Custom type _customTypes.Add(field.Type); } cmf.Attributes = MemberAttributes.Public; return cmf; } private static CodeMemberField CreateArrayMemberField(DnaField field) { Type t = Type.GetType(field.Type.ParseFType()); CodeMemberField cmf; // Parse all array dimensions var name = field.Name.ParseFName(); var dimensions = GetArrayDimensions(name); // Get clean field name (without array brackets) name = field.Name.Substring(0, field.Name.IndexOf('[')).ParseFName(); //Check if the type is a built-in type or a custom type if (t != null) cmf = new(t, name); //Built-in type else { cmf = new(field.Type, name); //Custom type _customTypes.Add(field.Type); } //Set the field attributes cmf.Attributes = MemberAttributes.Public; //Define the array type cmf.Type.ArrayElementType = new(field.Type.ParseFType() ?? field.Type); cmf.Type.ArrayRank = dimensions.Count; //Define the array initialization expression cmf.InitExpression = GenerateArrayInitExpression(cmf.Type, dimensions); return cmf; } private static CodeMemberField CreateListMemberField(DnaField field, DnaField lenght) { Type t = Type.GetType(field.Type.ParseFType()); CodeMemberField cmf; CodeTypeReference ctr = new(typeof(List<>)); //Check if the type is a built-in type or a custom type if (t != null) { //Built-in type ctr.TypeArguments.Add(t); cmf = new(ctr, field.Name.ParseFName()); } else { //Custom type ctr.TypeArguments.Add(new CodeTypeReference(field.Type)); cmf = new(ctr, field.Name.ParseFName()); } cmf.Attributes = MemberAttributes.Public; return cmf; } private static List GetArrayDimensions(string name) { var dimensions = new List(); int startIndex = 0; // Get all array dimensions while ((startIndex = name.IndexOf('[', startIndex)) != -1) { int endIndex = name.IndexOf(']', startIndex); string sizeStr = name.Substring(startIndex + 1, endIndex - startIndex - 1); if (int.TryParse(sizeStr, out int size)) { dimensions.Add(size); } startIndex = endIndex + 1; } return dimensions; } private static CodeExpression GenerateArrayInitExpression(CodeTypeReference type, IEnumerable dimensions) { var dimValues = dimensions as int[] ?? dimensions.ToArray(); string dims = string.Concat(dimValues.Take(dimValues.Count() - 1).Select(d => $"{d},")); dims += dimValues.Last(); return new CodeSnippetExpression($"new {type.BaseType}[{dims}]"); } private static CodeTypeConstructor GenerateStaticConstructor(CodeTypeDeclaration ctd) { CodeTypeConstructor ctc = new CodeTypeConstructor { Attributes = MemberAttributes.Static }; ctc.Statements.AddRange(ctd.Members .OfType() .Where(f => f.Type.ArrayRank > 0) .Select(f => { var dims = new List(); for (int i = 0; i < f.Type.ArrayRank; i++) { dims.Add(0); } return new CodeAssignStatement( new CFieldRefExp(new CThisRefExp(), f.Name), GenerateArrayInitExpression(f.Type, dims) ); }).ToArray()); return ctc; } private static CodeConstructor GenerateConstructor(DnaStruct type, CodeTypeDeclaration ctd) { //Create a normal constructor CodeConstructor cc = new CodeConstructor { Name = type.Type, Attributes = MemberAttributes.Public, ReturnType = new(type.Type) }; //Add the parameters to the constructor cc.Parameters.AddRange(ctd.Members .OfType() .Select(f => { var cpde = new CParamDeclExp(f.Type, f.Name) { Direction = FieldDirection.In }; return cpde; }).ToArray()); //Assign the parameters to the respective fields cc.Statements.AddRange(ctd.Members .OfType() .Select(f => new CodeAssignStatement( new CFieldRefExp(new CThisRefExp(), f.Name), new CArgRefExp(f.Name)) ).ToArray()); return cc; } private static CodeConstructor GenerateParameterlessConstructor(DnaStruct type, CodeTypeDeclaration ctd) { //Create a normal constructor CodeConstructor cc = new CodeConstructor { Name = type.Type, Attributes = MemberAttributes.Public, ReturnType = new(type.Type) }; //Assign the parameters to the respective fields cc.Statements.AddRange(ctd.Members .OfType() .Select(f => new CodeAssignStatement( new CFieldRefExp(new CThisRefExp(), f.Name), new CodeSnippetExpression("default")) ).ToArray()); return cc; } private static void SetupCcu(out CodeGeneratorOptions genOpts, out CSharpCodeProvider provider, out CodeCompileUnit ccu) { genOpts = new() { BlankLinesBetweenMembers = false, BracingStyle = "Block", ElseOnClosing = true, IndentString = " ", VerbatimOrder = true }; provider = new(); //var date = DateTime.Now.ToString(CultureInfo.InvariantCulture); CodeNamespace globalNs = new CodeNamespace(); //CodeComment comment = new CodeComment("Automatically generated by BlenderSharp at " + date, false); //globalNs.Comments.Add(new(comment)); globalNs.Imports.Add(new("System")); ccu = new(); ccu.Namespaces.Add(globalNs); } private static void OutputCodeFiles(CodeNamespace ns, bool outputExtraTypes = true) { string rootPath = Path.GetDirectoryName(GetOutputPath(ns, "")); if (!Path.Exists(rootPath)) Directory.CreateDirectory(rootPath!); SetupCcu(out var codeGeneratorOptions, out var provider, out var ccu); CodeNamespace tempNs = new CodeNamespace(ns.Name); tempNs.Imports.AddRange(ns.Imports.Cast().ToArray()); ccu.Namespaces.Add(tempNs); foreach (var type in ns.Types.OfType()) { tempNs.Types.Add(type); Log($"Writing out {(type.IsStruct ? "struct" : "class")} {type.Name}"); using var sw = new StreamWriter(GetOutputPath(ns, type.Name)); provider.GenerateCodeFromCompileUnit(ccu, sw, codeGeneratorOptions); tempNs.Types.Remove(type); } if (!outputExtraTypes) return; _customTypes.ExceptWith(ns.Types.OfType().Select(t => t.Name)); foreach (var type in _customTypes) { Log($"Creating empty struct for missing {type}"); var ctd = new CodeTypeDeclaration(type) { IsStruct = true, Attributes = MemberAttributes.Public }; tempNs.Types.Add(ctd); } using var finalsw = new StreamWriter(GetOutputPath(ns, "_ExtraTypes")); provider.GenerateCodeFromCompileUnit(ccu, finalsw, codeGeneratorOptions); } private static string GetOutputPath(CodeNamespace ns, string typeName) { return Path.Combine(OutPath, ns.Name, typeName + ".cs"); } } }