finish reflection based deserialisation (add for text codec)

Author: Angel-foxxo

Codecs/Binary.cs

@@ -392,7 +392,7 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in
 
                 if(matchedType && reflectionParams.AttemptReflection)
                 {
-                    var isElementDerived = IsElementDerived(classType);
+                    var isElementDerived = Element.IsElementDerived(classType);
                     if (isElementDerived && classType.Name == type)
                     {
                         Type derivedType = classType;
@@ -425,7 +425,8 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in
             // read attributes (or not, if we're deferred)
             foreach (var elem in dm.AllElements.ToArray())
             {
-                System.Diagnostics.Debug.Assert(!elem.Stub);
+                // assert if stub
+                Debug.Assert(!elem.Stub);
 
                 var num_attrs = Reader.ReadInt32();
 
@@ -798,28 +799,6 @@ void WriteAttribute(object value, bool in_array)
             }
         }
 
-        bool IsElementDerived(Type type)
-        {
-            var elementType = typeof(Element);
-
-            while (type.BaseType != elementType)
-            {
-                var baseType = type.BaseType;
-
-                if (baseType != null)
-                {
-                    type = baseType;
-                }
-                else
-                {
-                    return type == elementType ? true : false;
-                }
-
-            }
-
-            return type.BaseType == elementType ? true : false;
-        }
-
         class DmxBinaryWriter : BinaryWriter
         {
             public DmxBinaryWriter(Stream output)

Codecs/KeyValues2.cs

@@ -7,6 +7,8 @@
 using System.Runtime.CompilerServices;
 using System.Globalization;
 using System.Reflection;
+using System.Xml.Linq;
+using System.Collections;
 
 namespace Datamodel.Codecs
 {
@@ -292,7 +294,7 @@ void WriteAttribute(string name, Type type, object value, bool in_array)
                 else if (type == typeof(Matrix4x4))
                 {
                     var castValue = (Matrix4x4)value;
-                    var matrixString = 
+                    var matrixString =
                         $"{castValue.M11} {castValue.M12} {castValue.M13} {castValue.M14}" +
                         $" {castValue.M21} {castValue.M22} {castValue.M23} {castValue.M24}" +
                         $" {castValue.M31} {castValue.M32} {castValue.M33} {castValue.M34}" +
@@ -397,6 +399,44 @@ public void Encode(Datamodel dm, string encoding, int encoding_version, Stream s
         #endregion
 
         #region Decode
+
+        private class IntermediateData
+        {
+            // these store element refs while we process the elements, once were done
+            // we can go trough these and actually create the attributes
+            // and add the elements to lists
+            public Dictionary<Element, List<(string, Guid)>> PropertiesToAdd = new();
+            public Dictionary<IList, List<Guid>> ListRefs = new();
+
+            public void HandleElementProp(Element element, string attrName, Guid id)
+            {
+
+                PropertiesToAdd.TryGetValue(element, out var attrList);
+
+                if (attrList == null)
+                {
+                    attrList = new List<(string, Guid)>();
+                    PropertiesToAdd.Add(element, attrList);
+                }
+
+                attrList.Add((attrName, id));
+
+            }
+
+            public void HandleListRefs(IList list, Guid id)
+            {
+                ListRefs.TryGetValue(list, out var guidList);
+
+                if (guidList == null)
+                {
+                    guidList = new List<Guid>();
+                    ListRefs.Add(list, guidList);
+                }
+
+                guidList.Add(id);
+            }
+        }
+
         readonly StringBuilder TokenBuilder = new();
         int Line = 0;
         string Decode_NextToken()
@@ -441,31 +481,17 @@ string Decode_NextToken()
             }
         }
 
-        Element Decode_ParseElementId()
-        {
-            Element elem;
-            var id_s = Decode_NextToken();
-
-            if (string.IsNullOrEmpty(id_s))
-                elem = null;
-            else
-            {
-                Guid id = new(id_s);
-                elem = DM.AllElements[id];
-                elem ??= new Element(DM, id);
-            }
-            return elem;
-        }
-
-        Element Decode_ParseElement(string class_name)
+        Element Decode_ParseElement(string class_name, ReflectionParams reflectionParams, IntermediateData intermediateData)
         {
             string elem_class = class_name ?? Decode_NextToken();
             string elem_name = null;
             string elem_id = null;
             Element elem = null;
 
+            var types = CodecUtilities.GetReflectionTypes(reflectionParams);
+
             string next = Decode_NextToken();
-            if (next != "{") throw new CodecException(String.Format("Expected Element opener, got '{0}'.", next));
+            if (next != "{") throw new CodecException($"Expected Element opener, got '{next}'.");
             while (true)
             {
                 next = Decode_NextToken();
@@ -479,16 +505,41 @@ Element Decode_ParseElement(string class_name)
                 {
                     elem_id = Decode_NextToken();
                     var id = new Guid(elem_id);
-                    var local_element = DM.AllElements[id];
-                    if (local_element != null)
+                    if (elem_class != "$prefix_element$")
                     {
-                        elem = local_element;
-                        elem.Name = elem_name;
-                        elem.ClassName = elem_class;
-                        elem.Stub = false;
+                        var matchedType = types.TryGetValue(elem_class, out var classType);
+
+                        if (matchedType && reflectionParams.AttemptReflection)
+                        {
+                            var isElementDerived = Element.IsElementDerived(classType);
+                            if (isElementDerived && classType.Name == elem_class)
+                            {
+                                Type derivedType = classType;
+
+                                ConstructorInfo? constructor = typeof(Element).GetConstructor(
+                                    BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic,
+                                    null,
+                                    new Type[] { typeof(Datamodel), typeof(string), typeof(Guid), typeof(string) },
+                                    null
+                                );
+
+                                if (constructor == null)
+                                {
+                                    throw new InvalidOperationException("Failed to get constructor while attemption reflection based deserialisation");
+                                }
+
+                                object uninitializedObject = RuntimeHelpers.GetUninitializedObject(derivedType);
+                                constructor.Invoke(uninitializedObject, new object[] { DM, elem_name, new Guid(elem_id), elem_class });
+
+                                elem = (Element?)uninitializedObject;
+                            }
+                        }
+
+                        if (elem == null)
+                        {
+                            elem = new Element(DM, elem_name, new Guid(elem_id), elem_class);
+                        }
                     }
-                    else if (elem_class != "$prefix_element$")
-                        elem = new Element(DM, elem_name, new Guid(elem_id), elem_class);
 
                     continue;
                 }
@@ -501,19 +552,21 @@ Element Decode_ParseElement(string class_name)
                     continue;
                 }
 
-                if (elem == null)
-                    continue;
-
                 if (attr_type_s == "element")
                 {
-                    elem.Add(attr_name, Decode_ParseElementId());
+                    var id_s = Decode_NextToken();
+
+                    if (!string.IsNullOrEmpty(id_s))
+                    {
+                        intermediateData.HandleElementProp(elem, attr_name, new Guid(id_s));
+                    }
                     continue;
                 }
 
                 object attr_value = null;
 
                 if (attr_type == null)
-                    attr_value = Decode_ParseElement(attr_type_s);
+                    attr_value = Decode_ParseElement(attr_type_s, reflectionParams, intermediateData);
                 else if (attr_type_s.EndsWith("_array"))
                 {
                     var array = CodecUtilities.MakeList(attr_type, 5); // assume 5 items
@@ -527,15 +580,28 @@ Element Decode_ParseElement(string class_name)
                         if (next == "]") break;
 
                         if (next == "element") // Element ID reference
-                            array.Add(Decode_ParseElementId());
-                        else if (attr_type == typeof(Element)) // inline Element
-                            array.Add(Decode_ParseElement(next));
-                        else // normal value
-                            array.Add(Decode_ParseValue(attr_type, next));
+                        {
+                            var id_s = Decode_NextToken();
+
+                            if (!string.IsNullOrEmpty(id_s))
+                            {
+                                intermediateData.HandleListRefs(array, new Guid(id_s));
+                            }
+                        }
+                        // inline Element
+                        else if (attr_type == typeof(Element)) 
+                        {
+                            array.Add(Decode_ParseElement(next, reflectionParams, intermediateData));
+                        }
+                        // normal value
+                        else
+                        {
+                            array.Add(Decode_ParseValue(attr_type, next, reflectionParams, intermediateData));
+                        }
                     }
                 }
                 else
-                    attr_value = Decode_ParseValue(attr_type, Decode_NextToken());
+                    attr_value = Decode_ParseValue(attr_type, Decode_NextToken(), reflectionParams, intermediateData);
 
                 if (elem != null)
                     elem.Add(attr_name, attr_value);
@@ -545,15 +611,15 @@ Element Decode_ParseElement(string class_name)
             return elem;
         }
 
-        object Decode_ParseValue(Type type, string value)
+        object Decode_ParseValue(Type type, string value, ReflectionParams reflectionParams, IntermediateData intermediateData)
         {
             if (type == typeof(string))
                 return value;
 
             value = value.Trim();
 
             if (type == typeof(Element))
-                return Decode_ParseElement(value);
+                return Decode_ParseElement(value, reflectionParams, intermediateData);
             if (type == typeof(int))
                 return int.Parse(value, CultureInfo.InvariantCulture);
             else if (type == typeof(float))
@@ -562,11 +628,34 @@ object Decode_ParseValue(Type type, string value)
                 return byte.Parse(value, CultureInfo.InvariantCulture) == 1;
             else if (type == typeof(byte[]))
             {
+                // need to sanitise input here because for example when exporting map as txt,
+                // hammer will format the binary data to fit nicer on the screen by inserting two tabs
+                var sb = new StringBuilder(value.Length);
+                foreach (char c in value)
+                {
+                    switch (c)
+                    {
+                        case '\\':
+                        case '\r':
+                        case '\n':
+                        case '\t':
+                        case ' ':
+                            continue;
+                        default:
+                            sb.Append(c);
+                            break;
+                    }
+                }
+                value = sb.ToString();
+
                 byte[] result = new byte[value.Length / 2];
+
                 for (int i = 0; i * 2 < value.Length; i++)
                 {
-                    result[i] = byte.Parse(value.AsSpan(i * 2, 2), System.Globalization.NumberStyles.HexNumber, CultureInfo.InvariantCulture);
+                    var slice = value.AsSpan(i * 2, 2);
+                    result[i] = byte.Parse(slice, System.Globalization.NumberStyles.HexNumber, CultureInfo.InvariantCulture);
                 }
+
                 return result;
             }
             else if (type == typeof(TimeSpan))
@@ -611,6 +700,8 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in
             Line = 1;
             string next;
 
+            var intermediateData = new IntermediateData();
+
             while (true)
             {
                 try
@@ -619,11 +710,28 @@ public Datamodel Decode(string encoding, int encoding_version, string format, in
                 { break; }
 
                 try
-                { Decode_ParseElement(next); }
+                { Decode_ParseElement(next, reflectionParams, intermediateData); }
                 catch (Exception err)
                 { throw new CodecException($"KeyValues2 decode failed on line {Line}:\n\n{err.Message}", err); }
             }
 
+            foreach (var propList in intermediateData.PropertiesToAdd)
+            {
+                foreach (var prop in propList.Value)
+                {
+                    propList.Key.Add(prop.Item1, DM.AllElements[prop.Item2]);
+                }
+
+            }
+
+            foreach (var list in intermediateData.ListRefs)
+            {
+                foreach (var id in list.Value)
+                {
+                    list.Key.Add(DM.AllElements[id]);
+                }
+            }
+
             return DM;
         }
         #endregion

Datamodel.ElementList.cs

@@ -8,7 +8,7 @@
 
 namespace Datamodel
 {
-public partial class Datamodel
+    public partial class Datamodel
     {
         /// <summary>
         /// A collection of <see cref="Element"/>s owned by a single <see cref="Datamodel"/>.

Element.cs

@@ -381,6 +381,27 @@ public bool Equals(Element other)
         {
             return other != null && ID == other.ID;
         }
+
+        public static bool IsElementDerived(Type type)
+        {
+            var elementType = typeof(Element);
+
+            while (type.BaseType != elementType)
+            {
+                var baseType = type.BaseType;
+
+                if (baseType != null)
+                {
+                    type = baseType;
+                }
+                else
+                {
+                    return type == elementType ? true : false;
+                }
+            }
+
+            return type.BaseType == elementType ? true : false;
+        }
     }
 
     namespace TypeConverters