Fix dictionary serialization

Resolves #1869
Resolves #1782
Resolves #821
Closes #1701

Source/AssetRipper.AssemblyDumper/Passes/Pass002_RenameSubnodes.cs

@@ -359,7 +359,17 @@ public static class Pass002_RenameSubnodes
 		}
 		else if (node.TypeName == "ExposedReferenceTable")
 		{
-			node.TryRenameSubNode("m_References", isEditor ? "m_References_Editor" : "m_References_Release");
+			if (isEditor)
+			{
+				// Ensure yaml is emitted as a sequence, rather than a mapping.
+				// There does not seem to be any indication in the type trees about this,
+				// but nonetheless it is required for correct serialization.
+				// https://github.com/Unity-Technologies/Timeline-MessageMarker/blob/711db46387de66c746e9027090c2de786fe99855/Assets/TestScene.unity#L228
+				// https://github.com/AssetRipper/AssetRipper/issues/1667#issuecomment-2646056403
+				// This appears to be a unique case.
+				node.GetSubNodeByName("m_References").TypeName = "vector";
+			}
+			node.RenameSubNode("m_References", isEditor ? "m_References_Editor" : "m_References_Release");
 		}
 		else if (node.TypeName == "ExtensionPropertyValue")
 		{

Source/AssetRipper.Export.UnityProjects/YamlWalker.cs

@@ -48,14 +48,22 @@ public class YamlWalker : AssetWalker
 		"m_PrefabInstance",
 	};
 
+	private const string Data = "data";
 	private const string First = "first";
 	private const string Second = "second";
 	private Stack<YamlContext> ContextStack { get; } = new();
 	public bool ExportingAssetImporter { private get; set; }
+	public bool UseHyphensInDictionaries { get; set; } = true;
 	private YamlMappingNode? CurrentMappingNode => ContextStack.Peek().MappingNode;
 	private YamlSequenceNode? CurrentSequenceNode => ContextStack.Peek().SequenceNode;
 	private string? CurrentFieldName => ContextStack.Peek().FieldName;
 
+	public YamlWalker WithUnityVersion(UnityVersion version)
+	{
+		UseHyphensInDictionaries = version.GreaterThanOrEquals(5, 4);
+		return this;
+	}
+
 	public YamlDocument ExportYamlDocument(IUnityObjectBase asset, long exportID)
 	{
 		ContextStack.Clear();
@@ -223,7 +231,7 @@ public class YamlWalker : AssetWalker
 
 	public override bool EnterDictionary<TKey, TValue>(IReadOnlyCollection<KeyValuePair<TKey, TValue>> dictionary)
 	{
-		if (IsValidDictionaryKey<TKey>())
+		if (IsValidDictionaryKey<TKey>() || !UseHyphensInDictionaries)
 		{
 			return EnterMap();
 		}
@@ -235,7 +243,7 @@ public class YamlWalker : AssetWalker
 
 	public override void ExitDictionary<TKey, TValue>(IReadOnlyCollection<KeyValuePair<TKey, TValue>> dictionary)
 	{
-		if (IsValidDictionaryKey<TKey>())
+		if (IsValidDictionaryKey<TKey>() || !UseHyphensInDictionaries)
 		{
 			ExitMap();
 		}
@@ -253,6 +261,16 @@ public class YamlWalker : AssetWalker
 			Debug.Assert(CurrentSequenceNode is null);
 			Debug.Assert(CurrentFieldName is null);
 		}
+		else if (!UseHyphensInDictionaries)
+		{
+			Debug.Assert(CurrentMappingNode is not null);
+			Debug.Assert(CurrentSequenceNode is null);
+			Debug.Assert(CurrentFieldName is null);
+			YamlMappingNode node = new();
+			CurrentMappingNode.Add(Data, node);
+			ContextStack.Push(new(node));
+			ContextStack.Push(new(node, First));
+		}
 		else
 		{
 			Debug.Assert(CurrentMappingNode is null);

Source/AssetRipper.SourceGenerated.Extensions/AssetRipper.SourceGenerated.Extensions.csproj

@@ -13,7 +13,7 @@
 
 	<ItemGroup>
 		<PackageReference Include="AssetRipper.Checksum" Version="1.1.0" />
-		<PackageReference Include="AssetRipper.SourceGenerated" Version="1.3.5.1" />
+		<PackageReference Include="AssetRipper.SourceGenerated" Version="1.3.6" />
 	</ItemGroup>
 
 </Project>

Source/AssetRipper.Tests/Traversal/DefaultYamlWalkerTests.cs

@@ -1,9 +1,15 @@
 using AssetRipper.Assets;
 using AssetRipper.Assets.Metadata;
 using AssetRipper.Export.UnityProjects;
+using AssetRipper.Primitives;
 using AssetRipper.SourceGenerated.Classes.ClassID_114;
+using AssetRipper.SourceGenerated.Classes.ClassID_21;
+using AssetRipper.SourceGenerated.Classes.ClassID_320;
 using AssetRipper.SourceGenerated.Extensions;
+using AssetRipper.SourceGenerated.Subclasses.ColorRGBAf;
+using AssetRipper.SourceGenerated.Subclasses.FastPropertyName;
 using AssetRipper.SourceGenerated.Subclasses.StaticBatchInfo;
+using AssetRipper.SourceGenerated.Subclasses.UnityTexEnv;
 using AssetRipper.Yaml;
 using System.Globalization;
 
@@ -12,19 +18,12 @@ namespace AssetRipper.Tests.Traversal;
 internal class DefaultYamlWalkerTests
 {
 	[TestCaseSource(nameof(GetObjectTypes))]
-	[Ignore("Not investigated")]
+	public static void SerializedObjectIsConsistent(Type type, string yamlExpected)
-	public static void SerializedObjectIsConsistent(Type type, string yamlExpectedHyphen, string? yamlExpectedNoHyphen)
 	{
 		UnityObjectBase asset = AssetCreator.CreateUnsafe(type);
 		using (Assert.EnterMultipleScope())
 		{
-			AssertYamlGeneratedAsExpected(new DefaultYamlWalker(), asset, yamlExpectedHyphen);
+			string yamlActual = GenerateYaml(new DefaultYamlWalker(), asset);
-			AssertYamlGeneratedAsExpected(new YamlWalkerWithoutHyphens(), asset, /*yamlExpectedNoHyphen ??*/ yamlExpectedHyphen);
-		}
-
-		static void AssertYamlGeneratedAsExpected(YamlWalker yamlWalker, IUnityObjectBase asset, string yamlExpected)
-		{
-			string yamlActual = GenerateYaml(yamlWalker, asset);
 			Assert.That(yamlActual, Is.EqualTo(yamlExpected));
 		}
 	}
@@ -50,19 +49,19 @@ internal class DefaultYamlWalkerTests
 
 	private static object?[][] GetObjectTypes() =>
 	[
-		[typeof(ComponentListObject), ComponentListObject.Yaml, null],
+		[typeof(ComponentListObject), ComponentListObject.Yaml],
-		[typeof(DictionaryObject), DictionaryObject.Yaml, null],
+		[typeof(DictionaryObject), DictionaryObject.Yaml],
-		[typeof(GuidDictionaryObject), GuidDictionaryObject.Yaml, GuidDictionaryObject.YamlWithoutHyphens],
+		[typeof(GuidDictionaryObject), GuidDictionaryObject.Yaml],
-		[typeof(ListObject), ListObject.Yaml, null],
+		[typeof(ListObject), ListObject.Yaml],
-		[typeof(PairListObject), PairListObject.Yaml, null],
+		[typeof(PairListObject), PairListObject.Yaml],
-		[typeof(PairObject), PairObject.Yaml, null],
+		[typeof(PairObject), PairObject.Yaml],
-		[typeof(ParentObject), ParentObject.Yaml, null],
+		[typeof(ParentObject), ParentObject.Yaml],
-		[typeof(PrimitiveListObject), PrimitiveListObject.Yaml, null],
+		[typeof(PrimitiveListObject), PrimitiveListObject.Yaml],
-		[typeof(SerializedVersionObject), SerializedVersionObject.Yaml, null],
+		[typeof(SerializedVersionObject), SerializedVersionObject.Yaml],
-		[typeof(SimpleObject), SimpleObject.Yaml, null],
+		[typeof(SimpleObject), SimpleObject.Yaml],
-		[typeof(StringDictionaryObject), StringDictionaryObject.Yaml, StringDictionaryObject.YamlWithoutHyphens],
+		[typeof(StringDictionaryObject), StringDictionaryObject.Yaml],
-		[typeof(SubclassObject), SubclassObject.Yaml, null],
+		[typeof(SubclassObject), SubclassObject.Yaml],
-		[typeof(StaticSquaredDictionaryObject), StaticSquaredDictionaryObject.Yaml, null],
+		[typeof(StaticSquaredDictionaryObject), StaticSquaredDictionaryObject.Yaml],
 	];
 
 	private class DefaultYamlWalker : YamlWalker
@@ -77,10 +76,6 @@ internal class DefaultYamlWalkerTests
 		}
 	}
 
-	private sealed class YamlWalkerWithoutHyphens : DefaultYamlWalker
-	{
-	}
-
 	[Test]
 	public void MonoBehaviourStructureSerializationTest()
 	{
@@ -147,4 +142,213 @@ internal class DefaultYamlWalkerTests
 		string yamlActual = GenerateYaml(new DefaultYamlWalker(), [(monoBehaviour, 1), (monoBehaviour, 2)]);
 		Assert.That(yamlActual, Is.EqualTo(yamlExpected));
 	}
+
+	[Test]
+	public void MaterialSerializationTest_3_5()
+	{
+		const string yamlExpected = """
+			%YAML 1.1
+			%TAG !u! tag:unity3d.com,2011:
+			--- !u!0 &1
+			Material:
+			  serializedVersion: 3
+			  m_ObjectHideFlags: 0
+			  m_PrefabParentObject: {m_FileID: 0, m_PathID: 0}
+			  m_PrefabInternal: {m_FileID: 0, m_PathID: 0}
+			  m_Name:
+			  m_Shader: {m_FileID: 0, m_PathID: 0}
+			  m_SavedProperties:
+			    serializedVersion: 2
+			    m_TexEnvs:
+			      data:
+			        first:
+			          name: _MainTex
+			        second:
+			          m_Texture: {m_FileID: 0, m_PathID: 0}
+			          m_Scale: {x: 1, y: 1}
+			          m_Offset: {x: 0, y: 0}
+			    m_Floats: {}
+			    m_Colors:
+			      data:
+			        first:
+			          name: _Color
+			        second: {r: 1, g: 1, b: 1, a: 1}
+
+			""";
+		Material_3_5 material = AssetCreator.CreateUnsafe<Material_3_5>();
+
+		// Texture
+		{
+			(FastPropertyName name, UnityTexEnv_3_5 textureEnv) = material.SavedProperties_C21.TexEnvs_AssetDictionary_FastPropertyName_UnityTexEnv_3_5!.AddNew();
+			name.Name = "_MainTex";
+			textureEnv.Scale.X = 1f;
+			textureEnv.Scale.Y = 1f;
+		}
+
+		// Color
+		{
+			(FastPropertyName name, ColorRGBAf color) = material.SavedProperties_C21.Colors_AssetDictionary_FastPropertyName_ColorRGBAf!.AddNew();
+			name.Name = "_Color";
+			color.SetAsWhite();
+		}
+
+		string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(3, 5, 6)), material);
+		Assert.That(yamlActual, Is.EqualTo(yamlExpected));
+	}
+
+	[Test]
+	public void MaterialSerializationTest_5_3_8()
+	{
+		const string yamlExpected = """
+			%YAML 1.1
+			%TAG !u! tag:unity3d.com,2011:
+			--- !u!0 &1
+			Material:
+			  serializedVersion: 6
+			  m_ObjectHideFlags: 0
+			  m_PrefabParentObject: {m_FileID: 0, m_PathID: 0}
+			  m_PrefabInternal: {m_FileID: 0, m_PathID: 0}
+			  m_Name:
+			  m_Shader: {m_FileID: 0, m_PathID: 0}
+			  m_ShaderKeywords:
+			  m_LightmapFlags: 0
+			  m_CustomRenderQueue: 0
+			  stringTagMap: {}
+			  m_SavedProperties:
+			    serializedVersion: 2
+			    m_TexEnvs:
+			      data:
+			        first:
+			          name: _MainTex
+			        second:
+			          m_Texture: {m_FileID: 0, m_PathID: 0}
+			          m_Scale: {x: 1, y: 1}
+			          m_Offset: {x: 0, y: 0}
+			    m_Floats: {}
+			    m_Colors:
+			      data:
+			        first:
+			          name: _Color
+			        second: {r: 1, g: 1, b: 1, a: 1}
+
+			""";
+		Material_5_1 material = AssetCreator.CreateUnsafe<Material_5_1>();
+
+		// Texture
+		{
+			(FastPropertyName name, UnityTexEnv_5 textureEnv) = material.SavedProperties_C21.TexEnvs_AssetDictionary_FastPropertyName_UnityTexEnv_5!.AddNew();
+			name.Name = "_MainTex";
+			textureEnv.Scale.X = 1f;
+			textureEnv.Scale.Y = 1f;
+		}
+
+		// Color
+		{
+			(FastPropertyName name, ColorRGBAf color) = material.SavedProperties_C21.Colors_AssetDictionary_FastPropertyName_ColorRGBAf!.AddNew();
+			name.Name = "_Color";
+			color.SetAsWhite();
+		}
+
+		string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(5, 3, 8)), material);
+		Assert.That(yamlActual, Is.EqualTo(yamlExpected));
+	}
+
+	[Test]
+	public void MaterialSerializationTest_5_4()
+	{
+		const string yamlExpected = """
+			%YAML 1.1
+			%TAG !u! tag:unity3d.com,2011:
+			--- !u!0 &1
+			Material:
+			  serializedVersion: 6
+			  m_ObjectHideFlags: 0
+			  m_PrefabParentObject: {m_FileID: 0, m_PathID: 0}
+			  m_PrefabInternal: {m_FileID: 0, m_PathID: 0}
+			  m_Name:
+			  m_Shader: {m_FileID: 0, m_PathID: 0}
+			  m_ShaderKeywords:
+			  m_LightmapFlags: 0
+			  m_CustomRenderQueue: 0
+			  stringTagMap: {}
+			  m_SavedProperties:
+			    serializedVersion: 2
+			    m_TexEnvs:
+			    - first:
+			        name: _MainTex
+			      second:
+			        m_Texture: {m_FileID: 0, m_PathID: 0}
+			        m_Scale: {x: 1, y: 1}
+			        m_Offset: {x: 0, y: 0}
+			    m_Floats: {}
+			    m_Colors:
+			    - first:
+			        name: _Color
+			      second: {r: 1, g: 1, b: 1, a: 1}
+
+			""";
+		Material_5_1 material = AssetCreator.CreateUnsafe<Material_5_1>();
+
+		// Texture
+		{
+			(FastPropertyName name, UnityTexEnv_5 textureEnv) = material.SavedProperties_C21.TexEnvs_AssetDictionary_FastPropertyName_UnityTexEnv_5!.AddNew();
+			name.Name = "_MainTex";
+			textureEnv.Scale.X = 1f;
+			textureEnv.Scale.Y = 1f;
+		}
+
+		// Color
+		{
+			(FastPropertyName name, ColorRGBAf color) = material.SavedProperties_C21.Colors_AssetDictionary_FastPropertyName_ColorRGBAf!.AddNew();
+			name.Name = "_Color";
+			color.SetAsWhite();
+		}
+
+		string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(5, 4)), material);
+		Assert.That(yamlActual, Is.EqualTo(yamlExpected));
+	}
+
+	[Test]
+	public void PlayableDirectorSerializationTest()
+	{
+		// Based on:
+		// https://github.com/Unity-Technologies/Timeline-MessageMarker/blob/711db46387de66c746e9027090c2de786fe99855/Assets/TestScene.unity#L210-L231
+		const string yamlExpected = """
+			%YAML 1.1
+			%TAG !u! tag:unity3d.com,2011:
+			--- !u!0 &1
+			PlayableDirector:
+			  serializedVersion: 3
+			  m_ObjectHideFlags: 0
+			  m_CorrespondingSourceObject: {m_FileID: 0, m_PathID: 0}
+			  m_PrefabInstance: {m_FileID: 0, m_PathID: 0}
+			  m_PrefabAsset: {m_FileID: 0, m_PathID: 0}
+			  m_GameObject: {m_FileID: 0, m_PathID: 0}
+			  m_Enabled: 0
+			  m_PlayableAsset: {m_FileID: 0, m_PathID: 0}
+			  m_InitialState: 0
+			  m_WrapMode: 0
+			  m_DirectorUpdateMode: 0
+			  m_InitialTime: 0
+			  m_SceneBindings:
+			  - key: {m_FileID: 0, m_PathID: 0}
+			    value: {m_FileID: 0, m_PathID: 0}
+			  - key: {m_FileID: 0, m_PathID: 0}
+			    value: {m_FileID: 0, m_PathID: 0}
+			  m_ExposedReferences:
+			    m_References:
+			    - fc1441eaed6bd5f45a945cc3d2579dd6: {m_FileID: 0, m_PathID: 0}
+			    - ec546beecf692a4419584c0b9cc42a29: {m_FileID: 0, m_PathID: 0}
+
+			""";
+		PlayableDirector_2019 director = AssetCreator.CreateUnsafe<PlayableDirector_2019>();
+
+		director.SceneBindings_C320.AddNew();
+		director.SceneBindings_C320.AddNew();
+		director.ExposedReferences_C320.References_Editor.AddNew().Key = "fc1441eaed6bd5f45a945cc3d2579dd6";
+		director.ExposedReferences_C320.References_Editor.AddNew().Key = "ec546beecf692a4419584c0b9cc42a29";
+
+		string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(2019)), director);
+		Assert.That(yamlActual, Is.EqualTo(yamlExpected));
+	}
 }

Source/AssetRipper.Tests/Traversal/GuidDictionaryObject.cs

@@ -9,17 +9,6 @@ internal sealed class GuidDictionaryObject : CustomInjectedObjectBase
 	private readonly AssetDictionary<GUID, bool> guidDictionary = new();
 
 	public const string Yaml = """
-		%YAML 1.1
-		%TAG !u! tag:unity3d.com,2011:
-		--- !u!0 &1
-		GuidDictionaryObject:
-		  guidDictionary:
-		  - 00000000000000000000000000000000: 0
-		  - 00000000000000000000000000000000: 1
-
-		""";
-
-	public const string YamlWithoutHyphens = """
 		%YAML 1.1
 		%TAG !u! tag:unity3d.com,2011:
 		--- !u!0 &1

Source/AssetRipper.Tests/Traversal/PairListObject.cs

@@ -14,10 +14,8 @@ internal sealed class PairListObject : CustomInjectedObjectBase
 		--- !u!0 &1
 		PairListObject:
 		  list:
-		  - first: 
+		  - :
-		    second: 
+		  - _key: _value
-		  - first: _key
-		    second: _value
 
 		""";
 

Source/AssetRipper.Tests/Traversal/PairObject.cs

@@ -18,8 +18,7 @@ internal sealed class PairObject : CustomInjectedObjectBase
 		--- !u!0 &1
 		PairObject:
 		  pair:
-		    first: _key
+		    _key: _value
-		    second: _value
 
 		""";
 

Source/AssetRipper.Tests/Traversal/PrimitiveListObject.cs

@@ -13,13 +13,8 @@ internal sealed class PrimitiveListObject : CustomInjectedObjectBase
 		%TAG !u! tag:unity3d.com,2011:
 		--- !u!0 &1
 		PrimitiveListObject:
-		  emptyList: []
+		  emptyList:
-		  list:
+		  list: 0100000001000000020000000300000005000000
-		  - 1
-		  - 1
-		  - 2
-		  - 3
-		  - 5
 
 		""";
 

Source/AssetRipper.Tests/Traversal/StringDictionaryObject.cs

@@ -12,26 +12,6 @@ internal sealed class StringDictionaryObject : CustomInjectedObjectBase
 	private readonly AssetDictionary<Utf8String, UnityTexEnv_5> normalDictionary = new();
 
 	public const string Yaml = """
-		%YAML 1.1
-		%TAG !u! tag:unity3d.com,2011:
-		--- !u!0 &1
-		StringDictionaryObject:
-		  stringDictionary:
-		  - key1: value1
-		  - key2: value2
-		  normalDictionary:
-		  - _BumpMap:
-		      m_Texture: {m_FileID: 0, m_PathID: 0}
-		      m_Scale: {x: 1, y: 1}
-		      m_Offset: {x: 0, y: 0}
-		  - _DetailAlbedoMap:
-		      m_Texture: {m_FileID: 0, m_PathID: 0}
-		      m_Scale: {x: 1, y: 1}
-		      m_Offset: {x: 0, y: 0}
-
-		""";
-
-	public const string YamlWithoutHyphens = """
 		%YAML 1.1
 		%TAG !u! tag:unity3d.com,2011:
 		--- !u!0 &1