From 9ab4f12833a55ff38a3266b1ac76ebbaf69ba26e Mon Sep 17 00:00:00 2001 From: ds5678 <49847914+ds5678@users.noreply.github.com> Date: Sun, 26 Oct 2025 16:38:53 -0700 Subject: [PATCH] Fix dictionary serialization Resolves #1869 Resolves #1782 Resolves #821 Closes #1701 --- .../Passes/Pass002_RenameSubnodes.cs | 12 +- .../YamlWalker.cs | 22 +- ...etRipper.SourceGenerated.Extensions.csproj | 2 +- .../Traversal/DefaultYamlWalkerTests.cs | 256 ++++++++++++++++-- .../Traversal/GuidDictionaryObject.cs | 11 - .../Traversal/PairListObject.cs | 6 +- .../AssetRipper.Tests/Traversal/PairObject.cs | 3 +- .../Traversal/PrimitiveListObject.cs | 9 +- .../Traversal/StringDictionaryObject.cs | 20 -- 9 files changed, 267 insertions(+), 74 deletions(-) diff --git a/Source/AssetRipper.AssemblyDumper/Passes/Pass002_RenameSubnodes.cs b/Source/AssetRipper.AssemblyDumper/Passes/Pass002_RenameSubnodes.cs index 711053229..6cf09e071 100644 --- a/Source/AssetRipper.AssemblyDumper/Passes/Pass002_RenameSubnodes.cs +++ b/Source/AssetRipper.AssemblyDumper/Passes/Pass002_RenameSubnodes.cs @@ -359,7 +359,17 @@ public static class Pass002_RenameSubnodes } else if (node.TypeName == "ExposedReferenceTable") { - node.TryRenameSubNode("m_References", isEditor ? "m_References_Editor" : "m_References_Release"); + if (isEditor) + { + // Ensure yaml is emitted as a sequence, rather than a mapping. + // There does not seem to be any indication in the type trees about this, + // but nonetheless it is required for correct serialization. + // https://github.com/Unity-Technologies/Timeline-MessageMarker/blob/711db46387de66c746e9027090c2de786fe99855/Assets/TestScene.unity#L228 + // https://github.com/AssetRipper/AssetRipper/issues/1667#issuecomment-2646056403 + // This appears to be a unique case. + node.GetSubNodeByName("m_References").TypeName = "vector"; + } + node.RenameSubNode("m_References", isEditor ? "m_References_Editor" : "m_References_Release"); } else if (node.TypeName == "ExtensionPropertyValue") { diff --git a/Source/AssetRipper.Export.UnityProjects/YamlWalker.cs b/Source/AssetRipper.Export.UnityProjects/YamlWalker.cs index 1c77c66b8..48445ef2c 100644 --- a/Source/AssetRipper.Export.UnityProjects/YamlWalker.cs +++ b/Source/AssetRipper.Export.UnityProjects/YamlWalker.cs @@ -48,14 +48,22 @@ public class YamlWalker : AssetWalker "m_PrefabInstance", }; + private const string Data = "data"; private const string First = "first"; private const string Second = "second"; private Stack ContextStack { get; } = new(); public bool ExportingAssetImporter { private get; set; } + public bool UseHyphensInDictionaries { get; set; } = true; private YamlMappingNode? CurrentMappingNode => ContextStack.Peek().MappingNode; private YamlSequenceNode? CurrentSequenceNode => ContextStack.Peek().SequenceNode; private string? CurrentFieldName => ContextStack.Peek().FieldName; + public YamlWalker WithUnityVersion(UnityVersion version) + { + UseHyphensInDictionaries = version.GreaterThanOrEquals(5, 4); + return this; + } + public YamlDocument ExportYamlDocument(IUnityObjectBase asset, long exportID) { ContextStack.Clear(); @@ -223,7 +231,7 @@ public class YamlWalker : AssetWalker public override bool EnterDictionary(IReadOnlyCollection> dictionary) { - if (IsValidDictionaryKey()) + if (IsValidDictionaryKey() || !UseHyphensInDictionaries) { return EnterMap(); } @@ -235,7 +243,7 @@ public class YamlWalker : AssetWalker public override void ExitDictionary(IReadOnlyCollection> dictionary) { - if (IsValidDictionaryKey()) + if (IsValidDictionaryKey() || !UseHyphensInDictionaries) { ExitMap(); } @@ -253,6 +261,16 @@ public class YamlWalker : AssetWalker Debug.Assert(CurrentSequenceNode is null); Debug.Assert(CurrentFieldName is null); } + else if (!UseHyphensInDictionaries) + { + Debug.Assert(CurrentMappingNode is not null); + Debug.Assert(CurrentSequenceNode is null); + Debug.Assert(CurrentFieldName is null); + YamlMappingNode node = new(); + CurrentMappingNode.Add(Data, node); + ContextStack.Push(new(node)); + ContextStack.Push(new(node, First)); + } else { Debug.Assert(CurrentMappingNode is null); diff --git a/Source/AssetRipper.SourceGenerated.Extensions/AssetRipper.SourceGenerated.Extensions.csproj b/Source/AssetRipper.SourceGenerated.Extensions/AssetRipper.SourceGenerated.Extensions.csproj index a553f40c7..2e1f6b1d0 100644 --- a/Source/AssetRipper.SourceGenerated.Extensions/AssetRipper.SourceGenerated.Extensions.csproj +++ b/Source/AssetRipper.SourceGenerated.Extensions/AssetRipper.SourceGenerated.Extensions.csproj @@ -13,7 +13,7 @@ - + diff --git a/Source/AssetRipper.Tests/Traversal/DefaultYamlWalkerTests.cs b/Source/AssetRipper.Tests/Traversal/DefaultYamlWalkerTests.cs index 52663f669..968ca7f63 100644 --- a/Source/AssetRipper.Tests/Traversal/DefaultYamlWalkerTests.cs +++ b/Source/AssetRipper.Tests/Traversal/DefaultYamlWalkerTests.cs @@ -1,9 +1,15 @@ using AssetRipper.Assets; using AssetRipper.Assets.Metadata; using AssetRipper.Export.UnityProjects; +using AssetRipper.Primitives; using AssetRipper.SourceGenerated.Classes.ClassID_114; +using AssetRipper.SourceGenerated.Classes.ClassID_21; +using AssetRipper.SourceGenerated.Classes.ClassID_320; using AssetRipper.SourceGenerated.Extensions; +using AssetRipper.SourceGenerated.Subclasses.ColorRGBAf; +using AssetRipper.SourceGenerated.Subclasses.FastPropertyName; using AssetRipper.SourceGenerated.Subclasses.StaticBatchInfo; +using AssetRipper.SourceGenerated.Subclasses.UnityTexEnv; using AssetRipper.Yaml; using System.Globalization; @@ -12,19 +18,12 @@ namespace AssetRipper.Tests.Traversal; internal class DefaultYamlWalkerTests { [TestCaseSource(nameof(GetObjectTypes))] - [Ignore("Not investigated")] - public static void SerializedObjectIsConsistent(Type type, string yamlExpectedHyphen, string? yamlExpectedNoHyphen) + public static void SerializedObjectIsConsistent(Type type, string yamlExpected) { UnityObjectBase asset = AssetCreator.CreateUnsafe(type); using (Assert.EnterMultipleScope()) { - AssertYamlGeneratedAsExpected(new DefaultYamlWalker(), asset, yamlExpectedHyphen); - AssertYamlGeneratedAsExpected(new YamlWalkerWithoutHyphens(), asset, /*yamlExpectedNoHyphen ??*/ yamlExpectedHyphen); - } - - static void AssertYamlGeneratedAsExpected(YamlWalker yamlWalker, IUnityObjectBase asset, string yamlExpected) - { - string yamlActual = GenerateYaml(yamlWalker, asset); + string yamlActual = GenerateYaml(new DefaultYamlWalker(), asset); Assert.That(yamlActual, Is.EqualTo(yamlExpected)); } } @@ -50,19 +49,19 @@ internal class DefaultYamlWalkerTests private static object?[][] GetObjectTypes() => [ - [typeof(ComponentListObject), ComponentListObject.Yaml, null], - [typeof(DictionaryObject), DictionaryObject.Yaml, null], - [typeof(GuidDictionaryObject), GuidDictionaryObject.Yaml, GuidDictionaryObject.YamlWithoutHyphens], - [typeof(ListObject), ListObject.Yaml, null], - [typeof(PairListObject), PairListObject.Yaml, null], - [typeof(PairObject), PairObject.Yaml, null], - [typeof(ParentObject), ParentObject.Yaml, null], - [typeof(PrimitiveListObject), PrimitiveListObject.Yaml, null], - [typeof(SerializedVersionObject), SerializedVersionObject.Yaml, null], - [typeof(SimpleObject), SimpleObject.Yaml, null], - [typeof(StringDictionaryObject), StringDictionaryObject.Yaml, StringDictionaryObject.YamlWithoutHyphens], - [typeof(SubclassObject), SubclassObject.Yaml, null], - [typeof(StaticSquaredDictionaryObject), StaticSquaredDictionaryObject.Yaml, null], + [typeof(ComponentListObject), ComponentListObject.Yaml], + [typeof(DictionaryObject), DictionaryObject.Yaml], + [typeof(GuidDictionaryObject), GuidDictionaryObject.Yaml], + [typeof(ListObject), ListObject.Yaml], + [typeof(PairListObject), PairListObject.Yaml], + [typeof(PairObject), PairObject.Yaml], + [typeof(ParentObject), ParentObject.Yaml], + [typeof(PrimitiveListObject), PrimitiveListObject.Yaml], + [typeof(SerializedVersionObject), SerializedVersionObject.Yaml], + [typeof(SimpleObject), SimpleObject.Yaml], + [typeof(StringDictionaryObject), StringDictionaryObject.Yaml], + [typeof(SubclassObject), SubclassObject.Yaml], + [typeof(StaticSquaredDictionaryObject), StaticSquaredDictionaryObject.Yaml], ]; private class DefaultYamlWalker : YamlWalker @@ -77,10 +76,6 @@ internal class DefaultYamlWalkerTests } } - private sealed class YamlWalkerWithoutHyphens : DefaultYamlWalker - { - } - [Test] public void MonoBehaviourStructureSerializationTest() { @@ -147,4 +142,213 @@ internal class DefaultYamlWalkerTests string yamlActual = GenerateYaml(new DefaultYamlWalker(), [(monoBehaviour, 1), (monoBehaviour, 2)]); Assert.That(yamlActual, Is.EqualTo(yamlExpected)); } + + [Test] + public void MaterialSerializationTest_3_5() + { + const string yamlExpected = """ + %YAML 1.1 + %TAG !u! tag:unity3d.com,2011: + --- !u!0 &1 + Material: + serializedVersion: 3 + m_ObjectHideFlags: 0 + m_PrefabParentObject: {m_FileID: 0, m_PathID: 0} + m_PrefabInternal: {m_FileID: 0, m_PathID: 0} + m_Name: + m_Shader: {m_FileID: 0, m_PathID: 0} + m_SavedProperties: + serializedVersion: 2 + m_TexEnvs: + data: + first: + name: _MainTex + second: + m_Texture: {m_FileID: 0, m_PathID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + m_Floats: {} + m_Colors: + data: + first: + name: _Color + second: {r: 1, g: 1, b: 1, a: 1} + + """; + Material_3_5 material = AssetCreator.CreateUnsafe(); + + // Texture + { + (FastPropertyName name, UnityTexEnv_3_5 textureEnv) = material.SavedProperties_C21.TexEnvs_AssetDictionary_FastPropertyName_UnityTexEnv_3_5!.AddNew(); + name.Name = "_MainTex"; + textureEnv.Scale.X = 1f; + textureEnv.Scale.Y = 1f; + } + + // Color + { + (FastPropertyName name, ColorRGBAf color) = material.SavedProperties_C21.Colors_AssetDictionary_FastPropertyName_ColorRGBAf!.AddNew(); + name.Name = "_Color"; + color.SetAsWhite(); + } + + string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(3, 5, 6)), material); + Assert.That(yamlActual, Is.EqualTo(yamlExpected)); + } + + [Test] + public void MaterialSerializationTest_5_3_8() + { + const string yamlExpected = """ + %YAML 1.1 + %TAG !u! tag:unity3d.com,2011: + --- !u!0 &1 + Material: + serializedVersion: 6 + m_ObjectHideFlags: 0 + m_PrefabParentObject: {m_FileID: 0, m_PathID: 0} + m_PrefabInternal: {m_FileID: 0, m_PathID: 0} + m_Name: + m_Shader: {m_FileID: 0, m_PathID: 0} + m_ShaderKeywords: + m_LightmapFlags: 0 + m_CustomRenderQueue: 0 + stringTagMap: {} + m_SavedProperties: + serializedVersion: 2 + m_TexEnvs: + data: + first: + name: _MainTex + second: + m_Texture: {m_FileID: 0, m_PathID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + m_Floats: {} + m_Colors: + data: + first: + name: _Color + second: {r: 1, g: 1, b: 1, a: 1} + + """; + Material_5_1 material = AssetCreator.CreateUnsafe(); + + // Texture + { + (FastPropertyName name, UnityTexEnv_5 textureEnv) = material.SavedProperties_C21.TexEnvs_AssetDictionary_FastPropertyName_UnityTexEnv_5!.AddNew(); + name.Name = "_MainTex"; + textureEnv.Scale.X = 1f; + textureEnv.Scale.Y = 1f; + } + + // Color + { + (FastPropertyName name, ColorRGBAf color) = material.SavedProperties_C21.Colors_AssetDictionary_FastPropertyName_ColorRGBAf!.AddNew(); + name.Name = "_Color"; + color.SetAsWhite(); + } + + string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(5, 3, 8)), material); + Assert.That(yamlActual, Is.EqualTo(yamlExpected)); + } + + [Test] + public void MaterialSerializationTest_5_4() + { + const string yamlExpected = """ + %YAML 1.1 + %TAG !u! tag:unity3d.com,2011: + --- !u!0 &1 + Material: + serializedVersion: 6 + m_ObjectHideFlags: 0 + m_PrefabParentObject: {m_FileID: 0, m_PathID: 0} + m_PrefabInternal: {m_FileID: 0, m_PathID: 0} + m_Name: + m_Shader: {m_FileID: 0, m_PathID: 0} + m_ShaderKeywords: + m_LightmapFlags: 0 + m_CustomRenderQueue: 0 + stringTagMap: {} + m_SavedProperties: + serializedVersion: 2 + m_TexEnvs: + - first: + name: _MainTex + second: + m_Texture: {m_FileID: 0, m_PathID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + m_Floats: {} + m_Colors: + - first: + name: _Color + second: {r: 1, g: 1, b: 1, a: 1} + + """; + Material_5_1 material = AssetCreator.CreateUnsafe(); + + // Texture + { + (FastPropertyName name, UnityTexEnv_5 textureEnv) = material.SavedProperties_C21.TexEnvs_AssetDictionary_FastPropertyName_UnityTexEnv_5!.AddNew(); + name.Name = "_MainTex"; + textureEnv.Scale.X = 1f; + textureEnv.Scale.Y = 1f; + } + + // Color + { + (FastPropertyName name, ColorRGBAf color) = material.SavedProperties_C21.Colors_AssetDictionary_FastPropertyName_ColorRGBAf!.AddNew(); + name.Name = "_Color"; + color.SetAsWhite(); + } + + string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(5, 4)), material); + Assert.That(yamlActual, Is.EqualTo(yamlExpected)); + } + + [Test] + public void PlayableDirectorSerializationTest() + { + // Based on: + // https://github.com/Unity-Technologies/Timeline-MessageMarker/blob/711db46387de66c746e9027090c2de786fe99855/Assets/TestScene.unity#L210-L231 + const string yamlExpected = """ + %YAML 1.1 + %TAG !u! tag:unity3d.com,2011: + --- !u!0 &1 + PlayableDirector: + serializedVersion: 3 + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {m_FileID: 0, m_PathID: 0} + m_PrefabInstance: {m_FileID: 0, m_PathID: 0} + m_PrefabAsset: {m_FileID: 0, m_PathID: 0} + m_GameObject: {m_FileID: 0, m_PathID: 0} + m_Enabled: 0 + m_PlayableAsset: {m_FileID: 0, m_PathID: 0} + m_InitialState: 0 + m_WrapMode: 0 + m_DirectorUpdateMode: 0 + m_InitialTime: 0 + m_SceneBindings: + - key: {m_FileID: 0, m_PathID: 0} + value: {m_FileID: 0, m_PathID: 0} + - key: {m_FileID: 0, m_PathID: 0} + value: {m_FileID: 0, m_PathID: 0} + m_ExposedReferences: + m_References: + - fc1441eaed6bd5f45a945cc3d2579dd6: {m_FileID: 0, m_PathID: 0} + - ec546beecf692a4419584c0b9cc42a29: {m_FileID: 0, m_PathID: 0} + + """; + PlayableDirector_2019 director = AssetCreator.CreateUnsafe(); + + director.SceneBindings_C320.AddNew(); + director.SceneBindings_C320.AddNew(); + director.ExposedReferences_C320.References_Editor.AddNew().Key = "fc1441eaed6bd5f45a945cc3d2579dd6"; + director.ExposedReferences_C320.References_Editor.AddNew().Key = "ec546beecf692a4419584c0b9cc42a29"; + + string yamlActual = GenerateYaml(new DefaultYamlWalker().WithUnityVersion(new UnityVersion(2019)), director); + Assert.That(yamlActual, Is.EqualTo(yamlExpected)); + } } diff --git a/Source/AssetRipper.Tests/Traversal/GuidDictionaryObject.cs b/Source/AssetRipper.Tests/Traversal/GuidDictionaryObject.cs index 4671b6d41..c9d8d74aa 100644 --- a/Source/AssetRipper.Tests/Traversal/GuidDictionaryObject.cs +++ b/Source/AssetRipper.Tests/Traversal/GuidDictionaryObject.cs @@ -9,17 +9,6 @@ internal sealed class GuidDictionaryObject : CustomInjectedObjectBase private readonly AssetDictionary guidDictionary = new(); public const string Yaml = """ - %YAML 1.1 - %TAG !u! tag:unity3d.com,2011: - --- !u!0 &1 - GuidDictionaryObject: - guidDictionary: - - 00000000000000000000000000000000: 0 - - 00000000000000000000000000000000: 1 - - """; - - public const string YamlWithoutHyphens = """ %YAML 1.1 %TAG !u! tag:unity3d.com,2011: --- !u!0 &1 diff --git a/Source/AssetRipper.Tests/Traversal/PairListObject.cs b/Source/AssetRipper.Tests/Traversal/PairListObject.cs index d4ccb937f..33e9dcbb5 100644 --- a/Source/AssetRipper.Tests/Traversal/PairListObject.cs +++ b/Source/AssetRipper.Tests/Traversal/PairListObject.cs @@ -14,10 +14,8 @@ internal sealed class PairListObject : CustomInjectedObjectBase --- !u!0 &1 PairListObject: list: - - first: - second: - - first: _key - second: _value + - : + - _key: _value """; diff --git a/Source/AssetRipper.Tests/Traversal/PairObject.cs b/Source/AssetRipper.Tests/Traversal/PairObject.cs index 1536aef63..123d192fb 100644 --- a/Source/AssetRipper.Tests/Traversal/PairObject.cs +++ b/Source/AssetRipper.Tests/Traversal/PairObject.cs @@ -18,8 +18,7 @@ internal sealed class PairObject : CustomInjectedObjectBase --- !u!0 &1 PairObject: pair: - first: _key - second: _value + _key: _value """; diff --git a/Source/AssetRipper.Tests/Traversal/PrimitiveListObject.cs b/Source/AssetRipper.Tests/Traversal/PrimitiveListObject.cs index 17ae4e755..bc3ebbf8e 100644 --- a/Source/AssetRipper.Tests/Traversal/PrimitiveListObject.cs +++ b/Source/AssetRipper.Tests/Traversal/PrimitiveListObject.cs @@ -13,13 +13,8 @@ internal sealed class PrimitiveListObject : CustomInjectedObjectBase %TAG !u! tag:unity3d.com,2011: --- !u!0 &1 PrimitiveListObject: - emptyList: [] - list: - - 1 - - 1 - - 2 - - 3 - - 5 + emptyList: + list: 0100000001000000020000000300000005000000 """; diff --git a/Source/AssetRipper.Tests/Traversal/StringDictionaryObject.cs b/Source/AssetRipper.Tests/Traversal/StringDictionaryObject.cs index fc5db79cb..3981fc596 100644 --- a/Source/AssetRipper.Tests/Traversal/StringDictionaryObject.cs +++ b/Source/AssetRipper.Tests/Traversal/StringDictionaryObject.cs @@ -12,26 +12,6 @@ internal sealed class StringDictionaryObject : CustomInjectedObjectBase private readonly AssetDictionary normalDictionary = new(); public const string Yaml = """ - %YAML 1.1 - %TAG !u! tag:unity3d.com,2011: - --- !u!0 &1 - StringDictionaryObject: - stringDictionary: - - key1: value1 - - key2: value2 - normalDictionary: - - _BumpMap: - m_Texture: {m_FileID: 0, m_PathID: 0} - m_Scale: {x: 1, y: 1} - m_Offset: {x: 0, y: 0} - - _DetailAlbedoMap: - m_Texture: {m_FileID: 0, m_PathID: 0} - m_Scale: {x: 1, y: 1} - m_Offset: {x: 0, y: 0} - - """; - - public const string YamlWithoutHyphens = """ %YAML 1.1 %TAG !u! tag:unity3d.com,2011: --- !u!0 &1