Fix IndexOutOfRangeException when serializing/deserializing 4+ level deep...

Fix IndexOutOfRangeException when serializing/deserializing 4+ level deep nested types with polymorphism (#72261)

* Fix IndexOutOfRangeException when serializing/deserializing few level deep nested types with polymorphism

* move test to *.CustomTypeHierarchies.cs and add Peano roundtrip with up to 150 nesting

* Update src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.CustomTypeHierarchies.cs

* Update src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.CustomTypeHierarchies.cs

* Update src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.CustomTypeHierarchies.cs

* Update src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.CustomTypeHierarchies.cs

* Revert "Update src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.CustomTypeHierarchies.cs"

This reverts commit d7fbd896506f2f4b990025ebdfb90a3adacf7cd6.

* Fix JsonSerializerOptions mappings in JsonSerializerWrapper abstractions

* Additional testing
Co-authored-by: NEirik Tsarpalis <eirik.tsarpalis@gmail.com>

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Node.cs

@@ -123,7 +123,7 @@ public static partial class JsonSerializer
                 WriteUsingGeneratedSerializer(writer, value, jsonTypeInfo);
             }
 
-            return JsonNode.Parse(output.WrittenMemory.Span, options.GetNodeOptions());
+            return JsonNode.Parse(output.WrittenMemory.Span, options.GetNodeOptions(), options.GetDocumentOptions());
         }
 
         private static JsonNode? WriteNodeUsingSerializer<TValue>(in TValue value, JsonTypeInfo jsonTypeInfo)
@@ -138,7 +138,7 @@ public static partial class JsonSerializer
                 WriteUsingSerializer(writer, value, jsonTypeInfo);
             }
 
-            return JsonNode.Parse(output.WrittenMemory.Span, options.GetNodeOptions());
+            return JsonNode.Parse(output.WrittenMemory.Span, options.GetNodeOptions(), options.GetDocumentOptions());
         }
     }
 }

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs

@@ -127,7 +127,7 @@ private void EnsurePushCapacity()
             {
                 _stack = new WriteStackFrame[4];
             }
-            else if (_count - 1 == _stack.Length)
+            else if (_count - _indexOffset == _stack.Length)
             {
                 Array.Resize(ref _stack, 2 * _stack.Length);
             }

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/DomTests.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text.Json.Nodes;
 using Xunit;
 
@@ -184,6 +185,28 @@ public static void SerializeToNode()
             Assert.Equal(2, arrayProp[1].AsValue().GetValue<int>());
         }
 
+        [Theory]
+        [InlineData(5)]
+        [InlineData(32)]
+        [InlineData(70)] // default max depth is 64
+        public static void SerializeToNode_RespectsMaxDepth(int maxDepth)
+        {
+            var options = new JsonSerializerOptions { MaxDepth = maxDepth };
+
+            RecursiveClass value = RecursiveClass.FromInt(maxDepth);
+            JsonNode dom = JsonSerializer.SerializeToNode(value, options);
+
+            value = RecursiveClass.FromInt(maxDepth + 1);
+            Assert.Throws<JsonException>(() => JsonSerializer.SerializeToNode(value, options));
+        }
+
+        public class RecursiveClass
+        {
+            public RecursiveClass? Next { get; set; }
+            public static RecursiveClass FromInt(int depth) => depth == 0 ? null : new RecursiveClass { Next = FromInt(depth - 1) };
+            public static int ToInt(RecursiveClass value) => value is null ? 0 : 1 + ToInt(value.Next);
+        }
+
         [Fact]
         public static void SerializeToDocument_WithEscaping()
         {

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs

@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.IO;
-using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Text.Json.Nodes;
 using System.Text.Json.Serialization.Metadata;
@@ -241,7 +240,7 @@ private class ReaderWriterSerializerWrapper : JsonSerializerWrapper
             public override Task<string> SerializeWrapper(object value, Type inputType, JsonSerializerOptions options = null)
             {
                 using MemoryStream stream = new MemoryStream();
-                using (Utf8JsonWriter writer = new(stream))
+                using (Utf8JsonWriter writer = new(stream, OptionsHelpers.GetWriterOptions(options)))
                 {
                     JsonSerializer.Serialize(writer, value, inputType, options);
                 }
@@ -252,7 +251,7 @@ public override Task<string> SerializeWrapper(object value, Type inputType, Json
             public override Task<string> SerializeWrapper<T>(T value, JsonSerializerOptions options = null)
             {
                 using MemoryStream stream = new MemoryStream();
-                using (Utf8JsonWriter writer = new(stream))
+                using (Utf8JsonWriter writer = new(stream, OptionsHelpers.GetWriterOptions(options)))
                 {
                     JsonSerializer.Serialize<T>(writer, value, options);
                 }
@@ -263,7 +262,7 @@ public override Task<string> SerializeWrapper<T>(T value, JsonSerializerOptions
             public override Task<string> SerializeWrapper(object value, Type inputType, JsonSerializerContext context)
             {
                 using MemoryStream stream = new MemoryStream();
-                using (Utf8JsonWriter writer = new(stream))
+                using (Utf8JsonWriter writer = new(stream, OptionsHelpers.GetWriterOptions(context?.Options)))
                 {
                     JsonSerializer.Serialize(writer, value, inputType, context);
                 }
@@ -274,7 +273,7 @@ public override Task<string> SerializeWrapper(object value, Type inputType, Json
             public override Task<string> SerializeWrapper<T>(T value, JsonTypeInfo<T> jsonTypeInfo)
             {
                 using MemoryStream stream = new MemoryStream();
-                using (Utf8JsonWriter writer = new(stream))
+                using (Utf8JsonWriter writer = new(stream, OptionsHelpers.GetWriterOptions(jsonTypeInfo?.Options)))
                 {
                     JsonSerializer.Serialize(writer, value, jsonTypeInfo);
                 }
@@ -284,25 +283,25 @@ public override Task<string> SerializeWrapper<T>(T value, JsonTypeInfo<T> jsonTy
 
             public override Task<T> DeserializeWrapper<T>(string json, JsonSerializerOptions options = null)
             {
-                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json));
+                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json), OptionsHelpers.GetReaderOptions(options));
                 return Task.FromResult(JsonSerializer.Deserialize<T>(ref reader, options));
             }
 
             public override Task<object> DeserializeWrapper(string json, Type type, JsonSerializerOptions options = null)
             {
-                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json));
+                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json), OptionsHelpers.GetReaderOptions(options));
                 return Task.FromResult(JsonSerializer.Deserialize(ref reader, type, options));
             }
 
             public override Task<T> DeserializeWrapper<T>(string json, JsonTypeInfo<T> jsonTypeInfo)
             {
-                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json));
+                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json), OptionsHelpers.GetReaderOptions(jsonTypeInfo?.Options));
                 return Task.FromResult(JsonSerializer.Deserialize(ref reader, jsonTypeInfo));
             }
 
             public override Task<object> DeserializeWrapper(string json, Type type, JsonSerializerContext context)
             {
-                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json));
+                Utf8JsonReader reader = new(Encoding.UTF8.GetBytes(json), OptionsHelpers.GetReaderOptions(context?.Options));
                 return Task.FromResult(JsonSerializer.Deserialize(ref reader, type, context));
             }
         }
@@ -355,10 +354,10 @@ public override Task<T> DeserializeWrapper<T>(string json, JsonSerializerOptions
                 if (json is null)
                 {
                     // Emulate a null document for API validation tests.
-                    return Task.FromResult(JsonSerializer.Deserialize<T>(document: null));
+                    return Task.FromResult(JsonSerializer.Deserialize<T>(document: null, options));
                 }
 
-                using JsonDocument document = JsonDocument.Parse(json);
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(options));
                 return Task.FromResult(document.Deserialize<T>(options));
             }
 
@@ -367,10 +366,10 @@ public override Task<object> DeserializeWrapper(string json, Type type, JsonSeri
                 if (json is null)
                 {
                     // Emulate a null document for API validation tests.
-                    return Task.FromResult(JsonSerializer.Deserialize(document: null, type));
+                    return Task.FromResult(JsonSerializer.Deserialize(document: null, type, options));
                 }
 
-                using JsonDocument document = JsonDocument.Parse(json);
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(options));
                 return Task.FromResult(document.Deserialize(type, options));
             }
 
@@ -382,8 +381,8 @@ public override Task<T> DeserializeWrapper<T>(string json, JsonTypeInfo<T> jsonT
                     return Task.FromResult(JsonSerializer.Deserialize<T>(document: null, jsonTypeInfo));
                 }
 
-                using JsonDocument document = JsonDocument.Parse(json);
-                return Task.FromResult(document.Deserialize<T>(jsonTypeInfo));
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(jsonTypeInfo?.Options));
+                return Task.FromResult(document.Deserialize(jsonTypeInfo));
             }
 
             public override Task<object> DeserializeWrapper(string json, Type type, JsonSerializerContext context)
@@ -394,7 +393,7 @@ public override Task<object> DeserializeWrapper(string json, Type type, JsonSeri
                     return Task.FromResult(JsonSerializer.Deserialize(document: null, type, context));
                 }
 
-                using JsonDocument document = JsonDocument.Parse(json);
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(context?.Options));
                 return Task.FromResult(document.Deserialize(type, context));
             }
         }
@@ -404,31 +403,31 @@ private class ElementSerializerWrapper : JsonSerializerWrapper
             public override Task<string> SerializeWrapper(object value, Type inputType, JsonSerializerOptions options = null)
             {
                 JsonElement element = JsonSerializer.SerializeToElement(value, inputType, options);
-                return Task.FromResult(GetStringFromElement(element));
+                return Task.FromResult(GetStringFromElement(element, options));
             }
 
             public override Task<string> SerializeWrapper<T>(T value, JsonSerializerOptions options = null)
             {
                 JsonElement element = JsonSerializer.SerializeToElement(value, options);
-                return Task.FromResult(GetStringFromElement(element));
+                return Task.FromResult(GetStringFromElement(element, options));
             }
 
             public override Task<string> SerializeWrapper(object value, Type inputType, JsonSerializerContext context)
             {
                 JsonElement element = JsonSerializer.SerializeToElement(value, inputType, context);
-                return Task.FromResult(GetStringFromElement(element));
+                return Task.FromResult(GetStringFromElement(element, context?.Options));
             }
 
             public override Task<string> SerializeWrapper<T>(T value, JsonTypeInfo<T> jsonTypeInfo)
             {
                 JsonElement element = JsonSerializer.SerializeToElement(value, jsonTypeInfo);
-                return Task.FromResult(GetStringFromElement(element));
+                return Task.FromResult(GetStringFromElement(element, jsonTypeInfo?.Options));
             }
 
-            private string GetStringFromElement(JsonElement element)
+            private string GetStringFromElement(JsonElement element, JsonSerializerOptions options)
             {
                 using MemoryStream stream = new MemoryStream();
-                using (Utf8JsonWriter writer = new(stream))
+                using (Utf8JsonWriter writer = new(stream, OptionsHelpers.GetWriterOptions(options)))
                 {
                     element.WriteTo(writer);
                 }
@@ -437,25 +436,25 @@ private string GetStringFromElement(JsonElement element)
 
             public override Task<T> DeserializeWrapper<T>(string json, JsonSerializerOptions options = null)
             {
-                using JsonDocument document = JsonDocument.Parse(json);
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(options));
                 return Task.FromResult(document.RootElement.Deserialize<T>(options));
             }
 
             public override Task<object> DeserializeWrapper(string json, Type type, JsonSerializerOptions options = null)
             {
-                using JsonDocument document = JsonDocument.Parse(json);
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(options));
                 return Task.FromResult(document.RootElement.Deserialize(type, options));
             }
 
             public override Task<T> DeserializeWrapper<T>(string json, JsonTypeInfo<T> jsonTypeInfo)
             {
-                using JsonDocument document = JsonDocument.Parse(json);
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(jsonTypeInfo?.Options));
                 return Task.FromResult(document.RootElement.Deserialize<T>(jsonTypeInfo));
             }
 
             public override Task<object> DeserializeWrapper(string json, Type type, JsonSerializerContext context)
             {
-                using JsonDocument document = JsonDocument.Parse(json);
+                using JsonDocument document = JsonDocument.Parse(json, OptionsHelpers.GetDocumentOptions(context?.Options));
                 return Task.FromResult(document.RootElement.Deserialize(type, context));
             }
         }
@@ -521,10 +520,10 @@ public override Task<T> DeserializeWrapper<T>(string json, JsonSerializerOptions
                 if (json is null)
                 {
                     // Emulate a null node for API validation tests.
-                    return Task.FromResult(JsonSerializer.Deserialize<T>(node: null));
+                    return Task.FromResult(JsonSerializer.Deserialize<T>(node: null, options));
                 }
 
-                JsonNode node = JsonNode.Parse(json);
+                JsonNode node = JsonNode.Parse(json, OptionsHelpers.GetNodeOptions(options), OptionsHelpers.GetDocumentOptions(options));
                 return Task.FromResult(node.Deserialize<T>(options));
             }
 
@@ -533,10 +532,10 @@ public override Task<object> DeserializeWrapper(string json, Type type, JsonSeri
                 if (json is null)
                 {
                     // Emulate a null node for API validation tests.
-                    return Task.FromResult(JsonSerializer.Deserialize(node: null, type));
+                    return Task.FromResult(JsonSerializer.Deserialize(node: null, type, options));
                 }
 
-                JsonNode node = JsonNode.Parse(json);
+                JsonNode node = JsonNode.Parse(json, OptionsHelpers.GetNodeOptions(options), OptionsHelpers.GetDocumentOptions(options));
                 return Task.FromResult(node.Deserialize(type, options));
             }
 
@@ -548,7 +547,7 @@ public override Task<T> DeserializeWrapper<T>(string json, JsonTypeInfo<T> jsonT
                     return Task.FromResult(JsonSerializer.Deserialize(node: null, jsonTypeInfo));
                 }
 
-                JsonNode node = JsonNode.Parse(json);
+                JsonNode node = JsonNode.Parse(json, OptionsHelpers.GetNodeOptions(jsonTypeInfo?.Options), OptionsHelpers.GetDocumentOptions(jsonTypeInfo?.Options));
                 return Task.FromResult(node.Deserialize<T>(jsonTypeInfo));
             }
 
@@ -557,14 +556,68 @@ public override Task<object> DeserializeWrapper(string json, Type type, JsonSeri
                 if (json is null)
                 {
                     // Emulate a null document for API validation tests.
-                    return Task.FromResult(JsonSerializer.Deserialize(node: null, type));
+                    return Task.FromResult(JsonSerializer.Deserialize(node: null, type, context?.Options));
                 }
 
-                JsonNode node = JsonNode.Parse(json);
+                JsonNode node = JsonNode.Parse(json, OptionsHelpers.GetNodeOptions(context?.Options), OptionsHelpers.GetDocumentOptions(context?.Options));
                 return Task.FromResult(node.Deserialize(type, context));
             }
         }
 
+        private static class OptionsHelpers
+        {
+            public static JsonWriterOptions GetWriterOptions(JsonSerializerOptions? options)
+            {
+                return options is null
+                    ? default
+                    : new JsonWriterOptions
+                    {
+                        Encoder = options.Encoder,
+                        Indented = options.WriteIndented,
+                        MaxDepth = GetEffectiveMaxDepth(options),
+    #if !DEBUG
+                            SkipValidation = true
+    #endif
+                    };
+            }
+
+            public static JsonReaderOptions GetReaderOptions(JsonSerializerOptions? options)
+            {
+                return options is null
+                    ? default
+                    : new JsonReaderOptions
+                    {
+                        AllowTrailingCommas = options.AllowTrailingCommas,
+                        CommentHandling = options.ReadCommentHandling,
+                        MaxDepth = GetEffectiveMaxDepth(options),
+                    };
+            }
+
+            public static JsonDocumentOptions GetDocumentOptions(JsonSerializerOptions? options)
+            {
+                return options is null
+                    ? default
+                    : new JsonDocumentOptions
+                    {
+                        MaxDepth = GetEffectiveMaxDepth(options),
+                        AllowTrailingCommas = options.AllowTrailingCommas,
+                        CommentHandling = options.ReadCommentHandling
+                    };
+            }
+
+            public static JsonNodeOptions GetNodeOptions(JsonSerializerOptions? options)
+            {
+                return options is null
+                    ? default
+                    : new JsonNodeOptions
+                    {
+                        PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive
+                    };
+            }
+
+            private static int GetEffectiveMaxDepth(JsonSerializerOptions options) => options.MaxDepth == 0 ? 64 : options.MaxDepth;
+        }
+
         /// <summary>
         /// Maintains an index of equivalent JsonSerializerOptions instances with DefaultBufferSize = 1
         /// </summary>

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.CustomTypeHierarchies.cs

@@ -962,6 +962,124 @@ static object[] WrapArgs(PolymorphicClass_WithBaseTypeDiscriminator value, strin
                     => new object[] { value, expectedJson };
             }
         }
+
+        [Fact]
+        public async Task NestedPolymorphicClassesIncreaseReadAndWriteStackWhenNeeded()
+        {
+            // Regression test for https://github.com/dotnet/runtime/issues/71994
+            TestNode obj = new TestNodeList
+            {
+                Name = "testName",
+                Info = "1",
+                List = new List<TestNode>
+                {
+                    new TestLeaf
+                    {
+                        Name = "testName2"
+                    },
+
+                    new TestNodeList
+                    {
+                        Name = "testName3",
+                        Info = "2",
+                        List = new List<TestNode>
+                        {
+                            new TestNodeList
+                            {
+                                Name = "testName4",
+                                Info = "1"
+                            }
+                        }
+
+                    }
+                }
+            };
+
+            string json = await Serializer.SerializeWrapper(obj);
+            JsonTestHelper.AssertJsonEqual(
+                @"{
+                   ""$type"": ""NodeList"",
+                   ""Info"": ""1"",
+                   ""List"": [
+                     {
+                       ""$type"": ""Leaf"",
+                       ""Test"": null,
+                       ""Name"": ""testName2""
+                     },
+                     {
+                       ""$type"": ""NodeList"",
+                       ""Info"": ""2"",
+                       ""List"": [
+                         {
+                           ""$type"": ""NodeList"",
+                           ""Info"": ""1"",
+                           ""List"": null,
+                           ""Name"": ""testName4""
+                         }
+                       ],
+                       ""Name"": ""testName3""
+                     }
+                   ],
+                   ""Name"": ""testName""}", json);
+
+            TestNode deserialized = await Serializer.DeserializeWrapper<TestNode>(json);
+            obj.AssertEqualTo(deserialized);
+        }
+
+        [JsonDerivedType(typeof(TestNodeList), "NodeList")]
+        [JsonDerivedType(typeof(TestLeaf), "Leaf")]
+        abstract class TestNode
+        {
+            public string Name { get; set; }
+
+            public abstract void AssertEqualTo(TestNode other);
+        }
+
+        class TestNodeList : TestNode
+        {
+            public string Info { get; set; }
+
+            public IEnumerable<TestNode> List { get; set; }
+
+            public override void AssertEqualTo(TestNode other)
+            {
+                Assert.Equal(Name, other.Name);
+                Assert.IsType<TestNodeList>(other);
+                TestNodeList typedOther = (TestNodeList)other;
+                Assert.Equal(Info, typedOther.Info);
+                Assert.Equal(List is null, typedOther.List is null);
+
+                if (List is not null)
+                {
+                    using IEnumerator<TestNode> thisEnumerator = List.GetEnumerator();
+                    using IEnumerator<TestNode> otherEnumerator = typedOther.List.GetEnumerator();
+
+                    while (true)
+                    {
+                        bool hasNext = thisEnumerator.MoveNext();
+                        Assert.Equal(hasNext, otherEnumerator.MoveNext());
+
+                        if (!hasNext)
+                            break;
+
+                        thisEnumerator.Current.AssertEqualTo(otherEnumerator.Current);
+                    }
+                }
+            }
+        }
+
+        class TestLeaf : TestNode
+        {
+            public string Test { get; set; }
+
+            public override void AssertEqualTo(TestNode other)
+            {
+                Assert.Equal(Name, other.Name);
+                Assert.IsType<TestLeaf>(other);
+                TestLeaf typedOther = (TestLeaf)other;
+                Assert.Equal(Test, typedOther.Test);
+            }
+        }
         #endregion
 
         #region Polymorphic Class with Constructor
@@ -1762,6 +1880,26 @@ public async Task Peano_Deserialization(int expectedSize, string json)
             await TestMultiContextDeserialization<Peano>(json, expected);
         }
 
+        [Theory]
+        [InlineData(0)]
+        [InlineData(1)]
+        [InlineData(3)]
+        [InlineData(4)] // initial Write/ReadStack size
+        [InlineData(5)] // 1st Write/ReadStack resize
+        [InlineData(17)] // 2nd Write/ReadStack resize
+        [InlineData(33)] // 3rd Write/ReadStack resize
+        [InlineData(65)] // 4th Write/ReadStack resize
+        [InlineData(80)]
+        public async Task Peano_Roundtrip(int number)
+        {
+            JsonSerializerOptions options = new();
+            options.MaxDepth = number + 1;
+            Peano obj = Peano.FromInteger(number);
+            string json = await Serializer.SerializeWrapper(obj, options);
+            Peano deserialized = await Serializer.DeserializeWrapper<Peano>(json, options);
+            Assert.Equal(obj, deserialized);
+        }
+
         // A Peano representation for natural numbers
         [JsonDerivedType(typeof(Zero), "zero")]
         [JsonDerivedType(typeof(Succ), "succ")]

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj

@@ -5,6 +5,8 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <!-- SYSLIB0020: JsonSerializerOptions.IgnoreNullValues is obsolete -->
     <NoWarn>$(NoWarn);SYSLIB0020</NoWarn>
+    <!-- Disable analyzers warnings about JSON being misformatted in string literals -->
+    <NoWarn>$(NoWarn);JSON001</NoWarn>
 
     <!-- these tests depend on the pdb files. Causes test failures like:
           [FAIL] System.Text.Json.Tests.DebuggerTests.DefaultJsonElement -->