Merge pull request #16807 from heejaechang/guardEncoding

Guard against serialization failure of Encoding

src/Workspaces/Core/Desktop/Execution/DesktopReferenceSerializationServiceFactory.cs

@@ -4,6 +4,7 @@
 using System.Collections.Concurrent;
 using System.Composition;
 using System.IO;
+using System.Runtime.Serialization;
 using System.Runtime.Serialization.Formatters.Binary;
 using System.Text;
 using System.Threading;
@@ -47,26 +48,48 @@ public override void WriteTo(Encoding encoding, ObjectWriter writer, Cancellatio
 
                 cancellationToken.ThrowIfCancellationRequested();
 
+                var value = GetEncodingBytes(encoding);
+                if (value == null)
+                {
+                    // we couldn't serialize encoding, act like there is no encoding.
+                    base.WriteTo(encoding, writer, cancellationToken);
+                    return;
+                }
+
+                // write data out
                 writer.WriteByte(EncodingSerialization);
+                writer.WriteValue(value);
+            }
 
-                byte[] value;
-                if (!s_encodingCache.TryGetValue(encoding, out value))
+            private static byte[] GetEncodingBytes(Encoding encoding)
+            {
+                try
                 {
-                    // we don't have cache, cache it
-                    var formatter = new BinaryFormatter();
-                    using (var stream = SerializableBytes.CreateWritableStream())
+                    byte[] value;
+                    if (!s_encodingCache.TryGetValue(encoding, out value))
                     {
-                        // unfortunately, this is only way to properly clone encoding
-                        formatter.Serialize(stream, encoding);
-                        value = stream.ToArray();
-
-                        // add if not already exist. otherwise, noop
-                        s_encodingCache.TryAdd(encoding, value);
+                        // we don't have cache, cache it
+                        var formatter = new BinaryFormatter();
+                        using (var stream = SerializableBytes.CreateWritableStream())
+                        {
+                            // unfortunately, this is only way to properly clone encoding
+                            formatter.Serialize(stream, encoding);
+                            value = stream.ToArray();
+
+                            // add if not already exist. otherwise, noop
+                            s_encodingCache.TryAdd(encoding, value);
+                        }
                     }
-                }
 
-                // write data out
-                writer.WriteValue(value);
+                    return value;
+                }
+                catch (SerializationException)
+                {
+                    // even though Encoding is supposed to be serializable, 
+                    // not every Encoding follows the rule strictly.
+                    // in such as, behave like there was no encoding
+                    return null;
+                }
             }
 
             public override Encoding ReadEncodingFrom(ObjectReader reader, CancellationToken cancellationToken)

src/Workspaces/CoreTest/Execution/SnapshotSerializationTests.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.CodeStyle;
@@ -486,6 +487,52 @@ public async Task VBParseOptionsInCompilationOptions()
             Assert.NotNull(checksum);
         }
 
+        [Fact]
+        public void TestEncodingSerialization()
+        {
+            var hostServices = MefHostServices.Create(
+                MefHostServices.DefaultAssemblies.Add(typeof(Host.TemporaryStorageServiceFactory.TemporaryStorageService).Assembly));
+
+            var workspace = new AdhocWorkspace(hostServices);
+            var serializer = new Serializer(workspace);
+
+            // test with right serializable encoding
+            var sourceText = SourceText.From("Hello", Encoding.UTF8);
+            using (var stream = SerializableBytes.CreateWritableStream())
+            {
+                using (var objectWriter = new StreamObjectWriter(stream))
+                {
+                    serializer.Serialize(sourceText, objectWriter, CancellationToken.None);
+                }
+
+                stream.Position = 0;
+
+                using (var objectReader = StreamObjectReader.TryGetReader(stream))
+                {
+                    var newText = serializer.Deserialize<SourceText>(sourceText.GetWellKnownSynchronizationKind(), objectReader, CancellationToken.None);
+                    Assert.Equal(sourceText.ToString(), newText.ToString());
+                }
+            }
+
+            // test with wrong encoding that doesn't support serialization
+            sourceText = SourceText.From("Hello", new NotSerializableEncoding());
+            using (var stream = SerializableBytes.CreateWritableStream())
+            {
+                using (var objectWriter = new StreamObjectWriter(stream))
+                {
+                    serializer.Serialize(sourceText, objectWriter, CancellationToken.None);
+                }
+
+                stream.Position = 0;
+
+                using (var objectReader = StreamObjectReader.TryGetReader(stream))
+                {
+                    var newText = serializer.Deserialize<SourceText>(sourceText.GetWellKnownSynchronizationKind(), objectReader, CancellationToken.None);
+                    Assert.Equal(sourceText.ToString(), newText.ToString());
+                }
+            }
+        }
+
         private static async Task VerifyOptionSetsAsync(Workspace workspace, string language)
         {
             var assetBuilder = new CustomAssetBuilder(workspace);
@@ -671,5 +718,18 @@ public Assembly LoadFromPath(string fullPath)
                 return typeof(SharedAttribute).Assembly;
             }
         }
+
+        private class NotSerializableEncoding : Encoding
+        {
+            private readonly Encoding _real = Encoding.UTF8;
+
+            public override string WebName => _real.WebName;
+            public override int GetByteCount(char[] chars, int index, int count) => _real.GetByteCount(chars, index, count);
+            public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) => GetBytes(chars, charIndex, charCount, bytes, byteIndex);
+            public override int GetCharCount(byte[] bytes, int index, int count) => GetCharCount(bytes, index, count);
+            public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) => GetChars(bytes, byteIndex, byteCount, chars, charIndex);
+            public override int GetMaxByteCount(int charCount) => GetMaxByteCount(charCount);
+            public override int GetMaxCharCount(int byteCount) => GetMaxCharCount(byteCount);
+        }
     }
 }
\ No newline at end of file