Add System.Memory reference (#31162)

Also replaces some unsafe code with Span<T>.

build/Targets/Packages.props

@@ -184,6 +184,7 @@
     <SystemIOFileSystemVersion>4.3.0</SystemIOFileSystemVersion>
     <SystemIOFileSystemPrimitivesVersion>4.3.0</SystemIOFileSystemPrimitivesVersion>
     <SystemIOPipesAccessControlVersion>4.3.0</SystemIOPipesAccessControlVersion>
+    <SystemMemoryVersion>4.5.1</SystemMemoryVersion>
     <SystemReflectionMetadataVersion>1.6.0</SystemReflectionMetadataVersion>
     <SystemRuntimeCompilerServicesUnsafeVersion>4.5.0</SystemRuntimeCompilerServicesUnsafeVersion>
     <SystemTextEncodingCodePagesVersion>4.5.0</SystemTextEncodingCodePagesVersion>

src/Compilers/Core/CodeAnalysisTest/AnalyzerFileReferenceTests.cs

@@ -223,6 +223,8 @@ public class TestAnalyzer : DiagnosticAnalyzer
 
             dir.CopyFile(typeof(System.Reflection.Metadata.MetadataReader).Assembly.Location);
             dir.CopyFile(typeof(AppDomainUtils).Assembly.Location);
+            dir.CopyFile(typeof(Memory<>).Assembly.Location);
+            dir.CopyFile(typeof(System.Runtime.CompilerServices.Unsafe).Assembly.Location);
             var immutable = dir.CopyFile(typeof(ImmutableArray).Assembly.Location);
             var analyzer = dir.CopyFile(typeof(DiagnosticAnalyzer).Assembly.Location);
             var test = dir.CopyFile(typeof(FromFileLoader).Assembly.Location);

src/Compilers/Core/CodeAnalysisTest/StringTableTests.cs

 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
+using System;
 using System.Text;
 using Microsoft.CodeAnalysis.Text;
 using Roslyn.Utilities;
@@ -83,7 +84,7 @@ private unsafe static bool TestTextEqualsASCII(string str, string ascii)
         {
             fixed (byte* ptr = Encoding.ASCII.GetBytes(ascii))
             {
-                var ptrResult = StringTable.TextEqualsASCII(str, ptr, ascii.Length);
+                var ptrResult = StringTable.TextEqualsASCII(str, new ReadOnlySpan<byte>(ptr, ascii.Length));
                 var sbResult = StringTable.TextEquals(str, new StringBuilder(ascii));
                 var substrResult = StringTable.TextEquals(str, "xxx" + ascii + "yyy", 3, ascii.Length);
                 Assert.Equal(substrResult, sbResult);

src/Compilers/Core/Portable/InternalUtilities/Hash.cs

@@ -178,16 +178,15 @@ internal static int GetFNVHashCode(byte[] data)
         /// See http://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
         /// </summary>
         /// <param name="data">The sequence of bytes that are likely to be ASCII text.</param>
-        /// <param name="length">The length of the sequence.</param>
         /// <param name="isAscii">True if the sequence contains only characters in the ASCII range.</param>
         /// <returns>The FNV-1a hash of <paramref name="data"/></returns>
-        internal static unsafe int GetFNVHashCode(byte* data, int length, out bool isAscii)
+        internal static int GetFNVHashCode(ReadOnlySpan<byte> data, out bool isAscii)
         {
             int hashCode = Hash.FnvOffsetBias;
 
             byte asciiMask = 0;
 
-            for (int i = 0; i < length; i++)
+            for (int i = 0; i < data.Length; i++)
             {
                 byte b = data[i];
                 asciiMask |= b;

src/Compilers/Core/Portable/InternalUtilities/StringTable.cs

@@ -109,6 +109,7 @@ public void Free()
 
         internal string Add(char[] chars, int start, int len)
         {
+            var span = chars.AsSpan(start, len);
             var hashCode = Hash.GetFNVHashCode(chars, start, len);
 
             // capture array to avoid extra range checks
@@ -120,7 +121,7 @@ internal string Add(char[] chars, int start, int len)
             if (text != null && arr[idx].HashCode == hashCode)
             {
                 var result = arr[idx].Text;
-                if (StringTable.TextEquals(result, chars, start, len))
+                if (StringTable.TextEquals(result, span))
                 {
                     return result;
                 }
@@ -294,7 +295,7 @@ private static string FindSharedEntry(char[] chars, int start, int len, int hash
 
                 if (e != null)
                 {
-                    if (hash == hashCode && TextEquals(e, chars, start, len))
+                    if (hash == hashCode && TextEquals(e, chars.AsSpan(start, len)))
                     {
                         break;
                     }
@@ -349,7 +350,7 @@ private static string FindSharedEntry(string chars, int start, int len, int hash
             return e;
         }
 
-        private static unsafe string FindSharedEntryASCII(int hashCode, byte* asciiChars, int length)
+        private static string FindSharedEntryASCII(int hashCode, ReadOnlySpan<byte> asciiChars)
         {
             var arr = s_sharedTable;
             int idx = SharedIdxFromHash(hashCode);
@@ -364,7 +365,7 @@ private static unsafe string FindSharedEntryASCII(int hashCode, byte* asciiChars
 
                 if (e != null)
                 {
-                    if (hash == hashCode && TextEqualsASCII(e, asciiChars, length))
+                    if (hash == hashCode && TextEqualsASCII(e, asciiChars))
                     {
                         break;
                     }
@@ -580,29 +581,34 @@ private static string AddSharedSlow(int hashCode, StringBuilder builder)
             return text;
         }
 
-        internal static unsafe string AddSharedUTF8(byte* bytes, int byteCount)
+        internal static string AddSharedUTF8(ReadOnlySpan<byte> bytes)
         {
             bool isAscii;
-            int hashCode = Hash.GetFNVHashCode(bytes, byteCount, out isAscii);
+            int hashCode = Hash.GetFNVHashCode(bytes, out isAscii);
 
             if (isAscii)
             {
-                string shared = FindSharedEntryASCII(hashCode, bytes, byteCount);
+                string shared = FindSharedEntryASCII(hashCode, bytes);
                 if (shared != null)
                 {
                     return shared;
                 }
             }
 
-            return AddSharedSlow(hashCode, bytes, byteCount, isAscii);
+            return AddSharedSlow(hashCode, bytes, isAscii);
         }
 
-        private static unsafe string AddSharedSlow(int hashCode, byte* utf8Bytes, int byteCount, bool isAscii)
+        private static string AddSharedSlow(int hashCode, ReadOnlySpan<byte> utf8Bytes, bool isAscii)
         {
-            // TODO: This should be Encoding.UTF8.GetString (for better layering) but the unsafe variant isn't portable. 
-            //       The MetadataReader has code to light it up and even fall back to internal String.CreateStringFromEncoding
-            //       on .NET < 4.5.3. Use that instead of copying the light-up code here.
-            string text = System.Reflection.Metadata.MetadataStringDecoder.DefaultUTF8.GetString(utf8Bytes, byteCount);
+            string text;
+
+            unsafe
+            {
+                fixed (byte* bytes = &utf8Bytes.GetPinnableReference())
+                {
+                    text = Encoding.UTF8.GetString(bytes, utf8Bytes.Length);
+                }
+            }
 
             // Don't add non-ascii strings to table. The hashCode we have here is not correct and we won't find them again.
             // Non-ascii in UTF8-encoded parts of metadata (the only use of this at the moment) is assumed to be rare in 
@@ -705,21 +711,21 @@ internal static bool TextEquals(string array, StringBuilder text)
             return true;
         }
 
-        internal static unsafe bool TextEqualsASCII(string text, byte* ascii, int length)
+        internal static bool TextEqualsASCII(string text, ReadOnlySpan<byte> ascii)
         {
 #if DEBUG
-            for (var i = 0; i < length; i++)
+            for (var i = 0; i < ascii.Length; i++)
             {
-                Debug.Assert((ascii[i] & 0x80) == 0, "The byte* input to this method must be valid ASCII.");
+                Debug.Assert((ascii[i] & 0x80) == 0, $"The {nameof(ascii)} input to this method must be valid ASCII.");
             }
 #endif
 
-            if (length != text.Length)
+            if (ascii.Length != text.Length)
             {
                 return false;
             }
 
-            for (var i = 0; i < length; i++)
+            for (var i = 0; i < ascii.Length; i++)
             {
                 if (ascii[i] != text[i])
                 {
@@ -730,25 +736,7 @@ internal static unsafe bool TextEqualsASCII(string text, byte* ascii, int length
             return true;
         }
 
-        internal static bool TextEquals(string array, char[] text, int start, int length)
-        {
-            return array.Length == length && TextEqualsCore(array, text, start);
-        }
-
-        private static bool TextEqualsCore(string array, char[] text, int start)
-        {
-            // use array.Length to eliminate the range check
-            int s = start;
-            for (var i = 0; i < array.Length; i++)
-            {
-                if (array[i] != text[s])
-                {
-                    return false;
-                }
-                s++;
-            }
-
-            return true;
-        }
+        internal static bool TextEquals(string array, ReadOnlySpan<char> text)
+            => text.Equals(array.AsSpan(), StringComparison.Ordinal);
     }
 }

src/Compilers/Core/Portable/InternalUtilities/TextKeyedCache.cs

@@ -118,7 +118,7 @@ internal T FindItem(char[] chars, int start, int len, int hashCode)
 
             if (text != null && localSlot.HashCode == hashCode)
             {
-                if (StringTable.TextEquals(text, chars, start, len))
+                if (StringTable.TextEquals(text, chars.AsSpan(start, len)))
                 {
                     return localSlot.Item;
                 }
@@ -155,7 +155,7 @@ private SharedEntryValue FindSharedEntry(char[] chars, int start, int len, int h
 
                 if (e != null)
                 {
-                    if (hash == hashCode && StringTable.TextEquals(e.Text, chars, start, len))
+                    if (hash == hashCode && StringTable.TextEquals(e.Text, chars.AsSpan(start, len)))
                     {
                         break;
                     }

src/Compilers/Core/Portable/MetadataReader/PEModule.cs

@@ -3135,7 +3135,7 @@ private sealed class StringTableDecoder : MetadataStringDecoder
 
             public unsafe override string GetString(byte* bytes, int byteCount)
             {
-                return StringTable.AddSharedUTF8(bytes, byteCount);
+                return StringTable.AddSharedUTF8(new ReadOnlySpan<byte>(bytes, byteCount));
             }
         }
 

src/Compilers/Core/Portable/Microsoft.CodeAnalysis.csproj

@@ -57,6 +57,7 @@
 
     <PackageReference Include="Microsoft.DiaSymReader.Native" Version="$(MicrosoftDiaSymReaderNativeVersion)" PrivateAssets="all" />
     <PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
+    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
     <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
     <PackageReference Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" />
     <PackageReference Include="System.Text.Encoding.CodePages" Version="$(SystemTextEncodingCodePagesVersion)" />

src/Setup/DevDivInsertionFiles/DevDivInsertionFiles.csproj

@@ -44,10 +44,13 @@
     <ExpectedDependency Include="ICSharpCode.Decompiler" DevEnvLib="lib/netstandard2.0/ICSharpCode.Decompiler.dll" Optimize="false" />
     <ExpectedDependency Include="Microsoft.DiaSymReader" DevEnvLib="lib/netstandard1.1/Microsoft.DiaSymReader.dll"/>
     <ExpectedDependency Include="Microsoft.CodeAnalysis.Elfie" DevEnvLib="lib/net46/Microsoft.CodeAnalysis.Elfie.dll"/>
+    <ExpectedDependency Include="System.Buffers" DevEnvLib="lib/netstandard2.0/System.Buffers.dll" />
     <ExpectedDependency Include="System.Collections.Immutable" DevEnvLib="lib/netstandard2.0/System.Collections.Immutable.dll"/>
     <ExpectedDependency Include="System.Reflection.Metadata" DevEnvLib="lib/netstandard2.0/System.Reflection.Metadata.dll"/>
+    <ExpectedDependency Include="System.Memory" DevEnvLib="lib/netstandard2.0/System.Memory.dll" />
     <ExpectedDependency Include="System.Runtime.CompilerServices.Unsafe" DevEnvLib="lib/netstandard2.0/System.Runtime.CompilerServices.Unsafe.dll"/>
     <ExpectedDependency Include="System.Text.Encoding.CodePages" DevEnvLib="lib/netstandard2.0/System.Text.Encoding.CodePages.dll"/>
+    <ExpectedDependency Include="System.Numerics.Vectors" DevEnvLib="lib/netstandard2.0/System.Numerics.Vectors.dll" />
 
     <!-- 
       Do not optimize this binary for now.

src/Test/Utilities/Portable/Platform/Desktop/TestHelpers.cs

@@ -67,6 +67,8 @@ public class TestAnalyzer : DiagnosticAnalyzer
             dir.CopyFile(typeof(System.Reflection.Metadata.MetadataReader).Assembly.Location);
             var immutable = dir.CopyFile(typeof(ImmutableArray).Assembly.Location);
             var analyzer = dir.CopyFile(typeof(DiagnosticAnalyzer).Assembly.Location);
+            dir.CopyFile(typeof(Memory<>).Assembly.Location);
+            dir.CopyFile(typeof(System.Runtime.CompilerServices.Unsafe).Assembly.Location);
 
             var analyzerCompilation = CSharpCompilation.Create(
                 assemblyName,

src/Test/Utilities/Portable/Roslyn.Test.Utilities.csproj

@@ -25,6 +25,10 @@
        Condition="'$(TargetFramework)' == 'netcoreapp2.1'" />
 
   </ItemGroup>
+  <ItemGroup>
+    <!-- Needed to find the Unsafe.dll binary to lay out at runtime for the compiler when testing analyzers. -->
+    <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="$(SystemRuntimeCompilerServicesUnsafeVersion)" />
+  </ItemGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|AnyCPU'" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|AnyCPU'" />
   <ItemGroup>