Merge pull request #8875 from tmat/ShareDelete2

Allow compiler to write to output files if they were open with FileShare.Delete

src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs

@@ -26,13 +26,16 @@
 
 using static Roslyn.Test.Utilities.SharedResourceHelpers;
 using static Microsoft.CodeAnalysis.CommonDiagnosticAnalyzers;
+using System.IO.MemoryMappedFiles;
+using System.Reflection.Metadata;
 
 namespace Microsoft.CodeAnalysis.CSharp.CommandLine.UnitTests
 {
     public class CommandLineTests : CSharpTestBase
     {
-        private static readonly string s_CSharpCompilerExecutable = typeof(Microsoft.CodeAnalysis.CSharp.CommandLine.Csc).Assembly.Location;
+        private static readonly string s_CSharpCompilerExecutable = typeof(Csc).GetTypeInfo().Assembly.Location;
         private static readonly string s_defaultSdkDirectory = RuntimeEnvironment.GetRuntimeDirectory();
+        private static readonly string s_compilerVersion = typeof(Csc).GetTypeInfo().Assembly.GetCustomAttribute<AssemblyFileVersionAttribute>().Version;
 
         private readonly string _baseDirectory = TempRoot.Root;
 
@@ -6472,6 +6475,169 @@ private static void ValidateZeroes(string path, int count)
             }
         }
 
+        /// <summary>
+        /// When the output file is open with <see cref="FileShare.Read"/> | <see cref="FileShare.Delete"/>
+        /// the compiler should delete the file to unblock build while allowing the reader to continue 
+        /// reading the previous snapshot of the file content.
+        /// 
+        /// On Windows we can read the original data directly from the stream withotu creating a memory map. 
+        /// </summary>
+        [ConditionalFact(typeof(WindowsOnly))]
+        public void FileShareDeleteCompatibility_Windows()
+        {
+            var dir = Temp.CreateDirectory();
+            var libSrc = dir.CreateFile("Lib.cs").WriteAllText("class C { }");
+            var libDll = dir.CreateFile("Lib.dll").WriteAllText("DLL");
+            var libPdb = dir.CreateFile("Lib.pdb").WriteAllText("PDB");
+
+            var fsDll = new FileStream(libDll.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
+            var fsPdb = new FileStream(libPdb.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
+
+            var outWriter = new StringWriter(CultureInfo.InvariantCulture);
+            int exitCode = new MockCSharpCompiler(null, dir.Path, new[] { "/target:library", "/debug:full", libSrc.Path }).Run(outWriter);
+            if (exitCode != 0)
+            {
+                AssertEx.AssertEqualToleratingWhitespaceDifferences("", outWriter.ToString());
+            }
+
+            Assert.Equal(0, exitCode);
+
+            AssertEx.Equal(new byte[] { 0x4D, 0x5A }, ReadBytes(libDll.Path, 2));
+            AssertEx.Equal(new[] { (byte)'D', (byte)'L', (byte)'L' }, ReadBytes(fsDll, 3));
+
+            AssertEx.Equal(new byte[] { 0x4D, 0x69 }, ReadBytes(libPdb.Path, 2));
+            AssertEx.Equal(new[] { (byte)'P', (byte)'D', (byte)'B' }, ReadBytes(fsPdb, 3));
+
+            fsDll.Dispose();
+            fsPdb.Dispose();
+
+            AssertEx.Equal(new[] { "Lib.cs", "Lib.dll", "Lib.pdb" }, Directory.GetFiles(dir.Path).Select(p => Path.GetFileName(p)).Order());
+        }
+
+        /// <summary>
+        /// On Linux/Mac <see cref="FileShare.Delete"/> on its own doesn't do anything. 
+        /// We need to create the actual memory map. This works on Windows as well.
+        /// </summary>
+        [ConditionalFact(typeof(WindowsOnly)), WorkItem(8896, "https://github.com/dotnet/roslyn/issues/8896")]
+        public void FileShareDeleteCompatibility_Xplat()
+        {
+            var bytes = TestResources.MetadataTests.InterfaceAndClass.CSClasses01;
+            var mvid = ReadMvid(new MemoryStream(bytes));
+
+            var dir = Temp.CreateDirectory();
+            var libSrc = dir.CreateFile("Lib.cs").WriteAllText("class C { }");
+            var libDll = dir.CreateFile("Lib.dll").WriteAllBytes(bytes);
+            var libPdb = dir.CreateFile("Lib.pdb").WriteAllBytes(bytes);
+
+            var fsDll = new FileStream(libDll.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
+            var fsPdb = new FileStream(libPdb.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
+
+            var peDll = new PEReader(fsDll);
+            var pePdb = new PEReader(fsPdb);
+
+            // creates memory map view:
+            var imageDll = peDll.GetEntireImage();
+            var imagePdb = pePdb.GetEntireImage();
+
+            var output = ProcessUtilities.RunAndGetOutput(s_CSharpCompilerExecutable, $"/target:library /debug:portable {libSrc.Path}", startFolder: dir.ToString());
+            AssertEx.AssertEqualToleratingWhitespaceDifferences($@"
+Microsoft (R) Visual C# Compiler version {s_compilerVersion}
+Copyright (C) Microsoft Corporation. All rights reserved.", output);
+
+            // reading original content from the memory map: 
+            Assert.Equal(mvid, ReadMvid(new MemoryStream(imageDll.GetContent().ToArray())));
+            Assert.Equal(mvid, ReadMvid(new MemoryStream(imagePdb.GetContent().ToArray())));
+            
+            // reading original content directly from the streams: 
+            fsDll.Position = 0;
+            fsPdb.Position = 0;
+            Assert.Equal(mvid, ReadMvid(fsDll));
+            Assert.Equal(mvid, ReadMvid(fsPdb));
+
+            // reading new content from the file:
+            using (var fsNewDll = File.OpenRead(libDll.Path))
+            {
+                Assert.NotEqual(mvid, ReadMvid(fsNewDll));
+            }
+
+            // Portable PDB metadata signature:
+            AssertEx.Equal(new[] { (byte)'B', (byte)'S', (byte)'J', (byte)'B' }, ReadBytes(libPdb.Path, 4));
+
+            // dispose PEReaders (they dispose the underlying file streams)
+            peDll.Dispose();
+            pePdb.Dispose();
+
+            AssertEx.Equal(new[] { "Lib.cs", "Lib.dll", "Lib.pdb" }, Directory.GetFiles(dir.Path).Select(p => Path.GetFileName(p)).Order());
+
+            // files can be deleted now:
+            File.Delete(libSrc.Path);
+            File.Delete(libDll.Path);
+            File.Delete(libPdb.Path);
+
+            // directory can be deleted (should be empty):
+            Directory.Delete(dir.Path, recursive: false);
+        }
+
+        private static Guid ReadMvid(Stream stream)
+        {
+            using (var peReader = new PEReader(stream, PEStreamOptions.LeaveOpen))
+            {
+                var mdReader = peReader.GetMetadataReader();
+                return mdReader.GetGuid(mdReader.GetModuleDefinition().Mvid);
+            }
+        }
+
+        // Seems like File.SetAttributes(libDll.Path, FileAttributes.ReadOnly) doesn't restrict access to the file on Mac (Linux passes).
+        [ConditionalFact(typeof(WindowsOnly)), WorkItem(8939, "https://github.com/dotnet/roslyn/issues/8939")]
+        public void FileShareDeleteCompatibility_ReadOnlyFiles()
+        {
+            var dir = Temp.CreateDirectory();
+            var libSrc = dir.CreateFile("Lib.cs").WriteAllText("class C { }");
+            var libDll = dir.CreateFile("Lib.dll").WriteAllText("DLL");
+
+            File.SetAttributes(libDll.Path, FileAttributes.ReadOnly);
+
+            var fsDll = new FileStream(libDll.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
+
+            var outWriter = new StringWriter(CultureInfo.InvariantCulture);
+            int exitCode = new MockCSharpCompiler(null, dir.Path, new[] { "/target:library", libSrc.Path }).Run(outWriter);
+            Assert.Contains($"error CS2012: Cannot open '{libDll.Path}' for writing", outWriter.ToString());
+
+            AssertEx.Equal(new[] { (byte)'D', (byte)'L', (byte)'L' }, ReadBytes(libDll.Path, 3));
+            AssertEx.Equal(new[] { (byte)'D', (byte)'L', (byte)'L' }, ReadBytes(fsDll, 3));
+
+            fsDll.Dispose();
+
+            AssertEx.Equal(new[] { "Lib.cs", "Lib.dll" }, Directory.GetFiles(dir.Path).Select(p => Path.GetFileName(p)).Order());
+        }
+
+        [Fact]
+        public void FileShareDeleteCompatibility_ExistingDirectory()
+        {
+            var dir = Temp.CreateDirectory();
+            var libSrc = dir.CreateFile("Lib.cs").WriteAllText("class C { }");
+            var libDll = dir.CreateDirectory("Lib.dll");
+
+            var outWriter = new StringWriter(CultureInfo.InvariantCulture);
+            int exitCode = new MockCSharpCompiler(null, dir.Path, new[] { "/target:library", libSrc.Path }).Run(outWriter);
+            Assert.Contains($"error CS2012: Cannot open '{libDll.Path}' for writing", outWriter.ToString());
+        }
+
+        private byte[] ReadBytes(Stream stream, int count)
+        {
+            var buffer = new byte[count];
+            stream.Read(buffer, 0, count);
+            return buffer;
+        }
+
+        private byte[] ReadBytes(string path, int count)
+        {
+            using (var stream = File.OpenRead(path))
+            {
+                return ReadBytes(stream, count);
+            }
+        }
+
         [Fact]
         public void IOFailure_OpenOutputFile()
         {

src/Compilers/Core/Portable/CommandLine/CommonCompiler.CompilerEmitStreamProvider.cs

@@ -40,15 +40,71 @@ public override Stream CreateStream(DiagnosticBag diagnostics)
 
                 try
                 {
-                    return _streamToDispose = _compiler.FileOpen(_filePath, PortableShim.FileMode.Create, PortableShim.FileAccess.ReadWrite, PortableShim.FileShare.None);
+                    try
+                    {
+                        return OpenFileStream();
+                    }
+                    catch (IOException e)
+                    {
+                        // Other process is reading the file preventing us to write to it.
+                        // We attempt to rename and delete the file in case the reader opened it with FileShare.Delete flag that
+                        // allows the file to be deleted by other processes.
+                        //
+                        // Note that if the file is marked "readonly" or the current user doesn't have sufficient privileges
+                        // the exception thrown is UnauthorizedAccessException, not IOException, so we won't attempt to delete the file.
+
+                        try
+                        {
+                            const int eWin32SharingViolation = unchecked((int)0x80070020);
+
+                            if (PathUtilities.IsUnixLikePlatform)
+                            {
+                                // Unix & Mac are simple: just delete the file in the directory. 
+                                // The memory mapped content remains available for the reader.
+                                PortableShim.File.Delete(_filePath);
+                            }
+                            else if (e.HResult == eWin32SharingViolation)
+                            {
+                                // On Windows File.Delete only marks the file for deletion, but doens't remove it from the directory.
+                                var newFilePath = Path.Combine(Path.GetDirectoryName(_filePath), Guid.NewGuid().ToString() + "_" + Path.GetFileName(_filePath));
+
+                                // Try to rename the existing file. This fails unless the file is open with FileShare.Delete.
+                                PortableShim.File.Move(_filePath, newFilePath);
+
+                                // hide the renamed file:
+                                PortableShim.File.SetAttributes(newFilePath, PortableShim.FileAttributes.Hidden);
+
+                                // Mark the renamed file for deletion, so that it's deleted as soon as the current reader is finished reading it
+                                PortableShim.File.Delete(newFilePath);
+                            }
+                        }
+                        catch
+                        {
+                            // report the original exception
+                            ReportOpenFileDiagnostic(diagnostics, e);
+                            return null;
+                        }
+
+                        return OpenFileStream();
+                    }
                 }
                 catch (Exception e)
                 {
-                    var messageProvider = _compiler.MessageProvider;
-                    diagnostics.Add(messageProvider.CreateDiagnostic(messageProvider.ERR_CantOpenFileWrite, Location.None, _filePath, e.Message));
+                    ReportOpenFileDiagnostic(diagnostics, e);
                     return null;
                 }
             }
+
+            private Stream OpenFileStream()
+            {
+                return _streamToDispose = _compiler.FileOpen(_filePath, PortableShim.FileMode.Create, PortableShim.FileAccess.ReadWrite, PortableShim.FileShare.None);
+            }
+
+            private void ReportOpenFileDiagnostic(DiagnosticBag diagnostics, Exception e)
+            {
+                var messageProvider = _compiler.MessageProvider;
+                diagnostics.Add(messageProvider.CreateDiagnostic(messageProvider.ERR_CantOpenFileWrite, Location.None, _filePath, e.Message));
+            }
         }
     }
 }
\ No newline at end of file

src/Compilers/Core/Portable/FileSystem/PathUtilities.cs

@@ -17,7 +17,7 @@ internal static class PathUtilities
         internal static readonly string DirectorySeparatorStr = new string(DirectorySeparatorChar, 1);
         internal const char VolumeSeparatorChar = ':';
 
-        private static bool IsUnixLikePlatform
+        internal static bool IsUnixLikePlatform
         {
             get
             {

src/Compilers/Core/Portable/PortableShim.cs

@@ -45,6 +45,7 @@ internal static void Initialize()
             Touch(Directory.Type);
             Touch(Encoding.Type);
             Touch(Environment.Type);
+            Touch(FileAttributes.Type);
             Touch(File.Type);
             Touch(FileAccess.Type);
             Touch(FileMode.Type);
@@ -141,6 +142,17 @@ internal static class Path
                 .CreateDelegate(typeof(Func<string>));
         }
 
+        internal static class FileAttributes
+        {
+            private const string TypeName = "System.IO.FileAttributes";
+
+            internal static readonly Type Type = ReflectionUtilities.GetTypeFromEither(
+               contractName: $"{TypeName}, {CoreNames.System_IO_FileSystem_Primitives}",
+               desktopName: TypeName);
+
+            public static object Hidden = Enum.ToObject(Type, 2);
+        }
+
         internal static class File
         {
             internal const string TypeName = "System.IO.File";
@@ -174,6 +186,11 @@ internal static class File
                 .GetDeclaredMethod(nameof(Delete), new[] { typeof(string) })
                 .CreateDelegate<Action<string>>();
 
+            internal static readonly Action<string, string> Move = Type
+                .GetTypeInfo()
+                .GetDeclaredMethod(nameof(Move), new[] { typeof(string), typeof(string) })
+                .CreateDelegate<Action<string, string>>();
+
             internal static readonly Func<string, byte[]> ReadAllBytes = Type
                 .GetTypeInfo()
                 .GetDeclaredMethod(nameof(ReadAllBytes), paramTypes: new[] { typeof(string) })
@@ -183,6 +200,15 @@ internal static class File
                 .GetTypeInfo()
                 .GetDeclaredMethod(nameof(WriteAllBytes), paramTypes: new[] { typeof(string), typeof(byte[]) })
                 .CreateDelegate<Action<string, byte[]>>();
+
+            private static readonly MethodInfo SetAttributesMethod = Type
+                .GetTypeInfo()
+                .GetDeclaredMethod(nameof(SetAttributes), paramTypes: new[] { typeof(string), FileAttributes.Type });
+
+            public static void SetAttributes(string path, object attributes)
+            {
+                SetAttributesMethod.Invoke(null, new[] { path, attributes });
+            }
         }
 
         internal static class Directory

src/Test/Utilities/Desktop/ProcessUtilities.cs

@@ -127,7 +127,7 @@ public static string RunAndGetOutput(string exeFileName, string arguments = null
                 // might cause a deadlock.
                 result = process.StandardOutput.ReadToEnd();
                 process.WaitForExit();
-                Assert.Equal(expectedRetCode, process.ExitCode);
+                Assert.True(expectedRetCode == process.ExitCode, $"Unexpected exit code: {process.ExitCode} (expecting {expectedRetCode}). Process output: {result}");
             }
 
             return result;

src/VisualStudio/Core/Def/Implementation/EditAndContinue/VsENCRebuildableProjectImpl.cs

@@ -234,7 +234,7 @@ public int StartDebuggingPE()
                     try
                     {
                         InjectFault_MvidRead();
-                        _metadata = ModuleMetadata.CreateFromFile(outputPath);
+                        _metadata = ModuleMetadata.CreateFromStream(new FileStream(outputPath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete));
                         _metadata.GetModuleVersionId();
                     }
                     catch (FileNotFoundException)