LOH allocation fixes for strings and char arrays (#7972)

* Removed UnicodeFileAsLexbuf. Replaced it with StreamReaderAsLexbuf. No longer allocating string and chars as a result.

* Make these private

src/absil/illib.fs

@@ -1361,7 +1361,45 @@ module Shim =
                 directory.Contains("packages\\") || 
                 directory.Contains("lib/mono/")
 
-    let mutable FileSystem = DefaultFileSystem() :> IFileSystem 
+    let mutable FileSystem = DefaultFileSystem() :> IFileSystem
+
+    // The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure 
+    // uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold 
+    // plus 1 second. Once past the threshold the incremental builder will be able to retry asynchronously based
+    // on plain old timestamp checking.
+    //
+    // The sleep time of 50ms is chosen so that we can respond to the user more quickly for Intellisense operations.
+    //
+    // This is not run on the UI thread for VS but it is on a thread that must be stopped before Intellisense
+    // can return any result except for pending.
+    let private retryDelayMilliseconds = 50
+    let private numRetries = 60
+
+    let private getReader (filename, codePage: int option, retryLocked: bool) =
+        // Retry multiple times since other processes may be writing to this file.
+        let rec getSource retryNumber =
+          try 
+            // Use the .NET functionality to auto-detect the unicode encoding
+            let stream = FileSystem.FileStreamReadShim(filename) 
+            match codePage with 
+            | None -> new  StreamReader(stream,true)
+            | Some n -> new  StreamReader(stream,System.Text.Encoding.GetEncoding(n))
+          with 
+              // We can get here if the file is locked--like when VS is saving a file--we don't have direct
+              // access to the HRESULT to see that this is EONOACCESS.
+              | :? System.IO.IOException as err when retryLocked && err.GetType() = typeof<System.IO.IOException> -> 
+                   // This second check is to make sure the exception is exactly IOException and none of these for example:
+                   //   DirectoryNotFoundException 
+                   //   EndOfStreamException 
+                   //   FileNotFoundException 
+                   //   FileLoadException 
+                   //   PathTooLongException
+                   if retryNumber < numRetries then 
+                       System.Threading.Thread.Sleep (retryDelayMilliseconds)
+                       getSource (retryNumber + 1)
+                   else 
+                       reraise()
+        getSource 0
 
     type File with 
 
@@ -1374,3 +1412,6 @@ module Shim =
                 n <- n + stream.Read(buffer, n, len-n)
             buffer
 
+        static member OpenReaderAndRetry (filename, codepage, retryLocked)  =
+            getReader (filename, codepage, retryLocked)
+

src/fsharp/CompileOps.fs

@@ -3436,7 +3436,7 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp
         if verbose then dprintn ("Parsed "+shortFilename)
         Some input 
     with e -> (* errorR(Failure("parse failed")); *) errorRecovery e rangeStartup; None 
-            
+
             
 let ParseOneInputFile (tcConfig: TcConfig, lexResourceManager, conditionalCompilationDefines, filename, isLastCompiland, errorLogger, retryLocked) =
     try 
@@ -3445,7 +3445,8 @@ let ParseOneInputFile (tcConfig: TcConfig, lexResourceManager, conditionalCompil
             if not(FileSystem.SafeExists filename) then
                 error(Error(FSComp.SR.buildCouldNotFindSourceFile filename, rangeStartup))
             let isFeatureSupported featureId = tcConfig.langVersion.SupportsFeature featureId
-            let lexbuf = UnicodeLexing.UnicodeFileAsLexbuf(isFeatureSupported, filename, tcConfig.inputCodePage, retryLocked) 
+            use reader = File.OpenReaderAndRetry (filename, tcConfig.inputCodePage, retryLocked)
+            let lexbuf = UnicodeLexing.StreamReaderAsLexbuf(isFeatureSupported, reader)
             ParseOneInputLexbuf(tcConfig, lexResourceManager, conditionalCompilationDefines, lexbuf, filename, isLastCompiland, errorLogger)
        else error(Error(FSComp.SR.buildInvalidSourceFileExtension(SanitizeFileName filename tcConfig.implicitIncludeDir), rangeStartup))
     with e -> (* errorR(Failure("parse failed")); *) errorRecovery e rangeStartup; None 

src/fsharp/UnicodeLexing.fs

@@ -22,50 +22,15 @@ let FunctionAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, bufferF
 let SourceTextAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, sourceText) =
     LexBuffer<char>.FromSourceText(supportsFeature, sourceText)
 
-// The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure 
-// uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold 
-// plus 1 second. Once past the threshold the incremental builder will be able to retry asynchronously based
-// on plain old timestamp checking.
-//
-// The sleep time of 50ms is chosen so that we can respond to the user more quickly for Intellisense operations.
-//
-// This is not run on the UI thread for VS but it is on a thread that must be stopped before Intellisense
-// can return any result except for pending.
-let retryDelayMilliseconds = 50
-let numRetries = 60
-
-/// Standard utility to create a Unicode LexBuffer
-///
-/// One small annoyance is that LexBuffers and not IDisposable. This means 
-/// we can't just return the LexBuffer object, since the file it wraps wouldn't
-/// get closed when we're finished with the LexBuffer. Hence we return the stream,
-/// the reader and the LexBuffer. The caller should dispose the first two when done.
-let UnicodeFileAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, filename, codePage: int option, retryLocked: bool): Lexbuf =
-    // Retry multiple times since other processes may be writing to this file.
-    let rec getSource retryNumber =
-      try 
-        // Use the .NET functionality to auto-detect the unicode encoding
-        use stream = FileSystem.FileStreamReadShim(filename) 
-        use reader = 
-            match codePage with 
-            | None -> new  StreamReader(stream,true)
-            | Some n -> new  StreamReader(stream,System.Text.Encoding.GetEncoding(n)) 
-        reader.ReadToEnd()
-      with 
-          // We can get here if the file is locked--like when VS is saving a file--we don't have direct
-          // access to the HRESULT to see that this is EONOACCESS.
-          | :? System.IO.IOException as err when retryLocked && err.GetType() = typeof<System.IO.IOException> -> 
-               // This second check is to make sure the exception is exactly IOException and none of these for example:
-               //   DirectoryNotFoundException 
-               //   EndOfStreamException 
-               //   FileNotFoundException 
-               //   FileLoadException 
-               //   PathTooLongException
-               if retryNumber < numRetries then 
-                   System.Threading.Thread.Sleep (retryDelayMilliseconds)
-                   getSource (retryNumber + 1)
-               else 
-                   reraise()
-    let source = getSource 0
-    let lexbuf = LexBuffer<_>.FromChars(supportsFeature, source.ToCharArray())  
-    lexbuf
+let StreamReaderAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, reader: StreamReader) =
+    let mutable isFinished = false
+    FunctionAsLexbuf (supportsFeature, fun (chars, start, length) ->
+        if isFinished then 0
+        else
+            let nBytesRead = reader.Read(chars, start, length)
+            if nBytesRead = 0 then
+                isFinished <- true
+                0
+            else
+                nBytesRead
+    )

src/fsharp/UnicodeLexing.fsi

@@ -2,6 +2,7 @@
 
 module internal FSharp.Compiler.UnicodeLexing
 
+open System.IO
 open FSharp.Compiler.Features
 open FSharp.Compiler.Text
 open Microsoft.FSharp.Text
@@ -10,5 +11,7 @@ open Internal.Utilities.Text.Lexing
 type Lexbuf =  LexBuffer<char>
 val internal StringAsLexbuf: (Features.LanguageFeature -> bool) * string -> Lexbuf
 val public FunctionAsLexbuf: (Features.LanguageFeature -> bool) * (char [] * int * int -> int) -> Lexbuf
-val public UnicodeFileAsLexbuf: (Features.LanguageFeature -> bool) * string * int option * (*retryLocked*) bool -> Lexbuf
 val public SourceTextAsLexbuf: (Features.LanguageFeature -> bool) * ISourceText -> Lexbuf
+
+/// Will not dispose of the stream reader.
+val public StreamReaderAsLexbuf: (Features.LanguageFeature -> bool) * StreamReader -> Lexbuf

src/fsharp/fsi/fsi.fs

@@ -1677,8 +1677,8 @@ type internal FsiStdinLexerProvider
         CreateLexerForLexBuffer (Lexhelp.stdinMockFilename, lexbuf, errorLogger)
 
     // Create a new lexer to read an "included" script file
-    member __.CreateIncludedScriptLexer (sourceFileName, errorLogger) =
-        let lexbuf = UnicodeLexing.UnicodeFileAsLexbuf(isFeatureSupported, sourceFileName, tcConfigB.inputCodePage, (*retryLocked*)false)
+    member __.CreateIncludedScriptLexer (sourceFileName, reader, errorLogger) =
+        let lexbuf = UnicodeLexing.StreamReaderAsLexbuf(isFeatureSupported, reader)
         CreateLexerForLexBuffer (sourceFileName, lexbuf, errorLogger)
 
     // Create a new lexer to read a string
@@ -2037,7 +2037,8 @@ type internal FsiInteractionProcessor
         WithImplicitHome (tcConfigB, directoryName sourceFile)  (fun () ->
               // An included script file may contain maybe several interaction blocks.
               // We repeatedly parse and process these, until an error occurs.
-                let tokenizer = fsiStdinLexerProvider.CreateIncludedScriptLexer (sourceFile, errorLogger)
+                use reader = File.OpenReaderAndRetry (sourceFile, tcConfigB.inputCodePage, (*retryLocked*)false)
+                let tokenizer = fsiStdinLexerProvider.CreateIncludedScriptLexer (sourceFile, reader, errorLogger)
                 let rec run istate =
                     let istate,cont = processor.ParseAndExecOneSetOfInteractionsFromLexbuf ((fun f istate -> f ctok istate), istate, tokenizer, errorLogger)
                     match cont with Completed _ -> run istate | _ -> istate,cont