From 24407d5705c8baf46c40fd91bbb21f75b0790c8c Mon Sep 17 00:00:00 2001 From: Will Smith Date: Sun, 15 Dec 2019 18:47:10 -0800 Subject: [PATCH] LOH allocation fixes for strings and char arrays (#7972) * Removed UnicodeFileAsLexbuf. Replaced it with StreamReaderAsLexbuf. No longer allocating string and chars as a result. * Make these private --- src/absil/illib.fs | 43 +++++++++++++++++++++++++- src/fsharp/CompileOps.fs | 5 +-- src/fsharp/UnicodeLexing.fs | 59 ++++++++---------------------------- src/fsharp/UnicodeLexing.fsi | 5 ++- src/fsharp/fsi/fsi.fs | 7 +++-- 5 files changed, 65 insertions(+), 54 deletions(-) diff --git a/src/absil/illib.fs b/src/absil/illib.fs index 70e67a081..0a7d471e1 100644 --- a/src/absil/illib.fs +++ b/src/absil/illib.fs @@ -1361,7 +1361,45 @@ module Shim = directory.Contains("packages\\") || directory.Contains("lib/mono/") - let mutable FileSystem = DefaultFileSystem() :> IFileSystem + let mutable FileSystem = DefaultFileSystem() :> IFileSystem + + // The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure + // uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold + // plus 1 second. Once past the threshold the incremental builder will be able to retry asynchronously based + // on plain old timestamp checking. + // + // The sleep time of 50ms is chosen so that we can respond to the user more quickly for Intellisense operations. + // + // This is not run on the UI thread for VS but it is on a thread that must be stopped before Intellisense + // can return any result except for pending. + let private retryDelayMilliseconds = 50 + let private numRetries = 60 + + let private getReader (filename, codePage: int option, retryLocked: bool) = + // Retry multiple times since other processes may be writing to this file. + let rec getSource retryNumber = + try + // Use the .NET functionality to auto-detect the unicode encoding + let stream = FileSystem.FileStreamReadShim(filename) + match codePage with + | None -> new StreamReader(stream,true) + | Some n -> new StreamReader(stream,System.Text.Encoding.GetEncoding(n)) + with + // We can get here if the file is locked--like when VS is saving a file--we don't have direct + // access to the HRESULT to see that this is EONOACCESS. + | :? System.IO.IOException as err when retryLocked && err.GetType() = typeof -> + // This second check is to make sure the exception is exactly IOException and none of these for example: + // DirectoryNotFoundException + // EndOfStreamException + // FileNotFoundException + // FileLoadException + // PathTooLongException + if retryNumber < numRetries then + System.Threading.Thread.Sleep (retryDelayMilliseconds) + getSource (retryNumber + 1) + else + reraise() + getSource 0 type File with @@ -1374,3 +1412,6 @@ module Shim = n <- n + stream.Read(buffer, n, len-n) buffer + static member OpenReaderAndRetry (filename, codepage, retryLocked) = + getReader (filename, codepage, retryLocked) + diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index ab20bfcf1..3af0e04e2 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -3436,7 +3436,7 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp if verbose then dprintn ("Parsed "+shortFilename) Some input with e -> (* errorR(Failure("parse failed")); *) errorRecovery e rangeStartup; None - + let ParseOneInputFile (tcConfig: TcConfig, lexResourceManager, conditionalCompilationDefines, filename, isLastCompiland, errorLogger, retryLocked) = try @@ -3445,7 +3445,8 @@ let ParseOneInputFile (tcConfig: TcConfig, lexResourceManager, conditionalCompil if not(FileSystem.SafeExists filename) then error(Error(FSComp.SR.buildCouldNotFindSourceFile filename, rangeStartup)) let isFeatureSupported featureId = tcConfig.langVersion.SupportsFeature featureId - let lexbuf = UnicodeLexing.UnicodeFileAsLexbuf(isFeatureSupported, filename, tcConfig.inputCodePage, retryLocked) + use reader = File.OpenReaderAndRetry (filename, tcConfig.inputCodePage, retryLocked) + let lexbuf = UnicodeLexing.StreamReaderAsLexbuf(isFeatureSupported, reader) ParseOneInputLexbuf(tcConfig, lexResourceManager, conditionalCompilationDefines, lexbuf, filename, isLastCompiland, errorLogger) else error(Error(FSComp.SR.buildInvalidSourceFileExtension(SanitizeFileName filename tcConfig.implicitIncludeDir), rangeStartup)) with e -> (* errorR(Failure("parse failed")); *) errorRecovery e rangeStartup; None diff --git a/src/fsharp/UnicodeLexing.fs b/src/fsharp/UnicodeLexing.fs index c776a1f1f..70a253da1 100644 --- a/src/fsharp/UnicodeLexing.fs +++ b/src/fsharp/UnicodeLexing.fs @@ -22,50 +22,15 @@ let FunctionAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, bufferF let SourceTextAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, sourceText) = LexBuffer.FromSourceText(supportsFeature, sourceText) -// The choice of 60 retries times 50 ms is not arbitrary. The NTFS FILETIME structure -// uses 2 second resolution for LastWriteTime. We retry long enough to surpass this threshold -// plus 1 second. Once past the threshold the incremental builder will be able to retry asynchronously based -// on plain old timestamp checking. -// -// The sleep time of 50ms is chosen so that we can respond to the user more quickly for Intellisense operations. -// -// This is not run on the UI thread for VS but it is on a thread that must be stopped before Intellisense -// can return any result except for pending. -let retryDelayMilliseconds = 50 -let numRetries = 60 - -/// Standard utility to create a Unicode LexBuffer -/// -/// One small annoyance is that LexBuffers and not IDisposable. This means -/// we can't just return the LexBuffer object, since the file it wraps wouldn't -/// get closed when we're finished with the LexBuffer. Hence we return the stream, -/// the reader and the LexBuffer. The caller should dispose the first two when done. -let UnicodeFileAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, filename, codePage: int option, retryLocked: bool): Lexbuf = - // Retry multiple times since other processes may be writing to this file. - let rec getSource retryNumber = - try - // Use the .NET functionality to auto-detect the unicode encoding - use stream = FileSystem.FileStreamReadShim(filename) - use reader = - match codePage with - | None -> new StreamReader(stream,true) - | Some n -> new StreamReader(stream,System.Text.Encoding.GetEncoding(n)) - reader.ReadToEnd() - with - // We can get here if the file is locked--like when VS is saving a file--we don't have direct - // access to the HRESULT to see that this is EONOACCESS. - | :? System.IO.IOException as err when retryLocked && err.GetType() = typeof -> - // This second check is to make sure the exception is exactly IOException and none of these for example: - // DirectoryNotFoundException - // EndOfStreamException - // FileNotFoundException - // FileLoadException - // PathTooLongException - if retryNumber < numRetries then - System.Threading.Thread.Sleep (retryDelayMilliseconds) - getSource (retryNumber + 1) - else - reraise() - let source = getSource 0 - let lexbuf = LexBuffer<_>.FromChars(supportsFeature, source.ToCharArray()) - lexbuf +let StreamReaderAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, reader: StreamReader) = + let mutable isFinished = false + FunctionAsLexbuf (supportsFeature, fun (chars, start, length) -> + if isFinished then 0 + else + let nBytesRead = reader.Read(chars, start, length) + if nBytesRead = 0 then + isFinished <- true + 0 + else + nBytesRead + ) diff --git a/src/fsharp/UnicodeLexing.fsi b/src/fsharp/UnicodeLexing.fsi index 2478c7f85..a3f0fb8fb 100644 --- a/src/fsharp/UnicodeLexing.fsi +++ b/src/fsharp/UnicodeLexing.fsi @@ -2,6 +2,7 @@ module internal FSharp.Compiler.UnicodeLexing +open System.IO open FSharp.Compiler.Features open FSharp.Compiler.Text open Microsoft.FSharp.Text @@ -10,5 +11,7 @@ open Internal.Utilities.Text.Lexing type Lexbuf = LexBuffer val internal StringAsLexbuf: (Features.LanguageFeature -> bool) * string -> Lexbuf val public FunctionAsLexbuf: (Features.LanguageFeature -> bool) * (char [] * int * int -> int) -> Lexbuf -val public UnicodeFileAsLexbuf: (Features.LanguageFeature -> bool) * string * int option * (*retryLocked*) bool -> Lexbuf val public SourceTextAsLexbuf: (Features.LanguageFeature -> bool) * ISourceText -> Lexbuf + +/// Will not dispose of the stream reader. +val public StreamReaderAsLexbuf: (Features.LanguageFeature -> bool) * StreamReader -> Lexbuf diff --git a/src/fsharp/fsi/fsi.fs b/src/fsharp/fsi/fsi.fs index c5e71384f..60f53b396 100644 --- a/src/fsharp/fsi/fsi.fs +++ b/src/fsharp/fsi/fsi.fs @@ -1677,8 +1677,8 @@ type internal FsiStdinLexerProvider CreateLexerForLexBuffer (Lexhelp.stdinMockFilename, lexbuf, errorLogger) // Create a new lexer to read an "included" script file - member __.CreateIncludedScriptLexer (sourceFileName, errorLogger) = - let lexbuf = UnicodeLexing.UnicodeFileAsLexbuf(isFeatureSupported, sourceFileName, tcConfigB.inputCodePage, (*retryLocked*)false) + member __.CreateIncludedScriptLexer (sourceFileName, reader, errorLogger) = + let lexbuf = UnicodeLexing.StreamReaderAsLexbuf(isFeatureSupported, reader) CreateLexerForLexBuffer (sourceFileName, lexbuf, errorLogger) // Create a new lexer to read a string @@ -2037,7 +2037,8 @@ type internal FsiInteractionProcessor WithImplicitHome (tcConfigB, directoryName sourceFile) (fun () -> // An included script file may contain maybe several interaction blocks. // We repeatedly parse and process these, until an error occurs. - let tokenizer = fsiStdinLexerProvider.CreateIncludedScriptLexer (sourceFile, errorLogger) + use reader = File.OpenReaderAndRetry (sourceFile, tcConfigB.inputCodePage, (*retryLocked*)false) + let tokenizer = fsiStdinLexerProvider.CreateIncludedScriptLexer (sourceFile, reader, errorLogger) let rec run istate = let istate,cont = processor.ParseAndExecOneSetOfInteractionsFromLexbuf ((fun f istate -> f ctok istate), istate, tokenizer, errorLogger) match cont with Completed _ -> run istate | _ -> istate,cont -- GitLab