提交 1e8edefb 编写于 作者: W Will Smith 提交者: Phillip Carter

Fixed LOH byte array allocations in the IDE from metadata (#7971)

* Fixed LOH byte array allocations in the IDE. Unified memory handling for IL and FSharp metadata.

* minor cleanup

* minor cleanup

* rename EmitBytes to EmitByteMemory

* Fixed ByteArrayMemory

* fixing build

* Fixing build again

* Better ReadInt32/ReadUInt16

* ILResourceLocation.Local should use a ReadOnlyByteMemory

* Able to have a ReadOnlyStream

* Using OutOfRangeException according to feedback
上级 1528331f
......@@ -3,7 +3,14 @@
/// Byte arrays
namespace FSharp.Compiler.AbstractIL.Internal
open System
open System.IO
open System.IO.MemoryMappedFiles
open System.Runtime.InteropServices
open System.Runtime.CompilerServices
open FSharp.NativeInterop
#nowarn "9"
module internal Bytes =
let b0 n = (n &&& 0xFF)
......@@ -26,10 +33,286 @@ module internal Bytes =
Array.append (System.Text.Encoding.UTF8.GetBytes s) (ofInt32Array [| 0x0 |])
let stringAsUnicodeNullTerminated (s:string) =
Array.append (System.Text.Encoding.Unicode.GetBytes s) (ofInt32Array [| 0x0;0x0 |])
Array.append (System.Text.Encoding.Unicode.GetBytes s) (ofInt32Array [| 0x0;0x0 |])
[<AbstractClass>]
type ByteMemory () =
abstract Item: int -> byte with get, set
abstract Length: int
abstract ReadBytes: pos: int * count: int -> byte[]
abstract ReadInt32: pos: int -> int
abstract ReadUInt16: pos: int -> uint16
abstract ReadUtf8String: pos: int * count: int -> string
abstract Slice: pos: int * count: int -> ByteMemory
abstract CopyTo: Stream -> unit
abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit
abstract ToArray: unit -> byte[]
abstract AsStream: unit -> Stream
abstract AsReadOnlyStream: unit -> Stream
[<Sealed>]
type ByteArrayMemory(bytes: byte[], offset, length) =
inherit ByteMemory()
do
if length <= 0 || length > bytes.Length then
raise (ArgumentOutOfRangeException("length"))
if offset < 0 || (offset + length) > bytes.Length then
raise (ArgumentOutOfRangeException("offset"))
override _.Item
with get i = bytes.[offset + i]
and set i v = bytes.[offset + i] <- v
override _.Length = length
override _.ReadBytes(pos, count) =
Array.sub bytes (offset + pos) count
override _.ReadInt32 pos =
let finalOffset = offset + pos
(uint32 bytes.[finalOffset]) |||
((uint32 bytes.[finalOffset + 1]) <<< 8) |||
((uint32 bytes.[finalOffset + 2]) <<< 16) |||
((uint32 bytes.[finalOffset + 3]) <<< 24)
|> int
override _.ReadUInt16 pos =
let finalOffset = offset + pos
(uint16 bytes.[finalOffset]) |||
((uint16 bytes.[finalOffset + 1]) <<< 8)
override _.ReadUtf8String(pos, count) =
System.Text.Encoding.UTF8.GetString(bytes, offset + pos, count)
override _.Slice(pos, count) =
ByteArrayMemory(bytes, offset + pos, count) :> ByteMemory
override _.CopyTo stream =
stream.Write(bytes, offset, length)
override _.Copy(srcOffset, dest, destOffset, count) =
Array.blit bytes (offset + srcOffset) dest destOffset count
override _.ToArray() =
Array.sub bytes offset length
override _.AsStream() =
new MemoryStream(bytes, offset, length) :> Stream
override _.AsReadOnlyStream() =
new MemoryStream(bytes, offset, length, false) :> Stream
[<Sealed>]
type RawByteMemory(addr: nativeptr<byte>, length: int, hold: obj) =
inherit ByteMemory ()
let check i =
if i < 0 || i >= length then
raise (ArgumentOutOfRangeException("i"))
do
if length <= 0 then
raise (ArgumentOutOfRangeException("length"))
override _.Item
with get i =
check i
NativePtr.add addr i
|> NativePtr.read
and set i v =
check i
NativePtr.set addr i v
override _.Length = length
override _.ReadUtf8String(pos, count) =
check pos
check (pos + count - 1)
System.Text.Encoding.UTF8.GetString(NativePtr.add addr pos, count)
override _.ReadBytes(pos, count) =
check pos
check (pos + count - 1)
let res = Bytes.zeroCreate count
Marshal.Copy(NativePtr.toNativeInt addr + nativeint pos, res, 0, count)
res
override _.ReadInt32 pos =
check pos
check (pos + 3)
Marshal.ReadInt32(NativePtr.toNativeInt addr + nativeint pos)
override _.ReadUInt16 pos =
check pos
check (pos + 1)
uint16(Marshal.ReadInt16(NativePtr.toNativeInt addr + nativeint pos))
override _.Slice(pos, count) =
check pos
check (pos + count - 1)
RawByteMemory(NativePtr.add addr pos, count, hold) :> ByteMemory
override x.CopyTo stream =
use stream2 = x.AsStream()
stream2.CopyTo stream
override x.Copy(srcOffset, dest, destOffset, count) =
check srcOffset
Marshal.Copy(NativePtr.toNativeInt addr + nativeint srcOffset, dest, destOffset, count)
override _.ToArray() =
let res = Array.zeroCreate<byte> length
Marshal.Copy(NativePtr.toNativeInt addr, res, 0, res.Length)
res
override _.AsStream() =
new UnmanagedMemoryStream(addr, int64 length) :> Stream
override _.AsReadOnlyStream() =
new UnmanagedMemoryStream(addr, int64 length, int64 length, FileAccess.Read) :> Stream
[<Struct;NoEquality;NoComparison>]
type ReadOnlyByteMemory(bytes: ByteMemory) =
member _.Item with [<MethodImpl(MethodImplOptions.AggressiveInlining)>] get i = bytes.[i]
member _.Length with [<MethodImpl(MethodImplOptions.AggressiveInlining)>] get () = bytes.Length
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.ReadBytes(pos, count) = bytes.ReadBytes(pos, count)
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.ReadInt32 pos = bytes.ReadInt32 pos
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.ReadUInt16 pos = bytes.ReadUInt16 pos
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.ReadUtf8String(pos, count) = bytes.ReadUtf8String(pos, count)
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.Slice(pos, count) = bytes.Slice(pos, count) |> ReadOnlyByteMemory
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.CopyTo stream = bytes.CopyTo stream
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.Copy(srcOffset, dest, destOffset, count) = bytes.Copy(srcOffset, dest, destOffset, count)
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.ToArray() = bytes.ToArray()
[<MethodImpl(MethodImplOptions.AggressiveInlining)>]
member _.AsStream() = bytes.AsReadOnlyStream()
type ByteMemory with
member x.AsReadOnly() = ReadOnlyByteMemory x
static member CreateMemoryMappedFile(bytes: ReadOnlyByteMemory) =
let length = int64 bytes.Length
let mmf =
let mmf =
MemoryMappedFile.CreateNew(
null,
length,
MemoryMappedFileAccess.ReadWrite,
MemoryMappedFileOptions.None,
HandleInheritability.None)
use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite)
bytes.CopyTo stream
mmf
let accessor = mmf.CreateViewAccessor(0L, length, MemoryMappedFileAccess.ReadWrite)
let safeHolder =
{ new obj() with
override x.Finalize() =
(x :?> IDisposable).Dispose()
interface IDisposable with
member x.Dispose() =
GC.SuppressFinalize x
accessor.Dispose()
mmf.Dispose() }
RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, safeHolder)
static member FromFile(path, access, ?canShadowCopy: bool) =
let canShadowCopy = defaultArg canShadowCopy false
let memoryMappedFileAccess =
match access with
| FileAccess.Read -> MemoryMappedFileAccess.Read
| FileAccess.Write -> MemoryMappedFileAccess.Write
| _ -> MemoryMappedFileAccess.ReadWrite
let mmf, accessor, length =
let fileStream = File.Open(path, FileMode.Open, access, FileShare.Read)
let length = fileStream.Length
let mmf =
if canShadowCopy then
let mmf =
MemoryMappedFile.CreateNew(
null,
length,
MemoryMappedFileAccess.ReadWrite,
MemoryMappedFileOptions.None,
HandleInheritability.None)
use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite)
fileStream.CopyTo(stream)
fileStream.Dispose()
mmf
else
MemoryMappedFile.CreateFromFile(
fileStream,
null,
length,
memoryMappedFileAccess,
HandleInheritability.None,
leaveOpen=false)
mmf, mmf.CreateViewAccessor(0L, length, memoryMappedFileAccess), length
match access with
| FileAccess.Read when not accessor.CanRead -> failwith "Cannot read file"
| FileAccess.Write when not accessor.CanWrite -> failwith "Cannot write file"
| _ when not accessor.CanRead || not accessor.CanWrite -> failwith "Cannot read or write file"
| _ -> ()
let safeHolder =
{ new obj() with
override x.Finalize() =
(x :?> IDisposable).Dispose()
interface IDisposable with
member x.Dispose() =
GC.SuppressFinalize x
accessor.Dispose()
mmf.Dispose() }
RawByteMemory.FromUnsafePointer(accessor.SafeMemoryMappedViewHandle.DangerousGetHandle(), int length, safeHolder)
static member FromUnsafePointer(addr, length, hold: obj) =
RawByteMemory(NativePtr.ofNativeInt addr, length, hold) :> ByteMemory
static member FromArray(bytes, offset, length) =
ByteArrayMemory(bytes, offset, length) :> ByteMemory
static member FromArray bytes =
ByteArrayMemory.FromArray(bytes, 0, bytes.Length)
type internal ByteStream =
{ bytes: byte[]
{ bytes: ReadOnlyByteMemory
mutable pos: int
max: int }
member b.ReadByte() =
......@@ -38,18 +321,18 @@ type internal ByteStream =
b.pos <- b.pos + 1
res
member b.ReadUtf8String n =
let res = System.Text.Encoding.UTF8.GetString(b.bytes,b.pos,n)
let res = b.bytes.ReadUtf8String(b.pos,n)
b.pos <- b.pos + n; res
static member FromBytes (b:byte[],n,len) =
static member FromBytes (b: ReadOnlyByteMemory,n,len) =
if n < 0 || (n+len) > b.Length then failwith "FromBytes"
{ bytes = b; pos = n; max = n+len }
member b.ReadBytes n =
if b.pos + n > b.max then failwith "ReadBytes: end of stream"
let res = Bytes.sub b.bytes b.pos n
let res = b.bytes.Slice(b.pos, n)
b.pos <- b.pos + n
res
res
member b.Position = b.pos
#if LAZY_UNPICKLE
......@@ -108,6 +391,13 @@ type internal ByteBuffer =
Bytes.blit i 0 buf.bbArray buf.bbCurrent n
buf.bbCurrent <- newSize
member buf.EmitByteMemory (i:ReadOnlyByteMemory) =
let n = i.Length
let newSize = buf.bbCurrent + n
buf.Ensure newSize
i.Copy(0, buf.bbArray, buf.bbCurrent, n)
buf.bbCurrent <- newSize
member buf.EmitInt32AsUInt16 n =
let newSize = buf.bbCurrent + 2
buf.Ensure newSize
......
......@@ -3,6 +3,7 @@
/// Blobs of bytes, cross-compiling
namespace FSharp.Compiler.AbstractIL.Internal
open System.IO
open Internal.Utilities
open FSharp.Compiler.AbstractIL
......@@ -22,6 +23,85 @@ module internal Bytes =
val stringAsUnicodeNullTerminated: string -> byte[]
val stringAsUtf8NullTerminated: string -> byte[]
/// May be backed by managed or unmanaged memory, or memory mapped file.
[<AbstractClass>]
type internal ByteMemory =
abstract Item: int -> byte with get
abstract Length: int
abstract ReadBytes: pos: int * count: int -> byte[]
abstract ReadInt32: pos: int -> int
abstract ReadUInt16: pos: int -> uint16
abstract ReadUtf8String: pos: int * count: int -> string
abstract Slice: pos: int * count: int -> ByteMemory
abstract CopyTo: Stream -> unit
abstract Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit
abstract ToArray: unit -> byte[]
/// Get a stream representation of the backing memory.
/// Disposing this will not free up any of the backing memory.
abstract AsStream: unit -> Stream
/// Get a stream representation of the backing memory.
/// Disposing this will not free up any of the backing memory.
/// Stream cannot be written to.
abstract AsReadOnlyStream: unit -> Stream
[<Struct;NoEquality;NoComparison>]
type internal ReadOnlyByteMemory =
new: ByteMemory -> ReadOnlyByteMemory
member Item: int -> byte with get
member Length: int
member ReadBytes: pos: int * count: int -> byte[]
member ReadInt32: pos: int -> int
member ReadUInt16: pos: int -> uint16
member ReadUtf8String: pos: int * count: int -> string
member Slice: pos: int * count: int -> ReadOnlyByteMemory
member CopyTo: Stream -> unit
member Copy: srcOffset: int * dest: byte[] * destOffset: int * count: int -> unit
member ToArray: unit -> byte[]
member AsStream: unit -> Stream
type ByteMemory with
member AsReadOnly: unit -> ReadOnlyByteMemory
/// Create another ByteMemory object that has a backing memory mapped file based on another ByteMemory's contents.
static member CreateMemoryMappedFile: ReadOnlyByteMemory -> ByteMemory
/// Creates a ByteMemory object that has a backing memory mapped file from a file on-disk.
static member FromFile: path: string * FileAccess * ?canShadowCopy: bool -> ByteMemory
/// Creates a ByteMemory object that is backed by a raw pointer.
/// Use with care.
static member FromUnsafePointer: addr: nativeint * length: int * hold: obj -> ByteMemory
/// Creates a ByteMemory object that is backed by a byte array with the specified offset and length.
static member FromArray: bytes: byte[] * offset: int * length: int -> ByteMemory
/// Creates a ByteMemory object that is backed by a byte array.
static member FromArray: bytes: byte[] -> ByteMemory
/// Imperative buffers and streams of byte[]
[<Sealed>]
......@@ -31,6 +111,7 @@ type internal ByteBuffer =
member EmitIntsAsBytes : int[] -> unit
member EmitByte : byte -> unit
member EmitBytes : byte[] -> unit
member EmitByteMemory : ReadOnlyByteMemory -> unit
member EmitInt32 : int32 -> unit
member EmitInt64 : int64 -> unit
member FixupInt32 : pos: int -> value: int32 -> unit
......@@ -44,10 +125,10 @@ type internal ByteBuffer =
[<Sealed>]
type internal ByteStream =
member ReadByte : unit -> byte
member ReadBytes : int -> byte[]
member ReadBytes : int -> ReadOnlyByteMemory
member ReadUtf8String : int -> string
member Position : int
static member FromBytes : byte[] * start:int * length:int -> ByteStream
static member FromBytes : ReadOnlyByteMemory * start:int * length:int -> ByteStream
#if LAZY_UNPICKLE
member CloneAndSeek : int -> ByteStream
......
......@@ -2208,8 +2208,7 @@ type ILResourceAccess =
[<RequireQualifiedAccess>]
type ILResourceLocation =
| LocalIn of string * int * int
| LocalOut of byte[]
| Local of ReadOnlyByteMemory
| File of ILModuleRef * int32
| Assembly of ILAssemblyRef
......@@ -2223,9 +2222,7 @@ type ILResource =
/// Read the bytes from a resource local to an assembly
member r.GetBytes() =
match r.Location with
| ILResourceLocation.LocalIn (file, start, len) ->
File.ReadBinaryChunk(file, start, len)
| ILResourceLocation.LocalOut bytes -> bytes
| ILResourceLocation.Local bytes -> bytes
| _ -> failwith "GetBytes"
member x.CustomAttrs = x.CustomAttrsStored.GetCustomAttrs x.MetadataIndex
......@@ -4208,8 +4205,7 @@ and refs_of_exported_types s (tab: ILExportedTypesAndForwarders) = List.iter (re
and refs_of_resource_where s x =
match x with
| ILResourceLocation.LocalIn _ -> ()
| ILResourceLocation.LocalOut _ -> ()
| ILResourceLocation.Local _ -> ()
| ILResourceLocation.File (mref, _) -> refs_of_modref s mref
| ILResourceLocation.Assembly aref -> refs_of_assemblyRef s aref
......
......@@ -3,6 +3,7 @@
/// The "unlinked" view of .NET metadata and code. Central to the Abstract IL library
module public FSharp.Compiler.AbstractIL.IL
open FSharp.Compiler.AbstractIL.Internal
open System.Collections.Generic
open System.Reflection
......@@ -1405,11 +1406,9 @@ type ILResourceAccess =
[<RequireQualifiedAccess>]
type ILResourceLocation =
/// Represents a manifest resource that can be read from within the PE file
| LocalIn of string * int * int
/// Represents a manifest resource that is due to be written to the output PE file
| LocalOut of byte[]
internal
/// Represents a manifest resource that can be read or written to a PE file
| Local of ReadOnlyByteMemory
/// Represents a manifest resource in an associated file
| File of ILModuleRef * int32
......@@ -1429,7 +1428,7 @@ type ILResource =
MetadataIndex: int32 }
/// Read the bytes from a resource local to an assembly. Will fail for non-local resources.
member GetBytes : unit -> byte[]
member internal GetBytes : unit -> ReadOnlyByteMemory
member CustomAttrs: ILAttributes
......
......@@ -1024,8 +1024,7 @@ let goutput_resource env os r =
output_string os " { "
goutput_custom_attrs env os r.CustomAttrs
match r.Location with
| ILResourceLocation.LocalIn _
| ILResourceLocation.LocalOut _ ->
| ILResourceLocation.Local _ ->
output_string os " /* loc nyi */ "
| ILResourceLocation.File (mref, off) ->
output_string os " .file "
......
......@@ -19,6 +19,7 @@ open System.Runtime.InteropServices
open System.Text
open Internal.Utilities
open Internal.Utilities.Collections
open FSharp.NativeInterop
open FSharp.Compiler.AbstractIL.Internal
open FSharp.Compiler.AbstractIL.Internal.Support
open FSharp.Compiler.AbstractIL.Diagnostics
......@@ -29,6 +30,8 @@ open FSharp.Compiler.ErrorLogger
open FSharp.Compiler.Range
open System.Reflection
#nowarn "9"
let checking = false
let logging = false
let _ = if checking then dprintn "warning: ILBinaryReader.checking is on"
......@@ -104,125 +107,31 @@ let stats =
let GetStatistics() = stats
[<AbstractClass>]
/// An abstraction over how we access the contents of .NET binaries. May be backed by managed or unmanaged memory,
/// memory mapped file or by on-disk resources. These objects should never need explicit disposal - they must either
/// not hold resources of clean up after themselves when collected.
type BinaryView() =
/// Read a byte from the file
abstract ReadByte: addr: int -> byte
/// Read a chunk of bytes from the file
abstract ReadBytes: addr: int -> int -> byte[]
/// Read an Int32 from the file
abstract ReadInt32: addr: int -> int
type private BinaryView = ReadOnlyByteMemory
/// Read a UInt16 from the file
abstract ReadUInt16: addr: int -> uint16
/// Read a length of a UTF8 string from the file
abstract CountUtf8String: addr: int -> int
/// Read a UTF8 string from the file
abstract ReadUTF8String: addr: int -> string
/// An abstraction over how we access the contents of .NET binaries. May be backed by managed or unmanaged memory,
/// memory mapped file or by on-disk resources.
/// An abstraction over how we access the contents of .NET binaries.
type BinaryFile =
/// Return a BinaryView for temporary use which eagerly holds any necessary memory resources for the duration of its lifetime,
/// and is faster to access byte-by-byte. The returned BinaryView should _not_ be captured in a closure that outlives the
/// desired lifetime.
abstract GetView: unit -> BinaryView
/// A view over a raw pointer to memory
type RawMemoryView(obj: obj, start: nativeint, len: int) =
inherit BinaryView()
override m.ReadByte i =
if nativeint i + 1n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj
Marshal.ReadByte(start + nativeint i)
override m.ReadBytes i n =
if nativeint i + nativeint n > nativeint len then failwithf "RawMemoryView overrun, i = %d, n = %d, obj = %A" i n obj
let res = Bytes.zeroCreate n
Marshal.Copy(start + nativeint i, res, 0, n)
res
override m.ReadInt32 i =
if nativeint i + 4n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj
Marshal.ReadInt32(start + nativeint i)
override m.ReadUInt16 i =
if nativeint i + 2n > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj
uint16(Marshal.ReadInt16(start + nativeint i))
override m.CountUtf8String i =
if nativeint i > nativeint len then failwithf "RawMemoryView overrun, i = %d, obj = %A" i obj
let pStart = start + nativeint i
let mutable p = start
while Marshal.ReadByte p <> 0uy do
p <- p + 1n
int (p - pStart)
override m.ReadUTF8String i =
let n = m.CountUtf8String i
if nativeint i + nativeint n > nativeint len then failwithf "RawMemoryView overrun, i = %d, n = %d, obj = %A" i n obj
System.Runtime.InteropServices.Marshal.PtrToStringAnsi(start + nativeint i, n)
member __.HoldObj() = obj
/// Gives views over a raw chunk of memory, for example those returned to us by the memory manager in Roslyn's
/// Visual Studio integration. 'obj' must keep the memory alive. The object will capture it and thus also keep the memory alive for
/// the lifetime of this object.
type RawMemoryFile(fileName: string, obj: obj, addr: nativeint, length: int) =
do stats.rawMemoryFileCount <- stats.rawMemoryFileCount + 1
let view = RawMemoryView(obj, addr, length)
let view = ByteMemory.FromUnsafePointer(addr, length, obj).AsReadOnly()
member __.HoldObj() = obj // make sure we capture 'obj'
member __.FileName = fileName
interface BinaryFile with
override __.GetView() = view :>_
/// Read file from memory blocks
type ByteView(bytes: byte[]) =
inherit BinaryView()
override __.ReadByte addr = bytes.[addr]
override __.ReadBytes addr len = Array.sub bytes addr len
override __.CountUtf8String addr =
let mutable p = addr
while bytes.[p] <> 0uy do
p <- p + 1
p - addr
override bfv.ReadUTF8String addr =
let n = bfv.CountUtf8String addr
System.Text.Encoding.UTF8.GetString (bytes, addr, n)
override bfv.ReadInt32 addr =
let b0 = bfv.ReadByte addr
let b1 = bfv.ReadByte (addr+1)
let b2 = bfv.ReadByte (addr+2)
let b3 = bfv.ReadByte (addr+3)
int b0 ||| (int b1 <<< 8) ||| (int b2 <<< 16) ||| (int b3 <<< 24)
override bfv.ReadUInt16 addr =
let b0 = bfv.ReadByte addr
let b1 = bfv.ReadByte (addr+1)
uint16 b0 ||| (uint16 b1 <<< 8)
override __.GetView() = view
/// A BinaryFile backed by an array of bytes held strongly as managed memory
[<DebuggerDisplay("{FileName}")>]
type ByteFile(fileName: string, bytes: byte[]) =
let view = ByteView bytes
let view = ByteMemory.FromArray(bytes).AsReadOnly()
do stats.byteFileCount <- stats.byteFileCount + 1
member __.FileName = fileName
interface BinaryFile with
override bf.GetView() = view :> BinaryView
override bf.GetView() = view
/// Same as ByteFile but holds the bytes weakly. The bytes will be re-read from the backing file when a view is requested.
/// This is the default implementation used by F# Compiler Services when accessing "stable" binaries. It is not used
......@@ -261,11 +170,11 @@ type WeakByteFile(fileName: string, chunk: (int * int) option) =
tg
(ByteView strongBytes :> BinaryView)
ByteMemory.FromArray(strongBytes).AsReadOnly()
let seekReadByte (mdv: BinaryView) addr = mdv.ReadByte addr
let seekReadBytes (mdv: BinaryView) addr len = mdv.ReadBytes addr len
let seekReadByte (mdv: BinaryView) addr = mdv.[addr]
let seekReadBytes (mdv: BinaryView) addr len = mdv.ReadBytes(addr, len)
let seekReadInt32 (mdv: BinaryView) addr = mdv.ReadInt32 addr
let seekReadUInt16 (mdv: BinaryView) addr = mdv.ReadUInt16 addr
......@@ -1502,7 +1411,7 @@ let rvaToData (ctxt: ILMetadataReader) (pectxt: PEReader) nm rva =
let isSorted (ctxt: ILMetadataReader) (tab: TableName) = ((ctxt.sorted &&& (int64 1 <<< tab.Index)) <> int64 0x0)
// Note, pectxtEager and pevEager must not be captured by the results of this function
let rec seekReadModule (ctxt: ILMetadataReader) (pectxtEager: PEReader) pevEager peinfo ilMetadataVersion idx =
let rec seekReadModule (ctxt: ILMetadataReader) canReduceMemory (pectxtEager: PEReader) pevEager peinfo ilMetadataVersion idx =
let (subsys, subsysversion, useHighEntropyVA, ilOnly, only32, is32bitpreferred, only64, platform, isDll, alignVirt, alignPhys, imageBaseReal) = peinfo
let mdv = ctxt.mdfile.GetView()
let (_generation, nameIdx, _mvidIdx, _encidIdx, _encbaseidIdx) = seekReadModuleRow ctxt mdv idx
......@@ -1531,7 +1440,7 @@ let rec seekReadModule (ctxt: ILMetadataReader) (pectxtEager: PEReader) pevEager
PhysicalAlignment = alignPhys
ImageBase = imageBaseReal
MetadataVersion = ilMetadataVersion
Resources = seekReadManifestResources ctxt mdv pectxtEager pevEager }
Resources = seekReadManifestResources ctxt canReduceMemory mdv pectxtEager pevEager }
and seekReadAssemblyManifest (ctxt: ILMetadataReader) pectxt idx =
let mdview = ctxt.mdfile.GetView()
......@@ -3101,10 +3010,10 @@ and sigptrGetILNativeType ctxt bytes sigptr =
else sigptrGetZInt32 bytes sigptr
ILNativeType.Array (Some nt, Some(pnum, Some additive)), sigptr
else (ILNativeType.Empty, sigptr)
// Note, pectxtEager and pevEager must not be captured by the results of this function
// As a result, reading the resource offsets in the physical file is done eagerly to avoid holding on to any resources
and seekReadManifestResources (ctxt: ILMetadataReader) (mdv: BinaryView) (pectxtEager: PEReader) (pevEager: BinaryView) =
and seekReadManifestResources (ctxt: ILMetadataReader) canReduceMemory (mdv: BinaryView) (pectxtEager: PEReader) (pevEager: BinaryView) =
mkILResources
[ for i = 1 to ctxt.getNumRows TableNames.ManifestResource do
let (offset, flags, nameIdx, implIdx) = seekReadManifestResourceRow ctxt mdv i
......@@ -3117,10 +3026,14 @@ and seekReadManifestResources (ctxt: ILMetadataReader) (mdv: BinaryView) (pectxt
let start = pectxtEager.anyV2P ("resource", offset + pectxtEager.resourcesAddr)
let resourceLength = seekReadInt32 pevEager start
let offsetOfBytesFromStartOfPhysicalPEFile = start + 4
if pectxtEager.noFileOnDisk then
ILResourceLocation.LocalOut (seekReadBytes pevEager offsetOfBytesFromStartOfPhysicalPEFile resourceLength)
else
ILResourceLocation.LocalIn (ctxt.fileName, offsetOfBytesFromStartOfPhysicalPEFile, resourceLength)
let bytes =
let bytes = pevEager.Slice(offsetOfBytesFromStartOfPhysicalPEFile, resourceLength)
// If we are trying to reduce memory, create a memory mapped file based on the contents.
if canReduceMemory then
ByteMemory.CreateMemoryMappedFile bytes
else
ByteMemory.FromArray(bytes.ToArray())
ILResourceLocation.Local(bytes.AsReadOnly())
| ILScopeRef.Module mref -> ILResourceLocation.File (mref, offset)
| ILScopeRef.Assembly aref -> ILResourceLocation.Assembly aref
......@@ -3207,7 +3120,7 @@ let getPdbReader pdbDirPath fileName =
#endif
// Note, pectxtEager and pevEager must not be captured by the results of this function
let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, pectxtEager: PEReader, pevEager: BinaryView, pectxtCaptured, reduceMemoryUsage, ilGlobals) =
let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, pectxtEager: PEReader, pevEager, pectxtCaptured, reduceMemoryUsage, ilGlobals) =
let mdv = mdfile.GetView()
let magic = seekReadUInt16AsInt32 mdv metadataPhysLoc
if magic <> 0x5342 then failwith (fileName + ": bad metadata magic number: " + string magic)
......@@ -3584,7 +3497,7 @@ let openMetadataReader (fileName, mdfile: BinaryFile, metadataPhysLoc, peinfo, p
tableBigness=tableBigness }
ctxtH := Some ctxt
let ilModule = seekReadModule ctxt pectxtEager pevEager peinfo (System.Text.Encoding.UTF8.GetString (ilMetadataVersion, 0, ilMetadataVersion.Length)) 1
let ilModule = seekReadModule ctxt reduceMemoryUsage pectxtEager pevEager peinfo (System.Text.Encoding.UTF8.GetString (ilMetadataVersion, 0, ilMetadataVersion.Length)) 1
let ilAssemblyRefs = lazy [ for i in 1 .. getNumRows TableNames.AssemblyRef do yield seekReadAssemblyRef ctxt i ]
ilModule, ilAssemblyRefs
......@@ -3797,8 +3710,8 @@ let openPE (fileName, pefile, pdbDirPath, reduceMemoryUsage, ilGlobals, noFileOn
let ilModule, ilAssemblyRefs = openMetadataReader (fileName, pefile, metadataPhysLoc, peinfo, pectxt, pev, Some pectxt, reduceMemoryUsage, ilGlobals)
ilModule, ilAssemblyRefs, pdb
let openPEMetadataOnly (fileName, peinfo, pectxtEager, pev, mdfile: BinaryFile, reduceMemoryUsage, ilGlobals) =
openMetadataReader (fileName, mdfile, 0, peinfo, pectxtEager, pev, None, reduceMemoryUsage, ilGlobals)
let openPEMetadataOnly (fileName, peinfo, pectxtEager, pevEager, mdfile: BinaryFile, reduceMemoryUsage, ilGlobals) =
openMetadataReader (fileName, mdfile, 0, peinfo, pectxtEager, pevEager, None, reduceMemoryUsage, ilGlobals)
let ClosePdbReader pdb =
#if FX_NO_PDB_READER
......@@ -3992,13 +3905,12 @@ let OpenILModuleReader fileName opts =
// We do however care about avoiding locks on files that prevent their deletion during a
// multi-proc build. So use memory mapping, but only for stable files. Other files
// still use an in-memory ByteFile
let _disposer, pefile =
let pefile =
if alwaysMemoryMapFSC || stableFileHeuristicApplies fullPath then
createMemoryMapFile fullPath
let _, pefile = createMemoryMapFile fullPath
pefile
else
let pefile = createByteFileChunk opts fullPath None
let disposer = { new IDisposable with member __.Dispose() = () }
disposer, pefile
createByteFileChunk opts fullPath None
let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbDirPath, reduceMemoryUsage, opts.ilGlobals, false)
let ilModuleReader = new ILModuleReaderImpl(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb))
......
......@@ -2070,11 +2070,9 @@ let buildModuleFragment cenv emEnv (asmB: AssemblyBuilder) (modB: ModuleBuilder)
m.Resources.AsList |> List.iter (fun r ->
let attribs = (match r.Access with ILResourceAccess.Public -> ResourceAttributes.Public | ILResourceAccess.Private -> ResourceAttributes.Private)
match r.Location with
| ILResourceLocation.LocalIn (file, start, len) ->
let bytes = FileSystem.ReadAllBytesShim(file).[start .. start + len - 1]
modB.DefineManifestResourceAndLog (r.Name, new MemoryStream(bytes), attribs)
| ILResourceLocation.LocalOut bytes ->
modB.DefineManifestResourceAndLog (r.Name, new MemoryStream(bytes), attribs)
| ILResourceLocation.Local bytes ->
use stream = bytes.AsStream()
modB.DefineManifestResourceAndLog (r.Name, stream, attribs)
| ILResourceLocation.File (mr, _) ->
asmB.AddResourceFileAndLog (r.Name, mr.Name, attribs)
| ILResourceLocation.Assembly _ ->
......
......@@ -2689,7 +2689,7 @@ and GenEventPass3 cenv env (md: ILEventDef) =
let rec GetResourceAsManifestResourceRow cenv r =
let data, impl =
let embedManagedResources (bytes: byte[]) =
let embedManagedResources (bytes: ReadOnlyByteMemory) =
// Embedded managed resources must be word-aligned. However resource format is
// not specified in ECMA. Some mscorlib resources appear to be non-aligned - it seems it doesn't matter..
let offset = cenv.resources.Position
......@@ -2698,12 +2698,11 @@ let rec GetResourceAsManifestResourceRow cenv r =
let resourceSize = bytes.Length
cenv.resources.EmitPadding pad
cenv.resources.EmitInt32 resourceSize
cenv.resources.EmitBytes bytes
cenv.resources.EmitByteMemory bytes
Data (alignedOffset, true), (i_File, 0)
match r.Location with
| ILResourceLocation.LocalIn _ -> embedManagedResources (r.GetBytes())
| ILResourceLocation.LocalOut bytes -> embedManagedResources bytes
| ILResourceLocation.Local bytes -> embedManagedResources bytes
| ILResourceLocation.File (mref, offset) -> ULong offset, (i_File, GetModuleRefAsFileIdx cenv mref)
| ILResourceLocation.Assembly aref -> ULong 0x0, (i_AssemblyRef, GetAssemblyRefAsIdx cenv aref)
......
......@@ -1893,9 +1893,9 @@ type IRawFSharpAssemblyData =
/// in the language service
abstract TryGetILModuleDef: unit -> ILModuleDef option
/// The raw F# signature data in the assembly, if any
abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list
abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list
/// The raw F# optimization data in the assembly, if any
abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list
abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list
/// The table of type forwarders in the assembly
abstract GetRawTypeForwarders: unit -> ILExportedTypesAndForwarders
/// The identity of the module
......@@ -3590,7 +3590,7 @@ let IsReflectedDefinitionsResource (r: ILResource) =
let MakeILResource rName bytes =
{ Name = rName
Location = ILResourceLocation.LocalOut bytes
Location = ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly())
Access = ILResourceAccess.Public
CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs
MetadataIndex = NoMetadataIdx }
......@@ -3599,7 +3599,7 @@ let PickleToResource inMem file (g: TcGlobals) scope rName p x =
let file = PathMap.apply g.pathMap file
{ Name = rName
Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.LocalOut bytes)
Location = (let bytes = pickleObjWithDanglingCcus inMem file g scope p x in ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly()))
Access = ILResourceAccess.Public
CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs
MetadataIndex = NoMetadataIdx }
......@@ -3657,7 +3657,7 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR
let sigFileName = Path.ChangeExtension(filename, "sigdata")
if not (FileSystem.SafeExists sigFileName) then
error(Error(FSComp.SR.buildExpectedSigdataFile (FileSystem.GetFullPathShim sigFileName), m))
[ (ilShortAssemName, fun () -> FileSystem.ReadAllBytesShim sigFileName)]
[ (ilShortAssemName, fun () -> ByteMemory.FromFile(sigFileName, FileAccess.Read, canShadowCopy=true).AsReadOnly())]
else
sigDataReaders
sigDataReaders
......@@ -3672,7 +3672,7 @@ type RawFSharpAssemblyDataBackedByFileOnDisk (ilModule: ILModuleDef, ilAssemblyR
let optDataFile = Path.ChangeExtension(filename, "optdata")
if not (FileSystem.SafeExists optDataFile) then
error(Error(FSComp.SR.buildExpectedFileAlongSideFSharpCore(optDataFile, FileSystem.GetFullPathShim optDataFile), m))
[ (ilShortAssemName, (fun () -> FileSystem.ReadAllBytesShim optDataFile))]
[ (ilShortAssemName, (fun () -> ByteMemory.FromFile(optDataFile, FileAccess.Read, canShadowCopy=true).AsReadOnly()))]
else
optDataReaders
optDataReaders
......
......@@ -11,6 +11,7 @@ open FSharp.Compiler.AbstractIL
open FSharp.Compiler.AbstractIL.IL
open FSharp.Compiler.AbstractIL.ILBinaryReader
open FSharp.Compiler.AbstractIL.ILPdbWriter
open FSharp.Compiler.AbstractIL.Internal
open FSharp.Compiler.AbstractIL.Internal.Library
open FSharp.Compiler
open FSharp.Compiler.TypeChecker
......@@ -154,9 +155,9 @@ type IRawFSharpAssemblyData =
abstract HasAnyFSharpSignatureDataAttribute: bool
abstract HasMatchingFSharpSignatureDataAttribute: ILGlobals -> bool
/// The raw F# signature data in the assembly, if any
abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list
abstract GetRawFSharpSignatureData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list
/// The raw F# optimization data in the assembly, if any
abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> byte[])) list
abstract GetRawFSharpOptimizationData: range * ilShortAssemName: string * fileName: string -> (string * (unit -> ReadOnlyByteMemory)) list
/// The table of type forwarders in the assembly
abstract GetRawTypeForwarders: unit -> ILExportedTypesAndForwarders
/// The identity of the module
......
......@@ -284,10 +284,13 @@ let u_int32 st =
assert(b0 = 0xFF)
prim_u_int32 st
let u_bytes st =
let u_byte_memory st =
let n = (u_int32 st)
st.is.ReadBytes n
let u_bytes st =
(u_byte_memory st).ToArray()
let u_prim_string st =
let len = (u_int32 st)
st.is.ReadUtf8String len
......@@ -832,7 +835,7 @@ let check (ilscope: ILScopeRef) (inMap : NodeInTable<_, _>) =
// an identical copy of the source for the DLL containing the data being unpickled. A message will
// then be printed indicating the name of the item.
let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u (phase2bytes: byte[]) =
let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u (phase2bytes: ReadOnlyByteMemory) =
let st2 =
{ is = ByteStream.FromBytes (phase2bytes, 0, phase2bytes.Length)
iilscope= ilscope
......@@ -858,7 +861,7 @@ let unpickleObjWithDanglingCcus file ilscope (iILModule: ILModuleDef option) u (
(u_array u_encoded_pubpath)
(u_array u_encoded_nleref)
(u_array u_encoded_simpletyp)
u_bytes
u_byte_memory
st2
let ccuTab = new_itbl "iccus" (Array.map (CcuThunk.CreateDelayed) ccuNameTab)
let stringTab = new_itbl "istrings" (Array.map decode_string stringTab)
......
......@@ -142,7 +142,7 @@ val internal u_ty : unpickler<TType>
val internal unpickleCcuInfo : ReaderState -> PickledCcuInfo
/// Deserialize an arbitrary object which may have holes referring to other compilation units
val internal unpickleObjWithDanglingCcus : file:string -> viewedScope:ILScopeRef -> ilModule:ILModuleDef option -> ('T unpickler) -> byte[] -> PickledDataWithReferences<'T>
val internal unpickleObjWithDanglingCcus : file:string -> viewedScope:ILScopeRef -> ilModule:ILModuleDef option -> ('T unpickler) -> ReadOnlyByteMemory -> PickledDataWithReferences<'T>
......@@ -442,7 +442,9 @@ let EncodeInterfaceData(tcConfig: TcConfig, tcGlobals, exportRemapping, generate
let useDataFiles = (tcConfig.useOptimizationDataFile || tcGlobals.compilingFslib) && not isIncrementalBuild
if useDataFiles then
let sigDataFileName = (Filename.chopExtension outfile)+".sigdata"
File.WriteAllBytes(sigDataFileName, resource.GetBytes())
let bytes = resource.GetBytes()
use fileStream = File.Create(sigDataFileName, bytes.Length)
bytes.CopyTo fileStream
let resources =
[ resource ]
let sigAttr = mkSignatureDataVersionAttr tcGlobals (IL.parseILVersion Internal.Utilities.FSharpEnvironment.FSharpBinaryMetadataFormatRevision)
......@@ -892,7 +894,7 @@ module MainModuleBuilder =
[ ]
let reflectedDefinitionResource =
{ Name=reflectedDefinitionResourceName
Location = ILResourceLocation.LocalOut reflectedDefinitionBytes
Location = ILResourceLocation.Local(ByteMemory.FromArray(reflectedDefinitionBytes).AsReadOnly())
Access= ILResourceAccess.Public
CustomAttrsStored = storeILCustomAttrs emptyILCustomAttrs
MetadataIndex = NoMetadataIdx }
......@@ -936,7 +938,7 @@ module MainModuleBuilder =
let bytes = FileSystem.ReadAllBytesShim file
name, bytes, pub
yield { Name=name
Location=ILResourceLocation.LocalOut bytes
Location=ILResourceLocation.Local(ByteMemory.FromArray(bytes).AsReadOnly())
Access=pub
CustomAttrsStored=storeILCustomAttrs emptyILCustomAttrs
MetadataIndex = NoMetadataIdx }
......
......@@ -14,6 +14,7 @@ open FSharp.Compiler.Lib
open FSharp.Compiler.AbstractIL
open FSharp.Compiler.AbstractIL.IL
open FSharp.Compiler.AbstractIL.ILBinaryReader
open FSharp.Compiler.AbstractIL.Internal
open FSharp.Compiler.AbstractIL.Internal.Library
open FSharp.Compiler.CompileOps
open FSharp.Compiler.CompileOptions
......
......@@ -225,7 +225,7 @@ module CompileHelpers =
// Register the reflected definitions for the dynamically generated assembly
for resource in ilxMainModule.Resources.AsList do
if IsReflectedDefinitionsResource resource then
Quotations.Expr.RegisterReflectedDefinitions (assemblyBuilder, moduleBuilder.Name, resource.GetBytes())
Quotations.Expr.RegisterReflectedDefinitions (assemblyBuilder, moduleBuilder.Name, resource.GetBytes().ToArray())
// Save the result
assemblyBuilderRef := Some assemblyBuilder
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册