From 2dcdb466df440852f09364ebbaba267ae829b89f Mon Sep 17 00:00:00 2001 From: David Glassborow Date: Wed, 7 Jun 2017 01:05:47 +0100 Subject: [PATCH] Determinism of the F# Compiler #1042 (#2954) * Add IL output option for deterministic output Thread a new option through the options to the code writing the IL. For the moment just use a constant for deterministic timestamp, and use 0 as the timestamp in the MVID * Basic check to prevent wildcard version + deterministic at the same time * Hash the code, data and metadata, and use for mvid + timestamp * Fix fsc & fsci help tests by adding in new command line option for deterministic * tests for command line, wildcard versions, normal non-deterministic behaviour * Attempt to get CI to support timeout * Allow PRECMD to call FSI more than once, and use this for copying files and pausing to prevent race-condition * PortablePdbBuilder use consistent id If compiling deterministically, use deterministic id provider support in making a pdb builder. see https://github.com/dotnet/roslyn/blob/master/src/Compilers/Core/Portable/PEWriter/PeWriter.cs#L132 * Portable and Embedded pdb now deterministic * wip: fix up timestamp in pdb, but mvids still out of sequence * Fix breaking change from rebasing on master * Add notes about determinism with non portable pdbs Make the checks for finding embedded guids more strict and unique * Throw error if determinsitc build and non-portable pdb specified --- src/absil/ilwrite.fs | 64 +++++++++++++++---- src/absil/ilwrite.fsi | 1 + src/absil/ilwritepdb.fs | 16 ++++- src/absil/ilwritepdb.fsi | 2 +- src/fsharp/CompileOps.fs | 3 + src/fsharp/CompileOps.fsi | 2 + src/fsharp/CompileOptions.fs | 7 +- src/fsharp/FSComp.txt | 3 + src/fsharp/fsc.fs | 12 +++- .../fsc/determinism/binaryCompare.fsx | 22 +++++++ .../fsc/determinism/copyArtifacts.fsx | 8 +++ .../CompilerOptions/fsc/determinism/dummy.fs | 3 + .../CompilerOptions/fsc/determinism/env.lst | 30 +++++++++ .../fsc/determinism/portablePdbOnly.fs | 3 + .../determinism/specificVersionSpecifed.fs | 4 ++ .../determinism/wildcardVersionSpecifed.fs | 4 ++ .../fsc/help/help40.437.1033.bsl | 2 + .../fsi/exename/help40.437.1033.bsl | 2 + .../fsi/help/help-nologo.437.1033.bsl | 2 + .../fsi/help/help40-nologo.437.1033.bsl | 2 + .../fsi/help/help40.437.1033.bsl | 2 + tests/fsharpqa/Source/run.pl | 2 +- tests/fsharpqa/Source/test.lst | 1 + 23 files changed, 175 insertions(+), 22 deletions(-) create mode 100644 tests/fsharpqa/Source/CompilerOptions/fsc/determinism/binaryCompare.fsx create mode 100644 tests/fsharpqa/Source/CompilerOptions/fsc/determinism/copyArtifacts.fsx create mode 100644 tests/fsharpqa/Source/CompilerOptions/fsc/determinism/dummy.fs create mode 100644 tests/fsharpqa/Source/CompilerOptions/fsc/determinism/env.lst create mode 100644 tests/fsharpqa/Source/CompilerOptions/fsc/determinism/portablePdbOnly.fs create mode 100644 tests/fsharpqa/Source/CompilerOptions/fsc/determinism/specificVersionSpecifed.fs create mode 100644 tests/fsharpqa/Source/CompilerOptions/fsc/determinism/wildcardVersionSpecifed.fs diff --git a/src/absil/ilwrite.fs b/src/absil/ilwrite.fs index 102f50cca..3a9e86191 100644 --- a/src/absil/ilwrite.fs +++ b/src/absil/ilwrite.fs @@ -551,6 +551,7 @@ type MetadataTable = type cenv = { ilg: ILGlobals emitTailcalls: bool + deterministic: bool showTimes: bool desiredMetadataVersion: ILVersionInfo requiredDataFixups: (int32 * (int * bool)) list ref @@ -2903,9 +2904,15 @@ and newGuid (modul: ILModuleDef) = let m2 = hash modul.Name [| b0 m; b1 m; b2 m; b3 m; b0 m2; b1 m2; b2 m2; b3 m2; 0xa7uy; 0x45uy; 0x03uy; 0x83uy; b0 n; b1 n; b2 n; b3 n |] -and GetModuleAsRow cenv (modul: ILModuleDef) = +and deterministicGuid (modul: ILModuleDef) = + let n = 16909060 + let m = hash n + let m2 = hash modul.Name + [| b0 m; b1 m; b2 m; b3 m; b0 m2; b1 m2; b2 m2; b3 m2; 0xa7uy; 0x45uy; 0x03uy; 0x83uy; b0 n; b1 n; b2 n; b3 n |] + +and GetModuleAsRow (cenv:cenv) (modul: ILModuleDef) = // Store the generated MVID in the environment (needed for generating debug information) - let modulGuid = newGuid modul + let modulGuid = if cenv.deterministic then deterministicGuid modul else newGuid modul cenv.moduleGuid <- modulGuid UnsharedRow [| UShort (uint16 0x0) @@ -2954,11 +2961,12 @@ let GenModule (cenv : cenv) (modul: ILModuleDef) = GenTypeDefsPass4 [] cenv tds reportTime cenv.showTimes "Module Generation Pass 4" -let generateIL requiredDataFixups (desiredMetadataVersion,generatePdb, ilg : ILGlobals, emitTailcalls,showTimes) (m : ILModuleDef) cilStartAddress = +let generateIL requiredDataFixups (desiredMetadataVersion,generatePdb, ilg : ILGlobals, emitTailcalls, deterministic, showTimes) (m : ILModuleDef) cilStartAddress = let isDll = m.IsDLL let cenv = { emitTailcalls=emitTailcalls + deterministic = deterministic showTimes=showTimes ilg = ilg desiredMetadataVersion=desiredMetadataVersion @@ -3098,7 +3106,7 @@ module FileSystemUtilites = #endif () -let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls,showTimes) modul cilStartAddress = +let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls,deterministic,showTimes) modul cilStartAddress = // When we know the real RVAs of the data section we fixup the references for the FieldRVA table. // These references are stored as offsets into the metadata we return from this function @@ -3107,7 +3115,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls let next = cilStartAddress let strings,userStrings,blobs,guids,tables,entryPointToken,code,requiredStringFixups,data,resources,pdbData,mappings = - generateIL requiredDataFixups (desiredMetadataVersion,generatePdb,ilg,emitTailcalls,showTimes) modul cilStartAddress + generateIL requiredDataFixups (desiredMetadataVersion,generatePdb,ilg,emitTailcalls,deterministic,showTimes) modul cilStartAddress reportTime showTimes "Generated Tables and Code" let tableSize (tab: TableName) = tables.[tab.Index].Count @@ -3404,7 +3412,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls reportTime showTimes "Layout Metadata" - let metadata = + let metadata, guidStart = let mdbuf = ByteBuffer.Create 500000 mdbuf.EmitIntsAsBytes [| 0x42; 0x53; 0x4a; 0x42; // Magic signature @@ -3462,6 +3470,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls reportTime showTimes "Write Metadata User Strings"; // The GUID stream + let guidStart = mdbuf.Position Array.iter mdbuf.EmitBytes guids; // The blob stream @@ -3473,7 +3482,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls mdbuf.EmitIntAsByte 0x00; reportTime showTimes "Write Blob Stream"; // Done - close the buffer and return the result. - mdbuf.Close() + mdbuf.Close(), guidStart // Now we know the user string tables etc. we can fixup the @@ -3488,7 +3497,7 @@ let writeILMetadataAndCode (generatePdb,desiredMetadataVersion,ilg,emitTailcalls applyFixup32 code locInCode token reportTime showTimes "Fixup Metadata"; - entryPointToken,code, codePadding,metadata,data,resources,!requiredDataFixups,pdbData,mappings + entryPointToken,code, codePadding,metadata,data,resources,!requiredDataFixups,pdbData,mappings,guidStart //--------------------------------------------------------------------- // PHYSICAL METADATA+BLOBS --> PHYSICAL PE FORMAT @@ -3550,7 +3559,7 @@ let writeBytes (os: BinaryWriter) (chunk:byte[]) = os.Write(chunk,0,chunk.Length let writeBinaryAndReportMappings (outfile, ilg: ILGlobals, pdbfile: string option, signer: ILStrongNameSigner option, portablePDB, embeddedPDB, - embedAllSource, embedSourceList, sourceLink, emitTailcalls, showTimes, dumpDebugInfo) modul = + embedAllSource, embedSourceList, sourceLink, emitTailcalls, deterministic, showTimes, dumpDebugInfo ) modul = // Store the public key from the signer into the manifest. This means it will be written // to the binary and also acts as an indicator to leave space for delay sign @@ -3666,8 +3675,8 @@ let writeBinaryAndReportMappings (outfile, | Some v -> v | None -> failwith "Expected msorlib to have a version number" - let entryPointToken,code,codePadding,metadata,data,resources,requiredDataFixups,pdbData,mappings = - writeILMetadataAndCode ((pdbfile <> None), desiredMetadataVersion, ilg,emitTailcalls,showTimes) modul next + let entryPointToken,code,codePadding,metadata,data,resources,requiredDataFixups,pdbData,mappings,guidStart = + writeILMetadataAndCode ((pdbfile <> None), desiredMetadataVersion, ilg,emitTailcalls, deterministic, showTimes) modul next reportTime showTimes "Generated IL and metadata"; let _codeChunk,next = chunk code.Length next @@ -3703,7 +3712,7 @@ let writeBinaryAndReportMappings (outfile, let pdbOpt = match portablePDB with | true -> - let (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb embedAllSource embedSourceList sourceLink showTimes pdbData + let (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb embedAllSource embedSourceList sourceLink showTimes pdbData deterministic if embeddedPDB then Some (compressPortablePdbStream uncompressedLength contentId stream) else Some (pdbStream) | _ -> None @@ -3849,7 +3858,33 @@ let writeBinaryAndReportMappings (outfile, writeInt32AsUInt16 os 0x014c; // Machine - IMAGE_FILE_MACHINE_I386 writeInt32AsUInt16 os numSections; - writeInt32 os timestamp // date since 1970 + + let pdbData = + if deterministic then + // Hash code, data and metadata + use sha = System.Security.Cryptography.SHA1.Create() // IncrementalHash is core only + let hCode = sha.ComputeHash code + let hData = sha.ComputeHash data + let hMeta = sha.ComputeHash metadata + let final = [| hCode; hData; hMeta |] |> Array.collect id |> sha.ComputeHash + + // Confirm we have found the correct data and aren't corrupting the metadata + if metadata.[ guidStart..guidStart+3] <> [| 4uy; 3uy; 2uy; 1uy |] then failwith "Failed to find MVID" + if metadata.[ guidStart+12..guidStart+15] <> [| 4uy; 3uy; 2uy; 1uy |] then failwith "Failed to find MVID" + + // Update MVID guid in metadata + Array.blit final 0 metadata guidStart 16 + + // Use last 4 bytes for timestamp - High bit set, to stop tool chains becoming confused + let timestamp = int final.[16] ||| (int final.[17] <<< 8) ||| (int final.[18] <<< 16) ||| (int (final.[19] ||| 128uy) <<< 24) + writeInt32 os timestamp + // Update pdbData with new guid and timestamp. Portable and embedded PDBs don't need the ModuleID + // Full and PdbOnly aren't supported under deterministic builds currently, they rely on non-determinsitic Windows native code + { pdbData with ModuleID = final.[0..15] ; Timestamp = timestamp } + else + writeInt32 os timestamp // date since 1970 + pdbData + writeInt32 os 0x00; // Pointer to Symbol Table Always 0 // 00000090 writeInt32 os 0x00; // Number of Symbols Always 0 @@ -4277,12 +4312,13 @@ type options = sourceLink: string signer: ILStrongNameSigner option emitTailcalls : bool + deterministic : bool showTimes: bool dumpDebugInfo:bool } let WriteILBinary (outfile, (args: options), modul) = writeBinaryAndReportMappings (outfile, args.ilg, args.pdbfile, args.signer, args.portablePDB, args.embeddedPDB, args.embedAllSource, - args.embedSourceList, args.sourceLink, args.emitTailcalls, args.showTimes, args.dumpDebugInfo) modul + args.embedSourceList, args.sourceLink, args.emitTailcalls, args.deterministic, args.showTimes, args.dumpDebugInfo) modul |> ignore diff --git a/src/absil/ilwrite.fsi b/src/absil/ilwrite.fsi index 6ab71287d..89c6fd035 100644 --- a/src/absil/ilwrite.fsi +++ b/src/absil/ilwrite.fsi @@ -25,6 +25,7 @@ type options = sourceLink: string signer : ILStrongNameSigner option emitTailcalls: bool + deterministic: bool showTimes : bool dumpDebugInfo : bool } diff --git a/src/absil/ilwritepdb.fs b/src/absil/ilwritepdb.fs index 9b8790fbc..0c7d11223 100644 --- a/src/absil/ilwritepdb.fs +++ b/src/absil/ilwritepdb.fs @@ -219,7 +219,7 @@ let getRowCounts tableRowCounts = tableRowCounts |> Seq.iter(fun x -> builder.Add(x)) builder.MoveToImmutable() -let generatePortablePdb (embedAllSource:bool) (embedSourceList:string list) (sourceLink:string) showTimes (info:PdbData) = +let generatePortablePdb (embedAllSource:bool) (embedSourceList:string list) (sourceLink:string) showTimes (info:PdbData) isDeterministic = sortMethods showTimes info let externalRowCounts = getRowCounts info.TableRowCounts let docs = @@ -445,7 +445,19 @@ let generatePortablePdb (embedAllSource:bool) (embedSourceList:string list) (sou | None -> MetadataTokens.MethodDefinitionHandle(0) | Some x -> MetadataTokens.MethodDefinitionHandle(x) - let serializer = PortablePdbBuilder(metadata, externalRowCounts, entryPoint, null) + let deterministicIdProvider isDeterministic : System.Func, BlobContentId> = + match isDeterministic with + | false -> null + | true -> + let convert (content:IEnumerable) = + use sha = System.Security.Cryptography.SHA1.Create() // IncrementalHash is core only + let hash = content + |> Seq.map ( fun c -> c.GetBytes().Array |> sha.ComputeHash ) + |> Seq.collect id |> Array.ofSeq |> sha.ComputeHash + BlobContentId.FromHash(hash) + System.Func, BlobContentId>( convert ) + + let serializer = PortablePdbBuilder(metadata, externalRowCounts, entryPoint, deterministicIdProvider isDeterministic) let blobBuilder = new BlobBuilder() let contentId= serializer.Serialize(blobBuilder) let portablePdbStream = new MemoryStream() diff --git a/src/absil/ilwritepdb.fsi b/src/absil/ilwritepdb.fsi index d6a3f8d83..e2b310747 100644 --- a/src/absil/ilwritepdb.fsi +++ b/src/absil/ilwritepdb.fsi @@ -82,7 +82,7 @@ type idd = iddData: byte[]; iddChunk: BinaryChunk } -val generatePortablePdb : embedAllSource:bool -> embedSourceList:string list -> sourceLink: string -> showTimes:bool -> info:PdbData -> (int64 * BlobContentId * MemoryStream) +val generatePortablePdb : embedAllSource:bool -> embedSourceList:string list -> sourceLink: string -> showTimes:bool -> info:PdbData -> isDeterministic:bool -> (int64 * BlobContentId * MemoryStream) val compressPortablePdbStream : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> (int64 * BlobContentId * MemoryStream) val embedPortablePdbInfo : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> pdbChunk:BinaryChunk -> idd[] val writePortablePdbInfo : contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> idd[] diff --git a/src/fsharp/CompileOps.fs b/src/fsharp/CompileOps.fs index c6901a602..81bfd48be 100644 --- a/src/fsharp/CompileOps.fs +++ b/src/fsharp/CompileOps.fs @@ -2133,6 +2133,7 @@ type TcConfigBuilder = mutable optsOn : bool (* optimizations are turned on *) mutable optSettings : Optimizer.OptimizationSettings mutable emitTailcalls : bool + mutable deterministic : bool #if PREFERRED_UI_LANG mutable preferredUiLang: string option #endif @@ -2301,6 +2302,7 @@ type TcConfigBuilder = optsOn = false optSettings = Optimizer.OptimizationSettings.Defaults emitTailcalls = true + deterministic = false #if PREFERRED_UI_LANG preferredUiLang = None #endif @@ -2791,6 +2793,7 @@ type TcConfig private (data : TcConfigBuilder,validate:bool) = member x.doFinalSimplify = data.doFinalSimplify member x.optSettings = data.optSettings member x.emitTailcalls = data.emitTailcalls + member x.deterministic = data.deterministic #if PREFERRED_UI_LANG member x.preferredUiLang = data.preferredUiLang #endif diff --git a/src/fsharp/CompileOps.fsi b/src/fsharp/CompileOps.fsi index 52dda18fe..7dd1bb7ad 100755 --- a/src/fsharp/CompileOps.fsi +++ b/src/fsharp/CompileOps.fsi @@ -342,6 +342,7 @@ type TcConfigBuilder = mutable optsOn : bool mutable optSettings : Optimizer.OptimizationSettings mutable emitTailcalls : bool + mutable deterministic : bool #if PREFERRED_UI_LANG mutable preferredUiLang: string option #endif @@ -494,6 +495,7 @@ type TcConfig = member doFinalSimplify : bool member optSettings : Optimizer.OptimizationSettings member emitTailcalls : bool + member deterministic : bool #if PREFERRED_UI_LANG member preferredUiLang: string option #else diff --git a/src/fsharp/CompileOptions.fs b/src/fsharp/CompileOptions.fs index 978600850..d231b4380 100644 --- a/src/fsharp/CompileOptions.fs +++ b/src/fsharp/CompileOptions.fs @@ -420,7 +420,10 @@ let SetOptimizeSwitch (tcConfigB : TcConfigBuilder) switch = let SetTailcallSwitch (tcConfigB : TcConfigBuilder) switch = tcConfigB.emitTailcalls <- (switch = OptionSwitch.On) - + +let SetDeterministicSwitch (tcConfigB : TcConfigBuilder) switch = + tcConfigB.deterministic <- (switch = OptionSwitch.On) + let jitoptimizeSwitch (tcConfigB : TcConfigBuilder) switch = tcConfigB.optSettings <- { tcConfigB.optSettings with jitOptUser = Some (switch = OptionSwitch.On) } @@ -676,6 +679,8 @@ let codeGenerationFlags isFsi (tcConfigB : TcConfigBuilder) = Some (FSComp.SR.optsOptimize())) CompilerOption("tailcalls", tagNone, OptionSwitch (SetTailcallSwitch tcConfigB), None, Some (FSComp.SR.optsTailcalls())) + CompilerOption("deterministic", tagNone, OptionSwitch (SetDeterministicSwitch tcConfigB), None, + Some (FSComp.SR.optsDeterministic())) CompilerOption("crossoptimize", tagNone, OptionSwitch (crossOptimizeSwitch tcConfigB), None, Some (FSComp.SR.optsCrossoptimize())) ] diff --git a/src/fsharp/FSComp.txt b/src/fsharp/FSComp.txt index cf6601238..4031b64e9 100644 --- a/src/fsharp/FSComp.txt +++ b/src/fsharp/FSComp.txt @@ -859,6 +859,7 @@ optsDebugPM,"Emit debug information (Short form: -g)" optsDebug,"Specify debugging type: full, portable, embedded, pdbonly. ('%s' is the default if no debuggging type specified and enables attaching a debugger to a running program, 'portable' is a cross-platform format, 'embedded' is a cross-platform format embedded into the output file)." optsOptimize,"Enable optimizations (Short form: -O)" optsTailcalls,"Enable or disable tailcalls" +optsDeterministic,"Produce a deterministic assembly (including module version GUID and timestamp)" optsCrossoptimize,"Enable or disable cross-module optimizations" optsWarnaserrorPM,"Report all warnings as errors" optsWarnaserror,"Report specific warnings as errors" @@ -1137,6 +1138,8 @@ fscTooManyErrors,"Exiting - too many errors" 2022,pathIsInvalid,"Problem with filename '%s': Illegal characters in path." 2023,fscResxSourceFileDeprecated,"Passing a .resx file (%s) as a source file to the compiler is deprecated. Use resgen.exe to transform the .resx file into a .resources file to pass as a --resource option. If you are using MSBuild, this can be done via an item in the .fsproj project file." 2024,fscStaticLinkingNoProfileMismatches,"Static linking may not be used on an assembly referencing mscorlib (e.g. a .NET Framework assembly) when generating an assembly that references System.Runtime (e.g. a .NET Core or Portable assembly)." +2025,fscAssemblyWildcardAndDeterminism,"An %s specified version '%s', but this value is a wildcard, and you have requested a deterministic build, these are in conflict." +2026,fscDeterministicDebugRequiresPortablePdb,"Determinstic builds only support portable PDBs (--debug:portable or --debug:embedded)" 3000,etIllegalCharactersInNamespaceName,"Character '%s' is not allowed in provided namespace name '%s'" 3001,etNullOrEmptyMemberName,"The provided type '%s' returned a member with a null or empty member name" 3002,etNullMember,"The provided type '%s' returned a null member" diff --git a/src/fsharp/fsc.fs b/src/fsharp/fsc.fs index 626ca4931..ff848d5ff 100644 --- a/src/fsharp/fsc.fs +++ b/src/fsharp/fsc.fs @@ -249,6 +249,9 @@ let ProcessCommandLineFlags (tcConfigB: TcConfigBuilder, setProcessThreadLocals, if not (String.IsNullOrEmpty(tcConfigB.sourceLink)) then error(Error(FSComp.SR.optsSourceLinkRequirePortablePDBs(), rangeCmdArgs)) + if tcConfigB.deterministic && tcConfigB.debuginfo && (tcConfigB.portablePDB = false) then + error(Error(FSComp.SR.fscDeterministicDebugRequiresPortablePdb(), rangeCmdArgs)) + let inputFiles = List.rev !inputFilesRef // Check if we have a codepage from the console @@ -704,9 +707,11 @@ module AttributeHelpers = None // Try to find an AssemblyVersion attribute - let TryFindVersionAttribute g attrib attribName attribs = + let TryFindVersionAttribute g attrib attribName attribs deterministic = match TryFindStringAttribute g attrib attribs with | Some versionString -> + if deterministic && versionString.Contains("*") then + errorR(Error(FSComp.SR.fscAssemblyWildcardAndDeterminism(attribName, versionString), Range.rangeStartup)) try Some (IL.parseILVersion versionString) with e -> warning(Error(FSComp.SR.fscBadAssemblyVersion(attribName, versionString), Range.rangeStartup)) @@ -1810,7 +1815,7 @@ let main1(Args (ctok, tcGlobals, tcImports: TcImports, frameworkTcImports, gener // Try to find an AssemblyVersion attribute let assemVerFromAttrib = - match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs with + match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs tcConfig.deterministic with | Some v -> match tcConfig.version with | VersionNone -> Some v @@ -1896,7 +1901,7 @@ let main1OfAst (ctok, legacyReferenceResolver, openBinariesInMemory, assemblyNam // Try to find an AssemblyVersion attribute let assemVerFromAttrib = - match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs with + match AttributeHelpers.TryFindVersionAttribute tcGlobals "System.Reflection.AssemblyVersionAttribute" "AssemblyVersionAttribute" topAttrs.assemblyAttrs tcConfig.deterministic with | Some v -> match tcConfig.version with | VersionNone -> Some v @@ -2016,6 +2021,7 @@ let main4 dynamicAssemblyCreator (Args (ctok, tcConfig, errorLogger: ErrorLogger { ilg = tcGlobals.ilg pdbfile=pdbfile emitTailcalls = tcConfig.emitTailcalls + deterministic = tcConfig.deterministic showTimes = tcConfig.showTimes portablePDB = tcConfig.portablePDB embeddedPDB = tcConfig.embeddedPDB diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/binaryCompare.fsx b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/binaryCompare.fsx new file mode 100644 index 000000000..2147b88cb --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/binaryCompare.fsx @@ -0,0 +1,22 @@ +// sgiven 'dummy' and 'dummy2', and 'dummy.exe' and 'dummy.pdb' exist in the current directory +// this will check 'dummy.exe' and 'dummy2.exe are exactly the same, and the same for the 'pdb' files +// expects 1 arg: whether files should be exactly the same +let shouldBeSame = bool.Parse fsi.CommandLineArgs.[1] + +let areSame (first,second) = + let load = System.IO.File.ReadAllBytes + (load first) = (load second) + +let filePairsToCheck = + System.IO.Directory.EnumerateFiles(__SOURCE_DIRECTORY__, "dummy.*") + |> Seq.filter (fun s -> s.EndsWith(".fs") |> not ) // Don't check the source code + |> Seq.map (fun f -> f, f.Replace("dummy","dummy2")) + +let compareFiles pair = + let correct = areSame pair = shouldBeSame + if not correct then + printfn "Expected %s and %s to be %s" (fst pair) (snd pair) (if shouldBeSame then "same" else "different") + correct + +// Check all pairs of files are exactly the same +exit (if filePairsToCheck |> Seq.forall compareFiles then 0 else 1) diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/copyArtifacts.fsx b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/copyArtifacts.fsx new file mode 100644 index 000000000..9d59e2c90 --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/copyArtifacts.fsx @@ -0,0 +1,8 @@ +// Copy all dummy files (except the source) to a dummy2 version so we can compare them +System.IO.Directory.EnumerateFiles(__SOURCE_DIRECTORY__, "dummy.*") +|> Seq.filter (fun x -> x.EndsWith(".fs") |> not) +|> Seq.iter (fun filename -> System.IO.File.Copy(filename, filename.Replace("dummy", "dummy2"), true)) + +// pause a second at the end to deal with the potential race condiction of +// too quick compiles back to back ending up in same exe, even when non-deterministic +System.Threading.Thread.Sleep(1000) diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/dummy.fs b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/dummy.fs new file mode 100644 index 000000000..431026692 --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/dummy.fs @@ -0,0 +1,3 @@ +// #NoMT #CompilerOptions #Deterministic +module TestDeterminsticCompilation +exit 0 diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/env.lst b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/env.lst new file mode 100644 index 000000000..5767e939f --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/env.lst @@ -0,0 +1,30 @@ +# Sanity check - simply check that the option is valid + SOURCE=dummy.fs SCFLAGS="--deterministic" + SOURCE=dummy.fs SCFLAGS="--deterministic+" + SOURCE=dummy.fs SCFLAGS="--deterministic-" + SOURCE=dummy.fs SCFLAGS="--deterministic" FSIMODE=EXEC COMPILE_ONLY=1 + SOURCE=dummy.fs SCFLAGS="--deterministic+" FSIMODE=EXEC COMPILE_ONLY=1 + SOURCE=dummy.fs SCFLAGS="--deterministic-" FSIMODE=EXEC COMPILE_ONLY=1 + +# Confirm wildcard versions are not allowed + SOURCE=wildcardVersionSpecifed.fs SCFLAGS="--deterministic" + +# Confirm specific versions are allowed + SOURCE=specificVersionSpecifed.fs SCFLAGS="--deterministic" + +# Confirm in normal not-determinstic mode, the same file compiled twice leads to different exes & pdbs +# NOTE: we need to use the timeout because non-determinism in fsc is due to datetime, so we need to guarentee at least 1 sec difference + SOURCE=dummy.fs PRECMD="\$FSC_PIPE dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx false" + SOURCE=dummy.fs SCFLAGS="--debug:full" PRECMD="\$FSC_PIPE --debug:full dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx false" + SOURCE=dummy.fs SCFLAGS="--debug:pdbonly" PRECMD="\$FSC_PIPE --debug:pdbonly dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx false" + SOURCE=dummy.fs SCFLAGS="--debug:portable" PRECMD="\$FSC_PIPE --debug:portable dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx false" + SOURCE=dummy.fs SCFLAGS="--debug:embedded" PRECMD="\$FSC_PIPE --debug:embedded dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx false" + +# Confirm in determinstic mode, the same file compiled twice leads to exactly the same exe & pdbs + SOURCE=dummy.fs SCFLAGS="--deterministic" PRECMD="\$FSC_PIPE --deterministic dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx true" + SOURCE=dummy.fs SCFLAGS="--deterministic --debug:portable" PRECMD="\$FSC_PIPE --deterministic --debug:portable dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx true" + SOURCE=dummy.fs SCFLAGS="--deterministic --debug:embedded" PRECMD="\$FSC_PIPE --deterministic --debug:embedded dummy.fs && \$FSI_PIPE copyArtifacts.fsx" POSTCMD="\$FSI_PIPE --nologo --quiet --exec binaryCompare.fsx true" + +# Confirm only portable and embeded debug PDBs are supported under determinstic build, not full or pdbonly + SOURCE=portablePdbOnly.fs SCFLAGS="--deterministic --debug:full" + SOURCE=portablePdbOnly.fs SCFLAGS="--deterministic --debug:pdbonly" diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/portablePdbOnly.fs b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/portablePdbOnly.fs new file mode 100644 index 000000000..c1b99e6a4 --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/portablePdbOnly.fs @@ -0,0 +1,3 @@ +// #NoMT #CompilerOptions #Determinism +//Determinstic builds only support portable PDBs +exit 0 diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/specificVersionSpecifed.fs b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/specificVersionSpecifed.fs new file mode 100644 index 000000000..e9199e9a1 --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/specificVersionSpecifed.fs @@ -0,0 +1,4 @@ +// #NoMT #CompilerOptions #Determinism +// +[] +exit 0 diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/wildcardVersionSpecifed.fs b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/wildcardVersionSpecifed.fs new file mode 100644 index 000000000..10f82d79a --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/determinism/wildcardVersionSpecifed.fs @@ -0,0 +1,4 @@ +// #NoMT #CompilerOptions #Determinism +//this value is a wildcard, and you have requested a deterministic build, these are in conflict. +[] +exit 0 diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/help/help40.437.1033.bsl b/tests/fsharpqa/Source/CompilerOptions/fsc/help/help40.437.1033.bsl index fc2f14041..3ed355c37 100644 --- a/tests/fsharpqa/Source/CompilerOptions/fsc/help/help40.437.1033.bsl +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/help/help40.437.1033.bsl @@ -63,6 +63,8 @@ Copyright (c) Microsoft Corporation. All Rights Reserved. portable PDB file --optimize[+|-] Enable optimizations (Short form: -O) --tailcalls[+|-] Enable or disable tailcalls +--deterministic[+|-] Produce a deterministic assembly (including + module version GUID and timestamp) --crossoptimize[+|-] Enable or disable cross-module optimizations diff --git a/tests/fsharpqa/Source/CompilerOptions/fsi/exename/help40.437.1033.bsl b/tests/fsharpqa/Source/CompilerOptions/fsi/exename/help40.437.1033.bsl index d22dec69b..e55b4f21e 100644 --- a/tests/fsharpqa/Source/CompilerOptions/fsi/exename/help40.437.1033.bsl +++ b/tests/fsharpqa/Source/CompilerOptions/fsi/exename/help40.437.1033.bsl @@ -21,6 +21,8 @@ Usage: fsharpi [script.fsx []] into the output file). --optimize[+|-] Enable optimizations (Short form: -O) --tailcalls[+|-] Enable or disable tailcalls +--deterministic[+|-] Produce a deterministic assembly (including + module version GUID and timestamp) --crossoptimize[+|-] Enable or disable cross-module optimizations diff --git a/tests/fsharpqa/Source/CompilerOptions/fsi/help/help-nologo.437.1033.bsl b/tests/fsharpqa/Source/CompilerOptions/fsi/help/help-nologo.437.1033.bsl index e5d0c20f9..28a991afa 100644 --- a/tests/fsharpqa/Source/CompilerOptions/fsi/help/help-nologo.437.1033.bsl +++ b/tests/fsharpqa/Source/CompilerOptions/fsi/help/help-nologo.437.1033.bsl @@ -19,6 +19,8 @@ Usage: fsi.exe [script.fsx []] format). --optimize[+|-] Enable optimizations (Short form: -O) --tailcalls[+|-] Enable or disable tailcalls +--deterministic[+|-] Produce a deterministic assembly (including + module version GUID and timestamp) --crossoptimize[+|-] Enable or disable cross-module optimizations diff --git a/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40-nologo.437.1033.bsl b/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40-nologo.437.1033.bsl index 277713566..b7894f8a2 100644 --- a/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40-nologo.437.1033.bsl +++ b/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40-nologo.437.1033.bsl @@ -21,6 +21,8 @@ Usage: fsi.exe [script.fsx []] into the output file). --optimize[+|-] Enable optimizations (Short form: -O) --tailcalls[+|-] Enable or disable tailcalls +--deterministic[+|-] Produce a deterministic assembly (including + module version GUID and timestamp) --crossoptimize[+|-] Enable or disable cross-module optimizations diff --git a/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40.437.1033.bsl b/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40.437.1033.bsl index 8347077fe..754fb1e56 100644 --- a/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40.437.1033.bsl +++ b/tests/fsharpqa/Source/CompilerOptions/fsi/help/help40.437.1033.bsl @@ -23,6 +23,8 @@ Usage: fsi.exe [script.fsx []] into the output file). --optimize[+|-] Enable optimizations (Short form: -O) --tailcalls[+|-] Enable or disable tailcalls +--deterministic[+|-] Produce a deterministic assembly (including + module version GUID and timestamp) --crossoptimize[+|-] Enable or disable cross-module optimizations diff --git a/tests/fsharpqa/Source/run.pl b/tests/fsharpqa/Source/run.pl index ca55b3ac0..faabcdbc4 100644 --- a/tests/fsharpqa/Source/run.pl +++ b/tests/fsharpqa/Source/run.pl @@ -160,7 +160,7 @@ if (exists($ENV{PRECMD})) { # and it will expanded into $FSC_PIPE before invoking it $_ = $ENV{PRECMD}; s/^\$FSC_PIPE/$FSC_PIPE/; - s/^\$FSI_PIPE/$FSI_PIPE/; + s/\$FSI_PIPE/$FSI_PIPE/g; s/^\$FSI32_PIPE/$FSI32_PIPE/; s/\$ISCFLAGS/$ISCFLAGS/; s/^\$CSC_PIPE/$CSC_PIPE/; diff --git a/tests/fsharpqa/Source/test.lst b/tests/fsharpqa/Source/test.lst index 6345ef46f..d4301fedb 100644 --- a/tests/fsharpqa/Source/test.lst +++ b/tests/fsharpqa/Source/test.lst @@ -66,6 +66,7 @@ CompilerOptions01,NoMT CompilerOptions\fsi\nologo CompilerOptions01,NoMT CompilerOptions\fsi\subsystemversion CompilerOptions01,NoMT CompilerOptions\fsi\times CompilerOptions02,NoMT CompilerOptions\fsi\exename +CompilerOptions01,NoMT,Determinism CompilerOptions\fsc\determinism Conformance01 Conformance\BasicGrammarElements\Constants -- GitLab