Embed F# metadata trimming rules in compiled assemblies (#20527)

Author: T-Gro

docs/release-notes/.FSharp.Compiler.Service/11.0.100.md

@@ -13,6 +13,7 @@
 * Fix exponential (2^N) compile time in pattern matching with shared guards and partial active patterns. ([Issue #18425](https://github.com/dotnet/fsharp/issues/18425), [PR #20244](https://github.com/dotnet/fsharp/pull/20244))
 * Fix incorrect Debug lowering of inline builders that compose low-level resumable state machines. ([Issue #20466](https://github.com/dotnet/fsharp/issues/20466), [PR #20469](https://github.com/dotnet/fsharp/pull/20469))
 * Fix incorrect `StructLayout(Size = 1)` emission for data-less struct unions where the compiler-generated tag field makes the actual runtime size larger. ([PR #19759](https://github.com/dotnet/fsharp/pull/19759))
+* Embed F# metadata-removal rules before resource collection, preserve custom rules, and combine generated rules when linking assemblies. ([PR #20527](https://github.com/dotnet/fsharp/pull/20527))
 * Fix FS0750 "This construct may only be used within computation expressions" incorrectly raised for `let!`/`use!`/`do!` appearing in the right-hand side of a plain `let` binding inside a computation expression. The right-hand side is now desugared as a nested computation of the same builder whose result is bound with `let!`, keeping its bindings correctly scoped. ([Issue #19457](https://github.com/dotnet/fsharp/issues/19457), [PR #19868](https://github.com/dotnet/fsharp/pull/19868))
 * Fix `use` bindings for `IAsyncDisposable` in `dotnet fsi` task expressions by shipping the `net10.0` FSharp.Core in the SDK toolset. ([Issue #14454](https://github.com/dotnet/fsharp/issues/14454), [PR #20555](https://github.com/dotnet/fsharp/pull/20555))
 * Stop leaking a `System.Diagnostics.Metrics.MeterListener` per `Cache` in DEBUG builds. Each cache created a `CacheMetrics.CacheMetricsListener` (which starts a `MeterListener` registered in the process-global metrics registry) and never disposed it, so listeners accumulated for the lifetime of the process. Because every cache hit/miss/add published to all registered listeners, the per-operation cost grew linearly with the number of leaked listeners, so repeated checks (and Debug FCS test runs) slowed down over time. The per-cache `CacheMetricsListener` and the per-instance `cacheId` tag are removed; `DebugDisplay` and tests now read the existing name-aggregated stats populated by the single `ListenToAll` listener, so no per-cache listener is created and no per-operation cost is added. ([PR #19995](https://github.com/dotnet/fsharp/pull/19995))
@@ -88,6 +89,7 @@
 * Avoid duplicate parameter names in closure constructors. (Issue [#17692](https://github.com/dotnet/fsharp/issues/17692), [PR #19339](https://github.com/dotnet/fsharp/pull/19339))
 * Improve let-rec codegen: reorder bindings to allocate lambda closures before non-lambda values that reference them. ([PR #19339](https://github.com/dotnet/fsharp/pull/19339))
 * Fix `YieldFromFinal`/`ReturnFromFinal` being incorrectly called in non-tail positions (`for`, `use`, `use!`, `try/with` handler). ([Issue #19402](https://github.com/dotnet/fsharp/issues/19402), [PR #19403](https://github.com/dotnet/fsharp/pull/19403))
+* Preserve commas in embedded resource paths, including generated trimming substitutions under custom intermediate directories. ([PR #20527](https://github.com/dotnet/fsharp/pull/20527))
 * Fixed how the source ranges of warn directives are reported (as trivia) in the parser output (by not reporting leading spaces). ([Issue #19405](https://github.com/dotnet/fsharp/issues/19405), [PR #19408]((https://github.com/dotnet/fsharp/pull/19408)))
 * Fix UoM value type `ToString()` returning garbage values when `--checknulls+` is enabled, caused by double address-taking in codegen. ([Issue #19435](https://github.com/dotnet/fsharp/issues/19435), [PR #19440](https://github.com/dotnet/fsharp/pull/19440))
 * Fix accessibility and type-matching for extension method lookups. ([Issue #19349](https://github.com/dotnet/fsharp/issues/19349), [PR #19536](https://github.com/dotnet/fsharp/pull/19536))

src/Compiler/Driver/CompilerConfig.fs

@@ -1098,10 +1098,22 @@ type TcConfigBuilder =
         tcConfigB.pathMap <- tcConfigB.pathMap |> PathMap.addMapping oldPrefix newPrefix
 
     static member SplitCommandLineResourceInfo(ri: string) =
-        let p = ri.IndexOf ','
+        let quoteEnd =
+            if ri.StartsWithOrdinal("\"") then
+                ri.IndexOf('"', 1)
+            else
+                -1
+
+        let p = ri.IndexOf(',', quoteEnd + 1)
+        let file = if p <> -1 then ri.Substring(0, p) else ri
+
+        let file =
+            if quoteEnd > 0 && quoteEnd = file.Length - 1 then
+                file.Substring(1, file.Length - 2)
+            else
+                file
 
         if p <> -1 then
-            let file = String.sub ri 0 p
             let rest = String.sub ri (p + 1) (String.length ri - p - 1)
             let p = rest.IndexOf ','
 
@@ -1118,7 +1130,7 @@ type TcConfigBuilder =
             else
                 file, rest, ILResourceAccess.Public
         else
-            ri, FileSystemUtils.fileNameOfPath ri, ILResourceAccess.Public
+            file, FileSystemUtils.fileNameOfPath file, ILResourceAccess.Public
 
 //----------------------------------------------------------------------------
 // TcConfig

src/Compiler/Driver/CompilerConfig.fsi

@@ -567,6 +567,7 @@ type TcConfigBuilder =
 
     member AddPathMapping: oldPrefix: string * newPrefix: string -> unit
 
+    /// Parse file[,name[,access]], allowing a quoted file path to contain commas.
     static member SplitCommandLineResourceInfo: string -> string * string * ILResourceAccess
 
     // Directories to start probing in for native DLLs for FSI dynamic loading

src/Compiler/Driver/CompilerImports.fsi

@@ -45,6 +45,8 @@ val IsOptimizationDataResource: ILResource -> bool
 /// Determine if an IL resource attached to an F# assembly is an F# optimization data resource (data stream B)
 val IsOptimizationDataResourceB: ILResource -> bool
 
+val GetNameOfILModule: ILModuleDef -> string
+
 /// Determine if an IL resource attached to an F# assembly is an F# quotation data resource for reflected definitions
 val IsReflectedDefinitionsResource: ILResource -> bool
 

src/Compiler/Driver/ILLinkSubstitutions.fs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
+
+module internal FSharp.Compiler.ILLinkSubstitutions
+
+open System.Xml.Linq
+
+let names (assemblyName: string) =
+    seq {
+        for kind in [ "Signature"; "Optimization" ] do
+            for suffix in [ "Data"; "CompressedData"; "DataB"; "CompressedDataB"; "Info" ] do
+                yield $"FSharp{kind}{suffix}.{assemblyName}"
+    }
+
+let document (assemblyName: string) (names: seq<string>) =
+    let x = XName.Get
+    let assembly = XElement(x "assembly", XAttribute(x "fullname", assemblyName))
+
+    for name in names do
+        assembly.Add(XElement(x "resource", XAttribute(x "name", name), XAttribute(x "action", "remove"), ""))
+
+    XElement(x "linker", assembly)

src/Compiler/Driver/StaticLinking.fs

@@ -4,6 +4,7 @@
 module internal FSharp.Compiler.StaticLinking
 
 open System
+open System.Xml.Linq
 open Internal.Utilities.Collections
 open Internal.Utilities.Library
 open Internal.Utilities.Library.Extras
@@ -23,6 +24,57 @@ open FSharp.Compiler.TypedTree
 open FSharp.Compiler.TypeProviders
 #endif
 
+let linkMetadataResources assemblyName (resources: (string * ILResource) list) =
+    let xname = XName.Get
+    let prefixes = ILLinkSubstitutions.names "" |> Seq.toList
+
+    let generated, retained =
+        resources
+        |> List.splitChoose (fun (owner, resource) ->
+            match resource.Location with
+            | ILResourceLocation.Local _ when resource.Name.Equals("ILLink.Substitutions.xml", StringComparison.OrdinalIgnoreCase) ->
+                try
+                    use stream = resource.GetBytes().AsStream()
+                    let xml = XElement.Load stream
+
+                    let names =
+                        [
+                            for rule in xml.Descendants(xname "resource") do
+                                if rule.IsEmpty then
+                                    rule.Value <- ""
+
+                                for name in rule.Attributes(xname "name") do
+                                    if List.exists name.Value.StartsWithOrdinal prefixes then
+                                        name.Value
+                        ]
+
+                    match resource.CustomAttrs.AsList(), XNode.DeepEquals(xml, ILLinkSubstitutions.document owner names) with
+                    | [], true -> Choice1Of2(resource, names)
+                    | _ -> Choice2Of2(true, resource)
+                with :? System.Xml.XmlException ->
+                    Choice2Of2(true, resource)
+            | _ -> Choice2Of2(false, resource))
+
+    let resources = List.map snd retained
+
+    match generated with
+    | (resource, _) :: _ when not (List.exists fst retained) ->
+        let present = resources |> List.map _.Name |> Set.ofList
+
+        let names =
+            generated |> List.collect snd |> List.filter present.Contains |> List.distinct
+
+        resources
+        @ [
+            if not names.IsEmpty then
+                let xml = ILLinkSubstitutions.document assemblyName names
+
+                { resource with
+                    Location = ILResourceLocation.Local(ByteStorage.FromByteArray(System.Text.Encoding.UTF8.GetBytes(xml.ToString())))
+                }
+        ]
+    | _ -> resources
+
 // Handles TypeForwarding for the generated IL model
 type TypeForwarding(tcImports: TcImports) =
 
@@ -157,51 +209,30 @@ let StaticLinkILModules
             ]
 
         let savedResources =
-            let allResources =
-                [
-                    for ccu, m in dependentILModules do
-                        for r in m.Resources.AsList() do
-                            (ccu, r)
-                ]
-            // Don't save interface, optimization or resource definitions for provider-generated assemblies.
-            // These are "fake".
-            let isProvided (ccu: CcuThunk option) =
+            [
+                for ccu, m in dependentILModules do
+                    let provided =
 #if !NO_TYPEPROVIDERS
-                match ccu with
-                | Some c -> c.IsProviderGenerated
-                | None -> false
+                        ccu |> Option.exists _.IsProviderGenerated
 #else
-                ignore ccu
-                false
+                        ignore ccu
+                        false
 #endif
+                    for r in m.Resources.AsList() do
+                        let order, enabled =
+                            if IsSignatureDataResource r || IsSignatureDataResourceB r then
+                                0, tcConfig.GenerateSignatureData
+                            elif IsOptimizationDataResource r || IsOptimizationDataResourceB r then
+                                1, tcConfig.GenerateOptimizationData
+                            else
+                                2, true
 
-            // Save only the interface/optimization attributes of generated data
-            let intfDataResources, others =
-                allResources
-                |> List.partition (fun (_, r) -> IsSignatureDataResource r || IsSignatureDataResourceB r)
-
-            let intfDataResources =
-                [
-                    for ccu, r in intfDataResources do
-                        if tcConfig.GenerateSignatureData && not (isProvided ccu) then
-                            r
-                ]
-
-            let optDataResources, others =
-                others
-                |> List.partition (fun (_, r) -> IsOptimizationDataResource r || IsOptimizationDataResourceB r)
-
-            let optDataResources =
-                [
-                    for ccu, r in optDataResources do
-                        if tcConfig.GenerateOptimizationData && not (isProvided ccu) then
-                            r
-                ]
-
-            let otherResources = others |> List.map snd
-
-            let result = intfDataResources @ optDataResources @ otherResources
-            result
+                        if enabled && (order = 2 || not provided) then
+                            yield order, (GetNameOfILModule m, r)
+            ]
+            |> List.groupBy fst
+            |> List.sortBy fst
+            |> List.collect (snd >> List.map snd)
 
         let moduls = ilxMainModule :: (List.map snd dependentILModules)
 
@@ -248,7 +279,11 @@ let StaticLinkILModules
                                 ]
                         )
                     TypeDefs = mkILTypeDefs (topTypeDef :: List.concat normalTypeDefs)
-                    Resources = mkILResources (savedResources @ ilxMainModule.Resources.AsList())
+                    Resources =
+                        savedResources
+                        @ (ilxMainModule.Resources.AsList() |> List.map (fun r -> oldManifest.Name, r))
+                        |> linkMetadataResources oldManifest.Name
+                        |> mkILResources
                     NativeResources = savedNativeResources
                 }
 

src/Compiler/FSharp.Compiler.Service.fsproj

@@ -542,6 +542,7 @@
     <Compile Include="Driver\XmlDocFileWriter.fs" />
     <Compile Include="Driver\BinaryResourceFormats.fsi" />
     <Compile Include="Driver\BinaryResourceFormats.fs" />
+    <Compile Include="Driver\ILLinkSubstitutions.fs" />
     <Compile Include="Driver\StaticLinking.fsi" />
     <Compile Include="Driver\StaticLinking.fs" />
     <Compile Include="Driver\CreateILModule.fsi" />

src/Compiler/Utilities/FileSystem.fs

@@ -1093,9 +1093,11 @@ type ByteStorage(getByteMemory: unit -> ReadOnlyByteMemory) =
     static member FromByteMemory(bytes: ReadOnlyByteMemory) = ByteStorage(fun () -> bytes)
 
     static member FromByteMemoryAndCopy(bytes: ReadOnlyByteMemory, useBackingMemoryMappedFile: bool) =
+        let length = bytes.Length
+
         if useBackingMemoryMappedFile then
             match MemoryMappedFile.TryFromByteMemory(bytes) with
-            | Some mmf -> ByteStorage(fun () -> ByteMemory.FromMemoryMappedFile(mmf).AsReadOnly())
+            | Some mmf -> ByteStorage(fun () -> ByteMemory.FromMemoryMappedFile(mmf).Slice(0, length).AsReadOnly())
             | _ ->
                 let copiedBytes = ByteMemory.FromArray(bytes.ToArray()).AsReadOnly()
                 ByteStorage.FromByteMemory(copiedBytes)

src/FSharp.Build/FSharp.Build.fsproj

@@ -48,6 +48,7 @@
     <Compile Include="CreateFSharpManifestResourceName.fs" />
     <Compile Include="SubstituteText.fs" />
     <Compile Include="MapSourceRoots.fs" />
+    <Compile Include="../Compiler/Driver/ILLinkSubstitutions.fs" />
     <Compile Include="GenerateILLinkSubstitutions.fs" />
     <None Include="Microsoft.FSharp.Targets" CopyToOutputDirectory="PreserveNewest" />
     <None Include="Microsoft.Portable.FSharp.Targets" CopyToOutputDirectory="PreserveNewest" />

src/FSharp.Build/Fsc.fs

@@ -250,14 +250,16 @@ type public Fsc() as this =
 
         // Resources
         for item in resources do
+            let fileName =
+                if item.ItemSpec.IndexOf(',') >= 0 then
+                    $"\"{item.ItemSpec}\""
+                else
+                    item.ItemSpec
+
             match useStandardResourceNames with
             | true ->
-                builder.AppendSwitchIfNotNull(
-                    "--resource:",
-                    item.ItemSpec,
-                    [| item.GetMetadata("LogicalName"); item.GetMetadata("Access") |]
-                )
-            | false -> builder.AppendSwitchIfNotNull("--resource:", item.ItemSpec)
+                builder.AppendSwitchIfNotNull("--resource:", fileName, [| item.GetMetadata("LogicalName"); item.GetMetadata("Access") |])
+            | false -> builder.AppendSwitchIfNotNull("--resource:", fileName)
 
         // VersionFile
         builder.AppendSwitchIfNotNull("--versionfile:", versionFile)