Add support in comhost tooling to embed type libraries. (#50986)

* Add support in comhost tooling to embed type libraries.

Sdk work will still be needed to enable developers to embed tlbs in their comhosts.

* Cleanup.

* PR feedback. Update validation to throw specific exception types for specific errors so the SDK can accurately report errors to the user.

src/installer/managed/Microsoft.NET.HostModel/ComHost/ComHost.cs

@@ -2,7 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Runtime.InteropServices;
 using System.Text;
 
 namespace Microsoft.NET.HostModel.ComHost
@@ -19,10 +21,12 @@ public class ComHost
         /// <param name="comHostSourceFilePath">The path of Apphost template, which has the place holder</param>
         /// <param name="comHostDestinationFilePath">The destination path for desired location to place, including the file name</param>
         /// <param name="clsidmapFilePath">The path to the *.clsidmap file.</param>
+        /// <param name="typeLibraries">Resource ids for tlbs and paths to the tlb files to be embedded.</param>
         public static void Create(
             string comHostSourceFilePath,
             string comHostDestinationFilePath,
-            string clsidmapFilePath)
+            string clsidmapFilePath,
+            IReadOnlyDictionary<int, string> typeLibraries = null)
         {
             var destinationDirectory = new FileInfo(comHostDestinationFilePath).Directory.FullName;
             if (!Directory.Exists(destinationDirectory))
@@ -44,6 +48,26 @@ public class ComHost
             using (ResourceUpdater updater = new ResourceUpdater(comHostDestinationFilePath))
             {
                 updater.AddResource(clsidMapBytes, (IntPtr)ClsidmapResourceType, (IntPtr)ClsidmapResourceId);
+                if (typeLibraries is not null)
+                {
+                    foreach (var typeLibrary in typeLibraries)
+                    {
+                        if (!ResourceUpdater.IsIntResource((IntPtr)typeLibrary.Key))
+                        {
+                            throw new InvalidTypeLibraryIdException(typeLibrary.Value, typeLibrary.Key);
+                        }
+
+                        try
+                        {
+                            byte[] tlbFileBytes = File.ReadAllBytes(typeLibrary.Value);
+                            updater.AddResource(tlbFileBytes, "typelib", (IntPtr)typeLibrary.Key);
+                        }
+                        catch (FileNotFoundException ex)
+                        {
+                            throw new TypeLibraryDoesNotExistException(typeLibrary.Value, ex);
+                        }
+                    }
+                }
                 updater.Update();
             }
         }

src/installer/managed/Microsoft.NET.HostModel/ComHost/InvalidTypeLibraryIdException.cs

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.NET.HostModel.ComHost
+{
+    /// <summary>
+    /// The provided resource id for the type library is unsupported.
+    /// </summary>
+    public class InvalidTypeLibraryIdException : Exception
+    {
+        public InvalidTypeLibraryIdException(string path, int id)
+        {
+            Path = path;
+            Id = id;
+        }
+
+        public string Path { get; }
+
+        public int Id { get; }
+    }
+}

src/installer/managed/Microsoft.NET.HostModel/ComHost/TypeLibraryDoesNotExistException.cs

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.NET.HostModel.ComHost
+{
+    /// <summary>
+    /// The specified type library path does not exist.
+    /// </summary>
+    public class TypeLibraryDoesNotExistException : Exception
+    {
+        public TypeLibraryDoesNotExistException(string path, Exception innerException)
+            :base($"Type library '{path}' does not exist.", innerException)
+        {
+            Path = path;
+        }
+
+        public string Path { get; }
+    }
+}

src/installer/managed/Microsoft.NET.HostModel/ResourceUpdater.cs

@@ -44,6 +44,16 @@ private sealed class Kernel32
                                                      [MarshalAs(UnmanagedType.LPArray, SizeParamIndex=5)] byte[] lpData,
                                                      uint cbData);
 
+            // Update a resource with data from a managed byte[]
+            [DllImport(nameof(Kernel32), SetLastError=true)]
+            [return: MarshalAs(UnmanagedType.Bool)]
+            public static extern bool UpdateResource(SafeUpdateHandle hUpdate,
+                                                     string lpType,
+                                                     IntPtr lpName,
+                                                     ushort wLanguage,
+                                                     [MarshalAs(UnmanagedType.LPArray, SizeParamIndex=5)] byte[] lpData,
+                                                     uint cbData);
+
             [DllImport(nameof(Kernel32), SetLastError=true)]
             [return: MarshalAs(UnmanagedType.Bool)]
             public static extern bool EndUpdateResource(SafeUpdateHandle hUpdate,
@@ -277,7 +287,7 @@ public ResourceUpdater AddResourcesFromPEImage(string peFile)
             return this;
         }
 
-        private static bool IsIntResource(IntPtr lpType)
+        internal static bool IsIntResource(IntPtr lpType)
         {
             return ((uint)lpType >> 16) == 0;
         }
@@ -308,6 +318,32 @@ public ResourceUpdater AddResource(byte[] data, IntPtr lpType, IntPtr lpName)
             return this;
         }
 
+        /// <summary>
+        /// Add a language-neutral integer resource from a byte[] with
+        /// a particular type and name. This will not modify the
+        /// target until Update() is called.
+        /// Throws an InvalidOperationException if Update() was already called.
+        /// </summary>
+        public ResourceUpdater AddResource(byte[] data, string lpType, IntPtr lpName)
+        {
+            if (hUpdate.IsInvalid)
+            {
+                ThrowExceptionForInvalidUpdate();
+            }
+
+            if (!IsIntResource(lpName))
+            {
+                throw new ArgumentException("AddResource can only be used with integer resource names");
+            }
+
+            if (!Kernel32.UpdateResource(hUpdate, lpType, lpName, Kernel32.LangID_LangNeutral_SublangNeutral, data, (uint)data.Length))
+            {
+                ThrowExceptionForLastWin32Error();
+            }
+
+            return this;
+        }
+
         /// <summary>
         /// Write the pending resource updates to the target PE
         /// file. After this, the ResourceUpdater no longer maintains

src/installer/tests/Assets/TestProjects/ComLibrary/ComLibrary.cs

@@ -12,10 +12,16 @@ public class UserDefinedAttribute : Attribute
     {
     }
 
+    [ComVisible(true)]
+    [Guid("27293cc8-7933-4fdf-9fde-653cbf9b55df")]
+    public interface IServer
+    {
+    }
+
     [UserDefined]
     [ComVisible(true)]
     [Guid("438968CE-5950-4FBC-90B0-E64691350DF5")]
-    public class Server
+    public class Server : IServer
     {
         public Server()
         {
@@ -28,6 +34,12 @@ public class NotComVisible
     {
     }
 
+    [ComVisible(true)]
+    [Guid("f7199267-9821-4f5b-924b-ab5246b455cd")]
+    public interface INested
+    {
+    }
+
     [ComVisible(true)]
     [Guid("36e75747-aecd-43bf-9082-1a605889c762")]
     public class ComVisible
@@ -35,7 +47,7 @@ public class ComVisible
         [UserDefined]
         [ComVisible(true)]
         [Guid("c82e4585-58bd-46e0-a76d-c0b6975e5984")]
-        public class Nested
+        public class Nested : INested
         {
         }
     }
@@ -46,7 +58,7 @@ internal class ComVisibleNonPublic
     {
         [ComVisible(true)]
         [Guid("8a0a7085-aca4-4651-9878-ca42747e2206")]
-        public class Nested
+        public class Nested : INested
         {
         }
     }

src/installer/tests/HostActivation.Tests/NativeHosting/Comhost.cs

 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Generic;
 using System.IO;
 using System.Reflection.Metadata;
 using System.Runtime.InteropServices;
@@ -119,6 +120,38 @@ public void ActivateClass_ValidateIErrorInfoResult()
             }
         }
 
+        [Fact]
+        public void LoadTypeLibraries()
+        {
+            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                // COM activation is only supported on Windows
+                return;
+            }
+
+            using (var fixture = sharedState.ComLibraryFixture.Copy())
+            {
+                var comHost = Path.Combine(
+                    fixture.TestProject.BuiltApp.Location,
+                    $"{ fixture.TestProject.AssemblyName }.comhost.dll");
+
+                string[] args = {
+                    "comhost",
+                    "typelib",
+                    "2",
+                    comHost,
+                    sharedState.ClsidString
+                };
+                CommandResult result = sharedState.CreateNativeHostCommand(args, fixture.BuiltDotnet.BinPath)
+                    .Execute();
+
+                result.Should().Pass()
+                    .And.HaveStdOutContaining("Loading default type library succeeded.")
+                    .And.HaveStdOutContaining("Loading type library 1 succeeded.")
+                    .And.HaveStdOutContaining("Loading type library 2 succeeded.");
+            }
+        }
+
         public class SharedTestState : SharedTestStateBase
         {
             public string ComHostPath { get; }
@@ -150,14 +183,23 @@ public SharedTestState()
                     }
                 }
 
-                // Use the locally built comhost to create a comhost with the embedded .clsidmap 
+                // Use the locally built comhost to create a comhost with the embedded .clsidmap
                 ComHostPath = Path.Combine(
                     ComLibraryFixture.TestProject.BuiltApp.Location,
                     $"{ ComLibraryFixture.TestProject.AssemblyName }.comhost.dll");
+
+                // Include the test type libraries in the ComHost tests.
+                var typeLibraries = new Dictionary<int, string>
+                {
+                    { 1, Path.Combine(RepoDirectories.Artifacts, "corehost_test", "Server.tlb") },
+                    { 2, Path.Combine(RepoDirectories.Artifacts, "corehost_test", "Nested.tlb") }
+                };
+
                 ComHost.Create(
                     Path.Combine(RepoDirectories.HostArtifacts, "comhost.dll"),
                     ComHostPath,
-                    clsidMapPath);
+                    clsidMapPath,
+                    typeLibraries);
             }
 
             protected override void Dispose(bool disposing)

src/native/corehost/test/CMakeLists.txt

@@ -3,3 +3,6 @@ add_subdirectory(mockcoreclr)
 add_subdirectory(mockhostfxr)
 add_subdirectory(mockhostpolicy)
 add_subdirectory(nativehost)
+if (CLR_CMAKE_TARGET_WIN32)
+    add_subdirectory(typelibs)
+endif()

src/native/corehost/test/nativehost/comhost_test.cpp

@@ -6,6 +6,7 @@
 #include <iostream>
 #include <future>
 #include <pal.h>
+#include <oleauto.h>
 
 namespace
 {
@@ -67,12 +68,42 @@ namespace
         return pal::pal_utf8string(clsid_str, &clsidVect);
     }
 
+    void log_hresult(HRESULT hr, std::ostream &ss)
+    {
+        if (FAILED(hr))
+            ss << "(" << std::hex << std::showbase << hr << ")";
+    }
+
     void log_activation(const char *clsid, int activationNumber, int total, HRESULT hr, std::ostream &ss)
     {
         ss << "Activation of " << clsid << (FAILED(hr) ? " failed. " : " succeeded. ") << activationNumber << " of " << total;
+        log_hresult(hr, ss);
+        ss << std::endl;
+    }
+
+    HRESULT load_typelib(const pal::string_t &typelib_path)
+    {
+        HRESULT hr;
+        ITypeLib* typelib = nullptr;
+        hr = LoadTypeLibEx(typelib_path.c_str(), REGKIND_NONE, &typelib);
         if (FAILED(hr))
-            ss << "(" << std::hex << std::showbase << hr << ")";
+            return hr;
 
+        typelib->Release();
+        return hr;
+    }
+
+    void log_typelib_load(int typelib_id, HRESULT hr, std::ostream &ss)
+    {
+        ss << "Loading type library " << typelib_id << (FAILED(hr) ? " failed. " : " succeeded. ");
+        log_hresult(hr, ss);
+        ss << std::endl;
+    }
+
+    void log_default_typelib_load(HRESULT hr, std::ostream &ss)
+    {
+        ss << "Loading default type library" << (FAILED(hr) ? " failed. " : " succeeded. ");
+        log_hresult(hr, ss);
         ss << std::endl;
     }
 }
@@ -165,3 +196,26 @@ bool comhost_test::errorinfo(const pal::string_t &comhost_path, const pal::strin
 
     return true;
 }
+
+bool comhost_test::typelib(const pal::string_t &comhost_path, int count)
+{
+    HRESULT hr;
+
+    hr = load_typelib(comhost_path);
+    log_default_typelib_load(hr, std::cout);
+    if (FAILED(hr))
+        return false;
+
+    for (int i = 1; i < count + 1; i++)
+    {
+        // The path format for a non-default embedded TLB is 'C:\file\path\to.exe\\2' where '2' is the resource name of the tlb to load.
+        // See https://docs.microsoft.com/windows/win32/api/oleauto/nf-oleauto-loadtypelib#remarks for documentation on the path format.
+        pal::stringstream_t tlb_path;
+        tlb_path << comhost_path << '\\' << i;
+        hr = load_typelib(tlb_path.str());
+        log_typelib_load(i, hr, std::cout);
+        if (FAILED(hr))
+            return false;
+    }
+    return true;
+}

src/native/corehost/test/nativehost/comhost_test.h

@@ -10,4 +10,6 @@ namespace comhost_test
     bool concurrent(const pal::string_t &comhost_path, const pal::string_t &clsid_str, int count);
 
     bool errorinfo(const pal::string_t &comhost_path, const pal::string_t &clsid_str, int count);
+
+    bool typelib(const pal::string_t &comhost_path, int count);
 }

src/native/corehost/test/nativehost/nativehost.cpp

@@ -403,6 +403,10 @@ int main(const int argc, const pal::char_t *argv[])
         {
             success = comhost_test::errorinfo(comhost_path, clsid_str, count);
         }
+        else if (pal::strcmp(scenario, _X("typelib")) == 0)
+        {
+            success = comhost_test::typelib(comhost_path, count);
+        }
 
         return success ? EXIT_SUCCESS : EXIT_FAILURE;
     }

src/native/corehost/test/typelibs/CMakeLists.txt

+
+# Get the current list of definitions to pass to midl
+get_compile_definitions(MIDL_DEFINITIONS)
+get_include_directories(MIDL_INCLUDE_DIRECTORIES)
+find_program(MIDL midl.exe)
+
+function(compile_idl idl_file tlb_out)
+    # Compile IDL file using MIDL
+    set(IDL_SOURCE ${idl_file})
+    get_filename_component(IDL_NAME ${IDL_SOURCE} NAME_WE)
+    set(tlb_out_local "${CMAKE_CURRENT_BINARY_DIR}/${IDL_NAME}.tlb")
+    set("${tlb_out}" "${tlb_out_local}" PARENT_SCOPE)
+
+    add_custom_command(
+        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${IDL_NAME}_i.c ${CMAKE_CURRENT_BINARY_DIR}/${IDL_NAME}.h ${tlb_out_local}
+        COMMAND ${MIDL} ${MIDL_INCLUDE_DIRECTORIES}
+            /h ${CMAKE_CURRENT_BINARY_DIR}/${IDL_NAME}.h ${MIDL_DEFINITIONS}
+            /out ${CMAKE_CURRENT_BINARY_DIR}
+            /tlb ${tlb_out_local}
+            ${IDL_SOURCE}
+        DEPENDS ${IDL_SOURCE}
+        COMMENT "Compiling ${IDL_SOURCE}")
+endfunction()
+
+compile_idl(${CMAKE_CURRENT_SOURCE_DIR}/Server.idl Server_tlb)
+compile_idl(${CMAKE_CURRENT_SOURCE_DIR}/Nested.idl Nested_tlb)
+
+add_custom_target(typelibs ALL DEPENDS "${Server_tlb}" "${Nested_tlb}")
+
+install(FILES "${Server_tlb}" "${Nested_tlb}" DESTINATION corehost_test)

src/native/corehost/test/typelibs/Nested.idl

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+import "oaidl.idl";
+import "ocidl.idl";
+
+[
+    object,
+    uuid(f7199267-9821-4f5b-924b-ab5246b455cd)
+]
+interface INested : IDispatch
+{
+};
+
+[
+    uuid(f7c46a13-a1fc-4bf1-a61d-4502215c24e9)
+]
+library ComLibrary
+{
+    importlib("stdole2.tlb");
+    [
+        uuid(c82e4585-58bd-46e0-a76d-c0b6975e5984)
+    ]
+    coclass ComVisible_Nested
+    {
+        [default] interface INested;
+    }
+}

src/native/corehost/test/typelibs/Server.idl

+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+import "oaidl.idl";
+import "ocidl.idl";
+
+[
+    object,
+    uuid(27293cc8-7933-4fdf-9fde-653cbf9b55df)
+]
+interface IServer : IDispatch
+{
+};
+
+[
+    uuid(20151109-a0e8-46ae-b28e-8ff2c0e72166)
+]
+library ComLibrary
+{
+    importlib("stdole2.tlb");
+    [
+        uuid(438968CE-5950-4FBC-90B0-E64691350DF5)
+    ]
+    coclass Server
+    {
+        [default] interface IServer;
+    }
+}