[wasm] HTTP tests on NodeJS (#65303)

- Use `node-fetch` and `node-abort-controller` polyfills.
- Always use `node_fs` on node in `fetch_like` to get runtime assets.
- Disable tests that are not supported on node.

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Cancellation.cs

@@ -195,6 +195,7 @@ public async Task GetAsync_CancelDuringResponseBodyReceived_Buffered_TaskCancele
 
         [Theory]
         [MemberData(nameof(ThreeBools))]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/65429", typeof(PlatformDetection), nameof(PlatformDetection.IsNodeJS))]
         public async Task GetAsync_CancelDuringResponseBodyReceived_Unbuffered_TaskCanceledQuickly(bool chunkedTransfer, bool connectionClose, bool readOrCopyToAsync)
         {
             if (LoopbackServerFactory.Version >= HttpVersion20.Value && (chunkedTransfer || connectionClose))

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.cs

@@ -449,8 +449,11 @@ public async Task PostAsync_ManyDifferentRequestHeaders_SentCorrectly()
                     request.Headers.Referrer = new Uri("http://en.wikipedia.org/wiki/Main_Page");
                     request.Headers.TE.Add(new TransferCodingWithQualityHeaderValue("trailers"));
                     request.Headers.TE.Add(new TransferCodingWithQualityHeaderValue("deflate"));
-                    request.Headers.Trailer.Add("MyTrailer");
-                    request.Headers.TransferEncoding.Add(new TransferCodingHeaderValue("chunked"));
+                    if (PlatformDetection.IsNotNodeJS) 
+                    {
+                        request.Headers.Trailer.Add("MyTrailer");
+                        request.Headers.TransferEncoding.Add(new TransferCodingHeaderValue("chunked"));
+                    }
                     if (PlatformDetection.IsNotBrowser)
                     {
                         request.Headers.UserAgent.Add(new ProductInfoHeaderValue(new ProductHeaderValue("Mozilla", "5.0")));
@@ -465,8 +468,11 @@ public async Task PostAsync_ManyDifferentRequestHeaders_SentCorrectly()
                     request.Headers.Add("X-Requested-With", "XMLHttpRequest");
                     request.Headers.Add("DNT", "1 (Do Not Track Enabled)");
                     request.Headers.Add("X-Forwarded-For", "client1");
-                    request.Headers.Add("X-Forwarded-For", "proxy1");
-                    request.Headers.Add("X-Forwarded-For", "proxy2");
+                    if (PlatformDetection.IsNotNodeJS) 
+                    {
+                        request.Headers.Add("X-Forwarded-For", "proxy1");
+                        request.Headers.Add("X-Forwarded-For", "proxy2");
+                    }
                     request.Headers.Add("X-Forwarded-Host", "en.wikipedia.org:8080");
                     request.Headers.Add("X-Forwarded-Proto", "https");
                     request.Headers.Add("Front-End-Https", "https");
@@ -477,7 +483,10 @@ public async Task PostAsync_ManyDifferentRequestHeaders_SentCorrectly()
                     request.Headers.Add("X-UIDH", "...");
                     request.Headers.Add("X-Csrf-Token", "i8XNjC4b8KVok4uw5RftR38Wgp2BFwql");
                     request.Headers.Add("X-Request-ID", "f058ebd6-02f7-4d3f-942e-904344e8cde5");
-                    request.Headers.Add("X-Request-ID", "f058ebd6-02f7-4d3f-942e-904344e8cde5");
+                    if (PlatformDetection.IsNotNodeJS) 
+                    {
+                        request.Headers.Add("X-Request-ID", "f058ebd6-02f7-4d3f-942e-904344e8cde5");
+                    }
                     request.Headers.Add("X-Empty", "");
                     request.Headers.Add("X-Null", (string)null);
                     request.Headers.Add("X-Underscore_Name", "X-Underscore_Name");
@@ -515,7 +524,10 @@ public async Task PostAsync_ManyDifferentRequestHeaders_SentCorrectly()
                         Assert.Equal($"Basic {authSafeValue}", requestData.GetSingleHeaderValue("Proxy-Authorization"));
                         Assert.Equal("Mozilla/5.0", requestData.GetSingleHeaderValue("User-Agent"));
                         Assert.Equal("http://en.wikipedia.org/wiki/Main_Page", requestData.GetSingleHeaderValue("Referer"));
-                        Assert.Equal("MyTrailer", requestData.GetSingleHeaderValue("Trailer"));
+                        if (PlatformDetection.IsNotNodeJS) 
+                        {
+                            Assert.Equal("MyTrailer", requestData.GetSingleHeaderValue("Trailer"));
+                        }
                         Assert.Equal("1.0 fred, 1.1 example.com (Apache/1.1)", requestData.GetSingleHeaderValue("Via"));
                         Assert.Equal("1 (Do Not Track Enabled)", requestData.GetSingleHeaderValue("DNT"));
                     }
@@ -531,7 +543,15 @@ public async Task PostAsync_ManyDifferentRequestHeaders_SentCorrectly()
                     Assert.Equal("bytes=500-999", requestData.GetSingleHeaderValue("Range"));
                     Assert.Equal("199 - \"Miscellaneous warning\"", requestData.GetSingleHeaderValue("Warning"));
                     Assert.Equal("XMLHttpRequest", requestData.GetSingleHeaderValue("X-Requested-With"));
-                    Assert.Equal("client1, proxy1, proxy2", requestData.GetSingleHeaderValue("X-Forwarded-For"));
+                    if (PlatformDetection.IsNotNodeJS) 
+                    {
+                        Assert.Equal("client1, proxy1, proxy2", requestData.GetSingleHeaderValue("X-Forwarded-For"));
+                    }
+                    else
+                    {
+                        // node-fetch polyfill doesn't support combining multiple header values
+                        Assert.Equal("client1", requestData.GetSingleHeaderValue("X-Forwarded-For"));
+                    }
                     Assert.Equal("en.wikipedia.org:8080", requestData.GetSingleHeaderValue("X-Forwarded-Host"));
                     Assert.Equal("https", requestData.GetSingleHeaderValue("X-Forwarded-Proto"));
                     Assert.Equal("https", requestData.GetSingleHeaderValue("Front-End-Https"));
@@ -540,7 +560,15 @@ public async Task PostAsync_ManyDifferentRequestHeaders_SentCorrectly()
                     Assert.Equal("http://wap.samsungmobile.com/uaprof/SGH-I777.xml", requestData.GetSingleHeaderValue("X-Wap-Profile"));
                     Assert.Equal("...", requestData.GetSingleHeaderValue("X-UIDH"));
                     Assert.Equal("i8XNjC4b8KVok4uw5RftR38Wgp2BFwql", requestData.GetSingleHeaderValue("X-Csrf-Token"));
-                    Assert.Equal("f058ebd6-02f7-4d3f-942e-904344e8cde5, f058ebd6-02f7-4d3f-942e-904344e8cde5", requestData.GetSingleHeaderValue("X-Request-ID"));
+                    if (PlatformDetection.IsNotNodeJS) 
+                    {
+                        Assert.Equal("f058ebd6-02f7-4d3f-942e-904344e8cde5, f058ebd6-02f7-4d3f-942e-904344e8cde5", requestData.GetSingleHeaderValue("X-Request-ID"));
+                    }
+                    else 
+                    {
+                        // node-fetch polyfill doesn't support combining multiple header values
+                        Assert.Equal("f058ebd6-02f7-4d3f-942e-904344e8cde5", requestData.GetSingleHeaderValue("X-Request-ID"));
+                    }
                     Assert.Equal("", requestData.GetSingleHeaderValue("X-Null"));
                     Assert.Equal("", requestData.GetSingleHeaderValue("X-Empty"));
                     Assert.Equal("X-Underscore_Name", requestData.GetSingleHeaderValue("X-Underscore_Name"));
@@ -938,6 +966,7 @@ public async Task SendAsync_ReadFromSlowStreamingServer_PartialDataReturned()
         [InlineData(false, true)]
         [InlineData(false, false)]
         [InlineData(null, false)]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/65429", typeof(PlatformDetection), nameof(PlatformDetection.IsNodeJS))]
         public async Task ReadAsStreamAsync_HandlerProducesWellBehavedResponseStream(bool? chunked, bool enableWasmStreaming)
         {
             if (IsWinHttpHandler && UseVersion >= HttpVersion20.Value)

src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs

@@ -92,6 +92,8 @@ public static partial class PlatformDetection
         public static bool IsBrowserDomSupportedOrNotBrowser => IsNotBrowser || IsBrowserDomSupported;
         public static bool IsNotBrowserDomSupported => !IsBrowserDomSupported;
         public static bool IsWebSocketSupported => IsEnvironmentVariableTrue("IsWebSocketSupported");
+        public static bool IsNodeJS => IsEnvironmentVariableTrue("IsNodeJS");
+        public static bool IsNotNodeJS => !IsNodeJS;
         public static bool LocalEchoServerIsNotAvailable => !LocalEchoServerIsAvailable;
         public static bool LocalEchoServerIsAvailable => IsBrowser;
 

src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http1.cs

@@ -16,7 +16,7 @@ public class HttpClientHandlerTest_Http1 : HttpClientHandlerTestBase
     {
         public HttpClientHandlerTest_Http1(ITestOutputHelper output) : base(output) { }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotNodeJS))]
         public async Task SendAsync_HostHeader_First()
         {
             // RFC 7230  3.2.2.  Field Order

src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj

@@ -20,12 +20,35 @@
   </ItemGroup>
 
   <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <!-- This doesn't run on V8 because it lacks websocket support -->
     <Scenario>WasmTestOnBrowser</Scenario>
-    <TestArchiveTestsRoot>$(TestArchiveRoot)browseronly/</TestArchiveTestsRoot>
+    <TestArchiveTestsRoot>$(TestArchiveRoot)browserornodejs/</TestArchiveTestsRoot>
     <TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
     <DefineConstants>$(DefineConstants);TARGET_BROWSER</DefineConstants>
   </PropertyGroup>
 
+  <ItemGroup>
+    <WasmExtraFilesToDeploy Include="package.json" />
+    <WasmExtraFilesToDeploy Include="package-lock.json" />
+  </ItemGroup>
+
+  <Target Name="ProvideNpmRestoreScripts" BeforeTargets="GenerateRunScript">
+    <ItemGroup Condition="'$(OS)' != 'Windows_NT'">
+      <!-- WebSocket tests use self-signed certificates for wss protocol that are refused by NodeJS -->
+      <SetScriptCommands Include="if [[ &quot;$SCENARIO&quot; == &quot;WasmTestOnNodeJs&quot; || &quot;$SCENARIO&quot; == &quot;wasmtestonnodejs&quot; ]]; then export NODE_TLS_REJECT_UNAUTHORIZED=0; fi" />
+      <SetScriptCommands Include="if [[ &quot;$SCENARIO&quot; == &quot;WasmTestOnNodeJs&quot; || &quot;$SCENARIO&quot; == &quot;wasmtestonnodejs&quot; ]]; then export WasmXHarnessMonoArgs=&quot;$WasmXHarnessMonoArgs --setenv=NPM_MODULES=ws:WebSocket,node-fetch,node-abort-controller&quot;; fi" />
+      <!-- Restore NPM packages -->
+      <RunScriptCommands Include="if [[ &quot;$SCENARIO&quot; == &quot;WasmTestOnNodeJs&quot; || &quot;$SCENARIO&quot; == &quot;wasmtestonnodejs&quot; ]]; then npm ci; fi" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(OS)' == 'Windows_NT'">
+      <!-- WebSocket tests use self-signed certificates for wss protocol that are refused by NodeJS -->
+      <SetScriptCommands Include="if /I [%SCENARIO%]==[WasmTestOnNodeJS] ( set &quot;NODE_TLS_REJECT_UNAUTHORIZED=0&quot; )" />
+      <SetScriptCommands Include="if /I [%SCENARIO%]==[WasmTestOnNodeJS] ( set &quot;WasmXHarnessMonoArgs=%WasmXHarnessMonoArgs% --setenv=NPM_MODULES^=ws:WebSocket,node-fetch,node-abort-controller&quot; )" />
+      <!-- Restore NPM packages -->
+      <RunScriptCommands Include="if /I [%SCENARIO%]==[WasmTestOnNodeJS] ( call npm ci )" />
+    </ItemGroup>
+  </Target>
+
   <Import Condition="'$(TargetOS)' == 'Browser'" Project="$(CommonTestPath)System/Net/Prerequisites/LocalEchoServer.props" />
 
   <!-- Browser specific files -->

src/libraries/System.Net.Http/tests/FunctionalTests/package-lock.json

+{
+  "name": "system.net.websockets.client.tests",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "system.net.websockets.client.tests",
+      "dependencies": {
+        "node-abort-controller": "3.0.1",
+        "node-fetch": "2.6.7",
+        "ws": "8.4.0"
+      }
+    },
+    "node_modules/node-abort-controller": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz",
+      "integrity": "sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw=="
+    },
+    "node_modules/node-fetch": {
+      "version": "2.6.7",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
+      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+    },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
+    "node_modules/ws": {
+      "version": "8.4.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz",
+      "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": "^5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    }
+  },
+  "dependencies": {
+    "node-abort-controller": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz",
+      "integrity": "sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw=="
+    },
+    "node-fetch": {
+      "version": "2.6.7",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
+      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+      "requires": {
+        "whatwg-url": "^5.0.0"
+      }
+    },
+    "tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+    },
+    "webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+    },
+    "whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+      "requires": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
+    "ws": {
+      "version": "8.4.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz",
+      "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==",
+      "requires": {}
+    }
+  }
+}

src/libraries/System.Net.Http/tests/FunctionalTests/package.json

+{
+  "name": "system.net.websockets.client.tests",
+  "private": true,
+  "dependencies": {
+    "node-abort-controller": "3.0.1",
+    "node-fetch": "2.6.7",
+    "ws": "8.4.0"
+  }
+}

src/mono/wasm/runtime/polyfills.ts

@@ -5,10 +5,7 @@ let node_url: any | undefined = undefined;
 
 export async function fetch_like(url: string): Promise<Response> {
     try {
-        if (typeof (globalThis.fetch) === "function") {
-            return globalThis.fetch(url, { credentials: "same-origin" });
-        }
-        else if (ENVIRONMENT_IS_NODE) {
+        if (ENVIRONMENT_IS_NODE) {
             if (!node_fs) {
                 const node_require = await requirePromise;
                 node_url = node_require("url");
@@ -26,6 +23,9 @@ export async function fetch_like(url: string): Promise<Response> {
                 json: () => JSON.parse(arrayBuffer)
             };
         }
+        else if (typeof (globalThis.fetch) === "function") {
+            return globalThis.fetch(url, { credentials: "same-origin" });
+        }
         else if (typeof (read) === "function") {
             // note that it can't open files with unicode names, like Straße.xml
             // https://bugs.chromium.org/p/v8/issues/detail?id=12541

src/mono/wasm/test-main.js

@@ -316,6 +316,7 @@ function processArguments(incomingArguments) {
 
     // cheap way to let the testing infrastructure know we're running in a browser context (or not)
     setenv["IsBrowserDomSupported"] = is_browser.toString().toLowerCase();
+    setenv["IsNodeJS"] = is_node.toString().toLowerCase();
 
     console.log("Application arguments: " + incomingArguments.join(' '));
 
@@ -360,13 +361,23 @@ if (is_node) {
     const modulesToLoad = processedArguments.setenv["NPM_MODULES"];
     if (modulesToLoad) {
         modulesToLoad.split(',').forEach(module => {
-            const parts = module.split(':');
-
-            let message = `Loading npm '${parts[0]}'`;
-            const moduleExport = require(parts[0]);
-            if (parts.length == 2) {
-                message += ` and attaching to global as '${parts[1]}'.`;
-                globalThis[parts[1]] = moduleExport;
+            const { 0:moduleName, 1:globalAlias } = module.split(':');
+
+            let message = `Loading npm '${moduleName}'`;
+            let moduleExport = require(moduleName);
+            
+            if (globalAlias) {
+                message += ` and attaching to global as '${globalAlias}'`;
+                globalThis[globalAlias] = moduleExport;
+            } else if(moduleName == "node-fetch") {
+                message += ' and attaching to global';
+                globalThis.fetch = moduleExport.default;
+                globalThis.Headers = moduleExport.Headers;
+                globalThis.Request = moduleExport.Request;
+                globalThis.Response = moduleExport.Response;
+            } else if(moduleName == "node-abort-controller") {
+                message += ' and attaching to global';
+                globalThis.AbortController = moduleExport.AbortController;
             }
 
             console.log(message);