[wasm] Fix a race condition in adding payloads for helix (#65995)

* [wasm] Fix a race condition in adding payloads for helix

The issue shows up only in Wasm.Build.Tests for the
`TestUsingWorkloads!=true` case, as `clang --version` failing with:

```
/datadisks/disk1/work/A0100914/p/build/emsdk/upstream/bin/clang: error while loading shared libraries: libLLVM-13git.so: cannot open shared object file: No such file or directory (TaskId:231)
```

For this non-workloads testing case, we copy `emscripten` from the
system copy to the git checkout. And then use that directory for the
payload.

The build logs show that the missing file (`libLLVM-13git.so`) does get
copied as part of copying `/usr/local/emscripten/emsdk` to
`$(RepoRoot)/src/mono/wasm/emsdk`. But the file, and a few others seem
to be missing in the final helix payload.

We add `emsdk` to the payload for target path `build/emsdk`.

But a recent change also added `node` for this case with a target path
`build/emsdk/node`, with an overlapping path with `build/emsdk`. I
believe this is causing an issue where these directories are being
processed in parallel, and cause some files get missed.

This commit:

1. Add `node` only when needed (skip WBT for example);
2. Use a non-overlapping path for `node`, `build/emsdk-node`.

Fixes https://github.com/dotnet/runtime/issues/65956

* Change `_HelixLocalNodePath` evaluation order
Co-authored-by: NRadek Doulik <radekdoulik@gmail.com>

src/libraries/sendtohelix-wasm.targets

@@ -20,13 +20,22 @@
     <EmSdkDirForHelixPayload>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono', 'wasm', 'emsdk'))</EmSdkDirForHelixPayload>
 
     <NeedsWorkload Condition="'$(Scenario)' == 'BuildWasmApps'">true</NeedsWorkload>
-    <NeedsEMSDK Condition="'$(NeedsToBuildWasmAppsOnHelix)' == 'true' or ('$(Scenario)' == 'BuildWasmApps' and '$(TestUsingWorkloads)' != 'true')">true</NeedsEMSDK>
+    <NeedsEMSDK Condition="'$(NeedsToBuildWasmAppsOnHelix)' == 'true' or '$(Scenario)' == 'BuildWasmApps'">true</NeedsEMSDK>
     <NeedsEMSDKNode Condition="'$(Scenario)' == 'WasmTestOnNodeJs' or '$(Scenario)' == 'BuildWasmApps'">true</NeedsEMSDKNode>
     <NeedsToRunOnBrowser Condition="'$(Scenario)' == 'WasmTestOnBrowser' or '$(Scenario)' == 'BuildWasmApps'">true</NeedsToRunOnBrowser>
     <NeedsToRunOnBrowser Condition="'$(NeedsToRunOnBrowser)' == '' and '$(IsWasmDebuggerTests)' == 'true'">true</NeedsToRunOnBrowser>
 
     <IncludeXHarnessCli>true</IncludeXHarnessCli>
     <EnableXHarnessTelemetry>true</EnableXHarnessTelemetry>
+    <IncludeNodePayload Condition="'$(NeedsEMSDKNode)' == 'true' and '$(NeedsEMSDK)' != 'true'">true</IncludeNodePayload>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <_HelixLocalNodePath Condition="'$(NeedsEMSDKNode)' == 'true' and '$(WindowsShell)' != 'true'">$HELIX_CORRELATION_PAYLOAD/build/emsdk-node</_HelixLocalNodePath>
+    <_HelixLocalNodePath Condition="'$(NeedsEMSDKNode)' == 'true' and '$(WindowsShell)' == 'true'">%HELIX_CORRELATION_PAYLOAD%\build\emsdk-node</_HelixLocalNodePath>
+
+    <_HelixLocalNodePath Condition="'$(NeedsEMSDK)' == 'true' and '$(WindowsShell)' != 'true'">$HELIX_CORRELATION_PAYLOAD/build/emsdk/node</_HelixLocalNodePath>
+    <_HelixLocalNodePath Condition="'$(NeedsEMSDK)' == 'true' and '$(WindowsShell)' == 'true'">%HELIX_CORRELATION_PAYLOAD%\build\emsdk\node</_HelixLocalNodePath>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(WindowsShell)' != 'true'">
@@ -40,15 +49,6 @@
 
     <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="export PATH=$HELIX_CORRELATION_PAYLOAD/$(ChromeDriverDirName):$PATH" />
     <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="export PATH=$HELIX_CORRELATION_PAYLOAD/$(ChromiumDirName):$PATH" />
-
-    <!-- Fix symbolic links that are broken already on build machine and also in the correlation payload -->
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="export HELIX_NODEJS_VERSION=%24(ls $HELIX_CORRELATION_PAYLOAD/build/emsdk/node)" />
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="export HELIX_NODEJS_PATH=$HELIX_CORRELATION_PAYLOAD/build/emsdk/node/$HELIX_NODEJS_VERSION" />
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="export PATH=$HELIX_NODEJS_PATH/bin:$PATH" />
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="rm $HELIX_NODEJS_PATH/bin/npm" />
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="rm $HELIX_NODEJS_PATH/bin/npx" />
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="ln -s ../lib/node_modules/npm/bin/npm-cli.js $HELIX_NODEJS_PATH/bin/npm" />
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="ln -s ../lib/node_modules/npm/bin/npx-cli.js $HELIX_NODEJS_PATH/bin/npx" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(WindowsShell)' == 'true'">
@@ -62,12 +62,26 @@
 
     <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\$(ChromeDriverDirName)%3B%PATH%" />
     <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\$(ChromiumDirName)%3B%PATH%" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(NeedsEMSDKNode)' == 'true' and '$(WindowsShell)' != 'true'">
+    <!-- Fix symbolic links that are broken already on build machine and also in the correlation payload -->
+    <HelixPreCommand Include="export _HELIX_NODEJS_VERSION=%24(ls $(_HelixLocalNodePath))" />
+    <HelixPreCommand Include="export _HELIX_NODEJS_PATH=$(_HelixLocalNodePath)/$_HELIX_NODEJS_VERSION" />
+    <HelixPreCommand Include="export PATH=$_HELIX_NODEJS_PATH/bin:$PATH" />
+    <HelixPreCommand Include="rm $_HELIX_NODEJS_PATH/bin/npm" />
+    <HelixPreCommand Include="rm $_HELIX_NODEJS_PATH/bin/npx" />
+    <HelixPreCommand Include="ln -s ../lib/node_modules/npm/bin/npm-cli.js $_HELIX_NODEJS_PATH/bin/npm" />
+    <HelixPreCommand Include="ln -s ../lib/node_modules/npm/bin/npx-cli.js $_HELIX_NODEJS_PATH/bin/npx" />
+  </ItemGroup>
 
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="for /f %%i in ('dir %HELIX_CORRELATION_PAYLOAD%\build\emsdk\node /b') do set HELIX_NODEJS_VERSION=%%i" />
-    <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\build\emsdk\node\%HELIX_NODEJS_VERSION%\bin%3B%PATH%" />
+  <ItemGroup Condition="'$(NeedsEMSDKNode)' == 'true' and '$(WindowsShell)' == 'true'">
+    <HelixPreCommand Include="for /f %%i in ('dir $(_HelixLocalNodePath) /b') do set _HELIX_NODEJS_VERSION=%%i" />
+    <HelixPreCommand Include="set PATH=$(_HelixLocalNodePath)/%_HELIX_NODEJS_VERSION%/bin%3B%PATH%" />
   </ItemGroup>
 
   <PropertyGroup>
+
     <!--
       We are hosting the payloads for the WASM/browser on kestrel in the xharness process.
       We also run some network tests to this server and so, we are running it on both HTTP and HTTPS.
@@ -123,8 +137,9 @@
       <HelixCorrelationPayload Include="$(MonoTargetsTasksDir)"                  Destination="build/MonoTargetsTasks" />
     </ItemGroup>
 
-    <ItemGroup Condition="'$(NeedsEMSDKNode)' == 'true'">
-      <HelixCorrelationPayload Include="$(EmSdkDirForHelixPayload)node"          Destination="build/emsdk/node" />
+    <!-- copy node separately only if EMSDK is not being included -->
+    <ItemGroup Condition="'$(IncludeNodePayload)' == 'true'">
+      <HelixCorrelationPayload Include="$(EmSdkDirForHelixPayload)node"          Destination="build/emsdk-node" />
     </ItemGroup>
 
     <ItemGroup Condition="'$(Scenario)' == '' or '$(Scenario)' == 'normal' or '$(Scenario)' == 'WasmTestOnBrowser' or '$(Scenario)' == 'WasmTestOnNodeJS'">
@@ -154,7 +169,7 @@
     </PropertyGroup>
 
     <Message Condition="'$(NeedsEMSDK)' == 'true' or '$(NeedsEMSDKNode)' == 'true'" Importance="High" Text="Using emsdk: $(EmSdkDirForHelixPayload)" />
-    
+
     <ItemGroup Condition="'$(IsRunningLibraryTests)' == 'true'">
       <_WasmWorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip"                          Condition="'$(Scenario)' == 'WasmTestOnBrowser'" />
       <_WasmWorkItem Include="$(TestArchiveRoot)browserornodejs/**/*.zip"                      Condition="'$(Scenario)' == 'WasmTestOnBrowser'" />
@@ -208,7 +223,7 @@
   <!-- CI has emscripten provisioned in $(EMSDK_PATH) as `/usr/local/emscripten`. Because helix tasks will
        attempt to write a .payload file, we cannot use $(EMSDK_PATH) to package emsdk as a helix correlation
        payload. Instead, we copy over the files to a new directory `src/mono/wasm/emsdk` and use that. -->
-  <Target Name="StageEmSdkForHelix" Condition="('$(NeedsEMSDK)' == 'true' or '$(NeedsEMSDKNode)' == 'true') and !Exists($(EmSdkDirForHelixPayload))">
+  <Target Name="StageEmSdkForHelix" Condition="('$(NeedsEMSDK)' == 'true' or '$(IncludeNodePayload)' == 'true') and !Exists($(EmSdkDirForHelixPayload))">
     <Error Condition="'$(EMSDK_PATH)' == '' or !Exists($(EMSDK_PATH))"
            Text="Could not find emscripten sdk in EMSDK_PATH=$(EMSDK_PATH), needed to provision for running tests on helix" />