[wasm] Add Timezone data generator as build task (#39913)

Add two build tasks:
1. DownloadTimeZoneData 
2. CreateWasmBundle (given directory bundle files into a data archive) 

Directory.Build.props

@@ -51,11 +51,13 @@
     <AppleAppBuilderDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'AppleAppBuilder', 'Debug', '$(NetCoreAppCurrent)'))</AppleAppBuilderDir>
     <AndroidAppBuilderDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'AndroidAppBuilder', 'Debug', '$(NetCoreAppCurrent)'))</AndroidAppBuilderDir>
     <WasmAppBuilderDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmAppBuilder', 'Debug', '$(NetCoreAppCurrent)', 'publish'))</WasmAppBuilderDir>
+    <CreateWasmBundleDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'CreateWasmBundle', 'Debug', '$(NetCoreAppCurrent)', 'publish'))</CreateWasmBundleDir>
     <MonoAOTCompilerDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppCurrent)'))</MonoAOTCompilerDir>
   
     <AppleAppBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(AppleAppBuilderDir)', 'AppleAppBuilder.dll'))</AppleAppBuilderTasksAssemblyPath>
     <AndroidAppBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(AndroidAppBuilderDir)', 'AndroidAppBuilder.dll'))</AndroidAppBuilderTasksAssemblyPath>
     <WasmAppBuilderTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WasmAppBuilderDir)', 'WasmAppBuilder.dll'))</WasmAppBuilderTasksAssemblyPath>
+    <CreateCreateWasmBundlesAssemblyPath>$([MSBuild]::NormalizePath('$(CreateWasmBundleDir)', 'CreateWasmBundle.dll'))</CreateCreateWasmBundlesAssemblyPath>
     <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath>
   </PropertyGroup>
 

eng/testing/tests.mobile.targets

@@ -151,6 +151,26 @@
       AssemblySearchPaths="@(AssemblySearchPaths)" />
   </Target>
 
+  <UsingTask TaskName="DownloadTimeZoneData"
+             AssemblyFile="$(CreateCreateWasmBundlesAssemblyPath)" />
+  <Target Condition="'$(TargetOS)' == 'Browser'" Name="LoadTimeZone" >
+    <PropertyGroup>
+      <TimeZoneDataVersion>2020a</TimeZoneDataVersion>
+    </PropertyGroup>
+    <DownloadTimeZoneData
+      InputDirectory="$(BundleDir)obj/data/input"
+      OutputDirectory="$(BundleDir)obj/data/output"
+      Version="$(TimeZoneDataVersion)"/>
+  </Target>
+
+  <UsingTask TaskName="CreateWasmBundle"
+             AssemblyFile="$(CreateCreateWasmBundlesAssemblyPath)" />
+  <Target Condition="'$(TargetOS)' == 'Browser'" Name="BundleWasmTestData" DependsOnTargets="LoadTimeZone">
+    <CreateWasmBundle
+      InputDirectory="$(BundleDir)obj/data/output"
+      OutputFileName="$(MicrosoftNetCoreAppRuntimePackRidDir)dotnet.timezones.blat" />
+  </Target>
+
   <Target Name="AddTestRunnersToPublishedFiles"
           AfterTargets="ComputeResolvedFilesToPublishList">
     <ItemGroup>
@@ -166,5 +186,5 @@
   <Target Name="PublishTestAsSelfContained"
           Condition="'$(IsCrossTargetingBuild)' != 'true'"
           AfterTargets="Build"
-          DependsOnTargets="Publish;BundleTestAppleApp;BundleTestAndroidApp;BundleTestWasmApp;ArchiveTests" />
+          DependsOnTargets="Publish;BundleTestAppleApp;BundleTestAndroidApp;BundleTestWasmApp;BundleWasmTestData;ArchiveTests" />
 </Project>

src/mono/wasm/Makefile

@@ -17,7 +17,7 @@ NATIVE_DIR?=$(OBJDIR)/native/net5.0-Browser-$(CONFIG)-wasm
 BUILDS_BIN_DIR?=$(BINDIR)/native/net5.0-Browser-$(CONFIG)-wasm
 ICU_LIBDIR?=
 
-all: build-native timezone-data icu-data
+all: build-native icu-data
 
 #
 # EMSCRIPTEN SETUP
@@ -112,9 +112,6 @@ clean:
 # Helper targets
 .PHONY: runtime
 
-timezone-data:
-	cp runtime/dotnet.timezones.blat $(BUILDS_BIN_DIR)
-
 icu-data:
 	cp $(ICU_LIBDIR)/icudt.dat $(BUILDS_BIN_DIR)
 
@@ -136,6 +133,9 @@ test-runner:
 app-builder:
 	$(DOTNET) build $(TOP)/tools-local/tasks/mobile.tasks/WasmAppBuilder
 
+bundle-task:
+	$(DOTNET) build $(TOP)/tools-local/tasks/mobile.tasks/CreateWasmBundle
+
 run-tests-v8-%:
 	PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:Configuration=$(CONFIG) /p:JSEngine=V8 $(MSBUILD_ARGS)
 run-tests-sm-%:

src/mono/wasm/wasm.targets

@@ -35,10 +35,28 @@
                            OutputPath="$(WasmPInvokeTablePath)" />
   </Target>
 
+  <UsingTask TaskName="DownloadTimeZoneData" AssemblyFile="$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'CreateWasmBundle', 'Debug', '$(NetCoreAppCurrent)', 'publish', 'CreateWasmBundle.dll'))"/>
+  <Target Name="LoadTimeZones" DependsOnTargets="CheckEnv">
+    <DownloadTimeZoneData
+      InputDirectory="$(ArtifactsObjDir)wasm/data/input"
+      OutputDirectory="$(ArtifactsObjDir)wasm/data/output"
+      Version="2020a" />
+  </Target>
+
+  <UsingTask TaskName="CreateWasmBundle" AssemblyFile="$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'CreateWasmBundle', 'Debug', '$(NetCoreAppCurrent)', 'publish', 'CreateWasmBundle.dll'))"/>
+  <Target Name="BundleWasm" DependsOnTargets="CheckEnv;LoadTimeZones">
+    <PropertyGroup>
+      <TimeZonesDataPath>$(NativeBinDir)dotnet.timezones.blat</TimeZonesDataPath>
+    </PropertyGroup>
+    <CreateWasmBundle
+      InputDirectory="$(ArtifactsObjDir)wasm/data/output"
+      OutputFileName="$(TimeZonesDataPath)" />
+  </Target>
+
   <!-- This is a documented target that is invoked by developers in their innerloop work. -->
   <Target Name="BuildWasmRuntimes"
           AfterTargets="Build"
-          DependsOnTargets="BuildPInvokeTable">
+          DependsOnTargets="BuildPInvokeTable;BundleWasm">
     <Exec Command="make -C $(MonoProjectRoot)wasm all SHELL=/bin/bash BINDIR=$(ArtifactsBinDir) MONO_BIN_DIR=$(MonoArtifactsPath) OBJDIR=$(ArtifactsObjDir) BUILDS_BIN_DIR=$(NativeBinDir) NATIVE_DIR=$(ArtifactsObjDir)/native/$(NetCoreAppCurrent)-$(TargetOS)-$(Configuration)-$(TargetArchitecture) CONFIG=$(Configuration) PINVOKE_TABLE=$(WasmPInvokeTablePath) ICU_LIBDIR=$(PkgMicrosoft_NETCore_Runtime_ICU_Transport)/runtimes/browser-wasm/native/lib"
           IgnoreStandardErrorWarningFormat="true" />
 

tools-local/tasks/mobile.tasks/CreateWasmBundle/CreateWasmBundle.cs

+using System;
+using System.Buffers.Binary;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Net;
+using System.Reflection;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+public class CreateWasmBundle : Task
+{
+    [Required]
+    public string? InputDirectory { get; set; }
+
+    [Required]
+    public string? OutputFileName { get; set; } 
+
+    private (byte[] json_bytes, MemoryStream stream) EnumerateData() 
+    {
+        var indices = new List<object[]>();
+        var stream = new MemoryStream();
+        
+        var directoryInfo = new DirectoryInfo(InputDirectory!);
+        
+        foreach (var entry in directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories)) 
+        {
+            var relativePath = Path.GetRelativePath(InputDirectory!, entry.FullName);
+            indices.Add(new object[] { relativePath, entry.Length });
+
+            using (var readStream = entry.OpenRead())
+                readStream.CopyTo(stream);
+        }
+        
+        stream.Position = 0;
+        var jsonBytes = JsonSerializer.SerializeToUtf8Bytes(indices);
+        
+        return (jsonBytes, stream);
+    }
+
+    public override bool Execute()
+    {
+        if (!Directory.Exists(InputDirectory)) 
+        {
+            Log.LogError($"Input directory '{InputDirectory}' does not exist");
+            return false;
+        }
+
+        (byte[] json_bytes, MemoryStream stream) data = EnumerateData();
+        
+        using (var file = File.Open(OutputFileName!, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite))
+        {
+            var lengthBytes = new byte[4];
+            var magicBytes = Encoding.ASCII.GetBytes("talb");
+            BinaryPrimitives.WriteInt32LittleEndian(lengthBytes, data.json_bytes.Length);
+            file.Write(magicBytes);
+            file.Write(lengthBytes);
+            file.Write(data.json_bytes);
+            
+            data.stream.CopyTo(file);
+        }
+        
+        return true;
+    }
+}
+

tools-local/tasks/mobile.tasks/CreateWasmBundle/CreateWasmBundle.csproj

+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
+    <OutputType>Library</OutputType>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build" Version="$(RefOnlyMicrosoftBuildVersion)" />
+    <PackageReference Include="Microsoft.Build.Framework" Version="$(RefOnlyMicrosoftBuildFrameworkVersion)" />
+    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(RefOnlyMicrosoftBuildTasksCoreVersion)" />
+    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(RefOnlyMicrosoftBuildUtilitiesCoreVersion)" />
+  </ItemGroup>
+
+  <Target Name="PublishBuilder"
+          AfterTargets="Build"
+          DependsOnTargets="Publish" />
+</Project>

tools-local/tasks/mobile.tasks/CreateWasmBundle/DownloadTimeZoneData.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Net;
+using System.Reflection;
+using System.Diagnostics;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+public class DownloadTimeZoneData : Task
+{
+    [Required]
+    public string? InputDirectory { get; set; }
+
+    [Required]
+    public string? OutputDirectory { get; set; }
+
+    [Required]
+    public string? Version { get; set; }
+
+    private void DownloadTimeZoneDataSource() 
+    {
+        List<string> files = new List<string>() {"africa", "antarctica", "asia", "australasia", "etcetera", "europe", "northamerica", "southamerica", "zone1970.tab"};
+        using (var client = new WebClient())
+        {
+            Console.WriteLine("Downloading TimeZone data files");
+            foreach (var file in files) 
+            {
+                client.DownloadFile($"https://data.iana.org/time-zones/tzdb-{Version}/{file}", $"{Path.Combine(InputDirectory!, file)}");
+            }
+        }
+
+        files.Remove("zone1970.tab");        
+
+        using (Process process = new Process()) 
+        {
+            process.StartInfo.UseShellExecute = false;
+            process.StartInfo.FileName = "zic";
+            foreach (var f in files) 
+            {
+                process.StartInfo.Arguments = $"-d \"{OutputDirectory}\" \"{Path.Combine(InputDirectory!, f)}\"";
+                process.Start();
+                process.WaitForExit();
+            }
+        }
+        File.Copy(Path.Combine(InputDirectory!,"zone1970.tab"), Path.Combine(OutputDirectory!,"zone1970.tab"));
+    }
+
+    private void FilterTimeZoneData() 
+    {
+        //  Remove unnecessary timezone files 
+        foreach (var entry in new DirectoryInfo (OutputDirectory!).EnumerateFiles()) 
+        {
+            if (entry.Name != "zone1970.tab")
+                File.Delete(entry.FullName);
+        }
+    }
+
+    private void FilterZoneTab(string[] filters) 
+    {
+        var oldPath = Path.Combine(OutputDirectory!, "zone1970.tab");
+        var path = Path.Combine(OutputDirectory!, "zone.tab");
+        using (StreamReader sr = new StreamReader(oldPath))
+        using (StreamWriter sw = new StreamWriter(path))
+        {
+            string? line;
+            while ((line = sr.ReadLine()) != null) {
+                if (filters.Any(x => Regex.IsMatch(line, $@"\b{x}\b"))) 
+                {
+                    sw.WriteLine(line);
+                }
+            }
+        }
+        File.Delete(oldPath);
+    }
+
+    public override bool Execute() 
+    {
+
+        if (!Directory.Exists(InputDirectory))
+            Directory.CreateDirectory(InputDirectory!);
+        
+        if (!Directory.Exists(OutputDirectory))
+            Directory.CreateDirectory(OutputDirectory!);
+
+        DownloadTimeZoneDataSource();
+        
+        string[] filtered = new string[] { "America/Los_Angeles", "Australia/Sydney", "Europe/London", "Pacific/Tongatapu", 
+                                "America/Sao_Paulo", "Australia/Perth", "Africa/Nairobi", "Europe/Berlin",
+                                "Europe/Moscow", "Africa/Tripoli", "America/Argentina/Catamarca", "Europe/Lisbon",
+                                "America/St_Johns"};
+        
+        FilterTimeZoneData();
+        FilterZoneTab(filtered);
+
+        return true;
+    }
+}
\ No newline at end of file

tools-local/tasks/tasks.proj

@@ -7,6 +7,8 @@
                       Condition="'$(TargetOS)' != 'iOS' and '$(TargetOS)' != 'tvOS'" />
     <ProjectReference Remove="$(MSBuildThisFileDirectory)mobile.tasks\WasmAppBuilder\WasmAppBuilder.csproj"
                       Condition="'$(TargetOS)' != 'Browser'" />
+    <ProjectReference Remove="$(MSBuildThisFileDirectory)mobile.tasks\CreateWasmBundle\CreateWasmBundle.csproj"
+                      Condition="'$(TargetOS)' != 'Browser'" />
     <ProjectReference Remove="$(MSBuildThisFileDirectory)mobile.tasks\AotCompilerTask\MonoAOTCompiler.csproj"
                       Condition="'$(TargetsMobile)' != 'true'" />
   </ItemGroup>