Break out runtime lib to main and worker (#3771)

Co-authored-by: NBartek Iwańczuk <biwanczuk@gmail.com>

cli/build.rs

@@ -71,9 +71,14 @@ fn main() {
   let snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
   let mut custom_libs: HashMap<String, PathBuf> = HashMap::new();
   custom_libs.insert(
-    "lib.deno_runtime.d.ts".to_string(),
-    c.join("js/lib.deno_runtime.d.ts"),
+    "lib.deno_main.d.ts".to_string(),
+    c.join("js/lib.deno_main.d.ts"),
   );
+  custom_libs.insert(
+    "lib.deno_worker.d.ts".to_string(),
+    c.join("js/lib.deno_worker.d.ts"),
+  );
+  custom_libs.insert("lib.deno.d.ts".to_string(), c.join("js/lib.deno.d.ts"));
 
   let main_module_name =
     deno_typescript::compile_bundle(&bundle_path, root_names)

cli/compilers/ts.rs

@@ -165,6 +165,7 @@ fn req(
   let j = match (compiler_config.path, compiler_config.content) {
     (Some(config_path), Some(config_data)) => json!({
       "type": request_type as i32,
+      "target": "main",
       "rootNames": root_names,
       "outFile": out_file,
       "bundle": bundle,
@@ -173,6 +174,7 @@ fn req(
     }),
     _ => json!({
       "type": request_type as i32,
+      "target": "main",
       "rootNames": root_names,
       "outFile": out_file,
       "bundle": bundle,
@@ -617,6 +619,7 @@ pub fn runtime_compile_async<S: BuildHasher>(
 ) -> Pin<Box<CompilationResultFuture>> {
   let req_msg = json!({
     "type": msg::CompilerRequestType::RuntimeCompile as i32,
+    "target": "runtime",
     "rootName": root_name,
     "sources": sources,
     "options": options,

cli/js.rs

@@ -16,7 +16,8 @@ pub static COMPILER_SNAPSHOT_MAP: &[u8] =
 pub static COMPILER_SNAPSHOT_DTS: &[u8] =
   include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.d.ts"));
 
-pub static DENO_RUNTIME: &str = include_str!("js/lib.deno_runtime.d.ts");
+pub static DENO_NS_LIB: &str = include_str!("js/lib.deno.d.ts");
+pub static DENO_MAIN_LIB: &str = include_str!("js/lib.deno_main.d.ts");
 
 #[test]
 fn cli_snapshot() {

cli/js/compiler.ts

@@ -9,6 +9,7 @@ import { TranspileOnlyResult } from "./compiler_api.ts";
 import { oldProgram } from "./compiler_bootstrap.ts";
 import { setRootExports } from "./compiler_bundler.ts";
 import {
+  CompilerHostTarget,
   defaultBundlerOptions,
   defaultRuntimeCompileOptions,
   defaultTranspileOptions,
@@ -42,6 +43,7 @@ const self = globalThis;
 
 interface CompilerRequestCompile {
   type: CompilerRequestType.Compile;
+  target: CompilerHostTarget;
   rootNames: string[];
   // TODO(ry) add compiler config to this interface.
   // options: ts.CompilerOptions;
@@ -53,6 +55,7 @@ interface CompilerRequestCompile {
 
 interface CompilerRequestRuntimeCompile {
   type: CompilerRequestType.RuntimeCompile;
+  target: CompilerHostTarget;
   rootName: string;
   sources?: Record<string, string>;
   bundle?: boolean;
@@ -100,7 +103,14 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
       // `Compile` are requests from the internals to Deno, generated by both
       // the `run` and `bundle` sub command.
       case CompilerRequestType.Compile: {
-        const { bundle, config, configPath, outFile, rootNames } = request;
+        const {
+          bundle,
+          config,
+          configPath,
+          outFile,
+          rootNames,
+          target
+        } = request;
         util.log(">>> compile start", {
           rootNames,
           type: CompilerRequestType[request.type]
@@ -129,7 +139,11 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
         };
         const writeFile = createWriteFile(state);
 
-        const host = (state.host = new Host({ bundle, writeFile }));
+        const host = (state.host = new Host({
+          bundle,
+          target,
+          writeFile
+        }));
         let diagnostics: readonly ts.Diagnostic[] | undefined;
 
         // if there is a configuration supplied, we need to parse that
@@ -189,7 +203,7 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
         // `RuntimeCompile` are requests from a runtime user, both compiles and
         // bundles.  The process is similar to a request from the privileged
         // side, but also returns the output to the on message.
-        const { rootName, sources, options, bundle } = request;
+        const { rootName, sources, options, bundle, target } = request;
 
         util.log(">>> runtime compile start", {
           rootName,
@@ -216,7 +230,11 @@ self.bootstrapTsCompiler = function tsCompilerMain(): void {
         };
         const writeFile = createWriteFile(state);
 
-        const host = (state.host = new Host({ bundle, writeFile }));
+        const host = (state.host = new Host({
+          bundle,
+          target,
+          writeFile
+        }));
         const compilerOptions = [defaultRuntimeCompileOptions];
         if (options) {
           compilerOptions.push(convertCompilerOptions(options));

cli/js/compiler_bootstrap.ts

 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import { ASSETS, Host } from "./compiler_host.ts";
+import { ASSETS, CompilerHostTarget, Host } from "./compiler_host.ts";
 import { core } from "./core.ts";
 import * as dispatch from "./dispatch.ts";
 import { getAsset } from "./compiler_util.ts";
@@ -15,9 +15,28 @@ for (const [name, opId] of Object.entries(ops)) {
   (dispatch as any)[opName] = opId;
 }
 
-const host = new Host({ writeFile(): void {} });
+const host = new Host({
+  target: CompilerHostTarget.Main,
+  writeFile(): void {}
+});
 const options = host.getCompilationSettings();
 
+// This is a hacky way of adding our libs to the libs available in TypeScript()
+// as these are internal APIs of TypeScript which maintain valid libs
+/* eslint-disable @typescript-eslint/no-explicit-any */
+(ts as any).libs.push("deno_main", "deno_worker", "deno");
+(ts as any).libMap.set("deno_main", "lib.deno_main.d.ts");
+(ts as any).libMap.set("deno_worker", "lib.deno_worker.d.ts");
+(ts as any).libMap.set("deno", "lib.deno.d.ts");
+/* eslint-enable @typescript-eslint/no-explicit-any */
+
+// this pre-populates the cache at snapshot time of our library files, so they
+// are available in the future when needed.
+host.getSourceFile(`${ASSETS}/lib.deno_main.d.ts`, ts.ScriptTarget.ESNext);
+host.getSourceFile(`${ASSETS}/lib.deno_worker.d.ts`, ts.ScriptTarget.ESNext);
+host.getSourceFile(`${ASSETS}/lib.deno.d.ts`, ts.ScriptTarget.ESNext);
+host.getSourceFile(`${ASSETS}/lib.webworker.d.ts`, ts.ScriptTarget.ESNext);
+
 /** Used to generate the foundational AST for all other compilations, so it can
  * be cached as part of the snapshot and available to speed up startup */
 export const oldProgram = ts.createProgram({

cli/js/compiler_host.ts

@@ -6,8 +6,20 @@ import { cwd } from "./dir.ts";
 import { assert, notImplemented } from "./util.ts";
 import * as util from "./util.ts";
 
+/** Specifies the target that the host should use to inform the TypeScript
+ * compiler of what types should be used to validate the program against. */
+export enum CompilerHostTarget {
+  /** The main isolate library, where the main program runs. */
+  Main = "main",
+  /** The runtime API library. */
+  Runtime = "runtime",
+  /** The worker isolate library, where worker programs run. */
+  Worker = "worker"
+}
+
 export interface CompilerHostOptions {
   bundle?: boolean;
+  target: CompilerHostTarget;
   writeFile: WriteFileCallback;
 }
 
@@ -124,6 +136,8 @@ const ignoredCompilerOptions: readonly string[] = [
 export class Host implements ts.CompilerHost {
   private readonly _options = defaultCompileOptions;
 
+  private _target: CompilerHostTarget;
+
   private _writeFile: WriteFileCallback;
 
   private _getAsset(filename: string): SourceFile {
@@ -146,7 +160,8 @@ export class Host implements ts.CompilerHost {
 
   /** Provides the `ts.HostCompiler` interface for Deno. */
   constructor(options: CompilerHostOptions) {
-    const { bundle = false, writeFile } = options;
+    const { bundle = false, target, writeFile } = options;
+    this._target = target;
     this._writeFile = writeFile;
     if (bundle) {
       // options we need to change when we are generating a bundle
@@ -215,7 +230,13 @@ export class Host implements ts.CompilerHost {
   }
 
   getDefaultLibFileName(_options: ts.CompilerOptions): string {
-    return ASSETS + "/lib.deno_runtime.d.ts";
+    switch (this._target) {
+      case CompilerHostTarget.Main:
+      case CompilerHostTarget.Runtime:
+        return `${ASSETS}/lib.deno_main.d.ts`;
+      case CompilerHostTarget.Worker:
+        return `${ASSETS}/lib.deno_worker.d.ts`;
+    }
   }
 
   getNewLine(): string {

cli/js/lib.deno_worker.d.ts

+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+/// <reference no-default-lib="true" />
+/// <reference lib="deno" />
+/// <reference lib="webworker" />

cli/js/lib.web_assembly.d.ts

-// This follows the WebIDL at: https://webassembly.github.io/spec/js-api/
-// And follow on WebIDL at: https://webassembly.github.io/spec/web-api/
-
-/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any */
-
-declare namespace WebAssembly {
-  interface WebAssemblyInstantiatedSource {
-    module: Module;
-    instance: Instance;
-  }
-
-  /** Compiles a `WebAssembly.Module` from WebAssembly binary code.  This
-   * function is useful if it is necessary to a compile a module before it can
-   * be instantiated (otherwise, the `WebAssembly.instantiate()` function
-   * should be used). */
-  function compile(bufferSource: domTypes.BufferSource): Promise<Module>;
-
-  /** Compiles a `WebAssembly.Module` directly from a streamed underlying
-   * source. This function is useful if it is necessary to a compile a module
-   * before it can be instantiated (otherwise, the
-   * `WebAssembly.instantiateStreaming()` function should be used). */
-  function compileStreaming(
-    source: Promise<domTypes.Response>
-  ): Promise<Module>;
-
-  /** Takes the WebAssembly binary code, in the form of a typed array or
-   * `ArrayBuffer`, and performs both compilation and instantiation in one step.
-   * The returned `Promise` resolves to both a compiled `WebAssembly.Module` and
-   * its first `WebAssembly.Instance`. */
-  function instantiate(
-    bufferSource: domTypes.BufferSource,
-    importObject?: object
-  ): Promise<WebAssemblyInstantiatedSource>;
-
-  /** Takes an already-compiled `WebAssembly.Module` and returns a `Promise`
-   * that resolves to an `Instance` of that `Module`. This overload is useful if
-   * the `Module` has already been compiled. */
-  function instantiate(
-    module: Module,
-    importObject?: object
-  ): Promise<Instance>;
-
-  /** Compiles and instantiates a WebAssembly module directly from a streamed
-   * underlying source. This is the most efficient, optimized way to load wasm
-   * code. */
-  function instantiateStreaming(
-    source: Promise<domTypes.Response>,
-    importObject?: object
-  ): Promise<WebAssemblyInstantiatedSource>;
-
-  /** Validates a given typed array of WebAssembly binary code, returning
-   * whether the bytes form a valid wasm module (`true`) or not (`false`). */
-  function validate(bufferSource: domTypes.BufferSource): boolean;
-
-  type ImportExportKind = "function" | "table" | "memory" | "global";
-
-  interface ModuleExportDescriptor {
-    name: string;
-    kind: ImportExportKind;
-  }
-  interface ModuleImportDescriptor {
-    module: string;
-    name: string;
-    kind: ImportExportKind;
-  }
-
-  class Module {
-    constructor(bufferSource: domTypes.BufferSource);
-
-    /** Given a `Module` and string, returns a copy of the contents of all
-     * custom sections in the module with the given string name. */
-    static customSections(
-      moduleObject: Module,
-      sectionName: string
-    ): ArrayBuffer;
-
-    /** Given a `Module`, returns an array containing descriptions of all the
-     * declared exports. */
-    static exports(moduleObject: Module): ModuleExportDescriptor[];
-
-    /** Given a `Module`, returns an array containing descriptions of all the
-     * declared imports. */
-    static imports(moduleObject: Module): ModuleImportDescriptor[];
-  }
-
-  class Instance<T extends object = { [key: string]: any }> {
-    constructor(module: Module, importObject?: object);
-
-    /** An object containing as its members all the functions exported from the
-     * WebAssembly module instance, to allow them to be accessed and used by
-     * JavaScript. */
-    readonly exports: T;
-  }
-
-  interface MemoryDescriptor {
-    initial: number;
-    maximum?: number;
-  }
-
-  class Memory {
-    constructor(descriptor: MemoryDescriptor);
-
-    /** An accessor property that returns the buffer contained in the memory. */
-    readonly buffer: ArrayBuffer;
-
-    /** Increases the size of the memory instance by a specified number of
-     * WebAssembly pages (each one is 64KB in size). */
-    grow(delta: number): number;
-  }
-
-  type TableKind = "anyfunc";
-
-  interface TableDescriptor {
-    element: TableKind;
-    initial: number;
-    maximum?: number;
-  }
-
-  class Table {
-    constructor(descriptor: TableDescriptor);
-
-    /** Returns the length of the table, i.e. the number of elements. */
-    readonly length: number;
-
-    /** Accessor function — gets the element stored at a given index. */
-    get(index: number): (...args: any[]) => any;
-
-    /** Increases the size of the Table instance by a specified number of
-     * elements. */
-    grow(delta: number): number;
-
-    /** Sets an element stored at a given index to a given value. */
-    set(index: number, value: (...args: any[]) => any): void;
-  }
-
-  interface GlobalDescriptor {
-    value: string;
-    mutable?: boolean;
-  }
-
-  /** Represents a global variable instance, accessible from both JavaScript and
-   * importable/exportable across one or more `WebAssembly.Module` instances.
-   * This allows dynamic linking of multiple modules. */
-  class Global {
-    constructor(descriptor: GlobalDescriptor, value?: any);
-
-    /** Old-style method that returns the value contained inside the global
-     * variable. */
-    valueOf(): any;
-
-    /** The value contained inside the global variable — this can be used to
-     * directly set and get the global's value. */
-    value: any;
-  }
-
-  /** Indicates an error during WebAssembly decoding or validation */
-  class CompileError extends Error {
-    constructor(message: string, fileName?: string, lineNumber?: string);
-  }
-
-  /** Indicates an error during module instantiation (besides traps from the
-   * start function). */
-  class LinkError extends Error {
-    constructor(message: string, fileName?: string, lineNumber?: string);
-  }
-
-  /** Is thrown whenever WebAssembly specifies a trap. */
-  class RuntimeError extends Error {
-    constructor(message: string, fileName?: string, lineNumber?: string);
-  }
-}
-
-/* eslint-enable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any */

cli/lib.rs

@@ -146,7 +146,7 @@ fn create_worker_and_state(
 }
 
 fn types_command() {
-  println!("{}", crate::js::DENO_RUNTIME);
+  println!("{}\n{}", crate::js::DENO_NS_LIB, crate::js::DENO_MAIN_LIB);
 }
 
 fn print_cache_info(worker: MainWorker) {

deno_typescript/lib.rs

@@ -280,6 +280,7 @@ pub fn get_asset(name: &str) -> Option<&'static str> {
     "lib.esnext.bigint.d.ts" => inc!("lib.esnext.bigint.d.ts"),
     "lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"),
     "lib.esnext.symbol.d.ts" => inc!("lib.esnext.symbol.d.ts"),
+    "lib.webworker.d.ts" => inc!("lib.webworker.d.ts"),
     _ => None,
   }
 }