Remove ts_library_builder, maintain lib.deno_runtime.d.ts by hand (#2827)

cli/BUILD.gn

@@ -61,7 +61,6 @@ if (is_posix) {
 }
 
 ts_sources = [
-  "../js/assets.ts",
   "../js/base64.ts",
   "../js/blob.ts",
   "../js/body.ts",
@@ -98,6 +97,7 @@ ts_sources = [
   "../js/globals.ts",
   "../js/headers.ts",
   "../js/io.ts",
+  "../js/lib.deno_runtime.d.ts",
   "../js/lib.web_assembly.d.ts",
   "../js/link.ts",
   "../js/location.ts",
@@ -200,55 +200,16 @@ rust_test("cli_test") {
   env = [ "CARGO_PKG_VERSION=${deno_cargo_info.version}" ]
 }
 
-# Generates the core TypeScript type library for deno that will be
-# included in the runtime bundle
-run_node("deno_runtime_declaration") {
-  out_dir = target_gen_dir
-  sources = ts_sources
-  outputs = [
-    "$out_dir/lib/lib.deno_runtime.d.ts",
-  ]
-  inputs = ts_sources + [
-             "//tools/ts_library_builder/tsconfig.json",
-             "//tools/ts_library_builder/main.ts",
-             "//tools/ts_library_builder/build_library.ts",
-             "//tools/ts_library_builder/ast_util.ts",
-           ]
-  args = [
-    rebase_path("//node_modules/ts-node/dist/bin.js", root_build_dir),
-    "--project",
-    rebase_path("//tools/ts_library_builder/tsconfig.json"),
-    "--skip-ignore",
-    rebase_path("//tools/ts_library_builder/main.ts", root_build_dir),
-    "--basePath",
-    rebase_path("//", root_build_dir),
-    "--inline",
-    rebase_path("//js/lib.web_assembly.d.ts", root_build_dir),
-    "--buildPath",
-    rebase_path(root_build_dir, root_build_dir),
-    "--outFile",
-    rebase_path(outputs[0], root_build_dir),
-    "--silent",
-  ]
-  if (is_debug) {
-    args += [ "--debug" ]
-  }
-}
-
 bundle("main_bundle") {
+  sources = ts_sources
   out_dir = "$target_gen_dir/bundle/"
   out_name = "main"
-  deps = [
-    ":deno_runtime_declaration",
-  ]
 }
 
 bundle("compiler_bundle") {
+  sources = ts_sources
   out_dir = "$target_gen_dir/bundle/"
   out_name = "compiler"
-  deps = [
-    ":deno_runtime_declaration",
-  ]
 }
 
 # Generates $target_gen_dir/snapshot_deno.bin

cli/assets.rs

+static DENO_RUNTIME: &str = include_str!("../js/lib.deno_runtime.d.ts");
+
+macro_rules! inc {
+  ($e:expr) => {
+    Some(include_str!(concat!(
+      "../third_party/node_modules/typescript/lib/",
+      $e
+    )))
+  };
+}
+
+pub fn get_source_code(name: &str) -> Option<&'static str> {
+  match name {
+    "lib.deno_runtime.d.ts" => Some(DENO_RUNTIME),
+    "lib.es2015.collection.d.ts" => inc!("lib.es2015.collection.d.ts"),
+    "lib.es2015.core.d.ts" => inc!("lib.es2015.core.d.ts"),
+    "lib.es2015.d.ts" => inc!("lib.es2015.d.ts"),
+    "lib.es2015.generator.d.ts" => inc!("lib.es2015.generator.d.ts"),
+    "lib.es2015.iterable.d.ts" => inc!("lib.es2015.iterable.d.ts"),
+    "lib.es2015.promise.d.ts" => inc!("lib.es2015.promise.d.ts"),
+    "lib.es2015.proxy.d.ts" => inc!("lib.es2015.proxy.d.ts"),
+    "lib.es2015.reflect.d.ts" => inc!("lib.es2015.reflect.d.ts"),
+    "lib.es2015.symbol.d.ts" => inc!("lib.es2015.symbol.d.ts"),
+    "lib.es2015.symbol.wellknown.d.ts" => {
+      inc!("lib.es2015.symbol.wellknown.d.ts")
+    }
+    "lib.es2016.array.include.d.ts" => inc!("lib.es2016.array.include.d.ts"),
+    "lib.es2016.d.ts" => inc!("lib.es2016.d.ts"),
+    "lib.es2017.d.ts" => inc!("lib.es2017.d.ts"),
+    "lib.es2017.intl.d.ts" => inc!("lib.es2017.intl.d.ts"),
+    "lib.es2017.object.d.ts" => inc!("lib.es2017.object.d.ts"),
+    "lib.es2017.sharedmemory.d.ts" => inc!("lib.es2017.sharedmemory.d.ts"),
+    "lib.es2017.string.d.ts" => inc!("lib.es2017.string.d.ts"),
+    "lib.es2017.typedarrays.d.ts" => inc!("lib.es2017.typedarrays.d.ts"),
+    "lib.es2018.d.ts" => inc!("lib.es2018.d.ts"),
+    "lib.es2018.asynciterable.d.ts" => inc!("lib.es2018.asynciterable.d.ts"),
+    "lib.es2018.intl.d.ts" => inc!("lib.es2018.intl.d.ts"),
+    "lib.es2018.promise.d.ts" => inc!("lib.es2018.promise.d.ts"),
+    "lib.es2018.regexp.d.ts" => inc!("lib.es2018.regexp.d.ts"),
+    "lib.es2019.d.ts" => inc!("lib.es2019.d.ts"),
+    "lib.es2019.array.d.ts" => inc!("lib.es2019.array.d.ts"),
+    "lib.es2019.object.d.ts" => inc!("lib.es2019.object.d.ts"),
+    "lib.es2019.string.d.ts" => inc!("lib.es2019.string.d.ts"),
+    "lib.es2019.symbol.d.ts" => inc!("lib.es2019.symbol.d.ts"),
+    "lib.es2020.d.ts" => inc!("lib.es2020.d.ts"),
+    "lib.es2020.string.d.ts" => inc!("lib.es2020.string.d.ts"),
+    "lib.es2020.symbol.wellknown.d.ts" => {
+      inc!("lib.es2020.symbol.wellknown.d.ts")
+    }
+    "lib.es5.d.ts" => inc!("lib.es5.d.ts"),
+    "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"),
+    "lib.esnext.array.d.ts" => inc!("lib.esnext.array.d.ts"),
+    "lib.esnext.asynciterable.d.ts" => inc!("lib.esnext.asynciterable.d.ts"),
+    "lib.esnext.bigint.d.ts" => inc!("lib.esnext.bigint.d.ts"),
+    "lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"),
+    "lib.esnext.symbol.d.ts" => inc!("lib.esnext.symbol.d.ts"),
+    _ => None,
+  }
+}

cli/main.rs

@@ -18,6 +18,7 @@ extern crate serde_derive;
 extern crate url;
 
 mod ansi;
+mod assets;
 pub mod compilers;
 pub mod deno_dir;
 pub mod deno_error;
@@ -127,10 +128,7 @@ fn create_worker_and_state(
 }
 
 fn types_command() {
-  let content = include_str!(concat!(
-    env!("GN_OUT_DIR"),
-    "/gen/cli/lib/lib.deno_runtime.d.ts"
-  ));
+  let content = assets::get_source_code("lib.deno_runtime.d.ts").unwrap();
   println!("{}", content);
 }
 

cli/ops/compiler.rs

 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use super::dispatch_json::{Deserialize, JsonOp, Value};
+use crate::assets;
 use crate::state::ThreadSafeState;
 use crate::tokio_util;
 use deno::*;
@@ -66,3 +67,21 @@ pub fn op_fetch_source_file(
     "sourceCode": String::from_utf8(out.source_code).unwrap(),
   })))
 }
+
+#[derive(Deserialize)]
+struct FetchAssetArgs {
+  name: String,
+}
+
+pub fn op_fetch_asset(
+  _state: &ThreadSafeState,
+  args: Value,
+  _zero_copy: Option<PinnedBuf>,
+) -> Result<JsonOp, ErrBox> {
+  let args: FetchAssetArgs = serde_json::from_value(args)?;
+  if let Some(source_code) = assets::get_source_code(&args.name) {
+    Ok(JsonOp::Sync(json!(source_code)))
+  } else {
+    panic!("op_fetch_asset bad asset {}", args.name)
+  }
+}

cli/ops/mod.rs

@@ -80,6 +80,7 @@ pub const OP_READ_LINK: OpId = 53;
 pub const OP_TRUNCATE: OpId = 54;
 pub const OP_MAKE_TEMP_DIR: OpId = 55;
 pub const OP_CWD: OpId = 56;
+pub const OP_FETCH_ASSET: OpId = 57;
 
 pub fn dispatch(
   state: &ThreadSafeState,
@@ -293,6 +294,12 @@ pub fn dispatch(
       dispatch_json::dispatch(fs::op_make_temp_dir, state, control, zero_copy)
     }
     OP_CWD => dispatch_json::dispatch(fs::op_cwd, state, control, zero_copy),
+    OP_FETCH_ASSET => dispatch_json::dispatch(
+      compiler::op_fetch_asset,
+      state,
+      control,
+      zero_copy,
+    ),
     _ => panic!("bad op_id"),
   };
 

js/assets.ts

-// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-
-// eslint-disable-next-line @typescript-eslint/no-triple-slash-reference
-/// <reference path="./plugins.d.ts" />
-
-// There is a rollup plugin that will inline any module ending with `!string`
-
-// Generated default library
-import libDts from "gen/cli/lib/lib.deno_runtime.d.ts!string";
-
-// Static libraries
-import libEs2015Dts from "/third_party/node_modules/typescript/lib/lib.es2015.d.ts!string";
-import libEs2015CollectionDts from "/third_party/node_modules/typescript/lib/lib.es2015.collection.d.ts!string";
-import libEs2015CoreDts from "/third_party/node_modules/typescript/lib/lib.es2015.core.d.ts!string";
-import libEs2015GeneratorDts from "/third_party/node_modules/typescript/lib/lib.es2015.generator.d.ts!string";
-import libEs2015IterableDts from "/third_party/node_modules/typescript/lib/lib.es2015.iterable.d.ts!string";
-import libEs2015PromiseDts from "/third_party/node_modules/typescript/lib/lib.es2015.promise.d.ts!string";
-import libEs2015ProxyDts from "/third_party/node_modules/typescript/lib/lib.es2015.proxy.d.ts!string";
-import libEs2015ReflectDts from "/third_party/node_modules/typescript/lib/lib.es2015.reflect.d.ts!string";
-import libEs2015SymbolDts from "/third_party/node_modules/typescript/lib/lib.es2015.symbol.d.ts!string";
-import libEs2015SymbolWellknownDts from "/third_party/node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts!string";
-import libEs2016Dts from "/third_party/node_modules/typescript/lib/lib.es2016.d.ts!string";
-import libEs2016ArrayIncludeDts from "/third_party/node_modules/typescript/lib/lib.es2016.array.include.d.ts!string";
-import libEs2017Dts from "/third_party/node_modules/typescript/lib/lib.es2017.d.ts!string";
-import libEs2017IntlDts from "/third_party/node_modules/typescript/lib/lib.es2017.intl.d.ts!string";
-import libEs2017ObjectDts from "/third_party/node_modules/typescript/lib/lib.es2017.object.d.ts!string";
-import libEs2017SharedmemoryDts from "/third_party/node_modules/typescript/lib/lib.es2017.sharedmemory.d.ts!string";
-import libEs2017StringDts from "/third_party/node_modules/typescript/lib/lib.es2017.string.d.ts!string";
-import libEs2017TypedarraysDts from "/third_party/node_modules/typescript/lib/lib.es2017.typedarrays.d.ts!string";
-import libEs2018Dts from "/third_party/node_modules/typescript/lib/lib.es2018.d.ts!string";
-import libEs2018AsyncIterableDts from "/third_party/node_modules/typescript/lib/lib.es2018.asynciterable.d.ts!string";
-import libEs2018IntlDts from "/third_party/node_modules/typescript/lib/lib.es2018.intl.d.ts!string";
-import libEs2018PromiseDts from "/third_party/node_modules/typescript/lib/lib.es2018.promise.d.ts!string";
-import libEs2018RegexpDts from "/third_party/node_modules/typescript/lib/lib.es2018.regexp.d.ts!string";
-import libEs2019Dts from "/third_party/node_modules/typescript/lib/lib.es2019.d.ts!string";
-import libEs2019ArrayDts from "/third_party/node_modules/typescript/lib/lib.es2019.array.d.ts!string";
-import libEs2019ObjectDts from "/third_party/node_modules/typescript/lib/lib.es2019.object.d.ts!string";
-import libEs2019StringDts from "/third_party/node_modules/typescript/lib/lib.es2019.string.d.ts!string";
-import libEs2019SymbolDts from "/third_party/node_modules/typescript/lib/lib.es2019.symbol.d.ts!string";
-import libEs2020Dts from "/third_party/node_modules/typescript/lib/lib.es2020.d.ts!string";
-import libEs2020String from "/third_party/node_modules/typescript/lib/lib.es2020.string.d.ts!string";
-import libEs2020SymbolWellknownDts from "/third_party/node_modules/typescript/lib/lib.es2020.symbol.wellknown.d.ts!string";
-import libEs5Dts from "/third_party/node_modules/typescript/lib/lib.es5.d.ts!string";
-import libEsnextArrayDts from "/third_party/node_modules/typescript/lib/lib.esnext.array.d.ts!string";
-import libEsnextAsynciterablesDts from "/third_party/node_modules/typescript/lib/lib.esnext.asynciterable.d.ts!string";
-import libEsnextBigintDts from "/third_party/node_modules/typescript/lib/lib.esnext.bigint.d.ts!string";
-import libEsnextDts from "/third_party/node_modules/typescript/lib/lib.esnext.d.ts!string";
-import libEsnextIntlDts from "/third_party/node_modules/typescript/lib/lib.esnext.intl.d.ts!string";
-import libEsnextSymbolDts from "/third_party/node_modules/typescript/lib/lib.esnext.symbol.d.ts!string";
-
-// Default static libraries for all compile jobs
-const defaultAssets: { [key: string]: string } = {
-  "lib.es2015.collection.d.ts": libEs2015CollectionDts,
-  "lib.es2015.core.d.ts": libEs2015CoreDts,
-  "lib.es2015.d.ts": libEs2015Dts,
-  "lib.es2015.generator.d.ts": libEs2015GeneratorDts,
-  "lib.es2015.iterable.d.ts": libEs2015IterableDts,
-  "lib.es2015.promise.d.ts": libEs2015PromiseDts,
-  "lib.es2015.proxy.d.ts": libEs2015ProxyDts,
-  "lib.es2015.reflect.d.ts": libEs2015ReflectDts,
-  "lib.es2015.symbol.d.ts": libEs2015SymbolDts,
-  "lib.es2015.symbol.wellknown.d.ts": libEs2015SymbolWellknownDts,
-  "lib.es2016.array.include.d.ts": libEs2016ArrayIncludeDts,
-  "lib.es2016.d.ts": libEs2016Dts,
-  "lib.es2017.d.ts": libEs2017Dts,
-  "lib.es2017.intl.d.ts": libEs2017IntlDts,
-  "lib.es2017.object.d.ts": libEs2017ObjectDts,
-  "lib.es2017.sharedmemory.d.ts": libEs2017SharedmemoryDts,
-  "lib.es2017.string.d.ts": libEs2017StringDts,
-  "lib.es2017.typedarrays.d.ts": libEs2017TypedarraysDts,
-  "lib.es2018.d.ts": libEs2018Dts,
-  "lib.es2018.asynciterable.d.ts": libEs2018AsyncIterableDts,
-  "lib.es2018.intl.d.ts": libEs2018IntlDts,
-  "lib.es2018.promise.d.ts": libEs2018PromiseDts,
-  "lib.es2018.regexp.d.ts": libEs2018RegexpDts,
-  "lib.es2019.d.ts": libEs2019Dts,
-  "lib.es2019.array.d.ts": libEs2019ArrayDts,
-  "lib.es2019.object.d.ts": libEs2019ObjectDts,
-  "lib.es2019.string.d.ts": libEs2019StringDts,
-  "lib.es2019.symbol.d.ts": libEs2019SymbolDts,
-  "lib.es2020.d.ts": libEs2020Dts,
-  "lib.es2020.string.d.ts": libEs2020String,
-  "lib.es2020.symbol.wellknown.d.ts": libEs2020SymbolWellknownDts,
-  "lib.es5.d.ts": libEs5Dts,
-  "lib.esnext.d.ts": libEsnextDts,
-  "lib.esnext.array.d.ts": libEsnextArrayDts,
-  "lib.esnext.asynciterable.d.ts": libEsnextAsynciterablesDts,
-  "lib.esnext.bigint.d.ts": libEsnextBigintDts,
-  "lib.esnext.intl.d.ts": libEsnextIntlDts,
-  "lib.esnext.symbol.d.ts": libEsnextSymbolDts
-};
-
-// assests for normal compile jobs
-// @internal
-export const assetSourceCode: { [key: string]: string } = {
-  // Generated library
-  "lib.deno_runtime.d.ts": libDts,
-  ...defaultAssets
-};

js/compiler.ts

 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import * as ts from "typescript";
-import { assetSourceCode } from "./assets";
 import { bold, cyan, yellow } from "./colors";
 import { Console } from "./console";
 import { core } from "./core";
@@ -128,6 +127,10 @@ interface EmitResult {
   diagnostics?: Diagnostic;
 }
 
+function fetchAsset(name: string): string {
+  return sendSync(dispatch.OP_FETCH_ASSET, { name });
+}
+
 /** Ops to Rust to resolve and fetch a modules meta data. */
 function fetchSourceFile(specifier: string, referrer: string): SourceFile {
   util.log("compiler.fetchSourceFile", { specifier, referrer });
@@ -222,8 +225,7 @@ class Host implements ts.CompilerHost {
       const assetName = moduleName.includes(".")
         ? moduleName
         : `${moduleName}.d.ts`;
-      assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
-      const sourceCode = assetSourceCode[assetName];
+      const sourceCode = fetchAsset(assetName);
       const sourceFile = {
         moduleName,
         filename: specifier,

js/dispatch.ts

@@ -59,6 +59,7 @@ export const OP_READ_LINK = 53;
 export const OP_TRUNCATE = 54;
 export const OP_MAKE_TEMP_DIR = 55;
 export const OP_CWD = 56;
+export const OP_FETCH_ASSET = 57;
 
 export function asyncMsgFromRust(opId: number, ui8: Uint8Array): void {
   switch (opId) {

js/globals.ts

@@ -37,9 +37,6 @@ import { core } from "./core";
 // file are tracked and created as part of default library that is built into
 // Deno, we only need to declare the enough to compile Deno.
 declare global {
-  const console: consoleTypes.Console;
-  const setTimeout: typeof timers.setTimeout;
-
   interface CallSite {
     getThis(): unknown;
     getTypeName(): string;

rollup.config.js

@@ -29,56 +29,6 @@ const tsconfigOverride = {
   }
 };
 
-/** this is a rollup plugin which will look for imports ending with `!string` and resolve
- * them with a module that will inline the contents of the file as a string.  Needed to
- * support `js/assets.ts`.
- * @param {any} param0
- */
-function strings(
-  { include, exclude } = { include: undefined, exclude: undefined }
-) {
-  if (!include) {
-    throw new Error("include option must be passed");
-  }
-
-  const filter = createFilter(include, exclude);
-
-  return {
-    name: "strings",
-
-    /**
-     * @param {string} importee
-     */
-    resolveId(importee) {
-      if (importee.endsWith("!string")) {
-        // strip the `!string` from `importee`
-        importee = importee.slice(0, importee.lastIndexOf("!string"));
-        if (!importee.startsWith("gen/")) {
-          // this is a static asset which is located relative to the root of
-          // the source project
-          return path.resolve(path.join(__dirname, importee));
-        }
-        // this is an asset which has been generated, therefore it will be
-        // located within the build path
-        return path.resolve(path.join(process.cwd(), importee));
-      }
-    },
-
-    /**
-     * @param {any} code
-     * @param {string} id
-     */
-    transform(code, id) {
-      if (filter(id)) {
-        return {
-          code: `export default ${JSON.stringify(code)};`,
-          map: { mappings: "" }
-        };
-      }
-    }
-  };
-}
-
 const archNodeToDeno = {
   x64: "x64"
 };
@@ -191,15 +141,6 @@ export default function makeConfig(commandOptions) {
         module: mockPath
       }),
 
-      // Provides inlining of file contents for `js/assets.ts`
-      strings({
-        include: [
-          "*.d.ts",
-          `${__dirname}/**/*.d.ts`,
-          `${process.cwd()}/**/*.d.ts`
-        ]
-      }),
-
       // Allows rollup to resolve modules based on Node.js resolution
       nodeResolve(),
 

tests/types.out

 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-
-/// <reference no-default-lib="true" />
-/// <reference lib="esnext" />
+[WILDCARD]
 
 declare namespace Deno {
 [WILDCARD]

tools/target_test.py

@@ -34,16 +34,6 @@ class TestTarget(DenoTestCase):
     def test_core_http_benchmark(self):
         self._test("deno_core_http_bench_test")
 
-    def test_ts_library_builder(self):
-        result = run_output([
-            "node", "./node_modules/.bin/ts-node", "--project",
-            "tools/ts_library_builder/tsconfig.json",
-            "tools/ts_library_builder/test.ts"
-        ],
-                            quiet=True)
-        self.assertEqual(result.code, 0)
-        assert "ts_library_builder ok" in result.out
-
     def test_no_color(self):
         t = os.path.join(tests_path, "no_color.js")
         result = run_output([self.deno_exe, "run", t],

tools/ts_library_builder/README.md

-# ts_library_builder
-
-This tool allows us to produce a single TypeScript declaration file that
-describes the complete Deno runtime, including global variables and the built-in
-`Deno` global object. The output of this tool, `lib.deno_runtime.d.ts`, serves
-several purposes:
-
-1. It is passed to the TypeScript compiler `js/compiler.ts`, so that TypeScript
-   knows what types to expect and can validate code against the runtime
-   environment.
-2. It is outputted to stdout by `deno types`, so that users can easily have
-   access to the complete declaration file. Editors can use this in the future
-   to perform type checking.
-3. Because JSDocs are maintained, this serves as a simple documentation page for
-   Deno. We will use this file to generate HTML docs in the future.
-
-The tool depends upon a couple libraries:
-
-- [`ts-node`](https://www.npmjs.com/package/ts-node) to provide just in time
-  transpiling of TypeScript for the tool itself.
-- [`ts-morph`](https://www.npmjs.com/package/ts-morph) which provides a more
-  rational and functional interface to the TypeScript AST to make manipulations
-  easier.
-- [`prettier`](https://www.npmjs.com/package/prettier) and
-  [`@types/prettier`](https://www.npmjs.com/package/@types/prettier) to format
-  the output.
-
-## Design
-
-Ideally we wouldn't have to build this tool at all, and could simply use `tsc`
-to output this declaration file. While, `--emitDeclarationsOnly`, `--outFile`
-and `--module AMD` generates a single declaration file, it isn't clean. It was
-never designed for a library generation, where what is available in a runtime
-environment significantly differs from the code that creates that environment's
-structure.
-
-Therefore this tool injects some of the knowledge of what occurs in the Deno
-runtime environment as well as ensures that the output file is more clean and
-logical for an end user. In the deno runtime, code runs in a global scope that
-is defined in `js/global.ts`. This contains global scope items that one
-reasonably expects in a JavaScript runtime, like `console`. It also defines the
-global scope on a self-reflective `window` variable. There is currently only one
-module of Deno specific APIs which is available to the user. This is defined in
-`js/deno.ts`.
-
-This tool takes advantage of an experimental feature of TypeScript that items
-that are not really intended to be part of the public API are marked with a
-comment pragma of `@internal` and then are not emitted when generating type
-definitions. In addition TypeScript will _tree-shake_ any dependencies tied to
-that "hidden" API and elide them as well. This really helps keep the public API
-clean and as minimal as needed.
-
-In order to create the default type library, the process at a high-level looks
-like this:
-
-- We read in all of the runtime environment definition code into TypeScript AST
-  parser "project".
-- We emit the TypeScript type definitions only into another AST parser
-  "project".
-- We process the `deno` namespace/module, by "flattening" the type definition
-  file.
-  - We determine the exported symbols for `js/deno.ts`.
-  - We recurse over all imports/exports of the modules, only exporting those
-    symbols which are finally exported by `js/deno.ts`.
-  - We replace the import/export with the type information from the source file.
-  - This process assumes that all the modules that feed `js/deno.ts` will have a
-    public type API that does not have name conflicts.
-- We process the `js/globals.ts` file to generate the global namespace.
-  - We create a `Window` interface and a `global` scope augmentation namespace.
-  - We iterate over augmentations to the `window` variable declared in the file,
-    extract the type information and apply it to both a global variable
-    declaration and a property on the `Window` interface.
-  - We identify any type aliases in the module and declare them globally.
-- We take each namespace import to `js/globals.ts`, we resolve the emitted
-  declaration `.d.ts` file and create it as its own namespace within the global
-  scope. It is unsafe to just flatten these, because there is a high risk of
-  collisions, but also, it makes authoring the types easier within the generated
-  interface and variable declarations.
-- We then validate the resulting definition file and write it out to the
-  appropriate build path.
-
-## TODO
-
-- The tool does not _tree-shake_ when flattening imports. This means there are
-  extraneous types that get included that are not really needed and it means
-  that `gen/msg_generated.ts` has to be explicitly carved down.
-- Complete the tests... we have some coverage, but not a lot of what is in
-  `ast_util_test` which is being tested implicitly.

tools/ts_library_builder/ast_util.ts

-// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import { basename, dirname, join, relative } from "path";
-import { readFileSync } from "fs";
-import { EOL } from "os";
-import {
-  ExportDeclaration,
-  ImportDeclaration,
-  InterfaceDeclaration,
-  JSDoc,
-  Project,
-  PropertySignature,
-  SourceFile,
-  StatementedNode,
-  ts,
-  TypeAliasDeclaration,
-  TypeGuards,
-  VariableStatement,
-  VariableDeclarationKind
-} from "ts-morph";
-
-let silent = false;
-
-/** Logs a message to the console. */
-export function log(message: any = "", ...args: any[]): void {
-  if (!silent) {
-    console.log(message, ...args);
-  }
-}
-
-/** Sets the silent flag which impacts logging to the console. */
-export function setSilent(value = false): void {
-  silent = value;
-}
-
-/** Add a property to an interface */
-export function addInterfaceProperty(
-  interfaceDeclaration: InterfaceDeclaration,
-  name: string,
-  type: string,
-  jsdocs?: JSDoc[]
-): PropertySignature {
-  return interfaceDeclaration.addProperty({
-    name,
-    type,
-    docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText())
-  });
-}
-
-/** Add `@url` comment to node. */
-export function addSourceComment(
-  node: StatementedNode,
-  sourceFile: SourceFile,
-  rootPath: string
-): void {
-  node.insertStatements(
-    0,
-    `// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`
-  );
-}
-
-/** Add a declaration of a type alias to a node */
-export function addTypeAlias(
-  node: StatementedNode,
-  name: string,
-  type: string,
-  hasDeclareKeyword = false,
-  jsdocs?: JSDoc[]
-): TypeAliasDeclaration {
-  return node.addTypeAlias({
-    name,
-    type,
-    docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()),
-    hasDeclareKeyword
-  });
-}
-
-/** Add a declaration of an interface to a node */
-export function addInterfaceDeclaration(
-  node: StatementedNode,
-  interfaceDeclaration: InterfaceDeclaration
-) {
-  const interfaceStructure = interfaceDeclaration.getStructure();
-
-  return node.addInterface({
-    name: interfaceStructure.name,
-    properties: interfaceStructure.properties,
-    docs: interfaceStructure.docs,
-    hasDeclareKeyword: true
-  });
-}
-
-/** Add a declaration of a variable to a node */
-export function addVariableDeclaration(
-  node: StatementedNode,
-  name: string,
-  type: string,
-  isConst: boolean,
-  hasDeclareKeyword?: boolean,
-  jsdocs?: JSDoc[]
-): VariableStatement {
-  return node.addVariableStatement({
-    declarationKind: isConst
-      ? VariableDeclarationKind.Const
-      : VariableDeclarationKind.Let,
-    declarations: [{ name, type }],
-    docs: jsdocs && jsdocs.map(jsdoc => jsdoc.getText()),
-    hasDeclareKeyword
-  });
-}
-
-/** Copy one source file to the end of another source file. */
-export function appendSourceFile(
-  sourceFile: SourceFile,
-  targetSourceFile: SourceFile
-): void {
-  targetSourceFile.addStatements(`\n${sourceFile.print()}`);
-}
-
-/** Used when formatting diagnostics */
-const formatDiagnosticHost: ts.FormatDiagnosticsHost = {
-  getCurrentDirectory() {
-    return process.cwd();
-  },
-  getCanonicalFileName(path: string) {
-    return path;
-  },
-  getNewLine() {
-    return EOL;
-  }
-};
-
-/** Log diagnostics to the console with colour. */
-export function logDiagnostics(diagnostics: ts.Diagnostic[]): void {
-  if (diagnostics.length) {
-    console.log(
-      ts.formatDiagnosticsWithColorAndContext(diagnostics, formatDiagnosticHost)
-    );
-  }
-}
-
-/** Check diagnostics, and if any exist, exit the process */
-export function checkDiagnostics(project: Project, onlyFor?: string[]): void {
-  const program = project.getProgram();
-  const diagnostics = [
-    ...program.getGlobalDiagnostics(),
-    ...program.getSyntacticDiagnostics(),
-    ...program.getSemanticDiagnostics(),
-    ...program.getDeclarationDiagnostics()
-  ]
-    .filter(diagnostic => {
-      const sourceFile = diagnostic.getSourceFile();
-      return onlyFor && sourceFile
-        ? onlyFor.includes(sourceFile.getFilePath())
-        : true;
-    })
-    .map(diagnostic => diagnostic.compilerObject);
-
-  logDiagnostics(diagnostics);
-
-  if (diagnostics.length) {
-    process.exit(1);
-  }
-}
-
-function createDeclarationError(
-  msg: string,
-  declaration: ImportDeclaration | ExportDeclaration
-): Error {
-  return new Error(
-    `${msg}\n` +
-      `  In: "${declaration.getSourceFile().getFilePath()}"\n` +
-      `  Text: "${declaration.getText()}"`
-  );
-}
-
-export interface FlattenNamespaceOptions {
-  customSources?: { [sourceFilePath: string]: string };
-  debug?: boolean;
-  rootPath: string;
-  sourceFile: SourceFile;
-}
-
-/** Returns a string which indicates the source file as the source */
-export function getSourceComment(
-  sourceFile: SourceFile,
-  rootPath: string
-): string {
-  return `\n// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`;
-}
-
-/** Return a set of fully qualified symbol names for the files exports */
-function getExportedSymbols(sourceFile: SourceFile): Set<string> {
-  const exportedSymbols = new Set<string>();
-  const exportDeclarations = sourceFile.getExportDeclarations();
-  for (const exportDeclaration of exportDeclarations) {
-    const exportSpecifiers = exportDeclaration.getNamedExports();
-    for (const exportSpecifier of exportSpecifiers) {
-      const aliasedSymbol = exportSpecifier
-        .getSymbolOrThrow()
-        .getAliasedSymbol();
-      if (aliasedSymbol) {
-        exportedSymbols.add(aliasedSymbol.getFullyQualifiedName());
-      }
-    }
-  }
-  return exportedSymbols;
-}
-
-/** Take a namespace and flatten all exports. */
-export function flattenNamespace({
-  customSources,
-  debug,
-  rootPath,
-  sourceFile
-}: FlattenNamespaceOptions): string {
-  const sourceFiles = new Set<SourceFile>();
-  let output = "";
-  const exportedSymbols = getExportedSymbols(sourceFile);
-
-  function flattenDeclarations(
-    declaration: ImportDeclaration | ExportDeclaration
-  ): void {
-    const declarationSourceFile = declaration.getModuleSpecifierSourceFile();
-    if (declarationSourceFile) {
-      // eslint-disable-next-line @typescript-eslint/no-use-before-define
-      processSourceFile(declarationSourceFile);
-      declaration.remove();
-    }
-  }
-
-  function rectifyNodes(currentSourceFile: SourceFile): void {
-    currentSourceFile.forEachChild(node => {
-      if (TypeGuards.isAmbientableNode(node)) {
-        node.setHasDeclareKeyword(false);
-      }
-      if (TypeGuards.isExportableNode(node)) {
-        const nodeSymbol = node.getSymbol();
-        if (
-          nodeSymbol &&
-          !exportedSymbols.has(nodeSymbol.getFullyQualifiedName())
-        ) {
-          node.setIsExported(false);
-        }
-      }
-    });
-  }
-
-  function processSourceFile(
-    currentSourceFile: SourceFile
-  ): string | undefined {
-    if (sourceFiles.has(currentSourceFile)) {
-      return;
-    }
-    sourceFiles.add(currentSourceFile);
-
-    const currentSourceFilePath = currentSourceFile
-      .getFilePath()
-      .replace(/(\.d)?\.ts$/, "");
-    log("Process source file:", currentSourceFilePath);
-    if (customSources && currentSourceFilePath in customSources) {
-      log("  Using custom source.");
-      output += customSources[currentSourceFilePath];
-      return;
-    }
-
-    currentSourceFile.getImportDeclarations().forEach(flattenDeclarations);
-    currentSourceFile.getExportDeclarations().forEach(flattenDeclarations);
-
-    rectifyNodes(currentSourceFile);
-
-    output +=
-      (debug ? getSourceComment(currentSourceFile, rootPath) : "") +
-      currentSourceFile.print();
-  }
-
-  sourceFile.getExportDeclarations().forEach(exportDeclaration => {
-    const exportedSourceFile = exportDeclaration.getModuleSpecifierSourceFile();
-    if (exportedSourceFile) {
-      processSourceFile(exportedSourceFile);
-    } else {
-      throw createDeclarationError("Missing source file.", exportDeclaration);
-    }
-    exportDeclaration.remove();
-  });
-
-  rectifyNodes(sourceFile);
-
-  return (
-    output +
-    (debug ? getSourceComment(sourceFile, rootPath) : "") +
-    sourceFile.print()
-  );
-}
-
-interface InlineFilesOptions {
-  basePath: string;
-  debug?: boolean;
-  inline: string[];
-  targetSourceFile: SourceFile;
-}
-
-/** Inline files into the target source file. */
-export function inlineFiles({
-  basePath,
-  debug,
-  inline,
-  targetSourceFile
-}: InlineFilesOptions): void {
-  for (const filename of inline) {
-    const text = readFileSync(filename, {
-      encoding: "utf8"
-    });
-    targetSourceFile.addStatements(
-      debug
-        ? `\n// @url ${relative(basePath, filename)}\n\n${text}`
-        : `\n${text}`
-    );
-  }
-}
-
-/** Load a set of files into a file system host. */
-export function loadFiles(
-  project: Project,
-  filePaths: string[],
-  rebase?: string
-): void {
-  const fileSystem = project.getFileSystem();
-  for (const filePath of filePaths) {
-    const fileText = readFileSync(filePath, {
-      encoding: "utf8"
-    });
-    fileSystem.writeFileSync(
-      rebase ? join(rebase, basename(filePath)) : filePath,
-      fileText
-    );
-  }
-}
-
-/**
- * Load and write to a virtual file system all the default libs needed to
- * resolve types on project.
- */
-export function loadDtsFiles(
-  project: Project,
-  compilerOptions: ts.CompilerOptions
-): void {
-  const libSourcePath = dirname(ts.getDefaultLibFilePath(compilerOptions));
-  // TODO (@kitsonk) Add missing libs when ts-morph supports TypeScript 3.4
-  loadFiles(
-    project,
-    [
-      "lib.es2015.collection.d.ts",
-      "lib.es2015.core.d.ts",
-      "lib.es2015.d.ts",
-      "lib.es2015.generator.d.ts",
-      "lib.es2015.iterable.d.ts",
-      "lib.es2015.promise.d.ts",
-      "lib.es2015.proxy.d.ts",
-      "lib.es2015.reflect.d.ts",
-      "lib.es2015.symbol.d.ts",
-      "lib.es2015.symbol.wellknown.d.ts",
-      "lib.es2016.array.include.d.ts",
-      "lib.es2016.d.ts",
-      "lib.es2017.d.ts",
-      "lib.es2017.intl.d.ts",
-      "lib.es2017.object.d.ts",
-      "lib.es2017.sharedmemory.d.ts",
-      "lib.es2017.string.d.ts",
-      "lib.es2017.typedarrays.d.ts",
-      "lib.es2018.d.ts",
-      "lib.es2018.intl.d.ts",
-      "lib.es2018.promise.d.ts",
-      "lib.es5.d.ts",
-      "lib.esnext.d.ts",
-      "lib.esnext.array.d.ts",
-      "lib.esnext.asynciterable.d.ts",
-      "lib.esnext.intl.d.ts",
-      "lib.esnext.symbol.d.ts"
-    ].map(fileName => join(libSourcePath, fileName)),
-    "node_modules/typescript/lib/"
-  );
-}
-
-export interface NamespaceSourceFileOptions {
-  debug?: boolean;
-  namespace?: string;
-  namespaces: Set<string>;
-  rootPath: string;
-  sourceFileMap: Map<SourceFile, string>;
-}
-
-/**
- * Take a source file (`.d.ts`) and convert it to a namespace, resolving any
- * imports as their own namespaces.
- */
-export function namespaceSourceFile(
-  sourceFile: SourceFile,
-  {
-    debug,
-    namespace,
-    namespaces,
-    rootPath,
-    sourceFileMap
-  }: NamespaceSourceFileOptions
-): string {
-  if (sourceFileMap.has(sourceFile)) {
-    return "";
-  }
-  if (!namespace) {
-    namespace = sourceFile.getBaseNameWithoutExtension();
-  }
-  sourceFileMap.set(sourceFile, namespace);
-
-  sourceFile.forEachChild(node => {
-    if (TypeGuards.isAmbientableNode(node)) {
-      node.setHasDeclareKeyword(false);
-    }
-  });
-
-  // TODO need to properly unwrap this
-  const globalNamespace = sourceFile.getNamespace("global");
-  let globalNamespaceText = "";
-  if (globalNamespace) {
-    const structure = globalNamespace.getStructure();
-    if (structure.bodyText && typeof structure.bodyText === "string") {
-      globalNamespaceText = structure.bodyText;
-    } else {
-      throw new TypeError("Unexpected global declaration structure.");
-    }
-  }
-  if (globalNamespace) {
-    globalNamespace.remove();
-  }
-
-  const output = sourceFile
-    .getImportDeclarations()
-    .filter(declaration => {
-      const dsf = declaration.getModuleSpecifierSourceFile();
-      if (dsf == null) {
-        try {
-          const namespaceName = declaration
-            .getNamespaceImportOrThrow()
-            .getText();
-          if (!namespaces.has(namespaceName)) {
-            throw createDeclarationError(
-              "Already defined source file under different namespace.",
-              declaration
-            );
-          }
-        } catch (e) {
-          throw createDeclarationError(
-            `Unsupported import clause: ${e}`,
-            declaration
-          );
-        }
-        declaration.remove();
-      }
-      return dsf;
-    })
-    .map(declaration => {
-      if (
-        declaration.getNamedImports().length ||
-        !declaration.getNamespaceImport()
-      ) {
-        throw createDeclarationError("Unsupported import clause.", declaration);
-      }
-      const text = namespaceSourceFile(
-        declaration.getModuleSpecifierSourceFileOrThrow(),
-        {
-          debug,
-          namespace: declaration.getNamespaceImportOrThrow().getText(),
-          namespaces,
-          rootPath,
-          sourceFileMap
-        }
-      );
-      declaration.remove();
-      return text;
-    })
-    .join("\n");
-  sourceFile
-    .getExportDeclarations()
-    .forEach(declaration => declaration.remove());
-
-  namespaces.add(namespace);
-
-  return `${output}
-    ${globalNamespaceText || ""}
-
-    declare namespace ${namespace} {
-      ${debug ? getSourceComment(sourceFile, rootPath) : ""}
-      ${sourceFile.getText()}
-    }`;
-}
-
-/** Mirrors TypeScript's handling of paths */
-export function normalizeSlashes(path: string): string {
-  return path.replace(/\\/g, "/");
-}

tools/ts_library_builder/build_library.ts

-import { writeFileSync } from "fs";
-import { join } from "path";
-import * as prettier from "prettier";
-import {
-  InterfaceDeclaration,
-  ExpressionStatement,
-  NamespaceDeclarationKind,
-  Project,
-  SourceFile,
-  ts,
-  Type,
-  TypeGuards
-} from "ts-morph";
-import {
-  addInterfaceDeclaration,
-  addInterfaceProperty,
-  addSourceComment,
-  addTypeAlias,
-  addVariableDeclaration,
-  checkDiagnostics,
-  flattenNamespace,
-  getSourceComment,
-  inlineFiles,
-  loadDtsFiles,
-  loadFiles,
-  log,
-  logDiagnostics,
-  namespaceSourceFile,
-  normalizeSlashes,
-  setSilent
-} from "./ast_util";
-
-export interface BuildLibraryOptions {
-  /**
-   * The path to the root of the deno repository
-   */
-  basePath: string;
-
-  /**
-   * The path to the current build path
-   */
-  buildPath: string;
-
-  /**
-   * Denotes if the library should be built with debug information (comments
-   * that indicate the source of the types)
-   */
-  debug?: boolean;
-
-  /**
-   * An array of files that should be inlined into the library
-   */
-  inline?: string[];
-
-  /** An array of input files to be provided to the input project, relative to
-   * the basePath. */
-  inputs?: string[];
-
-  /**
-   * Path to globals file to be used I.E. `js/globals.ts`
-   */
-  additionalGlobals?: string[];
-
-  /**
-   * List of global variables to define as let instead of the default const.
-   */
-  declareAsLet?: string[];
-
-  /**
-   * The path to the output library
-   */
-  outFile: string;
-
-  /**
-   * Execute in silent mode or not
-   */
-  silent?: boolean;
-}
-
-const { ModuleKind, ModuleResolutionKind, ScriptTarget } = ts;
-
-/**
- * A preamble which is appended to the start of the library.
- */
-const libPreamble = `// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-
-/// <reference no-default-lib="true" />
-/// <reference lib="esnext" />
-
-`;
-
-interface FlattenOptions {
-  basePath: string;
-  customSources: { [filePath: string]: string };
-  filePath: string;
-  debug?: boolean;
-  declarationProject: Project;
-  globalInterfaceName?: string;
-  moduleName?: string;
-  namespaceName?: string;
-  targetSourceFile: SourceFile;
-}
-
-/** Flatten a module */
-export function flatten({
-  basePath,
-  customSources,
-  filePath,
-  debug,
-  declarationProject,
-  globalInterfaceName,
-  moduleName,
-  namespaceName,
-  targetSourceFile
-}: FlattenOptions): void {
-  // Flatten the source file into a single set of statements
-  const statements = flattenNamespace({
-    sourceFile: declarationProject.getSourceFileOrThrow(filePath),
-    rootPath: basePath,
-    customSources,
-    debug
-  });
-
-  // If a module name is specified create the module in the target file
-  if (moduleName) {
-    const namespace = targetSourceFile.addNamespace({
-      name: moduleName,
-      hasDeclareKeyword: true,
-      declarationKind: NamespaceDeclarationKind.Module
-    });
-
-    // Add the output of the flattening to the namespace
-    namespace.addStatements(statements);
-  }
-
-  if (namespaceName) {
-    const namespace = targetSourceFile.insertNamespace(0, {
-      name: namespaceName,
-      hasDeclareKeyword: true,
-      declarationKind: NamespaceDeclarationKind.Namespace
-    });
-
-    // Add the output of the flattening to the namespace
-    namespace.addStatements(statements);
-
-    if (globalInterfaceName) {
-      // Retrieve the global interface
-      const interfaceDeclaration = targetSourceFile.getInterfaceOrThrow(
-        globalInterfaceName
-      );
-
-      // Add the namespace to the global interface
-      addInterfaceProperty(
-        interfaceDeclaration,
-        namespaceName,
-        `typeof ${namespaceName}`
-      );
-    }
-  }
-}
-
-interface PrepareFileForMergeOptions {
-  globalVarName: string;
-  interfaceName: string;
-  targetSourceFile: SourceFile;
-}
-
-interface PrepareFileForMergeReturn {
-  interfaceDeclaration: InterfaceDeclaration;
-}
-
-export function prepareFileForMerge({
-  globalVarName,
-  interfaceName,
-  targetSourceFile
-}: PrepareFileForMergeOptions): PrepareFileForMergeReturn {
-  // Add the global object interface
-  const interfaceDeclaration = targetSourceFile.addInterface({
-    name: interfaceName,
-    hasDeclareKeyword: true
-  });
-
-  // Declare the global variable
-  addVariableDeclaration(
-    targetSourceFile,
-    globalVarName,
-    interfaceName,
-    true,
-    true
-  );
-
-  // Add self reference to the global variable
-  addInterfaceProperty(interfaceDeclaration, globalVarName, interfaceName);
-
-  return {
-    interfaceDeclaration
-  };
-}
-
-interface MergeGlobalOptions extends PrepareFileForMergeOptions {
-  basePath: string;
-  debug?: boolean;
-  declarationProject: Project;
-  filePath: string;
-  ignore?: string[];
-  inputProject: Project;
-  prepareReturn: PrepareFileForMergeReturn;
-  declareAsLet?: string[];
-}
-
-/** Take a module and merge it into the global scope */
-export function mergeGlobals({
-  basePath,
-  debug,
-  declarationProject,
-  filePath,
-  globalVarName,
-  ignore,
-  inputProject,
-  targetSourceFile,
-  declareAsLet,
-  prepareReturn: { interfaceDeclaration }
-}: MergeGlobalOptions): void {
-  // Retrieve source file from the input project
-  const sourceFile = inputProject.getSourceFileOrThrow(filePath);
-
-  // we are going to create a map of variables
-  const globalVariables = new Map<
-    string,
-    {
-      type: Type;
-      node: ExpressionStatement;
-    }
-  >();
-  const globalInterfaces: InterfaceDeclaration[] = [];
-
-  // For every augmentation of the global variable in source file, we want
-  // to extract the type and add it to the global variable map
-  sourceFile.forEachChild(node => {
-    if (TypeGuards.isExpressionStatement(node)) {
-      const firstChild = node.getFirstChild();
-      if (!firstChild) {
-        return;
-      }
-      if (TypeGuards.isBinaryExpression(firstChild)) {
-        const leftExpression = firstChild.getLeft();
-        if (
-          TypeGuards.isPropertyAccessExpression(leftExpression) &&
-          leftExpression.getExpression().getText() === globalVarName
-        ) {
-          const globalVarProperty = leftExpression.getName();
-          if (globalVarProperty !== globalVarName) {
-            globalVariables.set(globalVarProperty, {
-              type: firstChild.getType(),
-              node
-            });
-          }
-        }
-      }
-    } else if (TypeGuards.isInterfaceDeclaration(node) && node.isExported()) {
-      globalInterfaces.push(node);
-    }
-  });
-
-  // A set of source files that the types we are using are dependent on us
-  // importing
-  const dependentSourceFiles = new Set<SourceFile>();
-
-  // Create a global variable and add the property to the `Window` interface
-  // for each mutation of the `window` variable we observed in `globals.ts`
-  for (const [property, info] of globalVariables) {
-    if (!(ignore && ignore.includes(property))) {
-      const type = info.type.getText(info.node);
-      const typeSymbol = info.type.getSymbol();
-      if (typeSymbol) {
-        const valueDeclaration = typeSymbol.getValueDeclaration();
-        if (valueDeclaration) {
-          dependentSourceFiles.add(valueDeclaration.getSourceFile());
-        }
-      }
-      const isConst = !(declareAsLet && declareAsLet.includes(property));
-      addVariableDeclaration(targetSourceFile, property, type, isConst, true);
-      addInterfaceProperty(interfaceDeclaration, property, type);
-    }
-  }
-
-  // We need to copy over any type aliases
-  for (const typeAlias of sourceFile.getTypeAliases()) {
-    addTypeAlias(
-      targetSourceFile,
-      typeAlias.getName(),
-      typeAlias.getType().getText(sourceFile),
-      true
-    );
-  }
-
-  // We need to copy over any interfaces
-  for (const interfaceDeclaration of globalInterfaces) {
-    addInterfaceDeclaration(targetSourceFile, interfaceDeclaration);
-  }
-
-  // We need to ensure that we only namespace each source file once, so we
-  // will use this map for tracking that.
-  const sourceFileMap = new Map<SourceFile, string>();
-
-  // For each import declaration in source file we will want to convert the
-  // declaration source file into a namespace that exists within the merged
-  // namespace
-  const importDeclarations = sourceFile.getImportDeclarations();
-  const namespaces = new Set<string>();
-  for (const declaration of importDeclarations) {
-    const namespaceImport = declaration.getNamespaceImport();
-    if (namespaceImport) {
-      const declarationSourceFile = declaration.getModuleSpecifierSourceFile();
-      if (
-        declarationSourceFile &&
-        dependentSourceFiles.has(declarationSourceFile)
-      ) {
-        // the source file will resolve to the original `.ts` file, but the
-        // information we really want is in the emitted `.d.ts` file, so we will
-        // resolve to that file
-        const dtsFilePath = declarationSourceFile
-          .getFilePath()
-          .replace(/\.ts$/, ".d.ts");
-        const dtsSourceFile = declarationProject.getSourceFileOrThrow(
-          dtsFilePath
-        );
-        targetSourceFile.addStatements(
-          namespaceSourceFile(dtsSourceFile, {
-            debug,
-            namespace: namespaceImport.getText(),
-            namespaces,
-            rootPath: basePath,
-            sourceFileMap
-          })
-        );
-      }
-    }
-  }
-
-  if (debug) {
-    addSourceComment(targetSourceFile, sourceFile, basePath);
-  }
-}
-
-/**
- * Generate the runtime library for Deno and write it to the supplied out file
- * name.
- */
-export function main({
-  basePath,
-  buildPath,
-  inline,
-  inputs,
-  additionalGlobals,
-  declareAsLet,
-  debug,
-  outFile,
-  silent
-}: BuildLibraryOptions): void {
-  setSilent(silent);
-  log("-----");
-  log("build_lib");
-  log();
-  log(`basePath: "${basePath}"`);
-  log(`buildPath: "${buildPath}"`);
-  if (inline && inline.length) {
-    log("inline:");
-    for (const filename of inline) {
-      log(`  "${filename}"`);
-    }
-  }
-  if (inputs && inputs.length) {
-    log("inputs:");
-    for (const input of inputs) {
-      log(`  "${input}"`);
-    }
-  }
-  log(`debug: ${!!debug}`);
-  log(`outFile: "${outFile}"`);
-  log();
-
-  // the inputProject will take in the TypeScript files that are internal
-  // to Deno to be used to generate the library
-  const inputProject = new Project({
-    compilerOptions: {
-      baseUrl: basePath,
-      declaration: true,
-      emitDeclarationOnly: true,
-      lib: ["esnext"],
-      module: ModuleKind.ESNext,
-      moduleResolution: ModuleResolutionKind.NodeJs,
-      paths: {
-        "*": ["*", `${buildPath}/*`]
-      },
-      preserveConstEnums: true,
-      strict: true,
-      stripInternal: true,
-      target: ScriptTarget.ESNext
-    }
-  });
-
-  // Add the input files we will need to generate the declarations, `globals`
-  // plus any modules that are importable in the runtime need to be added here
-  // plus the `lib.esnext` which is used as the base library
-  if (inputs) {
-    inputProject.addExistingSourceFiles(
-      inputs.map(input => join(basePath, input))
-    );
-  }
-
-  // emit the project, which will be only the declaration files
-  const inputEmitResult = inputProject.emitToMemory();
-
-  log("Emitted input project.");
-
-  const inputDiagnostics = inputEmitResult
-    .getDiagnostics()
-    .map(d => d.compilerObject);
-  logDiagnostics(inputDiagnostics);
-  if (inputDiagnostics.length) {
-    console.error("\nDiagnostics present during input project emit.\n");
-    process.exit(1);
-  }
-
-  // the declaration project will be the target for the emitted files from
-  // the input project, these will be used to transfer information over to
-  // the final library file
-  const declarationProject = new Project({
-    compilerOptions: {
-      baseUrl: basePath,
-      lib: ["esnext"],
-      moduleResolution: ModuleResolutionKind.NodeJs,
-      paths: {
-        "*": ["*", `${buildPath}/*`]
-      },
-      strict: true,
-      target: ScriptTarget.ESNext
-    },
-    useVirtualFileSystem: true
-  });
-
-  // we don't want to add to the declaration project any of the original
-  // `.ts` source files, so we need to filter those out
-  const jsPath = normalizeSlashes(`${basePath}/js`);
-  const inputProjectFiles = inputProject
-    .getSourceFiles()
-    .map(sourceFile => sourceFile.getFilePath())
-    .filter(filePath => !filePath.startsWith(jsPath));
-  loadFiles(declarationProject, inputProjectFiles);
-
-  // now we add the emitted declaration files from the input project
-  for (const { filePath, text } of inputEmitResult.getFiles()) {
-    declarationProject.createSourceFile(filePath, text);
-  }
-
-  // the outputProject will contain the final library file we are looking to
-  // build
-  const outputProjectCompilerOptions: ts.CompilerOptions = {
-    baseUrl: buildPath,
-    lib: ["esnext"],
-    moduleResolution: ModuleResolutionKind.NodeJs,
-    strict: true,
-    target: ScriptTarget.ESNext
-  };
-
-  const outputProject = new Project({
-    compilerOptions: outputProjectCompilerOptions,
-    useVirtualFileSystem: true
-  });
-
-  // There are files we need to load into memory, so that the project "compiles"
-  loadDtsFiles(outputProject, outputProjectCompilerOptions);
-
-  // libDts is the final output file we are looking to build and we are not
-  // actually creating it, only in memory at this stage.
-  const libDTs = outputProject.createSourceFile(outFile);
-
-  // Deal with `js/deno.ts`
-
-  // Generate a object hash of substitutions of modules to use when flattening
-  const customSources = {};
-
-  const prepareForMergeOpts: PrepareFileForMergeOptions = {
-    globalVarName: "window",
-    interfaceName: "Window",
-    targetSourceFile: libDTs
-  };
-
-  const prepareReturn = prepareFileForMerge(prepareForMergeOpts);
-
-  mergeGlobals({
-    basePath,
-    debug,
-    declarationProject,
-    filePath: `${basePath}/js/globals.ts`,
-    inputProject,
-    ignore: ["Deno"],
-    declareAsLet,
-    ...prepareForMergeOpts,
-    prepareReturn
-  });
-
-  log(`Merged "globals" into global scope.`);
-
-  if (additionalGlobals) {
-    for (const additionalGlobal of additionalGlobals) {
-      mergeGlobals({
-        basePath,
-        debug,
-        declarationProject,
-        filePath: `${basePath}/${additionalGlobal}`,
-        inputProject,
-        ignore: ["Deno"],
-        declareAsLet,
-        ...prepareForMergeOpts,
-        prepareReturn
-      });
-    }
-
-    log(`Added additional "globals" into global scope.`);
-  }
-
-  flatten({
-    basePath,
-    customSources,
-    debug,
-    declarationProject,
-    filePath: `${basePath}/js/deno.d.ts`,
-    globalInterfaceName: "Window",
-    namespaceName: "Deno",
-    targetSourceFile: libDTs
-  });
-
-  log(`Created module "deno" and namespace Deno.`);
-
-  // Inline any files that were passed in, to be used to add additional libs
-  // which are not part of TypeScript.
-  if (inline && inline.length) {
-    inlineFiles({
-      basePath,
-      debug,
-      inline,
-      targetSourceFile: libDTs
-    });
-  }
-
-  // Add the preamble
-  libDTs.insertStatements(0, libPreamble);
-
-  // Check diagnostics
-  checkDiagnostics(outputProject);
-
-  // Output the final library file
-  libDTs.saveSync();
-  const libDTsText = prettier.format(
-    outputProject.getFileSystem().readFileSync(outFile, "utf8"),
-    { parser: "typescript" }
-  );
-  if (!silent) {
-    console.log(`Outputting library to: "${outFile}"`);
-    console.log(`  Length: ${libDTsText.length}`);
-  }
-  writeFileSync(outFile, libDTsText, { encoding: "utf8" });
-  if (!silent) {
-    console.log("-----");
-    console.log();
-  }
-}

tools/ts_library_builder/main.ts

-// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import * as path from "path";
-import { main as buildRuntimeLib } from "./build_library";
-
-// this is very simplistic argument parsing, just enough to integrate into
-// the build scripts, versus being very robust
-let basePath = process.cwd();
-let buildPath = path.join(basePath, "target", "debug");
-let outFile = path.join(buildPath, "gen", "lib", "lib.d.ts");
-let inline: string[] = [];
-let debug = false;
-let silent = false;
-
-process.argv.forEach((arg, i, argv) => {
-  switch (arg) {
-    case "--basePath":
-      basePath = path.resolve(argv[i + 1]);
-      break;
-    case "--buildPath":
-      buildPath = path.resolve(argv[i + 1]);
-      break;
-    case "--inline":
-      inline = argv[i + 1].split(",").map(filename => {
-        return path.resolve(filename);
-      });
-      break;
-    case "--outFile":
-      outFile = path.resolve(argv[i + 1]);
-      break;
-    case "--debug":
-      debug = true;
-      break;
-    case "--silent":
-      silent = true;
-      break;
-  }
-});
-
-buildRuntimeLib({
-  basePath,
-  buildPath,
-  debug,
-  inline,
-  inputs: [
-    "node_modules/typescript/lib/lib.esnext.d.ts",
-    "js/deno.ts",
-    "js/globals.ts"
-  ],
-  declareAsLet: ["onmessage"],
-  outFile,
-  silent
-});

tools/ts_library_builder/test.ts

-// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-// Run this manually with:
-//
-//  ./node_modules/.bin/ts-node --project tools/ts_library_builder/tsconfig.json tools/ts_library_builder/test.ts
-
-import * as assert from "assert";
-import { Project, ts } from "ts-morph";
-import { flatten, mergeGlobals, prepareFileForMerge } from "./build_library";
-import { inlineFiles, loadDtsFiles } from "./ast_util";
-
-const { ModuleKind, ModuleResolutionKind, ScriptTarget } = ts;
-
-/** setups and returns the fixtures for testing */
-// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-function setupFixtures() {
-  const basePath = process.cwd();
-  const buildPath = `${basePath}/tools/ts_library_builder/testdata`;
-  const outputFile = `${buildPath}/lib.output.d.ts`;
-  const inputProject = new Project({
-    compilerOptions: {
-      baseUrl: basePath,
-      declaration: true,
-      emitDeclarationOnly: true,
-      module: ModuleKind.ESNext,
-      moduleResolution: ModuleResolutionKind.NodeJs,
-      strict: true,
-      stripInternal: true,
-      target: ScriptTarget.ESNext
-    }
-  });
-  inputProject.addExistingSourceFiles([
-    `${buildPath}/globals.ts`,
-    `${buildPath}/api.ts`
-  ]);
-  const declarationProject = new Project({
-    compilerOptions: {},
-    useVirtualFileSystem: true
-  });
-  loadDtsFiles(declarationProject, {});
-  for (const { filePath, text } of inputProject.emitToMemory().getFiles()) {
-    declarationProject.createSourceFile(filePath, text);
-  }
-  const outputProject = new Project({
-    compilerOptions: {},
-    useVirtualFileSystem: true
-  });
-  loadDtsFiles(outputProject, {});
-  const outputSourceFile = outputProject.createSourceFile(outputFile);
-  const debug = true;
-
-  return {
-    basePath,
-    buildPath,
-    inputProject,
-    outputFile,
-    declarationProject,
-    outputProject,
-    outputSourceFile,
-    debug
-  };
-}
-
-function buildLibraryFlatten(): void {
-  const {
-    basePath,
-    buildPath,
-    debug,
-    declarationProject,
-    outputSourceFile: targetSourceFile
-  } = setupFixtures();
-
-  flatten({
-    basePath,
-    customSources: {},
-    debug,
-    declarationProject,
-    filePath: `${buildPath}/api.d.ts`,
-    moduleName: `"api"`,
-    namespaceName: "Api",
-    targetSourceFile
-  });
-
-  assert(targetSourceFile.getNamespace(`"api"`) != null);
-  assert(targetSourceFile.getNamespace("Api") != null);
-  assert.equal(targetSourceFile.getNamespaces().length, 2);
-  const moduleApi = targetSourceFile.getNamespaceOrThrow(`"api"`);
-  const functions = moduleApi.getFunctions();
-  assert.equal(functions[0].getName(), "foo");
-  assert.equal(
-    functions[0]
-      .getJsDocs()
-      .map(jsdoc => jsdoc.getInnerText())
-      .join("\n"),
-    "jsdoc for foo"
-  );
-  assert.equal(functions[1].getName(), "bar");
-  assert.equal(
-    functions[1]
-      .getJsDocs()
-      .map(jsdoc => jsdoc.getInnerText())
-      .join("\n"),
-    ""
-  );
-  assert.equal(functions.length, 2);
-  const classes = moduleApi.getClasses();
-  assert.equal(classes[0].getName(), "Foo");
-  assert.equal(classes.length, 1);
-  const variableDeclarations = moduleApi.getVariableDeclarations();
-  assert.equal(variableDeclarations[0].getName(), "arr");
-  assert.equal(variableDeclarations.length, 1);
-
-  const namespaceApi = targetSourceFile.getNamespaceOrThrow(`"api"`);
-  const functionsNs = namespaceApi.getFunctions();
-  assert.equal(functionsNs[0].getName(), "foo");
-  assert.equal(
-    functionsNs[0]
-      .getJsDocs()
-      .map(jsdoc => jsdoc.getInnerText())
-      .join("\n"),
-    "jsdoc for foo"
-  );
-  assert.equal(functionsNs[1].getName(), "bar");
-  assert.equal(
-    functionsNs[1]
-      .getJsDocs()
-      .map(jsdoc => jsdoc.getInnerText())
-      .join("\n"),
-    ""
-  );
-  assert.equal(functionsNs.length, 2);
-  const classesNs = namespaceApi.getClasses();
-  assert.equal(classesNs[0].getName(), "Foo");
-  assert.equal(classesNs.length, 1);
-  const variableDeclarationsNs = namespaceApi.getVariableDeclarations();
-  assert.equal(variableDeclarationsNs[0].getName(), "arr");
-  assert.equal(variableDeclarationsNs.length, 1);
-}
-
-function buildLibraryMerge(): void {
-  const {
-    basePath,
-    buildPath,
-    declarationProject,
-    debug,
-    inputProject,
-    outputSourceFile: targetSourceFile
-  } = setupFixtures();
-
-  const prepareForMergeOpts = {
-    globalVarName: "foobarbaz",
-    interfaceName: "FooBar",
-    targetSourceFile
-  };
-
-  const prepareReturn = prepareFileForMerge(prepareForMergeOpts);
-
-  mergeGlobals({
-    basePath,
-    declarationProject,
-    debug,
-    filePath: `${buildPath}/globals.ts`,
-    inputProject,
-    ...prepareForMergeOpts,
-    prepareReturn
-  });
-
-  assert(targetSourceFile.getNamespace("moduleC") != null);
-  assert(targetSourceFile.getNamespace("moduleD") != null);
-  assert(targetSourceFile.getNamespace("moduleE") != null);
-  assert(targetSourceFile.getNamespace("moduleF") != null);
-  assert.equal(targetSourceFile.getNamespaces().length, 4);
-  assert(targetSourceFile.getInterface("FooBar") != null);
-  assert.equal(targetSourceFile.getInterfaces().length, 2);
-  const variableDeclarations = targetSourceFile.getVariableDeclarations();
-  assert.equal(variableDeclarations[0].getType().getText(), `FooBar`);
-  assert.equal(variableDeclarations[1].getType().getText(), `moduleC.Bar`);
-  assert.equal(
-    variableDeclarations[2].getType().getText(),
-    `typeof moduleC.qat`
-  );
-  assert.equal(
-    variableDeclarations[3].getType().getText(),
-    `typeof moduleE.process`
-  );
-  assert.equal(
-    variableDeclarations[4].getType().getText(),
-    `typeof moduleD.reprocess`
-  );
-  assert.equal(
-    variableDeclarations[5].getType().getText(),
-    `typeof moduleC.Bar`
-  );
-  assert.equal(variableDeclarations.length, 6);
-  const typeAliases = targetSourceFile.getTypeAliases();
-  assert.equal(typeAliases[0].getName(), "Bar");
-  assert.equal(typeAliases[0].getType().getText(), "moduleC.Bar");
-  assert.equal(typeAliases.length, 1);
-  const exportedInterface = targetSourceFile.getInterfaceOrThrow("FizzBuzz");
-  const interfaceProperties = exportedInterface.getStructure().properties;
-  assert(interfaceProperties != null);
-  assert.equal(interfaceProperties!.length, 2);
-  assert.equal(interfaceProperties![0].name, "foo");
-  assert.equal(interfaceProperties![0].type, "string");
-  assert.equal(interfaceProperties![1].name, "bar");
-  assert.equal(interfaceProperties![1].type, "number");
-}
-
-function testInlineFiles(): void {
-  const {
-    basePath,
-    buildPath,
-    debug,
-    outputSourceFile: targetSourceFile
-  } = setupFixtures();
-
-  inlineFiles({
-    basePath,
-    debug,
-    inline: [`${buildPath}/lib.extra.d.ts`],
-    targetSourceFile
-  });
-
-  assert(targetSourceFile.getNamespace("Qat") != null);
-  const qatNamespace = targetSourceFile.getNamespaceOrThrow("Qat");
-  assert(qatNamespace.getClass("Foo") != null);
-}
-
-// TODO author unit tests for `ast_util.ts`
-
-function main(): void {
-  console.log("ts_library_builder buildLibraryFlatten");
-  buildLibraryFlatten();
-  console.log("ts_library_builder buildLibraryMerge");
-  buildLibraryMerge();
-  console.log("ts_library_builder testInlineFiles");
-  testInlineFiles();
-  console.log("ts_library_builder ok");
-}
-
-main();

tools/ts_library_builder/testdata/api.ts

-export { foo, bar } from "./moduleA";
-export { Foo } from "./moduleB";
-/** jsdoc for arr */
-export const arr: string[] = [];

tools/ts_library_builder/testdata/globals.ts

-import * as moduleC from "./moduleC";
-import * as moduleD from "./moduleD";
-import * as moduleE from "./moduleE";
-
-const foobarbaz: any = {};
-foobarbaz.bar = new moduleC.Bar();
-foobarbaz.qat = moduleC.qat;
-foobarbaz.process = moduleE.process;
-foobarbaz.reprocess = moduleD.reprocess;
-foobarbaz.Bar = moduleC.Bar;
-export type Bar = moduleC.Bar;
-export interface FizzBuzz {
-  foo: string;
-  bar: number;
-}

tools/ts_library_builder/testdata/lib.extra.d.ts

-// comment
-
-declare namespace Qat {
-  class Foo {
-    bar: string;
-  }
-}

tools/ts_library_builder/testdata/moduleA.ts

-/** jsdoc for foo */
-export function foo(a: string, b: string) {
-  console.log(a, b);
-}
-
-// no jsdoc for bar
-export async function bar(promise: Promise<void>): Promise<void> {
-  return promise.then(() => {});
-}

tools/ts_library_builder/testdata/moduleB.ts

-/** jsdoc about Foo */
-export class Foo {
-  private _foo = "foo";
-  /** jsdoc about Foo.log() */
-  log() {
-    console.log(this._foo);
-    return this._foo;
-  }
-}

tools/ts_library_builder/testdata/moduleC.ts

-/** jsdoc for Bar */
-export class Bar {
-  private _bar: string;
-  /** jsdoc for Bar.log() */
-  log() {
-    console.log(this._bar);
-    return this.log;
-  }
-}
-
-/**
- * jsdoc for qat
- * @param a jsdoc for qat(a)
- * @param b jsdoc for qat(b)
- */
-export function qat(a: string, b: string) {
-  return a + b;
-}

tools/ts_library_builder/testdata/moduleD.ts

-import * as moduleF from "./moduleF";
-
-export function reprocess(value: typeof moduleF.key) {
-  console.log(value);
-}

tools/ts_library_builder/testdata/moduleE.ts

-import * as moduleF from "./moduleF";
-
-export function process(value: typeof moduleF.key) {
-  console.log(value);
-}

tools/ts_library_builder/testdata/moduleF.ts

-export const key = "value";

tools/ts_library_builder/tsconfig.json

-{
-  "compilerOptions": {
-    "lib": ["esnext"],
-    "moduleResolution": "node",
-    "strict": true,
-    "target": "esnext"
-  },
-  "files": ["./build_library.ts"]
-}

tsconfig.json

@@ -21,6 +21,7 @@
   },
   "files": [
     "js/lib.web_assembly.d.ts",
+    "js/lib.deno_runtime.d.ts",
     "core/core.d.ts",
     "core/libdeno/libdeno.d.ts",
     "js/main.ts",