Add 'bundle' subcommand. (#2467)

cli/compiler.rs

@@ -50,16 +50,22 @@ impl ModuleMetaData {
 type CompilerConfig = Option<(String, Vec<u8>)>;
 
 /// Creates the JSON message send to compiler.ts's onmessage.
-fn req(root_names: Vec<String>, compiler_config: CompilerConfig) -> Buf {
+fn req(
+  root_names: Vec<String>,
+  compiler_config: CompilerConfig,
+  bundle: Option<String>,
+) -> Buf {
   let j = if let Some((config_path, config_data)) = compiler_config {
     json!({
       "rootNames": root_names,
+      "bundle": bundle,
       "configPath": config_path,
       "config": str::from_utf8(&config_data).unwrap(),
     })
   } else {
     json!({
       "rootNames": root_names,
+      "bundle": bundle,
     })
   };
   j.to_string().into_boxed_str().into_boxed_bytes()
@@ -82,6 +88,67 @@ pub fn get_compiler_config(
   }
 }
 
+pub fn bundle_async(
+  state: ThreadSafeState,
+  module_name: String,
+  out_file: String,
+) -> impl Future<Item = (), Error = Diagnostic> {
+  debug!(
+    "Invoking the compiler to bundle. module_name: {}",
+    module_name
+  );
+
+  let root_names = vec![module_name.clone()];
+  let compiler_config = get_compiler_config(&state, "typescript");
+  let req_msg = req(root_names, compiler_config, Some(out_file));
+
+  // Count how many times we start the compiler worker.
+  state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst);
+
+  let mut worker = Worker::new(
+    "TS".to_string(),
+    startup_data::compiler_isolate_init(),
+    // TODO(ry) Maybe we should use a separate state for the compiler.
+    // as was done previously.
+    state.clone(),
+  );
+  js_check(worker.execute("denoMain()"));
+  js_check(worker.execute("workerMain()"));
+  js_check(worker.execute("compilerMain()"));
+
+  let resource = worker.state.resource.clone();
+  let compiler_rid = resource.rid;
+  let first_msg_fut = resources::post_message_to_worker(compiler_rid, req_msg)
+    .then(move |_| worker)
+    .then(move |result| {
+      if let Err(err) = result {
+        // TODO(ry) Need to forward the error instead of exiting.
+        eprintln!("{}", err.to_string());
+        std::process::exit(1);
+      }
+      debug!("Sent message to worker");
+      let stream_future =
+        resources::get_message_stream_from_worker(compiler_rid).into_future();
+      stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
+    });
+
+  first_msg_fut.map_err(|_| panic!("not handled")).and_then(
+    move |maybe_msg: Option<Buf>| {
+      debug!("Received message from worker");
+
+      if let Some(msg) = maybe_msg {
+        let json_str = std::str::from_utf8(&msg).unwrap();
+        debug!("Message: {}", json_str);
+        if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
+          return Err(diagnostics);
+        }
+      }
+
+      Ok(())
+    },
+  )
+}
+
 pub fn compile_async(
   state: ThreadSafeState,
   module_meta_data: &ModuleMetaData,
@@ -95,7 +162,7 @@ pub fn compile_async(
 
   let root_names = vec![module_name.clone()];
   let compiler_config = get_compiler_config(&state, "typescript");
-  let req_msg = req(root_names, compiler_config);
+  let req_msg = req(root_names, compiler_config, None);
 
   // Count how many times we start the compiler worker.
   state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst);
@@ -197,7 +264,13 @@ mod tests {
         maybe_source_map: None,
       };
 
-      out = compile_sync(ThreadSafeState::mock(), &out).unwrap();
+      out = compile_sync(
+        ThreadSafeState::mock(vec![
+          String::from("./deno"),
+          String::from("hello.js"),
+        ]),
+        &out,
+      ).unwrap();
       assert!(
         out
           .maybe_output_code
@@ -210,8 +283,29 @@ mod tests {
   #[test]
   fn test_get_compiler_config_no_flag() {
     let compiler_type = "typescript";
-    let state = ThreadSafeState::mock();
+    let state = ThreadSafeState::mock(vec![
+      String::from("./deno"),
+      String::from("hello.js"),
+    ]);
     let out = get_compiler_config(&state, compiler_type);
     assert_eq!(out, None);
   }
+
+  #[test]
+  fn test_bundle_async() {
+    let specifier = "./tests/002_hello.ts";
+    use crate::worker;
+    let module_name = worker::root_specifier_to_url(specifier)
+      .unwrap()
+      .to_string();
+
+    let state = ThreadSafeState::mock(vec![
+      String::from("./deno"),
+      String::from("./tests/002_hello.ts"),
+      String::from("$deno$/bundle.js"),
+    ]);
+    let out =
+      bundle_async(state, module_name, String::from("$deno$/bundle.js"));
+    assert_eq!(tokio_util::block_on(out), Ok(()));
+  }
 }

cli/flags.rs

@@ -154,6 +154,16 @@ To get help on the another subcommands (run in this case):
 Includes versions of Deno, V8 JavaScript Engine, and the TypeScript
 compiler.",
         ),
+    ).subcommand(
+      SubCommand::with_name("bundle")
+        .setting(AppSettings::DisableVersion)
+        .about("Bundle module and dependnecies into single file")
+        .long_about(
+          "Fetch, compile, and output to a single file a module and its dependencies.
+"
+        )
+          .arg(Arg::with_name("source_file").takes_value(true).required(true))
+          .arg(Arg::with_name("out_file").takes_value(true).required(true)),
     ).subcommand(
       SubCommand::with_name("fetch")
         .setting(AppSettings::DisableVersion)
@@ -436,6 +446,7 @@ const PRETTIER_URL: &str = "https://deno.land/std@v0.7.0/prettier/main.ts";
 /// There is no "Help" subcommand because it's handled by `clap::App` itself.
 #[derive(Debug, PartialEq)]
 pub enum DenoSubcommand {
+  Bundle,
   Eval,
   Fetch,
   Info,
@@ -455,6 +466,13 @@ pub fn flags_from_vec(
   let mut flags = parse_flags(&matches.clone());
 
   let subcommand = match matches.subcommand() {
+    ("bundle", Some(bundle_match)) => {
+      flags.allow_write = true;
+      let source_file: &str = bundle_match.value_of("source_file").unwrap();
+      let out_file: &str = bundle_match.value_of("out_file").unwrap();
+      argv.extend(vec![source_file.to_string(), out_file.to_string()]);
+      DenoSubcommand::Bundle
+    }
     ("eval", Some(eval_match)) => {
       flags.allow_net = true;
       flags.allow_env = true;
@@ -1034,4 +1052,19 @@ mod tests {
     assert_eq!(subcommand, DenoSubcommand::Run);
     assert_eq!(argv, svec!["deno", "script.ts"]);
   }
+
+  #[test]
+  fn test_flags_from_vec_26() {
+    let (flags, subcommand, argv) =
+      flags_from_vec(svec!["deno", "bundle", "source.ts", "bundle.js"]);
+    assert_eq!(
+      flags,
+      DenoFlags {
+        allow_write: true,
+        ..DenoFlags::default()
+      }
+    );
+    assert_eq!(subcommand, DenoSubcommand::Bundle);
+    assert_eq!(argv, svec!["deno", "source.ts", "bundle.js"])
+  }
 }

cli/main.rs

@@ -41,6 +41,7 @@ mod tokio_write;
 pub mod version;
 pub mod worker;
 
+use crate::compiler::bundle_async;
 use crate::errors::RustOrJsError;
 use crate::progress::Progress;
 use crate::state::ThreadSafeState;
@@ -261,6 +262,26 @@ fn xeval_command(flags: DenoFlags, argv: Vec<String>) {
   tokio_util::run(main_future);
 }
 
+fn bundle_command(flags: DenoFlags, argv: Vec<String>) {
+  let (mut _worker, state) = create_worker_and_state(flags, argv);
+
+  let main_module = state.main_module().unwrap();
+  let main_url = root_specifier_to_url(&main_module).unwrap();
+  assert!(state.argv.len() >= 3);
+  let out_file = state.argv[2].clone();
+  debug!(">>>>> bundle_async START");
+  let bundle_future = bundle_async(state, main_url.to_string(), out_file)
+    .map_err(|e| {
+      debug!("diagnostics returned, exiting!");
+      eprintln!("\n{}", e.to_string());
+      std::process::exit(1);
+    }).and_then(move |_| {
+      debug!(">>>>> bundle_async END");
+      Ok(())
+    });
+  tokio_util::run(bundle_future);
+}
+
 fn run_repl(flags: DenoFlags, argv: Vec<String>) {
   let (mut worker, _state) = create_worker_and_state(flags, argv);
 
@@ -322,6 +343,7 @@ fn main() {
   });
 
   match subcommand {
+    DenoSubcommand::Bundle => bundle_command(flags, argv),
     DenoSubcommand::Eval => eval_command(flags, argv),
     DenoSubcommand::Fetch => fetch_or_info_command(flags, argv, false),
     DenoSubcommand::Info => fetch_or_info_command(flags, argv, true),

cli/state.rs

@@ -311,8 +311,7 @@ impl ThreadSafeState {
   }
 
   #[cfg(test)]
-  pub fn mock() -> ThreadSafeState {
-    let argv = vec![String::from("./deno"), String::from("hello.js")];
+  pub fn mock(argv: Vec<String>) -> ThreadSafeState {
     ThreadSafeState::new(
       flags::DenoFlags::default(),
       argv,
@@ -349,5 +348,8 @@ impl ThreadSafeState {
 #[test]
 fn thread_safe() {
   fn f<S: Send + Sync>(_: S) {}
-  f(ThreadSafeState::mock());
+  f(ThreadSafeState::mock(vec![
+    String::from("./deno"),
+    String::from("hello.js"),
+  ]));
 }

cli/worker.rs

@@ -280,7 +280,10 @@ mod tests {
   }
 
   fn create_test_worker() -> Worker {
-    let state = ThreadSafeState::mock();
+    let state = ThreadSafeState::mock(vec![
+      String::from("./deno"),
+      String::from("hello.js"),
+    ]);
     let mut worker =
       Worker::new("TEST".to_string(), startup_data::deno_isolate_init(), state);
     js_check(worker.execute("denoMain()"));

js/compiler.ts

 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import * as msg from "gen/cli/msg_generated";
+import * as ts from "typescript";
+
+import { assetSourceCode } from "./assets";
+import { bold, cyan, yellow } from "./colors";
+import { Console } from "./console";
 import { core } from "./core";
 import { Diagnostic, fromTypeScriptDiagnostic } from "./diagnostics";
-import * as flatbuffers from "./flatbuffers";
+import { cwd } from "./dir";
 import { sendSync } from "./dispatch";
-import { TextDecoder } from "./text_encoding";
-import * as ts from "typescript";
+import * as flatbuffers from "./flatbuffers";
 import * as os from "./os";
-import { bold, cyan, yellow } from "./colors";
-import { window } from "./window";
-import { postMessage, workerClose, workerMain } from "./workers";
-import { Console } from "./console";
+import { TextDecoder, TextEncoder } from "./text_encoding";
 import { assert, notImplemented } from "./util";
 import * as util from "./util";
-import { cwd } from "./dir";
-import { assetSourceCode } from "./assets";
+import { window } from "./window";
+import { postMessage, workerClose, workerMain } from "./workers";
+import { writeFileSync } from "./write_file";
 
 // Startup boilerplate. This is necessary because the compiler has its own
 // snapshot. (It would be great if we could remove these things or centralize
@@ -32,6 +34,7 @@ const OUT_DIR = "$deno$";
 /** The format of the work message payload coming from the privileged side */
 interface CompilerReq {
   rootNames: string[];
+  bundle?: string;
   // TODO(ry) add compiler config to this interface.
   // options: ts.CompilerOptions;
   configPath?: string;
@@ -116,6 +119,7 @@ interface EmitResult {
   diagnostics?: Diagnostic;
 }
 
+/** Ops to Rust to resolve and fetch a modules meta data. */
 function fetchModuleMetaData(
   specifier: string,
   referrer: string
@@ -151,7 +155,23 @@ function fetchModuleMetaData(
   };
 }
 
-/** For caching source map and compiled js */
+/** Utility function to turn the number of bytes into a human readable
+ * unit */
+function humanFileSize(bytes: number): string {
+  const thresh = 1000;
+  if (Math.abs(bytes) < thresh) {
+    return bytes + " B";
+  }
+  const units = ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+  let u = -1;
+  do {
+    bytes /= thresh;
+    ++u;
+  } while (Math.abs(bytes) >= thresh && u < units.length - 1);
+  return `${bytes.toFixed(1)} ${units[u]}`;
+}
+
+/** Ops to rest for caching source map and compiled js */
 function cache(extension: string, moduleId: string, contents: string): void {
   util.log("compiler.cache", moduleId);
   const builder = flatbuffers.createBuilder();
@@ -168,6 +188,21 @@ function cache(extension: string, moduleId: string, contents: string): void {
   assert(baseRes == null);
 }
 
+const encoder = new TextEncoder();
+
+/** Given a fileName and the data, emit the file to the file system. */
+function emitBundle(fileName: string, data: string): void {
+  // For internal purposes, when trying to emit to `$deno$` just no-op
+  if (fileName.startsWith("$deno$")) {
+    console.warn("skipping compiler.emitBundle", fileName);
+    return;
+  }
+  const encodedData = encoder.encode(data);
+  console.log(`Emitting bundle to "${fileName}"`);
+  writeFileSync(fileName, encodedData);
+  console.log(`${humanFileSize(encodedData.length)} emitted.`);
+}
+
 /** Returns the TypeScript Extension enum for a given media type. */
 function getExtension(
   fileName: string,
@@ -200,6 +235,46 @@ class Host implements ts.CompilerHost {
     target: ts.ScriptTarget.ESNext
   };
 
+  private _resolveModule(specifier: string, referrer: string): ModuleMetaData {
+    // Handle built-in assets specially.
+    if (specifier.startsWith(ASSETS)) {
+      const moduleName = specifier.split("/").pop()!;
+      const assetName = moduleName.includes(".")
+        ? moduleName
+        : `${moduleName}.d.ts`;
+      assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
+      const sourceCode = assetSourceCode[assetName];
+      return {
+        moduleName,
+        filename: specifier,
+        mediaType: msg.MediaType.TypeScript,
+        sourceCode
+      };
+    }
+    return fetchModuleMetaData(specifier, referrer);
+  }
+
+  /* Deno specific APIs */
+
+  /** Provides the `ts.HostCompiler` interface for Deno.
+   *
+   * @param _bundle Set to a string value to configure the host to write out a
+   *   bundle instead of caching individual files.
+   */
+  constructor(private _bundle?: string) {
+    if (this._bundle) {
+      // options we need to change when we are generating a bundle
+      const bundlerOptions: ts.CompilerOptions = {
+        module: ts.ModuleKind.AMD,
+        inlineSourceMap: true,
+        outDir: undefined,
+        outFile: `${OUT_DIR}/bundle.js`,
+        sourceMap: false
+      };
+      Object.assign(this._options, bundlerOptions);
+    }
+  }
+
   /** Take a configuration string, parse it, and use it to merge with the
    * compiler's configuration options.  The method returns an array of compiler
    * options which were ignored, or `undefined`.
@@ -234,17 +309,32 @@ class Host implements ts.CompilerHost {
     };
   }
 
+  /* TypeScript CompilerHost APIs */
+
+  fileExists(_fileName: string): boolean {
+    return notImplemented();
+  }
+
+  getCanonicalFileName(fileName: string): string {
+    // console.log("getCanonicalFileName", fileName);
+    return fileName;
+  }
+
   getCompilationSettings(): ts.CompilerOptions {
     util.log("getCompilationSettings()");
     return this._options;
   }
 
-  fileExists(_fileName: string): boolean {
-    return notImplemented();
+  getCurrentDirectory(): string {
+    return "";
   }
 
-  readFile(_fileName: string): string | undefined {
-    return notImplemented();
+  getDefaultLibFileName(_options: ts.CompilerOptions): string {
+    return ASSETS + "/lib.deno_runtime.d.ts";
+  }
+
+  getNewLine(): string {
+    return "\n";
   }
 
   getSourceFile(
@@ -266,47 +356,8 @@ class Host implements ts.CompilerHost {
     );
   }
 
-  getDefaultLibFileName(_options: ts.CompilerOptions): string {
-    return ASSETS + "/lib.deno_runtime.d.ts";
-  }
-
-  writeFile(
-    fileName: string,
-    data: string,
-    writeByteOrderMark: boolean,
-    onError?: (message: string) => void,
-    sourceFiles?: ReadonlyArray<ts.SourceFile>
-  ): void {
-    util.log("writeFile", fileName);
-    assert(sourceFiles != null && sourceFiles.length == 1);
-    const sourceFileName = sourceFiles![0].fileName;
-
-    if (fileName.endsWith(".map")) {
-      // Source Map
-      cache(".map", sourceFileName, data);
-    } else if (fileName.endsWith(".js") || fileName.endsWith(".json")) {
-      // Compiled JavaScript
-      cache(".js", sourceFileName, data);
-    } else {
-      assert(false, "Trying to cache unhandled file type " + fileName);
-    }
-  }
-
-  getCurrentDirectory(): string {
-    return "";
-  }
-
-  getCanonicalFileName(fileName: string): string {
-    // console.log("getCanonicalFileName", fileName);
-    return fileName;
-  }
-
-  useCaseSensitiveFileNames(): boolean {
-    return true;
-  }
-
-  getNewLine(): string {
-    return "\n";
+  readFile(_fileName: string): string | undefined {
+    return notImplemented();
   }
 
   resolveModuleNames(
@@ -335,23 +386,42 @@ class Host implements ts.CompilerHost {
     );
   }
 
-  private _resolveModule(specifier: string, referrer: string): ModuleMetaData {
-    // Handle built-in assets specially.
-    if (specifier.startsWith(ASSETS)) {
-      const moduleName = specifier.split("/").pop()!;
-      const assetName = moduleName.includes(".")
-        ? moduleName
-        : `${moduleName}.d.ts`;
-      assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
-      const sourceCode = assetSourceCode[assetName];
-      return {
-        moduleName,
-        filename: specifier,
-        mediaType: msg.MediaType.TypeScript,
-        sourceCode
-      };
+  useCaseSensitiveFileNames(): boolean {
+    return true;
+  }
+
+  writeFile(
+    fileName: string,
+    data: string,
+    writeByteOrderMark: boolean,
+    onError?: (message: string) => void,
+    sourceFiles?: ReadonlyArray<ts.SourceFile>
+  ): void {
+    util.log("writeFile", fileName);
+    try {
+      if (this._bundle) {
+        emitBundle(this._bundle, data);
+      } else {
+        assert(sourceFiles != null && sourceFiles.length == 1);
+        const sourceFileName = sourceFiles![0].fileName;
+
+        if (fileName.endsWith(".map")) {
+          // Source Map
+          cache(".map", sourceFileName, data);
+        } else if (fileName.endsWith(".js") || fileName.endsWith(".json")) {
+          // Compiled JavaScript
+          cache(".js", sourceFileName, data);
+        } else {
+          assert(false, "Trying to cache unhandled file type " + fileName);
+        }
+      }
+    } catch (e) {
+      if (onError) {
+        onError(String(e));
+      } else {
+        throw e;
+      }
     }
-    return fetchModuleMetaData(specifier, referrer);
   }
 }
 
@@ -360,12 +430,12 @@ class Host implements ts.CompilerHost {
 window.compilerMain = function compilerMain(): void {
   // workerMain should have already been called since a compiler is a worker.
   window.onmessage = ({ data }: { data: CompilerReq }): void => {
+    const { rootNames, configPath, config, bundle } = data;
+    const host = new Host(bundle);
+
     let emitSkipped = true;
     let diagnostics: ts.Diagnostic[] | undefined;
 
-    const { rootNames, configPath, config } = data;
-    const host = new Host();
-
     // if there is a configuration supplied, we need to parse that
     if (config && config.length && configPath) {
       const configResult = host.configure(configPath, config);
@@ -410,6 +480,9 @@ window.compilerMain = function compilerMain(): void {
 
       // We will only proceed with the emit if there are no diagnostics.
       if (diagnostics && diagnostics.length === 0) {
+        if (bundle) {
+          console.log(`Bundling "${bundle}"`);
+        }
         const emitResult = program.emit();
         emitSkipped = emitResult.emitSkipped;
         // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned