feat(cli cmd): deno xeval (#2260)

cli/BUILD.gn

@@ -115,6 +115,7 @@ ts_sources = [
   "../js/write_file.ts",
   "../js/performance.ts",
   "../js/version.ts",
+  "../js/xeval.ts",
   "../tsconfig.json",
 
   # Listing package.json and yarn.lock as sources ensures the bundle is rebuilt

cli/flags.rs

@@ -24,6 +24,8 @@ pub struct DenoFlags {
   pub no_prompts: bool,
   pub no_fetch: bool,
   pub v8_flags: Option<Vec<String>>,
+  pub xeval_replvar: Option<String>,
+  pub xeval_delim: Option<String>,
 }
 
 static ENV_VARIABLES_HELP: &str = "ENVIRONMENT VARIABLES:
@@ -193,6 +195,37 @@ Prettier dependencies on first run.
             .multiple(true)
             .required(true),
         ),
+    ).subcommand(
+      SubCommand::with_name("xeval")
+        .setting(AppSettings::DisableVersion)
+        .about("Eval a script on text segments from stdin")
+        .long_about(
+          "
+Eval a script on lines (or chunks split under delimiter) from stdin.
+
+Read from standard input and eval code on each whitespace-delimited
+string chunks.
+
+-I/--replvar optionally set variable name for input to be used in eval.
+Otherwise '$' will be used as default variable name.
+
+  cat /etc/passwd | deno xeval \"a = $.split(':'); if (a) console.log(a[0])\"
+  git branch | deno xeval -I 'line' \"if (line.startsWith('*')) console.log(line.slice(2))\"
+  cat LICENSE | deno xeval -d ' ' \"if ($ === 'MIT') console.log('MIT licensed')\"
+",
+        ).arg(
+          Arg::with_name("replvar")
+            .long("replvar")
+            .short("I")
+            .help("Set variable name to be used in eval, defaults to $")
+            .takes_value(true),
+        ).arg(
+          Arg::with_name("delim")
+            .long("delim")
+            .short("d")
+            .help("Set delimiter, defaults to newline")
+            .takes_value(true),
+        ).arg(Arg::with_name("code").takes_value(true).required(true)),
     ).subcommand(
       // this is a fake subcommand - it's used in conjunction with
       // AppSettings:AllowExternalSubcommand to treat it as an
@@ -281,6 +314,7 @@ pub enum DenoSubcommand {
   Repl,
   Run,
   Types,
+  Xeval,
 }
 
 pub fn flags_from_vec(
@@ -322,6 +356,17 @@ pub fn flags_from_vec(
       DenoSubcommand::Info
     }
     ("types", Some(_)) => DenoSubcommand::Types,
+    ("xeval", Some(eval_match)) => {
+      let code: &str = eval_match.value_of("code").unwrap();
+      flags.xeval_replvar =
+        Some(eval_match.value_of("replvar").unwrap_or("$").to_owned());
+      // Currently clap never escapes string,
+      // So -d "\n" won't expand to newline.
+      // Instead, do -d $'\n'
+      flags.xeval_delim = eval_match.value_of("delim").map(String::from);
+      argv.extend(vec![code.to_string()]);
+      DenoSubcommand::Xeval
+    }
     (script, Some(script_match)) => {
       argv.extend(vec![script.to_string()]);
       // check if there are any extra arguments that should
@@ -569,6 +614,25 @@ mod tests {
     assert_eq!(argv, svec!["deno", "script.ts"]);
   }
 
+  #[test]
+  fn test_flags_from_vec_15() {
+    let (flags, subcommand, argv) = flags_from_vec(svec![
+      "deno",
+      "xeval",
+      "-I",
+      "val",
+      "-d",
+      " ",
+      "console.log(val)"
+    ]);
+    let mut expected_flags = DenoFlags::default();
+    expected_flags.xeval_replvar = Some("val".to_owned());
+    expected_flags.xeval_delim = Some(" ".to_owned());
+    assert_eq!(flags, expected_flags);
+    assert_eq!(subcommand, DenoSubcommand::Xeval);
+    assert_eq!(argv, svec!["deno", "console.log(val)"]);
+  }
+
   #[test]
   fn test_set_flags_11() {
     let (flags, _, _) =

cli/main.rs

@@ -208,6 +208,31 @@ fn eval_command(flags: DenoFlags, argv: Vec<String>) {
   tokio_util::run(main_future);
 }
 
+fn xeval_command(flags: DenoFlags, argv: Vec<String>) {
+  let xeval_replvar = flags.xeval_replvar.clone().unwrap();
+  let (mut worker, state) = create_worker_and_state(flags, argv);
+  let xeval_source = format!(
+    "window._xevalWrapper = async function ({}){{
+        {}
+      }}",
+    &xeval_replvar, &state.argv[1]
+  );
+
+  let main_future = lazy(move || {
+    // Setup runtime.
+    js_check(worker.execute(&xeval_source));
+    js_check(worker.execute("denoMain()"));
+    worker
+      .then(|result| {
+        js_check(result);
+        Ok(())
+      }).map_err(|(err, _worker): (RustOrJsError, Worker)| {
+        print_err_and_exit(err)
+      })
+  });
+  tokio_util::run(main_future);
+}
+
 fn run_repl(flags: DenoFlags, argv: Vec<String>) {
   let (mut worker, _state) = create_worker_and_state(flags, argv);
 
@@ -275,5 +300,6 @@ fn main() {
     DenoSubcommand::Repl => run_repl(flags, argv),
     DenoSubcommand::Run => run_script(flags, argv),
     DenoSubcommand::Types => types_command(),
+    DenoSubcommand::Xeval => xeval_command(flags, argv),
   }
 }

cli/msg.fbs

@@ -175,6 +175,7 @@ table StartRes {
   deno_version: string;
   v8_version: string;
   no_color: bool;
+  xeval_delim: string;
 }
 
 table CompilerConfig {

cli/ops.rs

@@ -341,6 +341,12 @@ fn op_start(
 
   let main_module = state.main_module().map(|m| builder.create_string(&m));
 
+  let xeval_delim = state
+    .flags
+    .xeval_delim
+    .clone()
+    .map(|m| builder.create_string(&m));
+
   let inner = msg::StartRes::create(
     &mut builder,
     &msg::StartResArgs {
@@ -354,6 +360,7 @@ fn op_start(
       deno_version: Some(deno_version_off),
       no_color: !ansi::use_color(),
       exec_path: Some(exec_path),
+      xeval_delim,
       ..Default::default()
     },
   );

js/main.ts

@@ -9,7 +9,9 @@ import { assert, log } from "./util";
 import * as os from "./os";
 import { args } from "./deno";
 import { replLoop } from "./repl";
+import { xevalMain, XevalFunc } from "./xeval";
 import { setVersions } from "./version";
+import { window } from "./window";
 import { setLocation } from "./location";
 
 // builtin modules
@@ -43,7 +45,9 @@ export default function denoMain(name?: string): void {
   log("args", args);
   Object.freeze(args);
 
-  if (!mainModule) {
+  if (window["_xevalWrapper"] !== undefined) {
+    xevalMain(window["_xevalWrapper"] as XevalFunc, startResMsg.xevalDelim());
+  } else if (!mainModule) {
     replLoop();
   }
 }

js/xeval.ts

+import { Buffer } from "./buffer";
+import { stdin } from "./files";
+import { TextEncoder, TextDecoder } from "./text_encoding";
+import { Reader } from "./io";
+
+export type XevalFunc = (v: string) => void;
+
+async function writeAll(buffer: Buffer, arr: Uint8Array): Promise<void> {
+  let bytesWritten = 0;
+  while (bytesWritten < arr.length) {
+    try {
+      const nwritten = await buffer.write(arr.subarray(bytesWritten));
+      bytesWritten += nwritten;
+    } catch {
+      return;
+    }
+  }
+}
+
+// TODO(kevinkassimo): Move this utility to deno_std.
+// Import from there once doable.
+// Read from reader until EOF and emit string chunks separated
+// by the given delimiter.
+async function* chunks(
+  reader: Reader,
+  delim: string
+): AsyncIterableIterator<string> {
+  const inputBuffer = new Buffer();
+  const inspectArr = new Uint8Array(1024);
+  const encoder = new TextEncoder();
+  const decoder = new TextDecoder();
+  // Avoid unicode problems
+  const delimArr = encoder.encode(delim);
+
+  // Record how far we have gone with delimiter matching.
+  let nextMatchIndex = 0;
+  while (true) {
+    const rr = await reader.read(inspectArr);
+    if (rr.nread < 0) {
+      // Silently fail.
+      break;
+    }
+    const sliceRead = inspectArr.subarray(0, rr.nread);
+    // Remember how far we have scanned through inspectArr.
+    let nextSliceStartIndex = 0;
+    for (let i = 0; i < sliceRead.length; i++) {
+      if (sliceRead[i] == delimArr[nextMatchIndex]) {
+        // One byte matches with delimiter, move 1 step forward.
+        nextMatchIndex++;
+      } else {
+        // Match delimiter failed. Start from beginning.
+        nextMatchIndex = 0;
+      }
+      // A complete match is found.
+      if (nextMatchIndex === delimArr.length) {
+        nextMatchIndex = 0; // Reset delim match index.
+        const sliceToJoin = sliceRead.subarray(nextSliceStartIndex, i + 1);
+        // Record where to start next chunk when a subsequent match is found.
+        nextSliceStartIndex = i + 1;
+        // Write slice to buffer before processing, since potentially
+        // part of the delimiter is stored in the buffer.
+        await writeAll(inputBuffer, sliceToJoin);
+
+        let readyBytes = inputBuffer.bytes();
+        inputBuffer.reset();
+        // Remove delimiter from buffer bytes.
+        readyBytes = readyBytes.subarray(
+          0,
+          readyBytes.length - delimArr.length
+        );
+        let readyChunk = decoder.decode(readyBytes);
+        yield readyChunk;
+      }
+    }
+    // Write all unprocessed chunk to buffer for future inspection.
+    await writeAll(inputBuffer, sliceRead.subarray(nextSliceStartIndex));
+    if (rr.eof) {
+      // Flush the remainder unprocessed chunk.
+      const lastChunk = inputBuffer.toString();
+      yield lastChunk;
+      break;
+    }
+  }
+}
+
+export async function xevalMain(
+  xevalFunc: XevalFunc,
+  delim_: string | null
+): Promise<void> {
+  if (!delim_) {
+    delim_ = "\n";
+  }
+  for await (const chunk of chunks(stdin, delim_)) {
+    // Ignore empty chunks.
+    if (chunk.length > 0) {
+      xevalFunc(chunk);
+    }
+  }
+}

tests/030_xeval.out

+A
+B
+C

tests/030_xeval.test

+args: xeval console.log($.toUpperCase())
+input: a\nb\n\nc
+output: tests/030_xeval.out

tests/031_xeval_replvar.out

+A
+B
+C

tests/031_xeval_replvar.test

+args: xeval -I val console.log(val.toUpperCase());
+input: a\nb\n\nc
+output: tests/031_xeval_replvar.out

tests/032_xeval_delim.out

+A
+B
+C

tests/032_xeval_delim.test

+args: xeval -d DELIM console.log($.toUpperCase());
+input: aDELIMbDELIMDELIMc
+output: tests/032_xeval_delim.out

tools/integration_tests.py

@@ -65,6 +65,12 @@ def integration_tests(deno_exe, test_filter=None):
 
         stderr = subprocess.STDOUT if check_stderr else open(os.devnull, 'w')
 
+        stdin_input = (test.get("input",
+                                "").strip().decode("string_escape").replace(
+                                    "\r\n", "\n"))
+
+        has_stdin_input = len(stdin_input) > 0
+
         output_abs = os.path.join(root_path, test.get("output", ""))
         with open(output_abs, 'r') as f:
             expected_out = f.read()
@@ -73,8 +79,20 @@ def integration_tests(deno_exe, test_filter=None):
         sys.stdout.flush()
         actual_code = 0
         try:
-            actual_out = subprocess.check_output(
-                cmd, universal_newlines=True, stderr=stderr)
+            if has_stdin_input:
+                # Provided stdin
+                proc = subprocess.Popen(
+                    cmd,
+                    stdin=subprocess.PIPE,
+                    stdout=subprocess.PIPE,
+                    stderr=stderr)
+                actual_out, _ = proc.communicate(stdin_input)
+                actual_out = actual_out.replace("\r\n", "\n")
+            else:
+                # No stdin sent
+                actual_out = subprocess.check_output(
+                    cmd, universal_newlines=True, stderr=stderr)
+
         except subprocess.CalledProcessError as e:
             actual_code = e.returncode
             actual_out = e.output