Add repl (#998)

- Running repl from js side.
- Add tests for repl behavior.
- Handle ctrl-C and ctrl-D.

BUILD.gn

@@ -63,6 +63,7 @@ main_extern = [
   "$rust_build:rand",
   "$rust_build:remove_dir_all",
   "$rust_build:ring",
+  "$rust_build:rustyline",
   "$rust_build:tempfile",
   "$rust_build:tokio",
   "$rust_build:tokio_executor",
@@ -114,6 +115,7 @@ ts_sources = [
   "js/remove.ts",
   "js/rename.ts",
   "js/resources.ts",
+  "js/repl.ts",
   "js/stat.ts",
   "js/symlink.ts",
   "js/text_encoding.ts",

Cargo.toml

@@ -23,6 +23,7 @@ libc = "0.2.43"
 log = "0.4.6"
 rand = "0.5.5"
 remove_dir_all = "0.5.1"
+rustyline = "2.1.0"
 ring = "0.13.2"
 tempfile = "3.0.4"
 tokio = "0.1.11"

build_extra/rust/BUILD.gn

@@ -11,6 +11,51 @@ import("rust.gni")
 crates = "//third_party/rust_crates"
 registry_github = "$crates/registry/src/github.com-1ecc6299db9ec823/"
 
+rust_crate("nix") {
+  source_root = "$registry_github/nix-0.11.0/src/lib.rs"
+  extern = [
+    ":cfg_if",
+    ":libc",
+    ":void",
+    ":bitflags",
+  ]
+}
+
+rust_crate("rustyline") {
+  source_root = "$registry_github/rustyline-2.1.0/src/lib.rs"
+  extern = [
+    ":dirs",
+    ":libc",
+    ":log",
+    ":memchr",
+    ":nix",
+    ":unicode_segmentation",
+    ":unicode_width",
+    ":utf8parse",
+    ":winapi",
+  ]
+}
+
+rust_crate("bitflags") {
+  source_root = "$registry_github/bitflags-1.0.4/src/lib.rs"
+}
+
+rust_crate("unicode_segmentation") {
+  source_root = "$registry_github/unicode-segmentation-1.2.1/src/lib.rs"
+}
+
+rust_crate("memchr") {
+  source_root = "$registry_github/memchr-2.1.0/src/lib.rs"
+  extern = [
+    ":cfg_if",
+    ":libc",
+  ]
+}
+
+rust_crate("utf8parse") {
+  source_root = "$registry_github/utf8parse-0.1.1/src/lib.rs"
+}
+
 rust_crate("libc") {
   source_root = "$registry_github/libc-0.2.43/src/lib.rs"
   features = [ "use_std" ]
@@ -127,6 +172,7 @@ rust_crate("winapi") {
     "knownfolders",
     "ktmtypes",
     "libloaderapi",
+    "limits",
     "lsalookup",
     "minwinbase",
     "minwindef",
@@ -167,6 +213,7 @@ rust_crate("winapi") {
     "winnt",
     "winreg",
     "winsock2",
+    "winuser",
     "ws2def",
     "ws2ipdef",
     "ws2tcpip",

js/main.ts

@@ -8,6 +8,7 @@ import { libdeno } from "./libdeno";
 import { args } from "./deno";
 import { sendSync, handleAsyncMsgFromRust } from "./dispatch";
 import { promiseErrorExaminer, promiseRejectHandler } from "./promise_util";
+import { replLoop } from "./repl";
 import { version } from "typescript";
 
 function sendStart(): msg.StartRes {
@@ -77,13 +78,13 @@ export default function denoMain() {
   }
   log("args", args);
   Object.freeze(args);
-
   const inputFn = args[0];
-  if (!inputFn) {
-    console.log("No input script specified.");
-    os.exit(1);
-  }
 
   compiler.recompile = startResMsg.recompileFlag();
-  compiler.run(inputFn, `${cwd}/`);
+
+  if (inputFn) {
+    compiler.run(inputFn, `${cwd}/`);
+  } else {
+    replLoop();
+  }
 }

js/repl.ts

+// Copyright 2018 the Deno authors. All rights reserved. MIT license.
+import * as msg from "gen/msg_generated";
+import * as flatbuffers from "./flatbuffers";
+import { assert } from "./util";
+import * as deno from "./deno";
+import { close } from "./files";
+import * as dispatch from "./dispatch";
+import { exit } from "./os";
+import { window } from "./globals";
+
+function startRepl(historyFile: string): number {
+  const builder = flatbuffers.createBuilder();
+  const historyFile_ = builder.createString(historyFile);
+
+  msg.ReplStart.startReplStart(builder);
+  msg.ReplStart.addHistoryFile(builder, historyFile_);
+  const inner = msg.ReplStart.endReplStart(builder);
+
+  const baseRes = dispatch.sendSync(builder, msg.Any.ReplStart, inner);
+  assert(baseRes != null);
+  assert(msg.Any.ReplStartRes === baseRes!.innerType());
+  const innerRes = new msg.ReplStartRes();
+  assert(baseRes!.inner(innerRes) != null);
+  const rid = innerRes.rid();
+  return rid;
+}
+
+// @internal
+export function readline(rid: number, prompt: string): string {
+  const builder = flatbuffers.createBuilder();
+  const prompt_ = builder.createString(prompt);
+  msg.ReplReadline.startReplReadline(builder);
+  msg.ReplReadline.addRid(builder, rid);
+  msg.ReplReadline.addPrompt(builder, prompt_);
+  const inner = msg.ReplReadline.endReplReadline(builder);
+
+  // TODO use async?
+  const baseRes = dispatch.sendSync(builder, msg.Any.ReplReadline, inner);
+
+  assert(baseRes != null);
+  assert(msg.Any.ReplReadlineRes === baseRes!.innerType());
+  const innerRes = new msg.ReplReadlineRes();
+  assert(baseRes!.inner(innerRes) != null);
+  const line = innerRes.line();
+  assert(line !== null);
+  return line || "";
+}
+
+// @internal
+export function replLoop(): void {
+  window.deno = deno; // FIXME use a new scope (rather than window).
+
+  const historyFile = "deno_history.txt";
+  const prompt = "> ";
+
+  const rid = startRepl(historyFile);
+
+  let line = "";
+  while (true) {
+    try {
+      line = readline(rid, prompt);
+      line = line.trim();
+    } catch (err) {
+      if (err.message === "EOF") {
+        break;
+      }
+      console.error(err);
+      exit(1);
+    }
+    if (!line) {
+      continue;
+    }
+    if (line === ".exit") {
+      break;
+    }
+    try {
+      const result = eval.call(window, line); // FIXME use a new scope.
+      console.log(result);
+    } catch (err) {
+      if (err instanceof Error) {
+        console.error(`${err.constructor.name}: ${err.message}`);
+      } else {
+        console.error("Thrown:", err);
+      }
+    }
+  }
+
+  close(rid);
+}

src/main.rs

@@ -8,6 +8,7 @@ extern crate libc;
 extern crate rand;
 extern crate remove_dir_all;
 extern crate ring;
+extern crate rustyline;
 extern crate tempfile;
 extern crate tokio;
 extern crate tokio_executor;
@@ -35,6 +36,7 @@ pub mod msg;
 pub mod msg_util;
 pub mod ops;
 pub mod permissions;
+mod repl;
 pub mod resources;
 pub mod snapshot;
 mod tokio_util;

src/msg.fbs

@@ -24,6 +24,10 @@ union Any {
   Rename,
   Readlink,
   ReadlinkRes,
+  ReplStart,
+  ReplStartRes,
+  ReplReadline,
+  ReplReadlineRes,
   Resources,
   ResourcesRes,
   Symlink,
@@ -273,6 +277,24 @@ table ReadlinkRes {
   path: string;
 }
 
+table ReplStart {
+  history_file: string;
+  // TODO add config
+}
+
+table ReplStartRes {
+  rid: int;
+}
+
+table ReplReadline {
+  rid: int;
+  prompt: string;
+}
+
+table ReplReadlineRes {
+  line: string;
+}
+
 table Resources {}
 
 table Resource {

src/ops.rs

@@ -20,6 +20,7 @@ use futures::Poll;
 use hyper;
 use hyper::rt::{Future, Stream};
 use remove_dir_all::remove_dir_all;
+use repl;
 use resources::table_entries;
 use std;
 use std::fs;
@@ -96,6 +97,8 @@ pub fn dispatch(
       msg::Any::Read => op_read,
       msg::Any::Remove => op_remove,
       msg::Any::Rename => op_rename,
+      msg::Any::ReplReadline => op_repl_readline,
+      msg::Any::ReplStart => op_repl_start,
       msg::Any::Resources => op_resources,
       msg::Any::SetEnv => op_set_env,
       msg::Any::Shutdown => op_shutdown,
@@ -1086,6 +1089,75 @@ fn op_read_link(
   })
 }
 
+fn op_repl_start(
+  state: &Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
+  let inner = base.inner_as_repl_start().unwrap();
+  let cmd_id = base.cmd_id();
+  let history_file = String::from(inner.history_file().unwrap());
+
+  debug!("op_repl_start {}", history_file);
+  let history_path = repl::history_path(&state.dir, &history_file);
+  let repl = repl::Repl::new(history_path);
+  let resource = resources::add_repl(repl);
+
+  let builder = &mut FlatBufferBuilder::new();
+  let inner = msg::ReplStartRes::create(
+    builder,
+    &msg::ReplStartResArgs { rid: resource.rid },
+  );
+  ok_future(serialize_response(
+    cmd_id,
+    builder,
+    msg::BaseArgs {
+      inner: Some(inner.as_union_value()),
+      inner_type: msg::Any::ReplStartRes,
+      ..Default::default()
+    },
+  ))
+}
+
+fn op_repl_readline(
+  _state: &Arc<IsolateState>,
+  base: &msg::Base,
+  data: &'static mut [u8],
+) -> Box<Op> {
+  assert_eq!(data.len(), 0);
+  let inner = base.inner_as_repl_readline().unwrap();
+  let cmd_id = base.cmd_id();
+  let rid = inner.rid();
+  let prompt = inner.prompt().unwrap().to_owned();
+  debug!("op_repl_readline {} {}", rid, prompt);
+
+  // Ignore this clippy warning until this issue is addressed:
+  // https://github.com/rust-lang-nursery/rust-clippy/issues/1684
+  #[cfg_attr(feature = "cargo-clippy", allow(redundant_closure_call))]
+  Box::new(futures::future::result((move || {
+    let line = resources::readline(rid, &prompt)?;
+
+    let builder = &mut FlatBufferBuilder::new();
+    let line_off = builder.create_string(&line);
+    let inner = msg::ReplReadlineRes::create(
+      builder,
+      &msg::ReplReadlineResArgs {
+        line: Some(line_off),
+      },
+    );
+    Ok(serialize_response(
+      cmd_id,
+      builder,
+      msg::BaseArgs {
+        inner: Some(inner.as_union_value()),
+        inner_type: msg::Any::ReplReadlineRes,
+        ..Default::default()
+      },
+    ))
+  })()))
+}
+
 fn op_truncate(
   state: &Arc<IsolateState>,
   base: &msg::Base,

src/repl.rs

+// Copyright 2018 the Deno authors. All rights reserved. MIT license.
+extern crate rustyline;
+
+use rustyline::error::ReadlineError::Interrupted;
+
+use msg::ErrorKind;
+use std::error::Error;
+
+use deno_dir::DenoDir;
+use errors::new as deno_error;
+use errors::DenoResult;
+use std::path::PathBuf;
+use std::process::exit;
+
+#[cfg(not(windows))]
+use rustyline::Editor;
+
+// Work around the issue that on Windows, `struct Editor` does not implement the
+// `Send` trait, because it embeds a windows HANDLE which is a type alias for
+// *mut c_void. This value isn't actually a pointer and there's nothing that
+// can be mutated through it, so hack around it. TODO: a prettier solution.
+#[cfg(windows)]
+use std::ops::{Deref, DerefMut};
+
+#[cfg(windows)]
+struct Editor<T: rustyline::Helper> {
+  inner: rustyline::Editor<T>,
+}
+
+#[cfg(windows)]
+unsafe impl<T: rustyline::Helper> Send for Editor<T> {}
+
+#[cfg(windows)]
+impl<T: rustyline::Helper> Editor<T> {
+  pub fn new() -> Editor<T> {
+    Editor {
+      inner: rustyline::Editor::<T>::new(),
+    }
+  }
+}
+
+#[cfg(windows)]
+impl<T: rustyline::Helper> Deref for Editor<T> {
+  type Target = rustyline::Editor<T>;
+
+  fn deref(&self) -> &rustyline::Editor<T> {
+    &self.inner
+  }
+}
+
+#[cfg(windows)]
+impl<T: rustyline::Helper> DerefMut for Editor<T> {
+  fn deref_mut(&mut self) -> &mut rustyline::Editor<T> {
+    &mut self.inner
+  }
+}
+
+pub struct Repl {
+  editor: Editor<()>,
+  history_file: PathBuf,
+}
+
+impl Repl {
+  pub fn new(history_file: PathBuf) -> Repl {
+    let mut repl = Repl {
+      editor: Editor::<()>::new(),
+      history_file,
+    };
+
+    repl.load_history();
+    repl
+  }
+
+  fn load_history(&mut self) -> () {
+    debug!("Loading REPL history: {:?}", self.history_file);
+    self
+      .editor
+      .load_history(&self.history_file.to_str().unwrap())
+      .map_err(|e| debug!("Unable to load history file: {:?} {}", self.history_file, e))
+      // ignore this error (e.g. it occurs on first load)
+      .unwrap_or(())
+  }
+
+  fn save_history(&mut self) -> DenoResult<()> {
+    self
+      .editor
+      .save_history(&self.history_file.to_str().unwrap())
+      .map(|_| debug!("Saved REPL history to: {:?}", self.history_file))
+      .map_err(|e| {
+        eprintln!("Unable to save REPL history: {:?} {}", self.history_file, e);
+        deno_error(ErrorKind::Other, e.description().to_string())
+      })
+  }
+
+  pub fn readline(&mut self, prompt: &str) -> DenoResult<String> {
+    self
+      .editor
+      .readline(&prompt)
+      .map(|line| {
+        self.editor.add_history_entry(line.as_ref());
+        line
+      }).map_err(|e| match e {
+        Interrupted => {
+          self.save_history().unwrap();
+          exit(1)
+        }
+        e => deno_error(ErrorKind::Other, e.description().to_string()),
+      })
+  }
+}
+
+impl Drop for Repl {
+  fn drop(&mut self) {
+    self.save_history().unwrap();
+  }
+}
+
+pub fn history_path(dir: &DenoDir, history_file: &str) -> PathBuf {
+  let mut p: PathBuf = dir.root.clone();
+  p.push(history_file);
+  p
+}

src/resources.rs

@@ -10,7 +10,10 @@
 
 #[cfg(unix)]
 use eager_unix as eager;
+use errors::bad_resource;
 use errors::DenoError;
+use errors::DenoResult;
+use repl::Repl;
 use tokio_util;
 use tokio_write;
 
@@ -56,6 +59,7 @@ enum Repr {
   FsFile(tokio::fs::File),
   TcpListener(tokio::net::TcpListener),
   TcpStream(tokio::net::TcpStream),
+  Repl(Repl),
 }
 
 pub fn table_entries() -> Vec<(i32, String)> {
@@ -85,6 +89,7 @@ fn inspect_repr(repr: &Repr) -> String {
     Repr::FsFile(_) => "fsFile",
     Repr::TcpListener(_) => "tcpListener",
     Repr::TcpStream(_) => "tcpStream",
+    Repr::Repl(_) => "repl",
   };
 
   String::from(h_repr)
@@ -150,10 +155,7 @@ impl AsyncRead for Resource {
         Repr::FsFile(ref mut f) => f.poll_read(buf),
         Repr::Stdin(ref mut f) => f.poll_read(buf),
         Repr::TcpStream(ref mut f) => f.poll_read(buf),
-        Repr::Stdout(_) | Repr::Stderr(_) => {
-          panic!("Cannot read from stdout/stderr")
-        }
-        Repr::TcpListener(_) => panic!("Cannot read"),
+        _ => panic!("Cannot read"),
       },
     }
   }
@@ -180,8 +182,7 @@ impl AsyncWrite for Resource {
         Repr::Stdout(ref mut f) => f.poll_write(buf),
         Repr::Stderr(ref mut f) => f.poll_write(buf),
         Repr::TcpStream(ref mut f) => f.poll_write(buf),
-        Repr::Stdin(_) => panic!("Cannot write to stdin"),
-        Repr::TcpListener(_) => panic!("Cannot write"),
+        _ => panic!("Cannot write"),
       },
     }
   }
@@ -221,6 +222,26 @@ pub fn add_tcp_stream(stream: tokio::net::TcpStream) -> Resource {
   Resource { rid }
 }
 
+pub fn add_repl(repl: Repl) -> Resource {
+  let rid = new_rid();
+  let mut tg = RESOURCE_TABLE.lock().unwrap();
+  let r = tg.insert(rid, Repr::Repl(repl));
+  assert!(r.is_none());
+  Resource { rid }
+}
+
+pub fn readline(rid: ResourceId, prompt: &str) -> DenoResult<String> {
+  let mut table = RESOURCE_TABLE.lock().unwrap();
+  let maybe_repr = table.get_mut(&rid);
+  match maybe_repr {
+    Some(Repr::Repl(ref mut r)) => {
+      let line = r.readline(&prompt)?;
+      Ok(line)
+    }
+    _ => Err(bad_resource()),
+  }
+}
+
 pub fn lookup(rid: ResourceId) -> Option<Resource> {
   let table = RESOURCE_TABLE.lock().unwrap();
   table.get(&rid).map(|_| Resource { rid })

third_party @ d1447e63

-Subproject commit 96d35734a47e5b63d98ba7f7cbd01dfe4cbc435f
+Subproject commit d1447e6375ebddf590f1cd87219dadeca51cfec1

tools/repl_test.py

+# Copyright 2018 the Deno authors. All rights reserved. MIT license.
+import os
+from subprocess import PIPE, Popen
+import sys
+from time import sleep
+
+from util import build_path, executable_suffix, green_ok
+
+
+class Repl(object):
+    def __init__(self, deno_exe):
+        self.deno_exe = deno_exe
+        self.warm_up()
+
+    def input(self, *lines, **kwargs):
+        exit_ = kwargs.pop("exit", True)
+        p = Popen([self.deno_exe], stdout=PIPE, stderr=PIPE, stdin=PIPE)
+        try:
+            for line in lines:
+                p.stdin.write(line.encode("utf-8") + b'\n')
+            if exit_:
+                p.stdin.write(b'deno.exit(0)\n')
+            else:
+                sleep(1)  # wait to be killed by js
+            out, err = p.communicate()
+        except Exception as e:  # Should this be CalledProcessError?
+            p.kill()
+            p.wait()
+            raise
+        retcode = p.poll()
+        # Ignore Windows CRLF (\r\n).
+        return out.replace('\r\n', '\n'), err.replace('\r\n', '\n'), retcode
+
+    def warm_up(self):
+        # This may output an error message about the history file (ignore it).
+        self.input("")
+
+    def test_function(self):
+        out, err, code = self.input("deno.writeFileSync")
+        assertEqual(out, '[Function: writeFileSync]\n')
+        assertEqual(err, '')
+        assertEqual(code, 0)
+
+    def test_console_log(self):
+        out, err, code = self.input("console.log('hello')", "'world'")
+        assertEqual(out, 'hello\nundefined\nworld\n')
+        assertEqual(err, '')
+        assertEqual(code, 0)
+
+    def test_variable(self):
+        out, err, code = self.input("var a = 123;", "a")
+        assertEqual(out, 'undefined\n123\n')
+        assertEqual(err, '')
+        assertEqual(code, 0)
+
+    def test_settimeout(self):
+        out, err, code = self.input(
+            "setTimeout(() => { console.log('b'); deno.exit(0); }, 10)",
+            "'a'",
+            exit=False)
+        assertEqual(out, '1\na\nb\n')
+        assertEqual(err, '')
+        assertEqual(code, 0)
+
+    def test_reference_error(self):
+        out, err, code = self.input("not_a_variable")
+        assertEqual(out, '')
+        assertEqual(err, 'ReferenceError: not_a_variable is not defined\n')
+        assertEqual(code, 0)
+
+    def test_syntax_error(self):
+        out, err, code = self.input("syntax error")
+        assertEqual(out, '')
+        assertEqual(err, "SyntaxError: Unexpected identifier\n")
+        assertEqual(code, 0)
+
+    def test_type_error(self):
+        out, err, code = self.input("console()")
+        assertEqual(out, '')
+        assertEqual(err, 'TypeError: console is not a function\n')
+        assertEqual(code, 0)
+
+    def test_exit_command(self):
+        out, err, code = self.input(".exit", "'ignored'", exit=False)
+        assertEqual(out, '')
+        assertEqual(err, '')
+        assertEqual(code, 0)
+
+    def run(self):
+        print('repl_test.py')
+        test_names = [name for name in dir(self) if name.startswith("test_")]
+        for t in test_names:
+            self.__getattribute__(t)()
+            sys.stdout.write(".")
+            sys.stdout.flush()
+        print(' {}\n'.format(green_ok()))
+
+
+def assertEqual(left, right):
+    if left != right:
+        raise AssertionError("{} != {}".format(repr(left), repr(right)))
+
+
+def repl_tests(deno_exe):
+    Repl(deno_exe).run()
+
+
+if __name__ == "__main__":
+    deno_exe = os.path.join(build_path(), "deno" + executable_suffix)
+    repl_tests(deno_exe)

tools/test.py

@@ -11,6 +11,7 @@ from util import build_path, enable_ansi_colors, executable_suffix, run, rmtree
 from unit_tests import unit_tests
 from util_test import util_test
 from benchmark_test import benchmark_test
+from repl_test import repl_tests
 import subprocess
 import http_server
 
@@ -67,6 +68,8 @@ def main(argv):
         from permission_prompt_test import permission_prompt_test
         permission_prompt_test(deno_exe)
 
+    repl_tests(deno_exe)
+
     rmtree(deno_dir)
 
     deno_dir_test(deno_exe, deno_dir)