Re-implement init scripts in core (#1958)

Re-enables arm64 CI test

.travis.yml

@@ -131,8 +131,7 @@ jobs:
       - sudo apt -yq install qemu qemu-user binfmt-support qemu-user-binfmt
       - sudo ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/ld-linux-aarch64.so.1
       - export QEMU_LD_PREFIX=/usr/aarch64-linux-gnu
-      # TODO(ry) Temp disabling arm test to make progress on core integration.
-      #- $CARGO_TARGET_DIR/aarch64-unknown-linux-gnu/release/deno tests/002_hello.ts
+      - $CARGO_TARGET_DIR/aarch64-unknown-linux-gnu/release/deno tests/002_hello.ts
       # - DENO_BUILD_MODE=release ./tools/test.py $CARGO_TARGET_DIR/aarch64-unknown-linux-gnu/release TODO(afinch7): Get the tests working
 
     - name: "cargo release linux x86_64"

core/http_bench.rs

@@ -99,8 +99,13 @@ pub type HttpBenchOp = dyn Future<Item = i32, Error = std::io::Error> + Send;
 struct HttpBench();
 
 impl Behavior for HttpBench {
-  fn startup_snapshot(&mut self) -> Option<deno_buf> {
-    None
+  fn startup_data(&mut self) -> Option<StartupData> {
+    let js_source = include_str!("http_bench.js");
+
+    Some(StartupData::Script(StartupScript {
+      source: js_source.to_string(),
+      filename: "http_bench.js".to_string(),
+    }))
   }
 
   fn resolve(&mut self, _specifier: &str, _referrer: deno_mod) -> deno_mod {
@@ -164,15 +169,12 @@ impl Behavior for HttpBench {
 }
 
 fn main() {
-  let js_source = include_str!("http_bench.js");
-
   let main_future = lazy(move || {
-    let mut isolate = deno_core::Isolate::new(HttpBench());
-
     // TODO currently isolate.execute() must be run inside tokio, hence the
     // lazy(). It would be nice to not have that contraint. Probably requires
     // using v8::MicrotasksPolicy::kExplicit
-    js_check(isolate.execute("http_bench.js", js_source));
+    let isolate = deno_core::Isolate::new(HttpBench());
+
     isolate.then(|r| {
       js_check(r);
       Ok(())

core/isolate.rs

@@ -42,11 +42,26 @@ impl Future for PendingOp {
   }
 }
 
+/// Stores a script used to initalize a Isolate
+pub struct StartupScript {
+  pub source: String,
+  pub filename: String,
+}
+
+/// Represents data used to initialize isolate at startup
+/// either a binary snapshot or a javascript source file
+/// in the form of the StartupScript struct.
+pub enum StartupData {
+  Script(StartupScript),
+  Snapshot(deno_buf),
+}
+
 /// Defines the behavior of an Isolate.
 pub trait Behavior {
-  /// Called exactly once when an Isolate is created to retrieve the startup
-  /// snapshot.
-  fn startup_snapshot(&mut self) -> Option<deno_buf>;
+  /// Allow for a behavior to define the snapshot or script used at
+  /// startup to initalize the isolate. Called exactly once when an
+  /// Isolate is created.
+  fn startup_data(&mut self) -> Option<StartupData>;
 
   /// Called during mod_instantiate() to resolve imports.
   fn resolve(&mut self, specifier: &str, referrer: deno_mod) -> deno_mod;
@@ -96,9 +111,15 @@ impl<B: Behavior> Isolate<B> {
     let shared = SharedQueue::new(RECOMMENDED_SIZE);
 
     let needs_init = true;
+    // Seperate into Option values for eatch startup type
+    let (startup_snapshot, startup_script) = match behavior.startup_data() {
+      Some(StartupData::Snapshot(d)) => (Some(d), None),
+      Some(StartupData::Script(d)) => (None, Some(d)),
+      None => (None, None),
+    };
     let config = libdeno::deno_config {
       will_snapshot: 0,
-      load_snapshot: match behavior.startup_snapshot() {
+      load_snapshot: match startup_snapshot {
         Some(s) => s,
         None => libdeno::deno_buf::empty(),
       },
@@ -107,14 +128,24 @@ impl<B: Behavior> Isolate<B> {
     };
     let libdeno_isolate = unsafe { libdeno::deno_new(config) };
 
-    Self {
+    let mut core_isolate = Self {
       libdeno_isolate,
       behavior,
       shared,
       needs_init,
       pending_ops: Vec::new(),
       polled_recently: false,
-    }
+    };
+
+    // If we want to use execute this has to happen here sadly.
+    match startup_script {
+      Some(s) => core_isolate
+        .execute(s.filename.as_str(), s.source.as_str())
+        .unwrap(),
+      None => {}
+    };
+
+    core_isolate
   }
 
   /// Executes a bit of built-in JavaScript to provide Deno._sharedQueue.
@@ -475,7 +506,7 @@ mod tests {
   }
 
   impl Behavior for TestBehavior {
-    fn startup_snapshot(&mut self) -> Option<deno_buf> {
+    fn startup_data(&mut self) -> Option<StartupData> {
       None
     }
 

src/cli.rs

@@ -3,7 +3,6 @@
 #![allow(dead_code)]
 
 use crate::errors::DenoResult;
-use crate::isolate_init::IsolateInit;
 use crate::isolate_state::IsolateState;
 use crate::ops;
 use crate::permissions::DenoPermissions;
@@ -11,6 +10,7 @@ use deno_core::deno_buf;
 use deno_core::deno_mod;
 use deno_core::Behavior;
 use deno_core::Op;
+use deno_core::StartupData;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
@@ -21,19 +21,19 @@ pub type Buf = Box<[u8]>;
 
 /// Implements deno_core::Behavior for the main Deno command-line.
 pub struct Cli {
-  init: IsolateInit,
+  startup_data: Option<StartupData>,
   pub state: Arc<IsolateState>,
   pub permissions: Arc<DenoPermissions>, // TODO(ry) move to IsolateState
 }
 
 impl Cli {
   pub fn new(
-    init: IsolateInit,
+    startup_data: Option<StartupData>,
     state: Arc<IsolateState>,
     permissions: DenoPermissions,
   ) -> Self {
     Self {
-      init,
+      startup_data,
       state,
       permissions: Arc::new(permissions),
     }
@@ -66,8 +66,8 @@ impl Cli {
 }
 
 impl Behavior for Cli {
-  fn startup_snapshot(&mut self) -> Option<deno_buf> {
-    self.init.snapshot.take()
+  fn startup_data(&mut self) -> Option<StartupData> {
+    self.startup_data.take()
   }
 
   fn resolve(&mut self, specifier: &str, referrer: deno_mod) -> deno_mod {

src/compiler.rs

 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use crate::cli::Buf;
-use crate::isolate_init;
 use crate::isolate_state::IsolateState;
 use crate::msg;
 use crate::permissions::{DenoPermissions, PermissionAccessor};
 use crate::resources;
 use crate::resources::Resource;
 use crate::resources::ResourceId;
+use crate::startup_data;
 use crate::workers;
 use futures::Future;
 use serde_json;
@@ -48,7 +48,7 @@ impl ModuleMetaData {
 
 fn lazy_start(parent_state: &IsolateState) -> Resource {
   let mut cell = C_RID.lock().unwrap();
-  let isolate_init = isolate_init::compiler_isolate_init();
+  let startup_data = startup_data::compiler_isolate_init();
   let permissions = DenoPermissions {
     allow_read: PermissionAccessor::from(true),
     allow_write: PermissionAccessor::from(true),
@@ -58,7 +58,7 @@ fn lazy_start(parent_state: &IsolateState) -> Resource {
 
   let rid = cell.get_or_insert_with(|| {
     let resource = workers::spawn(
-      isolate_init,
+      Some(startup_data),
       parent_state,
       "compilerMain()".to_string(),
       permissions,

src/isolate.rs

@@ -181,7 +181,6 @@ fn fetch_module_meta_data_and_maybe_compile(
 mod tests {
   use super::*;
   use crate::flags;
-  use crate::isolate_init::IsolateInit;
   use crate::permissions::DenoPermissions;
   use crate::tokio_util;
   use futures::future::lazy;
@@ -199,12 +198,8 @@ mod tests {
 
     let state = Arc::new(IsolateState::new(flags, rest_argv, None));
     let state_ = state.clone();
-    let init = IsolateInit {
-      snapshot: None,
-      init_script: None,
-    };
     tokio_util::run(lazy(move || {
-      let cli = Cli::new(init, state.clone(), DenoPermissions::default());
+      let cli = Cli::new(None, state.clone(), DenoPermissions::default());
       let mut isolate = Isolate::new(cli);
       if let Err(err) = isolate.execute_mod(&filename, false) {
         eprintln!("execute_mod err {:?}", err);
@@ -226,12 +221,8 @@ mod tests {
 
     let state = Arc::new(IsolateState::new(flags, rest_argv, None));
     let state_ = state.clone();
-    let init = IsolateInit {
-      snapshot: None,
-      init_script: None,
-    };
     tokio_util::run(lazy(move || {
-      let cli = Cli::new(init, state.clone(), DenoPermissions::default());
+      let cli = Cli::new(None, state.clone(), DenoPermissions::default());
       let mut isolate = Isolate::new(cli);
       if let Err(err) = isolate.execute_mod(&filename, false) {
         eprintln!("execute_mod err {:?}", err);

src/main.rs

@@ -19,7 +19,6 @@ mod global_timer;
 mod http_body;
 mod http_util;
 pub mod isolate;
-pub mod isolate_init;
 pub mod isolate_state;
 pub mod js_errors;
 pub mod modules;
@@ -30,6 +29,7 @@ pub mod permissions;
 mod repl;
 pub mod resolve_addr;
 pub mod resources;
+mod startup_data;
 mod tokio_util;
 mod tokio_write;
 pub mod version;
@@ -110,9 +110,9 @@ fn main() {
 
   let state = Arc::new(IsolateState::new(flags, rest_argv, None));
   let state_ = state.clone();
-  let isolate_init = isolate_init::deno_isolate_init();
+  let startup_data = startup_data::deno_isolate_init();
   let permissions = permissions::DenoPermissions::from_flags(&state.flags);
-  let cli = Cli::new(isolate_init, state_, permissions);
+  let cli = Cli::new(Some(startup_data), state_, permissions);
   let mut isolate = Isolate::new(cli);
 
   let main_future = lazy(move || {

src/ops.rs

@@ -1880,7 +1880,7 @@ fn op_worker_post_message(
 mod tests {
   use super::*;
   use crate::cli::Cli;
-  use crate::isolate_init::IsolateInit;
+  use crate::isolate_state::IsolateState;
   use crate::permissions::{DenoPermissions, PermissionAccessor};
 
   #[test]
@@ -1893,14 +1893,7 @@ mod tests {
       allow_run: PermissionAccessor::from(true),
       ..Default::default()
     };
-    let cli = Cli::new(
-      IsolateInit {
-        snapshot: None,
-        init_script: None,
-      },
-      state,
-      permissions,
-    );
+    let cli = Cli::new(None, state, permissions);
     let builder = &mut FlatBufferBuilder::new();
     let fetch_msg_args = msg::FetchModuleMetaDataArgs {
       specifier: Some(builder.create_string("./somefile")),
@@ -1934,14 +1927,7 @@ mod tests {
       allow_run: PermissionAccessor::from(true),
       ..Default::default()
     };
-    let cli = Cli::new(
-      IsolateInit {
-        snapshot: None,
-        init_script: None,
-      },
-      state,
-      permissions,
-    );
+    let cli = Cli::new(None, state, permissions);
     let builder = &mut FlatBufferBuilder::new();
     let fetch_msg_args = msg::FetchModuleMetaDataArgs {
       specifier: Some(builder.create_string("./somefile")),
@@ -1975,14 +1961,7 @@ mod tests {
       allow_run: PermissionAccessor::from(true),
       ..Default::default()
     };
-    let cli = Cli::new(
-      IsolateInit {
-        snapshot: None,
-        init_script: None,
-      },
-      state,
-      permissions,
-    );
+    let cli = Cli::new(None, state, permissions);
     let builder = &mut FlatBufferBuilder::new();
     let fetch_msg_args = msg::FetchModuleMetaDataArgs {
       specifier: Some(builder.create_string("./somefile")),
@@ -2015,14 +1994,7 @@ mod tests {
       allow_net: PermissionAccessor::from(true),
       ..Default::default()
     };
-    let cli = Cli::new(
-      IsolateInit {
-        snapshot: None,
-        init_script: None,
-      },
-      state,
-      permissions,
-    );
+    let cli = Cli::new(None, state, permissions);
     let builder = &mut FlatBufferBuilder::new();
     let fetch_msg_args = msg::FetchModuleMetaDataArgs {
       specifier: Some(builder.create_string("./somefile")),

src/isolate_init.rs → src/startup_data.rs

 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use deno_core::deno_buf;
+use deno_core::{StartupData, StartupScript};
 
-pub struct IsolateInitScript {
-  pub source: String,
-  pub filename: String,
-}
-
-pub struct IsolateInit {
-  pub snapshot: Option<deno_buf>,
-  pub init_script: Option<IsolateInitScript>,
-}
-
-pub fn deno_isolate_init() -> IsolateInit {
+pub fn deno_isolate_init() -> StartupData {
   if cfg!(feature = "no-snapshot-init") {
     debug!("Deno isolate init without snapshots.");
     #[cfg(not(feature = "check-only"))]
@@ -20,13 +11,10 @@ pub fn deno_isolate_init() -> IsolateInit {
     #[cfg(feature = "check-only")]
     let source_bytes = vec![];
 
-    IsolateInit {
-      snapshot: None,
-      init_script: Some(IsolateInitScript {
-        filename: "gen/bundle/main.js".to_string(),
-        source: std::str::from_utf8(source_bytes).unwrap().to_string(),
-      }),
-    }
+    StartupData::Script(StartupScript {
+      filename: "gen/bundle/main.js".to_string(),
+      source: std::str::from_utf8(source_bytes).unwrap().to_string(),
+    })
   } else {
     debug!("Deno isolate init with snapshots.");
     #[cfg(not(any(feature = "check-only", feature = "no-snapshot-init")))]
@@ -36,15 +24,12 @@ pub fn deno_isolate_init() -> IsolateInit {
     let data = vec![];
 
     unsafe {
-      IsolateInit {
-        snapshot: Some(deno_buf::from_raw_parts(data.as_ptr(), data.len())),
-        init_script: None,
-      }
+      StartupData::Snapshot(deno_buf::from_raw_parts(data.as_ptr(), data.len()))
     }
   }
 }
 
-pub fn compiler_isolate_init() -> IsolateInit {
+pub fn compiler_isolate_init() -> StartupData {
   if cfg!(feature = "no-snapshot-init") {
     debug!("Deno isolate init without snapshots.");
     #[cfg(not(feature = "check-only"))]
@@ -53,13 +38,10 @@ pub fn compiler_isolate_init() -> IsolateInit {
     #[cfg(feature = "check-only")]
     let source_bytes = vec![];
 
-    IsolateInit {
-      snapshot: None,
-      init_script: Some(IsolateInitScript {
-        filename: "gen/bundle/compiler.js".to_string(),
-        source: std::str::from_utf8(source_bytes).unwrap().to_string(),
-      }),
-    }
+    StartupData::Script(StartupScript {
+      filename: "gen/bundle/compiler.js".to_string(),
+      source: std::str::from_utf8(source_bytes).unwrap().to_string(),
+    })
   } else {
     debug!("Deno isolate init with snapshots.");
     #[cfg(not(any(feature = "check-only", feature = "no-snapshot-init")))]
@@ -69,10 +51,7 @@ pub fn compiler_isolate_init() -> IsolateInit {
     let data = vec![];
 
     unsafe {
-      IsolateInit {
-        snapshot: Some(deno_buf::from_raw_parts(data.as_ptr(), data.len())),
-        init_script: None,
-      }
+      StartupData::Snapshot(deno_buf::from_raw_parts(data.as_ptr(), data.len()))
     }
   }
 }

src/workers.rs

@@ -3,7 +3,6 @@ use crate::cli::Buf;
 use crate::cli::Cli;
 use crate::flags::DenoFlags;
 use crate::isolate::Isolate;
-use crate::isolate_init::IsolateInit;
 use crate::isolate_state::IsolateState;
 use crate::isolate_state::WorkerChannels;
 use crate::js_errors::JSErrorColor;
@@ -11,6 +10,7 @@ use crate::permissions::DenoPermissions;
 use crate::resources;
 use crate::tokio_util;
 use deno_core::JSError;
+use deno_core::StartupData;
 use futures::future::lazy;
 use futures::sync::mpsc;
 use futures::sync::oneshot;
@@ -26,7 +26,7 @@ pub struct Worker {
 
 impl Worker {
   pub fn new(
-    init: IsolateInit,
+    startup_data: Option<StartupData>,
     flags: DenoFlags,
     argv: Vec<String>,
     permissions: DenoPermissions,
@@ -40,7 +40,7 @@ impl Worker {
     let state =
       Arc::new(IsolateState::new(flags, argv, Some(internal_channels)));
 
-    let cli = Cli::new(init, state, permissions);
+    let cli = Cli::new(startup_data, state, permissions);
     let isolate = Isolate::new(cli);
 
     let worker = Worker { isolate };
@@ -62,7 +62,7 @@ impl Future for Worker {
 }
 
 pub fn spawn(
-  init: IsolateInit,
+  startup_data: Option<StartupData>,
   state: &IsolateState,
   js_source: String,
   permissions: DenoPermissions,
@@ -81,7 +81,7 @@ pub fn spawn(
     .spawn(move || {
       tokio_util::run(lazy(move || {
         let (mut worker, external_channels) =
-          Worker::new(init, flags, argv, permissions);
+          Worker::new(startup_data, flags, argv, permissions);
         let resource = resources::add_worker(external_channels);
         p.send(resource.clone()).unwrap();
 
@@ -113,13 +113,13 @@ pub fn spawn(
 #[cfg(test)]
 mod tests {
   use super::*;
-  use crate::isolate_init;
+  use crate::startup_data;
 
   #[test]
   fn test_spawn() {
-    let isolate_init = isolate_init::compiler_isolate_init();
+    let startup_data = startup_data::compiler_isolate_init();
     let resource = spawn(
-      isolate_init,
+      Some(startup_data),
       &IsolateState::mock(),
       r#"
       onmessage = function(e) {
@@ -154,9 +154,9 @@ mod tests {
 
   #[test]
   fn removed_from_resource_table_on_close() {
-    let isolate_init = isolate_init::compiler_isolate_init();
+    let startup_data = startup_data::compiler_isolate_init();
     let resource = spawn(
-      isolate_init,
+      Some(startup_data),
       &IsolateState::mock(),
       "onmessage = () => close();".into(),
       DenoPermissions::default(),