feat(unstable): Support data: urls (#5157)

Cargo.lock

@@ -417,6 +417,7 @@ dependencies = [
  "nix",
  "notify",
  "os_pipe",
+ "percent-encoding",
  "rand 0.7.3",
  "regex",
  "reqwest",

cli/Cargo.toml

@@ -52,6 +52,7 @@ libc = "0.2.74"
 log = "0.4.11"
 env_logger = "0.7.1"
 notify = "5.0.0-pre.3"
+percent-encoding = "2.1.0"
 rand = "0.7.3"
 regex = "1.3.9"
 reqwest = { version = "0.10.7", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] }

cli/disk_cache.rs

@@ -103,6 +103,9 @@ impl DiskCache {
 
         out = out.join(remaining_components);
       }
+      "data" => {
+        out.push(crate::checksum::gen(&[url.as_str().as_bytes()]));
+      }
       scheme => {
         unimplemented!(
           "Don't know how to create cache name for scheme: {}",

cli/file_fetcher.rs

@@ -100,7 +100,7 @@ impl SourceFileCache {
   }
 }
 
-const SUPPORTED_URL_SCHEMES: [&str; 3] = ["http", "https", "file"];
+const SUPPORTED_URL_SCHEMES: [&str; 4] = ["http", "https", "file", "data"];
 
 #[derive(Clone)]
 pub struct SourceFileFetcher {
@@ -278,6 +278,7 @@ impl SourceFileFetcher {
   ) -> Result<Option<SourceFile>, ErrBox> {
     let url_scheme = module_url.scheme();
     let is_local_file = url_scheme == "file";
+    let is_data_url = url_scheme == "data";
     SourceFileFetcher::check_if_supported_scheme(&module_url)?;
 
     // Local files are always fetched from disk bypassing cache entirely.
@@ -285,6 +286,10 @@ impl SourceFileFetcher {
       return self.fetch_local_file(&module_url, permissions).map(Some);
     }
 
+    if is_data_url {
+      return extract_data_url(module_url).map(Some);
+    }
+
     self.fetch_cached_remote_source(&module_url, 10)
   }
 
@@ -309,6 +314,7 @@ impl SourceFileFetcher {
   ) -> Result<SourceFile, ErrBox> {
     let url_scheme = module_url.scheme();
     let is_local_file = url_scheme == "file";
+    let is_data_url = url_scheme == "data";
     SourceFileFetcher::check_if_supported_scheme(&module_url)?;
 
     // Local files are always fetched from disk bypassing cache entirely.
@@ -316,6 +322,10 @@ impl SourceFileFetcher {
       return self.fetch_local_file(&module_url, permissions);
     }
 
+    if is_data_url {
+      return extract_data_url(module_url);
+    }
+
     // The file is remote, fail if `no_remote` is true.
     if no_remote {
       let e = std::io::Error::new(
@@ -552,6 +562,36 @@ impl SourceFileFetcher {
   }
 }
 
+fn extract_data_url(url: &Url) -> Result<SourceFile, ErrBox> {
+  assert_eq!(url.scheme(), "data");
+  let url_content = &url.as_str()[5..];
+  let mut part_iterator = url_content.splitn(2, ',');
+
+  let media_type_str = part_iterator.next().unwrap();
+  let data = if let Some(d) = part_iterator.next() {
+    d
+  } else {
+    return Err(ErrBox::new("URIError", "Malformed data url, missing comma"));
+  };
+
+  let filename = PathBuf::new();
+  let (media_type, charset) = map_content_type(&filename, Some(media_type_str));
+  let is_base64 = media_type_str.rsplit(';').any(|v| v == "base64");
+  let bytes = if is_base64 {
+    base64::decode(data)?
+  } else {
+    percent_encoding::percent_decode_str(data).collect::<Vec<u8>>()
+  };
+
+  Ok(SourceFile {
+    url: url.clone(),
+    filename,
+    types_header: None,
+    media_type,
+    source_code: TextDocument::new(bytes, charset),
+  })
+}
+
 pub fn map_file_extension(path: &Path) -> msg::MediaType {
   match path.extension() {
     None => msg::MediaType::Unknown,

cli/global_state.rs

@@ -118,6 +118,7 @@ impl GlobalState {
       self.file_fetcher.clone(),
       maybe_import_map,
       permissions.clone(),
+      self.flags.unstable,
       is_dyn_import,
       false,
     );

cli/info.rs

@@ -34,6 +34,7 @@ impl ModuleDepInfo {
       global_state.file_fetcher.clone(),
       global_state.maybe_import_map.clone(),
       Permissions::allow_all(),
+      global_state.flags.unstable,
       false,
       true,
     );

cli/main.rs

@@ -478,6 +478,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), ErrBox> {
     global_state.file_fetcher.clone(),
     global_state.maybe_import_map.clone(),
     Permissions::allow_all(),
+    flags.unstable,
     false,
     false,
   );

cli/module_graph.rs

@@ -7,6 +7,7 @@ use crate::file_fetcher::SourceFileFetcher;
 use crate::import_map::ImportMap;
 use crate::msg::MediaType;
 use crate::permissions::Permissions;
+use crate::state::exit_unstable;
 use crate::swc_util::Location;
 use crate::tsc::pre_process_file;
 use crate::tsc::ImportDesc;
@@ -44,20 +45,31 @@ fn err_with_location(e: ErrBox, maybe_location: Option<&Location>) -> ErrBox {
 }
 
 /// Disallow http:// imports from modules loaded over https://
+/// Disallow any imports from modules loaded with data:
 fn validate_no_downgrade(
   module_specifier: &ModuleSpecifier,
   maybe_referrer: Option<&ModuleSpecifier>,
   maybe_location: Option<&Location>,
 ) -> Result<(), ErrBox> {
   if let Some(referrer) = maybe_referrer.as_ref() {
-    if let "https" = referrer.as_url().scheme() {
-      if let "http" = module_specifier.as_url().scheme() {
-        let e = ErrBox::new("PermissionDenied",
-          "Modules loaded over https:// are not allowed to import modules over http://"
+    match referrer.as_url().scheme() {
+      "https" => {
+        if let "http" = module_specifier.as_url().scheme() {
+          let e = ErrBox::new("PermissionDenied",
+                              "Modules loaded over https:// are not allowed to import modules over http://"
+          );
+          return Err(err_with_location(e, maybe_location));
+        };
+      }
+      "data" => {
+        let e = ErrBox::new(
+          "PermissionDenied",
+          "Modules loaded using data URL are not allowed to import other modules",
         );
         return Err(err_with_location(e, maybe_location));
-      };
-    };
+      }
+      _ => {}
+    }
   };
 
   Ok(())
@@ -75,7 +87,7 @@ fn validate_no_file_from_remote(
       "http" | "https" => {
         let specifier_url = module_specifier.as_url();
         match specifier_url.scheme() {
-          "http" | "https" => {}
+          "http" | "https" | "data" => {}
           _ => {
             let e = ErrBox::new(
               "PermissionDenied",
@@ -257,6 +269,7 @@ pub struct ModuleGraphLoader {
   pending_downloads: FuturesUnordered<SourceFileFuture>,
   has_downloaded: HashSet<ModuleSpecifier>,
   graph: ModuleGraph,
+  is_unstable: bool,
   is_dyn_import: bool,
   analyze_dynamic_imports: bool,
 }
@@ -266,6 +279,7 @@ impl ModuleGraphLoader {
     file_fetcher: SourceFileFetcher,
     maybe_import_map: Option<ImportMap>,
     permissions: Permissions,
+    is_unstable: bool,
     is_dyn_import: bool,
     analyze_dynamic_imports: bool,
   ) -> Self {
@@ -276,6 +290,7 @@ impl ModuleGraphLoader {
       pending_downloads: FuturesUnordered::new(),
       has_downloaded: HashSet::new(),
       graph: ModuleGraph::new(),
+      is_unstable,
       is_dyn_import,
       analyze_dynamic_imports,
     }
@@ -405,6 +420,10 @@ impl ModuleGraphLoader {
       return Ok(());
     }
 
+    if !self.is_unstable && module_specifier.as_url().scheme() == "data" {
+      exit_unstable("data imports");
+    }
+
     validate_no_downgrade(
       &module_specifier,
       maybe_referrer.as_ref(),
@@ -600,6 +619,7 @@ mod tests {
       global_state.file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      global_state.flags.unstable,
       false,
       false,
     );
@@ -873,7 +893,7 @@ fn test_pre_process_file() {
   let source = r#"
 // This comment is placed to make sure that directives are parsed
 // even when they start on non-first line
-  
+
 /// <reference lib="dom" />
 /// <reference types="./type_reference.d.ts" />
 /// <reference path="./type_reference/dep.ts" />
@@ -888,7 +908,7 @@ import * as qat from "./type_definitions/qat.ts";
 
 console.log(foo);
 console.log(fizz);
-console.log(qat.qat);  
+console.log(qat.qat);
 "#;
 
   let (imports, references) =

cli/state.rs

@@ -300,6 +300,7 @@ impl State {
         self.check_read(Path::new(&path))?;
         Ok(())
       }
+      "data" => Ok(()),
       _ => unreachable!(),
     }
   }

cli/tests/data_import_invalid.js

+import _invalid from "data:";

cli/tests/data_import_invalid.out

+error: Malformed data url, missing comma
+Imported from [WILDCARD]
\ No newline at end of file

cli/tests/data_import_origin_upgrade.js

+// export default from "https://deno.land/std/version.ts";
+import _upgrade from "data:application/javascript;base64,ZXhwb3J0IGRlZmF1bHQgZnJvbSAiaHR0cHM6Ly9kZW5vLmxhbmQvc3RkL3ZlcnNpb24udHMiOw==";

cli/tests/data_import_origin_upgrade.out

+error: Modules loaded using data URL are not allowed to import other modules
+Imported from [WILDCARD]
\ No newline at end of file

cli/tests/data_import_test.js

+import { assertEquals } from "../../std/testing/asserts.ts";
+
+// export const value = 'Successful import'; export default value;
+import data1 from "data:application/javascript;base64,ZXhwb3J0IGNvbnN0IHZhbHVlID0gJ1N1Y2Nlc3NmdWwgaW1wb3J0JzsgZXhwb3J0IGRlZmF1bHQgdmFsdWU7";
+
+Deno.test("static base64 data url import", () => {
+  assertEquals(data1, "Successful import");
+});
+
+Deno.test("dynamic base64 data url import", async () => {
+  const data2 = await import(
+    // export const leet = 1337
+    "data:application/javascript;base64,ZXhwb3J0IGNvbnN0IGxlZXQgPSAxMzM3"
+  );
+  assertEquals(data2.leet, 1337);
+});
+
+Deno.test("dynamic percent-encoding data url import", async () => {
+  const data3 = await import(
+    // export const value = 42;
+    "data:application/javascript,export%20const%20value%20%3D%2042%3B"
+  );
+  assertEquals(data3.value, 42);
+});
+
+Deno.test("dynamic base64 typescript data url import", async () => {
+  const data2 = await import(
+    // export const leet: number = 1337;
+    "data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGxlZXQ6IG51bWJlciA9IDEzMzc7"
+  );
+  assertEquals(data2.leet, 1337);
+});
+
+Deno.test("spawn worker with data url", async () => {
+  let resolve, timeout;
+  const promise = new Promise((res, rej) => {
+    resolve = res;
+    timeout = setTimeout(() => rej("Worker timed out"), 2000);
+  });
+
+  const worker = new Worker(
+    "data:application/javascript," +
+      encodeURIComponent("self.onmessage = () => self.postMessage('Worker');"),
+    { type: "module" },
+  );
+
+  worker.onmessage = (m) => {
+    if (m.data === "Worker") {
+      resolve();
+    }
+  };
+
+  worker.postMessage();
+
+  await promise;
+
+  clearTimeout(timeout);
+  worker.terminate();
+});

cli/tests/data_import_test.out

+[WILDCARD]
+test result: ok. 5 passed; [WILDCARD]
+

cli/tests/integration_tests.rs

@@ -2394,6 +2394,23 @@ itest!(info_type_import {
   output: "info_type_import.out",
 });
 
+itest!(data_import {
+  args: "test --reload --unstable data_import_test.js",
+  output: "data_import_test.out",
+});
+
+itest!(data_import_invalid {
+  args: "test --reload --unstable data_import_invalid.js",
+  output: "data_import_invalid.out",
+  exit_code: 1,
+});
+
+itest!(data_import_origin_upgrade {
+  args: "test --reload --unstable data_import_origin_upgrade.js",
+  output: "data_import_origin_upgrade.out",
+  exit_code: 1,
+});
+
 #[test]
 fn cafile_env_fetch() {
   use url::Url;

cli/tests/unsupported_dynamic_import_scheme.out

@@ -2,4 +2,5 @@ error: Uncaught TypeError: Unsupported scheme "xxx" for module "xxx:". Supported
     "http",
     "https",
     "file",
+    "data",
 ]

cli/tsc.rs

@@ -679,6 +679,7 @@ impl TsCompiler {
       self.file_fetcher.clone(),
       global_state.maybe_import_map.clone(),
       permissions.clone(),
+      global_state.flags.unstable,
       false,
       true,
     );
@@ -1157,6 +1158,7 @@ async fn create_runtime_module_graph(
     global_state.file_fetcher.clone(),
     None,
     permissions,
+    global_state.flags.unstable,
     false,
     false,
   );
@@ -1676,6 +1678,7 @@ mod tests {
       file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      mock_state.flags.unstable,
       false,
       false,
     );
@@ -1752,6 +1755,7 @@ mod tests {
       file_fetcher.clone(),
       None,
       Permissions::allow_all(),
+      mock_state.flags.unstable,
       false,
       false,
     );