refactor: internal runtime code TS support (denoland#17672)

This is a proof of concept for being able to snapshot TypeScript files. Currently only a single runtime file is authored in TypeScript - "runtime/js/01_version.ts". Not needed infrastructure was removed from "core/snapshot_util.rs". --------- Co-authored-by: Bartek Iwańczuk <[email protected]>
RaptorFX-JS · Feb 8, 2023 · 286e5d0 · 286e5d0
1 parent bef5041
commit 286e5d0
Show file tree

Hide file tree

Showing 14 changed files with 261 additions and 113 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bench_util/js_runtime.rs b/bench_util/js_runtime.rs
@@ -11,7 +11,7 @@ pub fn create_js_runtime(setup: impl FnOnce() -> Vec<Extension>) -> JsRuntime {
   JsRuntime::new(RuntimeOptions {
     extensions_with_js: setup(),
     module_loader: Some(std::rc::Rc::new(
-      deno_core::InternalModuleLoader::new(None),
+      deno_core::InternalModuleLoader::default(),
     )),
     ..Default::default()
   })

diff --git a/cli/build.rs b/cli/build.rs
@@ -4,8 +4,10 @@ use std::env;
 use std::path::Path;
 use std::path::PathBuf;
 
+use deno_core::include_js_files_dir;
 use deno_core::snapshot_util::*;
 use deno_core::Extension;
+use deno_core::ExtensionFileSource;
 use deno_runtime::deno_cache::SqliteBackedCache;
 use deno_runtime::permissions::PermissionsContainer;
 use deno_runtime::*;
@@ -15,6 +17,7 @@ mod ts {
   use crate::deno_webgpu_get_declaration;
   use deno_core::error::custom_error;
   use deno_core::error::AnyError;
+  use deno_core::include_js_files_dir;
   use deno_core::op;
   use deno_core::OpState;
   use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES;
@@ -32,11 +35,7 @@ mod ts {
     specifier: String,
   }
 
-  pub fn create_compiler_snapshot(
-    snapshot_path: PathBuf,
-    files: Vec<PathBuf>,
-    cwd: &Path,
-  ) {
+  pub fn create_compiler_snapshot(snapshot_path: PathBuf, cwd: &Path) {
     // libs that are being provided by op crates.
     let mut op_crate_libs = HashMap::new();
     op_crate_libs.insert("deno.cache", deno_cache::get_declaration());
@@ -252,36 +251,42 @@ mod ts {
       }
     }
 
+    let tsc_extension = Extension::builder("deno_tsc")
+      .ops(vec![
+        op_build_info::decl(),
+        op_cwd::decl(),
+        op_exists::decl(),
+        op_is_node_file::decl(),
+        op_load::decl(),
+        op_script_version::decl(),
+      ])
+      .js(include_js_files_dir! {
+        dir "tsc",
+        "00_typescript.js",
+        "99_main_compiler.js",
+      })
+      .state(move |state| {
+        state.put(op_crate_libs.clone());
+        state.put(build_libs.clone());
+        state.put(path_dts.clone());
+
+        Ok(())
+      })
+      .build();
+
     create_snapshot(CreateSnapshotOptions {
       cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
       snapshot_path,
       startup_snapshot: None,
-      extensions: vec![Extension::builder("deno_tsc")
-        .ops(vec![
-          op_build_info::decl(),
-          op_cwd::decl(),
-          op_exists::decl(),
-          op_is_node_file::decl(),
-          op_load::decl(),
-          op_script_version::decl(),
-        ])
-        .state(move |state| {
-          state.put(op_crate_libs.clone());
-          state.put(build_libs.clone());
-          state.put(path_dts.clone());
-
-          Ok(())
-        })
-        .build()],
-      extensions_with_js: vec![],
-      additional_files: files,
-      additional_esm_files: vec![],
+      extensions: vec![],
+      extensions_with_js: vec![tsc_extension],
       compression_cb: Some(Box::new(|vec, snapshot_slice| {
         vec.extend_from_slice(
           &zstd::bulk::compress(snapshot_slice, 22)
             .expect("snapshot compression failed"),
         );
       })),
+      snapshot_module_load_cb: None,
     });
   }
 
@@ -307,7 +312,7 @@ mod ts {
   }
 }
 
-fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec<PathBuf>) {
+fn create_cli_snapshot(snapshot_path: PathBuf) {
   let extensions: Vec<Extension> = vec![
     deno_webidl::init(),
     deno_console::init(),
@@ -338,14 +343,23 @@ fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec<PathBuf>) {
     deno_flash::init::<PermissionsContainer>(false), // No --unstable
   ];
 
+  let mut esm_files = include_js_files_dir!(
+    dir "js",
+    "40_testing.js",
+  );
+  esm_files.push(ExtensionFileSource {
+    specifier: "runtime/js/99_main.js".to_string(),
+    code: deno_runtime::js::SOURCE_CODE_FOR_99_MAIN_JS,
+  });
+  let extensions_with_js =
+    vec![Extension::builder("cli").esm(esm_files).build()];
+
   create_snapshot(CreateSnapshotOptions {
     cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
     snapshot_path,
     startup_snapshot: Some(deno_runtime::js::deno_isolate_init()),
     extensions,
-    extensions_with_js: vec![],
-    additional_files: vec![],
-    additional_esm_files: esm_files,
+    extensions_with_js,
     compression_cb: Some(Box::new(|vec, snapshot_slice| {
       lzzzz::lz4_hc::compress_to_vec(
         snapshot_slice,
@@ -354,6 +368,7 @@ fn create_cli_snapshot(snapshot_path: PathBuf, esm_files: Vec<PathBuf>) {
       )
       .expect("snapshot compression failed");
     })),
+    snapshot_module_load_cb: None,
   })
 }
 
@@ -450,13 +465,10 @@ fn main() {
   let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
 
   let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
-  let js_files = get_js_files(env!("CARGO_MANIFEST_DIR"), "tsc", None);
-  ts::create_compiler_snapshot(compiler_snapshot_path, js_files, &c);
+  ts::create_compiler_snapshot(compiler_snapshot_path, &c);
 
   let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
-  let mut esm_files = get_js_files(env!("CARGO_MANIFEST_DIR"), "js", None);
-  esm_files.push(deno_runtime::js::get_99_main());
-  create_cli_snapshot(cli_snapshot_path, esm_files);
+  create_cli_snapshot(cli_snapshot_path);
 
   #[cfg(target_os = "windows")]
   {

diff --git a/core/extensions.rs b/core/extensions.rs
@@ -6,6 +6,7 @@ use std::rc::Rc;
 use std::task::Context;
 use v8::fast_api::FastFunction;
 
+#[derive(Clone, Debug)]
 pub struct ExtensionFileSource {
   pub specifier: String,
   pub code: &'static str,
@@ -244,8 +245,9 @@ impl ExtensionBuilder {
     }
   }
 }
-/// Helps embed JS files in an extension. Returns Vec<(&'static str, &'static str)>
-/// representing the filename and source code.
+
+/// Helps embed JS files in an extension. Returns a vector of
+/// `ExtensionFileSource`, that represent the filename and source code.
 ///
 /// Example:
 /// ```ignore
@@ -265,3 +267,29 @@ macro_rules! include_js_files {
     ]
   };
 }
+
+/// Helps embed JS files in an extension. Returns a vector of
+/// `ExtensionFileSource`, that represent the filename and source code.
+/// Additional "dir" option is required, that specifies which directory in the
+/// crate root contains the listed files. "dir" option will be prepended to
+/// each file name.
+///
+/// Example:
+/// ```ignore
+/// include_js_files_dir!(
+///   dir "example",
+///   "01_hello.js",
+///   "02_goodbye.js",
+/// )
+/// ```
+#[macro_export]
+macro_rules! include_js_files_dir {
+  (dir $dir:literal, $($file:literal,)+) => {
+    vec![
+      $($crate::ExtensionFileSource {
+        specifier: concat!($dir, "/", $file).to_string(),
+        code: include_str!(concat!($dir, "/", $file)),
+      },)+
+    ]
+  };
+}
diff --git a/core/lib.rs b/core/lib.rs
@@ -75,6 +75,7 @@ pub use crate::module_specifier::ModuleSpecifier;
 pub use crate::module_specifier::DUMMY_SPECIFIER;
 pub use crate::modules::FsModuleLoader;
 pub use crate::modules::InternalModuleLoader;
+pub use crate::modules::InternalModuleLoaderCb;
 pub use crate::modules::ModuleId;
 pub use crate::modules::ModuleLoader;
 pub use crate::modules::ModuleSource;

diff --git a/core/modules.rs b/core/modules.rs
@@ -2,6 +2,7 @@
 
 use crate::bindings;
 use crate::error::generic_error;
+use crate::extensions::ExtensionFileSource;
 use crate::module_specifier::ModuleSpecifier;
 use crate::resolve_import;
 use crate::resolve_url;
@@ -312,13 +313,38 @@ pub(crate) fn resolve_helper(
   loader.resolve(specifier, referrer, kind)
 }
 
-pub struct InternalModuleLoader(Rc<dyn ModuleLoader>);
+/// Function that can be passed to the `InternalModuleLoader` that allows to
+/// transpile sources before passing to V8.
+pub type InternalModuleLoaderCb =
+  Box<dyn Fn(&ExtensionFileSource) -> Result<String, Error>>;
+
+pub struct InternalModuleLoader {
+  module_loader: Rc<dyn ModuleLoader>,
+  esm_sources: Vec<ExtensionFileSource>,
+  maybe_load_callback: Option<InternalModuleLoaderCb>,
+}
+
+impl Default for InternalModuleLoader {
+  fn default() -> Self {
+    Self {
+      module_loader: Rc::new(NoopModuleLoader),
+      esm_sources: vec![],
+      maybe_load_callback: None,
+    }
+  }
+}
 
 impl InternalModuleLoader {
-  pub fn new(module_loader: Option<Rc<dyn ModuleLoader>>) -> Self {
-    InternalModuleLoader(
-      module_loader.unwrap_or_else(|| Rc::new(NoopModuleLoader)),
-    )
+  pub fn new(
+    module_loader: Option<Rc<dyn ModuleLoader>>,
+    esm_sources: Vec<ExtensionFileSource>,
+    maybe_load_callback: Option<InternalModuleLoaderCb>,
+  ) -> Self {
+    InternalModuleLoader {
+      module_loader: module_loader.unwrap_or_else(|| Rc::new(NoopModuleLoader)),
+      esm_sources,
+      maybe_load_callback,
+    }
   }
 }
 
@@ -343,7 +369,7 @@ impl ModuleLoader for InternalModuleLoader {
       }
     }
 
-    self.0.resolve(specifier, referrer, kind)
+    self.module_loader.resolve(specifier, referrer, kind)
   }
 
   fn load(
@@ -352,7 +378,46 @@ impl ModuleLoader for InternalModuleLoader {
     maybe_referrer: Option<ModuleSpecifier>,
     is_dyn_import: bool,
   ) -> Pin<Box<ModuleSourceFuture>> {
-    self.0.load(module_specifier, maybe_referrer, is_dyn_import)
+    if module_specifier.scheme() != "internal" {
+      return self.module_loader.load(
+        module_specifier,
+        maybe_referrer,
+        is_dyn_import,
+      );
+    }
+
+    let specifier = module_specifier.to_string();
+    let maybe_file_source = self
+      .esm_sources
+      .iter()
+      .find(|file_source| file_source.specifier == module_specifier.as_str());
+
+    if let Some(file_source) = maybe_file_source {
+      let result = if let Some(load_callback) = &self.maybe_load_callback {
+        load_callback(file_source)
+      } else {
+        Ok(file_source.code.to_string())
+      };
+
+      return async move {
+        let code = result?;
+        let source = ModuleSource {
+          code: code.into_bytes().into_boxed_slice(),
+          module_type: ModuleType::JavaScript,
+          module_url_specified: specifier.clone(),
+          module_url_found: specifier.clone(),
+        };
+        Ok(source)
+      }
+      .boxed_local();
+    }
+
+    async move {
+      Err(generic_error(format!(
+        "Cannot find internal module source for specifier {specifier}"
+      )))
+    }
+    .boxed_local()
   }
 
   fn prepare_load(
@@ -366,7 +431,7 @@ impl ModuleLoader for InternalModuleLoader {
       return async { Ok(()) }.boxed_local();
     }
 
-    self.0.prepare_load(
+    self.module_loader.prepare_load(
       op_state,
       module_specifier,
       maybe_referrer,
@@ -2639,7 +2704,7 @@ if (import.meta.url != 'file:///main_with_code.js') throw Error();
 
   #[test]
   fn internal_module_loader() {
-    let loader = InternalModuleLoader::new(None);
+    let loader = InternalModuleLoader::default();
     assert!(loader
       .resolve("internal:foo", "internal:bar", ResolutionKind::Import)
       .is_ok());