[Turbopack] include async loaders in chunking instead of separating it (

#75052) ### What? Chunk async loaders together with normal chunk items to avoid multiple chunkings and small chunks
vercel · Jan 24, 2025 · 2c1f68e · 2c1f68e
1 parent 410250b
commit 2c1f68e
Show file tree

Hide file tree

Showing 24 changed files with 60 additions and 67 deletions.
diff --git a/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs b/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs
@@ -163,7 +163,6 @@ pub async fn make_chunk_group(
         })
         .try_join()
         .await?;
-    let has_async_loaders = !async_loaders.is_empty();
     let async_loader_chunk_items =
         async_loaders
             .iter()
@@ -180,14 +179,8 @@ pub async fn make_chunk_group(
         .map(|&loader| loader.references())
         .try_join()
         .await?;
-    let async_loader_external_module_references = Vc::cell(
-        async_loader_references
-            .iter()
-            .flat_map(|references| references.iter().copied())
-            .collect(),
-    );
 
-    let traced_output_assets = traced_modules
+    let mut referenced_output_assets = traced_modules
         .into_iter()
         .map(|module| async move {
             Ok(ResolvedVc::upcast(
@@ -203,7 +196,7 @@ pub async fn make_chunk_group(
         .try_join()
         .await?;
 
-    let chunk_items = all_modules
+    let mut chunk_items = all_modules
         .iter()
         .map(|(module, async_info)| {
             Either::Left(async move {
@@ -234,34 +227,21 @@ pub async fn make_chunk_group(
         .try_join()
         .await?;
 
+    chunk_items.extend(async_loader_chunk_items);
+    referenced_output_assets.reserve(async_loader_references.iter().map(|r| r.len()).sum());
+    referenced_output_assets.extend(async_loader_references.into_iter().flatten());
+
     // Pass chunk items to chunking algorithm
-    let mut chunks = make_chunks(
+    let chunks = make_chunks(
         module_graph,
         *chunking_context,
         Vc::cell(chunk_items),
         "".into(),
-        Vc::cell(traced_output_assets),
+        Vc::cell(referenced_output_assets),
     )
     .await?
     .clone_value();
 
-    if has_async_loaders {
-        // Pass async chunk loaders to chunking algorithm
-        // We want them to be separate since they are specific to this chunk group due
-        // to available chunk items differing
-        let async_loader_chunks = make_chunks(
-            module_graph,
-            *chunking_context,
-            Vc::cell(async_loader_chunk_items.into_iter().collect()),
-            "async-loader-".into(),
-            async_loader_external_module_references,
-        )
-        .await?;
-
-        // concatenate chunks
-        chunks.extend(async_loader_chunks.iter().copied());
-    }
-
     Ok(MakeChunkGroupResult {
         chunks,
         availability_info,

diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking/dev.rs b/turbopack/crates/turbopack-core/src/chunk/chunking/dev.rs
@@ -39,16 +39,29 @@ pub async fn app_vendors_split(
     mut name: String,
     split_context: &mut SplitContext<'_>,
 ) -> Result<()> {
+    let mut chunk_group_specific_chunk_items = Vec::new();
     let mut app_chunk_items = Vec::new();
     let mut vendors_chunk_items = Vec::new();
     for item in chunk_items {
-        let ChunkItemWithInfo { asset_ident, .. } = &item;
-        if is_app_code(asset_ident) {
+        let ChunkItemWithInfo {
+            asset_ident,
+            module,
+            ..
+        } = &item;
+        if module.is_none() {
+            // This happens for async module loaders.
+            // We want them to be in a separate chunk.
+            chunk_group_specific_chunk_items.push(item);
+        } else if is_app_code(asset_ident) {
             app_chunk_items.push(item);
         } else {
             vendors_chunk_items.push(item);
         }
     }
+    if !chunk_group_specific_chunk_items.is_empty() {
+        let mut name = format!("{}-specific", name);
+        make_chunk(chunk_group_specific_chunk_items, &mut name, split_context).await?;
+    }
     let mut remaining = Vec::new();
     let mut key = format!("{}-app", name);
     if !handle_split_group(

diff --git a/...-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_aba1a0.js b/...-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_aba1a0.js
@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_aba1a0.js",
+    {},
+    {"otherChunks":["output/b1abf_turbopack-tests_tests_snapshot_basic-tree-shake_dynamic-import_input_lib_44a9b7.js","output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_92a5f4.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic-tree-shake/dynamic-import/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime
diff --git a/..._dynamic-import_input_index_d5474b.js.map → ..._dynamic-import_input_index_aba1a0.js.map b/..._dynamic-import_input_index_d5474b.js.map → ..._dynamic-import_input_index_aba1a0.js.map
diff --git a/...-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_d5474b.js b/...-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_d5474b.js
diff --git a/...utput/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_index_3567a8.js b/...utput/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_index_3567a8.js
diff --git a/...utput/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_index_c2e2cc.js b/...utput/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_index_c2e2cc.js
@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_index_c2e2cc.js",
+    {},
+    {"otherChunks":["output/4e721_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_import_46e42b.js","output/turbopack_crates_turbopack-tests_tests_snapshot_basic_async_chunk_input_463663._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime
diff --git a/...sic_async_chunk_input_index_3567a8.js.map → ...sic_async_chunk_input_index_c2e2cc.js.map b/...sic_async_chunk_input_index_3567a8.js.map → ...sic_async_chunk_input_index_c2e2cc.js.map
diff --git a/...opack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/output/index.entry.js b/...opack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/output/index.entry.js
diff --git a/...t/4e721_crates_turbopack-tests_tests_snapshot_basic_top-level-await_input_index_b01f59.js b/...t/4e721_crates_turbopack-tests_tests_snapshot_basic_top-level-await_input_index_b01f59.js
diff --git a/...t/4e721_crates_turbopack-tests_tests_snapshot_basic_top-level-await_input_index_f3a578.js b/...t/4e721_crates_turbopack-tests_tests_snapshot_basic_top-level-await_input_index_f3a578.js
@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/4e721_crates_turbopack-tests_tests_snapshot_basic_top-level-await_input_index_f3a578.js",
+    {},
+    {"otherChunks":["output/b1abf_turbopack-tests_tests_snapshot_basic_top-level-await_input_UserAPI_ba56f9.js","output/4e721_crates_turbopack-tests_tests_snapshot_basic_top-level-await_input_9acd43._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/top-level-await/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime
diff --git a/...top-level-await_input_index_b01f59.js.map → ...top-level-await_input_index_f3a578.js.map b/...top-level-await_input_index_b01f59.js.map → ...top-level-await_input_index_f3a578.js.map
diff --git a/...hot_css_css-modules_input_index_ae0851.js → ...hot_css_css-modules_input_index_b09cf6.js b/...hot_css_css-modules_input_index_ae0851.js → ...hot_css_css-modules_input_index_b09cf6.js
diff --git a/...css_css-modules_input_index_ae0851.js.map → ...css_css-modules_input_index_b09cf6.js.map b/...css_css-modules_input_index_ae0851.js.map → ...css_css-modules_input_index_b09cf6.js.map
diff --git a/...ts_snapshot_css_css_input_index_ec14ee.js → ...ts_snapshot_css_css_input_index_7c78ad.js b/...ts_snapshot_css_css_input_index_ec14ee.js → ...ts_snapshot_css_css_input_index_7c78ad.js
diff --git a/...napshot_css_css_input_index_ec14ee.js.map → ...napshot_css_css_input_index_7c78ad.js.map b/...napshot_css_css_input_index_ec14ee.js.map → ...napshot_css_css_input_index_7c78ad.js.map
diff --git a/...hot_imports_dynamic_input_index_41b5f3.js → ...hot_imports_dynamic_input_index_fb4a0c.js b/...hot_imports_dynamic_input_index_41b5f3.js → ...hot_imports_dynamic_input_index_fb4a0c.js
@@ -1,6 +1,6 @@
 (globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
-    "output/4e721_crates_turbopack-tests_tests_snapshot_imports_dynamic_input_index_41b5f3.js",
+    "output/4e721_crates_turbopack-tests_tests_snapshot_imports_dynamic_input_index_fb4a0c.js",
     {},
-    {"otherChunks":["output/4e721_crates_turbopack-tests_tests_snapshot_imports_dynamic_input_index_23ea6f.js","output/4e721_crates_turbopack-tests_tests_snapshot_imports_dynamic_input_vercel_mjs_a2d40e._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/dynamic/input/index.js [test] (ecmascript)"]}
+    {"otherChunks":["output/4e721_crates_turbopack-tests_tests_snapshot_imports_dynamic_input_vercel_mjs_a2d40e._.js","output/4e721_crates_turbopack-tests_tests_snapshot_imports_dynamic_input_index_23ea6f.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/dynamic/input/index.js [test] (ecmascript)"]}
 ]);
 // Dummy runtime
diff --git a/...imports_dynamic_input_index_41b5f3.js.map → ...imports_dynamic_input_index_fb4a0c.js.map b/...imports_dynamic_input_index_41b5f3.js.map → ...imports_dynamic_input_index_fb4a0c.js.map
diff --git a/...output/b1abf_turbopack-tests_tests_snapshot_imports_ignore-comments_input_index_b5a38b.js b/...output/b1abf_turbopack-tests_tests_snapshot_imports_ignore-comments_input_index_b5a38b.js
diff --git a/...output/b1abf_turbopack-tests_tests_snapshot_imports_ignore-comments_input_index_e44b7c.js b/...output/b1abf_turbopack-tests_tests_snapshot_imports_ignore-comments_input_index_e44b7c.js
@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/b1abf_turbopack-tests_tests_snapshot_imports_ignore-comments_input_index_e44b7c.js",
+    {},
+    {"otherChunks":["output/b1abf_turbopack-tests_tests_snapshot_imports_ignore-comments_input_vercel_mjs_bc04b7._.js","output/4e721_crates_turbopack-tests_tests_snapshot_imports_ignore-comments_input_26ea62._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/ignore-comments/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime
diff --git a/...ignore-comments_input_index_b5a38b.js.map → ...ignore-comments_input_index_e44b7c.js.map b/...ignore-comments_input_index_b5a38b.js.map → ...ignore-comments_input_index_e44b7c.js.map
diff --git a/...put/b1abf_turbopack-tests_tests_snapshot_imports_static-and-dynamic_input_index_8101ed.js b/...put/b1abf_turbopack-tests_tests_snapshot_imports_static-and-dynamic_input_index_8101ed.js
@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/b1abf_turbopack-tests_tests_snapshot_imports_static-and-dynamic_input_index_8101ed.js",
+    {},
+    {"otherChunks":["output/4c35f_tests_snapshot_imports_static-and-dynamic_input_vercel_mjs_cbd419._.js","output/4e721_crates_turbopack-tests_tests_snapshot_imports_static-and-dynamic_input_ab9cac._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/static-and-dynamic/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime
diff --git a/...tic-and-dynamic_input_index_aeac15.js.map → ...tic-and-dynamic_input_index_8101ed.js.map b/...tic-and-dynamic_input_index_aeac15.js.map → ...tic-and-dynamic_input_index_8101ed.js.map
diff --git a/...put/b1abf_turbopack-tests_tests_snapshot_imports_static-and-dynamic_input_index_aeac15.js b/...put/b1abf_turbopack-tests_tests_snapshot_imports_static-and-dynamic_input_index_aeac15.js