Turbopack: fix sourcemaps of scopehoisted comments (#80987)

There was a bug in the source mappings for scope hoisted comments. They actually contained invalid mappings, which was only caught by the invalid mapping landing exactly on an invalid multibyte character boundary, which triggered a debug assertion:
![Bildschirmfoto 2025-06-27 um 10 01 15](https://github.com/user-attachments/assets/90cd7771-72c2-41ea-bc54-05b3be5f17f7)

What's happening is:
1. You do `comments.add_leading` with a `Comment` containing the real span
2. Traverse the AST and encode all bytepos in the soans
3. codegen, the bytepos in the AST are encoded, but the ones in the comments (Comment.span) are not (i.e. via `comments.take_leading`

Now, they are all consistently encoded and correctly mapped:
![Bildschirmfoto 2025-06-27 um 10 02 47](https://github.com/user-attachments/assets/90b4bb58-e946-4781-b498-d88668193107)

Author: mischnic

turbopack/crates/turbopack-ecmascript/src/lib.rs

@@ -2416,6 +2416,21 @@ enum CodeGenResultCommentsConsumable<'a> {
 unsafe impl Send for CodeGenResultComments {}
 unsafe impl Sync for CodeGenResultComments {}
 
+/// All BytePos in Spans in the AST are encoded correctly in [`merge_modules`], but the Comments
+/// also contain spans. These also need to be encoded so that all pos in `mappings` are consistently
+/// encoded.
+fn encode_module_into_comment_span(
+    modules_header_width: u32,
+    module: usize,
+    mut comment: Comment,
+) -> Comment {
+    comment.span.lo =
+        CodeGenResultComments::encode_bytepos(modules_header_width, module as u32, comment.span.lo);
+    comment.span.hi =
+        CodeGenResultComments::encode_bytepos(modules_header_width, module as u32, comment.span.hi);
+    comment
+}
+
 impl Comments for CodeGenResultCommentsConsumable<'_> {
     fn add_leading(&self, _pos: BytePos, _cmt: Comment) {
         unimplemented!()
@@ -2459,7 +2474,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
             } => {
                 let (module, pos) =
                     CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
-                comments[module].take_leading(pos)
+                comments[module].take_leading(pos).map(|comments| {
+                    comments
+                        .into_iter()
+                        .map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
+                        .collect()
+                })
             }
             Self::Empty => None,
         }
@@ -2477,7 +2497,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
             } => {
                 let (module, pos) =
                     CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
-                comments[module].get_leading(pos)
+                comments[module].get_leading(pos).map(|comments| {
+                    comments
+                        .into_iter()
+                        .map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
+                        .collect()
+                })
             }
             Self::Empty => None,
         }
@@ -2528,7 +2553,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
             } => {
                 let (module, pos) =
                     CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
-                comments[module].take_trailing(pos)
+                comments[module].take_trailing(pos).map(|comments| {
+                    comments
+                        .into_iter()
+                        .map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
+                        .collect()
+                })
             }
             Self::Empty => None,
         }
@@ -2546,7 +2576,12 @@ impl Comments for CodeGenResultCommentsConsumable<'_> {
             } => {
                 let (module, pos) =
                     CodeGenResultComments::decode_bytepos(*modules_header_width, pos);
-                comments[module].get_leading(pos)
+                comments[module].get_leading(pos).map(|comments| {
+                    comments
+                        .into_iter()
+                        .map(|c| encode_module_into_comment_span(*modules_header_width, module, c))
+                        .collect()
+                })
             }
             Self::Empty => None,
         }

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/collect-segment-data.js

@@ -0,0 +1,8 @@
+import { jsx as _jsx } from "./jsx-runtime";
+// eslint-disable-next-line import/no-extraneous-dependencies
+const createFromReadableStream = 123; //import { createFromReadableStream } from 'react-server-dom-webpack/client.edge';
+// eslint-disable-next-line import/no-extraneous-dependencies
+const prerender = 123; // import { unstable_prerender as prerender } from 'react-server-dom-webpack/static.edge';
+const streamFromBuffer=1,streamToBuffer=1; // import { streamFromBuffer, streamToBuffer } from '../stream-utils/node-web-streams-helper';
+const waitAtLeastOneReactRenderTask=1; //import { waitAtLeastOneReactRenderTask } from '../../lib/scheduler';
+// import  './segment-value-encoding';

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/entry-base.js

@@ -0,0 +1,2 @@
+import './params'
+import './collect-segment-data'

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index.js

@@ -0,0 +1,21 @@
+// "entry-base.js [app-rsc] (ecmascript) <module evaluation>" with
+// ["reflect-utils.js [app-rsc] (ecmascript)",
+// "params.js [app-rsc] (ecmascript)",
+// "segment-value-encoding.js [app-rsc] (ecmascript)",
+// "collect-segment-data.js [app-rsc] (ecmascript)",
+// "entry-base.js [app-rsc] (ecmascript) <locals>"]
+
+// "entry-base.js [test] (ecmascript) <module evaluation>" with
+// ["reflect-utils.js [test] (ecmascript)",
+// "params.js [test] (ecmascript)",
+// "segment-value-encoding.js [test] (ecmascript)",
+//  "collect-segment-data.js [test] (ecmascript)",
+//  "entry-base.js [test] (ecmascript) <locals>"]
+
+if (Date.now() > 0) {
+  require('./index1.js')
+}
+if (Date.now() > 0) {
+  require('./index2.js')
+}
+

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index1.js

@@ -0,0 +1 @@
+import "./entry-base.js";

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index2.js

@@ -0,0 +1 @@
+import "./entry-base.js";

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/jsx-runtime.js

@@ -0,0 +1 @@
+module.exports = {};

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/params.js

@@ -0,0 +1 @@
+import './reflect-utils'

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/reflect-utils.js

@@ -0,0 +1,9 @@
+// This regex will have fast negatives meaning valid identifiers may not pass
+// this test. However this is only used during static generation to provide hints
+// about why a page bailed out of some or all prerendering and we can use bracket notation
+// for example while `ಠ_ಠ` is a valid identifier it's ok to print `searchParams['ಠ_ಠ']`
+// even if this would have been fine too `searchParams.ಠ_ಠ`
+const isDefinitelyAValidIdentifier = /s/;
+export function describeStringPropertyAccess(target, prop) {}
+export function describeHasCheckingStringProperty(target, prop) {}
+export const wellKnownProperties = new Set([])

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/options.json

@@ -0,0 +1,4 @@
+{
+  "treeShakingMode": "reexports-only",
+  "scopeHoisting": true
+}

turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/output/4e721_crates_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_795dc95a._.js

@@ -0,0 +1,6 @@
+(globalThis.TURBOPACK = globalThis.TURBOPACK || []).push([
+    "output/b1abf_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_index_a93cc521.js",
+    {},
+    {"otherChunks":["output/4e721_crates_turbopack-tests_tests_snapshot_source_maps_merged-unicode_input_795dc95a._.js"],"runtimeModuleIds":["[project]/turbopack/crates/turbopack-tests/tests/snapshot/source_maps/merged-unicode/input/index.js [test] (ecmascript)"]}
+]);
+// Dummy runtime