Skip to content

Commit 1fb754d

Browse files
mischnichuozhi
mischnic
authored and
huozhi
committed
Turbopack: trace worker_threads worker entry (#85734)
Closes PACK-5688 Closes #84766 1. Trace `new require("worker_threads").Worker` calls 2. ignore syntax errors in all traced files This does lead to a build error currently with cargo run --bin turbopack-nft -- --show-issues --graph bench/app-router-server/node_modules/thread-stream/index.js: ``` error - [analysis] [workspace]/node_modules/.pnpm/[email protected]/node_modules/thread-stream/test/syntax-error.mjs [workspace]/node_modules/.pnpm/[email protected]/node_modules/thread-stream/test/syntax-error.mjs:2:6 Parsing ecmascript source code failed 1 | // this is a syntax error + v 2 + import + ^ 3 | Expected 'from', got '<eof>' ``` Pulled in by this partly dynamic `new Worker()` call: https://github.com/pinojs/thread-stream/blob/82e011281b8895bf5e8f70008e0242927fcaf0cb/index.js#L53-L55 (Related to #84408, where this was previously fixed for extensionless files such as `LICENSE`)
1 parent d9505ee commit 1fb754d

File tree

17 files changed

+504
-145
lines changed

17 files changed

+504
-145
lines changed

‎turbopack/crates/turbopack-core/src/reference_type.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,7 @@ pub enum WorkerReferenceSubType {
290290
WebWorker,
291291
SharedWorker,
292292
ServiceWorker,
293+
NodeWorker,
293294
Custom(u8),
294295
Undefined,
295296
}

‎turbopack/crates/turbopack-ecmascript/src/analyzer/mod.rs‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1788,15 +1788,19 @@ impl JsValue {
17881788
"module",
17891789
"The Node.js `module` module: https://nodejs.org/api/module.html",
17901790
),
1791-
WellKnownObjectKind::ChildProcess | WellKnownObjectKind::ChildProcessDefault => (
1791+
WellKnownObjectKind::WorkerThreadsModule | WellKnownObjectKind::WorkerThreadsModuleDefault => (
1792+
"worker_threads",
1793+
"The Node.js `worker_threads` module: https://nodejs.org/api/worker_threads.html",
1794+
),
1795+
WellKnownObjectKind::ChildProcessModule | WellKnownObjectKind::ChildProcessModuleDefault => (
17921796
"child_process",
17931797
"The Node.js child_process module: https://nodejs.org/api/child_process.html",
17941798
),
17951799
WellKnownObjectKind::OsModule | WellKnownObjectKind::OsModuleDefault => (
17961800
"os",
17971801
"The Node.js os module: https://nodejs.org/api/os.html",
17981802
),
1799-
WellKnownObjectKind::NodeProcess => (
1803+
WellKnownObjectKind::NodeProcessModule => (
18001804
"process",
18011805
"The Node.js process module: https://nodejs.org/api/process.html",
18021806
),
@@ -1954,6 +1958,10 @@ impl JsValue {
19541958
"load/loadSync".to_string(),
19551959
"require('@grpc/proto-loader').load(filepath, { includeDirs: [root] }) https://github.com/grpc/grpc-node"
19561960
),
1961+
WellKnownFunctionKind::NodeWorkerConstructor => (
1962+
"Worker".to_string(),
1963+
"The Node.js worker_threads Worker constructor: https://nodejs.org/api/worker_threads.html#worker_threads_class_worker"
1964+
),
19571965
WellKnownFunctionKind::WorkerConstructor => (
19581966
"Worker".to_string(),
19591967
"The standard Worker constructor: https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker"
@@ -3468,11 +3476,13 @@ pub enum WellKnownObjectKind {
34683476
ModuleModuleDefault,
34693477
UrlModule,
34703478
UrlModuleDefault,
3471-
ChildProcess,
3472-
ChildProcessDefault,
3479+
WorkerThreadsModule,
3480+
WorkerThreadsModuleDefault,
3481+
ChildProcessModule,
3482+
ChildProcessModuleDefault,
34733483
OsModule,
34743484
OsModuleDefault,
3475-
NodeProcess,
3485+
NodeProcessModule,
34763486
NodeProcessArgv,
34773487
NodeProcessEnv,
34783488
NodePreGyp,
@@ -3492,9 +3502,10 @@ impl WellKnownObjectKind {
34923502
Self::PathModule => Some(&["path"]),
34933503
Self::FsModule => Some(&["fs"]),
34943504
Self::UrlModule => Some(&["url"]),
3495-
Self::ChildProcess => Some(&["child_process"]),
3505+
Self::ChildProcessModule => Some(&["child_process"]),
34963506
Self::OsModule => Some(&["os"]),
3497-
Self::NodeProcess => Some(&["process"]),
3507+
Self::WorkerThreadsModule => Some(&["worker_threads"]),
3508+
Self::NodeProcessModule => Some(&["process"]),
34983509
Self::NodeProcessArgv => Some(&["process", "argv"]),
34993510
Self::NodeProcessEnv => Some(&["process", "env"]),
35003511
Self::NodeBuffer => Some(&["Buffer"]),
@@ -3623,6 +3634,8 @@ pub enum WellKnownFunctionKind {
36233634
NodeResolveFrom,
36243635
NodeProtobufLoad,
36253636
WorkerConstructor,
3637+
// The worker_threads Worker class
3638+
NodeWorkerConstructor,
36263639
URLConstructor,
36273640
}
36283641

@@ -3797,7 +3810,7 @@ pub mod test_utils {
37973810
),
37983811
"define" => JsValue::WellKnownFunction(WellKnownFunctionKind::Define),
37993812
"URL" => JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor),
3800-
"process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcess),
3813+
"process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessModule),
38013814
"Object" => JsValue::WellKnownObject(WellKnownObjectKind::GlobalObject),
38023815
"Buffer" => JsValue::WellKnownObject(WellKnownObjectKind::NodeBuffer),
38033816
_ => v.into_unknown(true, "unknown global"),

‎turbopack/crates/turbopack-ecmascript/src/analyzer/well_known.rs‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -609,13 +609,18 @@ async fn well_known_object_member(
609609
WellKnownObjectKind::UrlModule | WellKnownObjectKind::UrlModuleDefault => {
610610
url_module_member(kind, prop)
611611
}
612-
WellKnownObjectKind::ChildProcess | WellKnownObjectKind::ChildProcessDefault => {
613-
child_process_module_member(kind, prop)
612+
WellKnownObjectKind::WorkerThreadsModule
613+
| WellKnownObjectKind::WorkerThreadsModuleDefault => {
614+
worker_threads_module_member(kind, prop)
614615
}
616+
WellKnownObjectKind::ChildProcessModule
617+
| WellKnownObjectKind::ChildProcessModuleDefault => child_process_module_member(kind, prop),
615618
WellKnownObjectKind::OsModule | WellKnownObjectKind::OsModuleDefault => {
616619
os_module_member(kind, prop)
617620
}
618-
WellKnownObjectKind::NodeProcess => node_process_member(prop, compile_time_info).await?,
621+
WellKnownObjectKind::NodeProcessModule => {
622+
node_process_member(prop, compile_time_info).await?
623+
}
619624
WellKnownObjectKind::NodePreGyp => node_pre_gyp(prop),
620625
WellKnownObjectKind::NodeExpressApp => express(prop),
621626
WellKnownObjectKind::NodeProtobufLoader => protobuf_loader(prop),
@@ -762,7 +767,7 @@ fn url_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
762767
(.., Some("pathToFileURL")) => {
763768
JsValue::WellKnownFunction(WellKnownFunctionKind::PathToFileUrl)
764769
}
765-
(WellKnownObjectKind::UrlModuleDefault, Some("default")) => {
770+
(WellKnownObjectKind::UrlModule, Some("default")) => {
766771
JsValue::WellKnownObject(WellKnownObjectKind::UrlModuleDefault)
767772
}
768773
_ => JsValue::unknown(
@@ -776,6 +781,27 @@ fn url_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
776781
}
777782
}
778783

784+
fn worker_threads_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
785+
match (kind, prop.as_str()) {
786+
(.., Some("Worker")) => {
787+
JsValue::WellKnownFunction(WellKnownFunctionKind::NodeWorkerConstructor)
788+
}
789+
(WellKnownObjectKind::WorkerThreadsModule, Some("default")) => {
790+
JsValue::WellKnownObject(WellKnownObjectKind::WorkerThreadsModuleDefault)
791+
}
792+
_ => JsValue::unknown(
793+
JsValue::member(
794+
Box::new(JsValue::WellKnownObject(
795+
WellKnownObjectKind::WorkerThreadsModule,
796+
)),
797+
Box::new(prop),
798+
),
799+
true,
800+
"unsupported property on Node.js worker_threads module",
801+
),
802+
}
803+
}
804+
779805
fn child_process_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsValue {
780806
let prop_str = prop.as_str();
781807
match (kind, prop_str) {
@@ -785,13 +811,15 @@ fn child_process_module_member(kind: WellKnownObjectKind, prop: JsValue) -> JsVa
785811
))
786812
}
787813
(.., Some("fork")) => JsValue::WellKnownFunction(WellKnownFunctionKind::ChildProcessFork),
788-
(WellKnownObjectKind::ChildProcess, Some("default")) => {
789-
JsValue::WellKnownObject(WellKnownObjectKind::ChildProcessDefault)
814+
(WellKnownObjectKind::ChildProcessModule, Some("default")) => {
815+
JsValue::WellKnownObject(WellKnownObjectKind::ChildProcessModuleDefault)
790816
}
791817

792818
_ => JsValue::unknown(
793819
JsValue::member(
794-
Box::new(JsValue::WellKnownObject(WellKnownObjectKind::ChildProcess)),
820+
Box::new(JsValue::WellKnownObject(
821+
WellKnownObjectKind::ChildProcessModule,
822+
)),
795823
Box::new(prop),
796824
),
797825
true,
@@ -843,7 +871,9 @@ async fn node_process_member(
843871
Some("env") => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessEnv),
844872
_ => JsValue::unknown(
845873
JsValue::member(
846-
Box::new(JsValue::WellKnownObject(WellKnownObjectKind::NodeProcess)),
874+
Box::new(JsValue::WellKnownObject(
875+
WellKnownObjectKind::NodeProcessModule,
876+
)),
847877
Box::new(prop),
848878
),
849879
true,

‎turbopack/crates/turbopack-ecmascript/src/parse.rs‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -176,15 +176,9 @@ pub async fn parse(
176176
ty = display(&ty)
177177
);
178178

179-
match parse_internal(
180-
source,
181-
ty,
182-
transforms,
183-
is_external_tracing && matches!(ty, EcmascriptModuleAssetType::EcmascriptExtensionless),
184-
inline_helpers,
185-
)
186-
.instrument(span)
187-
.await
179+
match parse_internal(source, ty, transforms, is_external_tracing, inline_helpers)
180+
.instrument(span)
181+
.await
188182
{
189183
Ok(result) => Ok(result),
190184
Err(error) => Err(error.context(format!(

‎turbopack/crates/turbopack-ecmascript/src/references/mod.rs‎

Lines changed: 73 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ use crate::{
151151
},
152152
ident::IdentReplacement,
153153
member::MemberReplacement,
154-
node::PackageJsonReference,
154+
node::{FilePathModuleReference, PackageJsonReference},
155155
require_context::{RequireContextAssetReference, RequireContextMap},
156156
type_issue::SpecifiedModuleTypeIssue,
157157
},
@@ -1646,8 +1646,27 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
16461646
.await
16471647
};
16481648

1649-
let make_issue_source =
1649+
let get_traced_project_dir = async || -> Result<FileSystemPath> {
1650+
// readFileSync("./foo") should always be relative to the project root, but this is
1651+
// dangerous inside of node_modules as it can cause a lot of false positives in the tracing,
1652+
// if some package does `path.join(dynamic)`, it would include everything from the project
1653+
// root as well.
1654+
//
1655+
// Also, when there's no cwd set (i.e. in a tracing-specific module context, as we shouldn't
1656+
// assume a `process.cwd()` for all of node_modules), fallback to the source file directory.
1657+
// This still allows relative file accesses, just not from the project root.
1658+
if allow_project_root_tracing
1659+
&& let Some(cwd) = compile_time_info.environment().cwd().owned().await?
1660+
{
1661+
Ok(cwd)
1662+
} else {
1663+
Ok(source.ident().path().await?.parent())
1664+
}
1665+
};
1666+
1667+
let get_issue_source =
16501668
|| IssueSource::from_swc_offsets(source, span.lo.to_u32(), span.hi.to_u32());
1669+
16511670
if new {
16521671
match func {
16531672
JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor) => {
@@ -1726,9 +1745,54 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
17261745
// Ignore (e.g. dynamic parameter or string literal), just as Webpack does
17271746
return Ok(());
17281747
}
1748+
JsValue::WellKnownFunction(WellKnownFunctionKind::NodeWorkerConstructor)
1749+
if analysis.analyze_mode.is_tracing() =>
1750+
{
1751+
// Only for tracing, not for bundling (yet?)
1752+
let args = linked_args(args).await?;
1753+
if !args.is_empty() {
1754+
let pat = js_value_to_pattern(&args[0]);
1755+
if !pat.has_constant_parts() {
1756+
let (args, hints) = explain_args(&args);
1757+
handler.span_warn_with_code(
1758+
span,
1759+
&format!("new Worker({args}) is very dynamic{hints}",),
1760+
DiagnosticId::Lint(
1761+
errors::failed_to_analyze::ecmascript::NEW_WORKER.to_string(),
1762+
),
1763+
);
1764+
if ignore_dynamic_requests {
1765+
return Ok(());
1766+
}
1767+
}
1768+
analysis.add_reference(
1769+
FilePathModuleReference::new(
1770+
origin.asset_context(),
1771+
get_traced_project_dir().await?,
1772+
Pattern::new(pat),
1773+
collect_affecting_sources,
1774+
get_issue_source(),
1775+
)
1776+
.to_resolved()
1777+
.await?,
1778+
);
1779+
return Ok(());
1780+
}
1781+
let (args, hints) = explain_args(&args);
1782+
handler.span_warn_with_code(
1783+
span,
1784+
&format!("new Worker({args}) is not statically analyze-able{hints}",),
1785+
DiagnosticId::Error(
1786+
errors::failed_to_analyze::ecmascript::FS_METHOD.to_string(),
1787+
),
1788+
);
1789+
// Ignore (e.g. dynamic parameter or string literal)
1790+
return Ok(());
1791+
}
17291792
_ => {}
17301793
}
17311794

1795+
// linked_args wasn't called, so manually add the closure effects
17321796
for arg in args {
17331797
if let EffectArg::Closure(_, block) = arg {
17341798
add_effects(block.effects);
@@ -1737,24 +1801,6 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
17371801
return Ok(());
17381802
}
17391803

1740-
let get_traced_project_dir = async || -> Result<FileSystemPath> {
1741-
// readFileSync("./foo") should always be relative to the project root, but this is
1742-
// dangerous inside of node_modules as it can cause a lot of false positives in the tracing,
1743-
// if some package does `path.join(dynamic)`, it would include everything from the project
1744-
// root as well.
1745-
//
1746-
// Also, when there's no cwd set (i.e. in a tracing-specific module context, as we shouldn't
1747-
// assume a `process.cwd()` for all of node_modules), fallback to the source file directory.
1748-
// This still allows relative file accesses, just not from the project root.
1749-
if allow_project_root_tracing
1750-
&& let Some(cwd) = compile_time_info.environment().cwd().owned().await?
1751-
{
1752-
Ok(cwd)
1753-
} else {
1754-
Ok(source.ident().path().await?.parent())
1755-
}
1756-
};
1757-
17581804
match func {
17591805
JsValue::Alternatives {
17601806
total_nodes: _,
@@ -1988,7 +2034,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
19882034
get_traced_project_dir().await?,
19892035
Pattern::new(pat),
19902036
collect_affecting_sources,
1991-
make_issue_source(),
2037+
get_issue_source(),
19922038
)
19932039
.to_resolved()
19942040
.await?,
@@ -2041,7 +2087,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
20412087
DirAssetReference::new(
20422088
get_traced_project_dir().await?,
20432089
Pattern::new(pat),
2044-
make_issue_source(),
2090+
get_issue_source(),
20452091
)
20462092
.to_resolved()
20472093
.await?,
@@ -2085,7 +2131,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
20852131
DirAssetReference::new(
20862132
get_traced_project_dir().await?,
20872133
Pattern::new(pat),
2088-
make_issue_source(),
2134+
get_issue_source(),
20892135
)
20902136
.to_resolved()
20912137
.await?,
@@ -2376,7 +2422,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
23762422
DirAssetReference::new(
23772423
get_traced_project_dir().await?,
23782424
Pattern::new(abs_pattern),
2379-
make_issue_source(),
2425+
get_issue_source(),
23802426
)
23812427
.to_resolved()
23822428
.await?,
@@ -2443,7 +2489,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
24432489
DirAssetReference::new(
24442490
get_traced_project_dir().await?,
24452491
Pattern::new(abs_pattern),
2446-
make_issue_source(),
2492+
get_issue_source(),
24472493
)
24482494
.to_resolved()
24492495
.await?,
@@ -2512,7 +2558,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
25122558
DirAssetReference::new(
25132559
context_dir.clone(),
25142560
Pattern::new(Pattern::Constant(dir.into())),
2515-
make_issue_source(),
2561+
get_issue_source(),
25162562
)
25172563
.to_resolved()
25182564
})
@@ -3147,7 +3193,7 @@ async fn value_visitor_inner(
31473193
),
31483194
"define" => JsValue::WellKnownFunction(WellKnownFunctionKind::Define),
31493195
"URL" => JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor),
3150-
"process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcess),
3196+
"process" => JsValue::WellKnownObject(WellKnownObjectKind::NodeProcessModule),
31513197
"Object" => JsValue::WellKnownObject(WellKnownObjectKind::GlobalObject),
31523198
"Buffer" => JsValue::WellKnownObject(WellKnownObjectKind::NodeBuffer),
31533199
_ => return Ok((v, false)),

0 commit comments

Comments
 (0)