Turbopack: shard amount need to grow quadratic to cpu count to keep propability of conflicts constant

Author: sokra

turbopack/crates/turbo-tasks-backend/src/backend/mod.rs

@@ -134,6 +134,10 @@ pub struct BackendOptions {
     /// Enables the backing storage.
     pub storage_mode: Option<StorageMode>,
 
+    /// Number of tokio worker threads. It will be used to compute the shard amount of parallel
+    /// datastructures. If `None`, it will use the available parallelism.
+    pub num_workers: Option<usize>,
+
     /// Avoid big preallocations for faster startup. Should only be used for testing purposes.
     pub small_preallocation: bool,
 }
@@ -144,6 +148,7 @@ impl Default for BackendOptions {
             dependency_tracking: true,
             active_tracking: true,
             storage_mode: Some(StorageMode::ReadWrite),
+            num_workers: None,
             small_preallocation: false,
         }
     }
@@ -235,6 +240,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
             options.active_tracking = false;
         }
         let small_preallocation = options.small_preallocation;
+        let num_workers = options.num_workers;
         let next_task_id = backing_storage
             .next_free_task_id()
             .expect("Failed to get task id");
@@ -256,7 +262,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
             task_cache: BiMap::new(),
             transient_tasks: FxDashMap::default(),
             local_is_partial: AtomicBool::new(next_task_id != TaskId::MIN),
-            storage: Storage::new(small_preallocation),
+            storage: Storage::new(num_workers, small_preallocation),
             in_progress_operations: AtomicUsize::new(0),
             snapshot_request: Mutex::new(SnapshotRequest::new()),
             operations_suspended: Condvar::new(),

turbopack/crates/turbo-tasks-backend/src/backend/storage.rs

@@ -616,17 +616,19 @@ pub struct Storage {
 }
 
 impl Storage {
-    pub fn new(small_preallocation: bool) -> Self {
+    pub fn new(num_worker: Option<usize>, small_preallocation: bool) -> Self {
         let map_capacity: usize = if small_preallocation {
             1024
         } else {
             1024 * 1024
         };
         let modified_capacity: usize = if small_preallocation { 0 } else { 1024 };
-        let shard_factor: usize = if small_preallocation { 4 } else { 64 };
+        let shard_factor: usize = if small_preallocation { 1 } else { 16 };
 
-        let shard_amount =
-            (available_parallelism().map_or(4, |v| v.get()) * shard_factor).next_power_of_two();
+        let num_workers =
+            num_worker.unwrap_or_else(|| available_parallelism().map_or(4, |v| v.get()));
+
+        let shard_amount = (num_workers * num_workers * shard_factor).next_power_of_two();
 
         Self {
             snapshot_mode: AtomicBool::new(false),

turbopack/crates/turbo-tasks-backend/tests/test_config.trs

@@ -7,7 +7,11 @@
   std::fs::create_dir_all(&path).unwrap();
   turbo_tasks::TurboTasks::new(
     turbo_tasks_backend::TurboTasksBackend::new(
-      turbo_tasks_backend::BackendOptions::default(),
+      turbo_tasks_backend::BackendOptions {
+        num_workers: Some(2),
+        small_preallocation: true,
+        ..Default::default()
+      },
       turbo_tasks_backend::default_backing_storage(
         path.as_path(),
         &turbo_tasks_backend::GitVersionInfo {

turbopack/crates/turbo-tasks-fetch/tests/test_config.trs

@@ -7,7 +7,11 @@
   std::fs::create_dir_all(&path).unwrap();
   turbo_tasks::TurboTasks::new(
     turbo_tasks_backend::TurboTasksBackend::new(
-      turbo_tasks_backend::BackendOptions::default(),
+      turbo_tasks_backend::BackendOptions {
+        num_workers: Some(2),
+        small_preallocation: true,
+        ..Default::default()
+      },
       turbo_tasks_backend::default_backing_storage(
         path.as_path(),
         &turbo_tasks_backend::GitVersionInfo {

turbopack/crates/turbo-tasks-macros-tests/tests/test_config.trs

@@ -1,7 +1,11 @@
 |_name, _initial| {
   turbo_tasks::TurboTasks::new(
     turbo_tasks_backend::TurboTasksBackend::new(
-      turbo_tasks_backend::BackendOptions::default(),
+      turbo_tasks_backend::BackendOptions {
+        num_workers: Some(2),
+        small_preallocation: true,
+        ..Default::default()
+      },
       turbo_tasks_backend::noop_backing_storage(),
     )
   )