feat: Load proc-macros asynchronously

crates/base-db/src/change.rs

@@ -6,14 +6,15 @@ use std::{fmt, sync::Arc};
 use salsa::Durability;
 use vfs::FileId;
 
-use crate::{CrateGraph, SourceDatabaseExt, SourceRoot, SourceRootId};
+use crate::{CrateGraph, ProcMacros, SourceDatabaseExt, SourceRoot, SourceRootId};
 
 /// Encapsulate a bunch of raw `.set` calls on the database.
 #[derive(Default)]
 pub struct Change {
     pub roots: Option<Vec<SourceRoot>>,
     pub files_changed: Vec<(FileId, Option<Arc<String>>)>,
     pub crate_graph: Option<CrateGraph>,
+    pub proc_macros: Option<ProcMacros>,
 }
 
 impl fmt::Debug for Change {
@@ -49,6 +50,10 @@ impl Change {
         self.crate_graph = Some(graph);
     }
 
+    pub fn set_proc_macros(&mut self, proc_macros: ProcMacros) {
+        self.proc_macros = Some(proc_macros);
+    }
+
     pub fn apply(self, db: &mut dyn SourceDatabaseExt) {
         let _p = profile::span("RootDatabase::apply_change");
         if let Some(roots) = self.roots {
@@ -73,6 +78,9 @@ impl Change {
         if let Some(crate_graph) = self.crate_graph {
             db.set_crate_graph_with_durability(Arc::new(crate_graph), Durability::HIGH)
         }
+        if let Some(proc_macros) = self.proc_macros {
+            db.set_proc_macros_with_durability(Arc::new(proc_macros), Durability::HIGH)
+        }
     }
 }
 

crates/base-db/src/fixture.rs

@@ -12,8 +12,8 @@ use vfs::{file_set::FileSet, VfsPath};
 use crate::{
     input::{CrateName, CrateOrigin, LangCrateOrigin},
     Change, CrateDisplayName, CrateGraph, CrateId, Dependency, Edition, Env, FileId, FilePosition,
-    FileRange, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, SourceDatabaseExt,
-    SourceRoot, SourceRootId,
+    FileRange, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacros,
+    SourceDatabaseExt, SourceRoot, SourceRootId,
 };
 
 pub const WORKSPACE: SourceRootId = SourceRootId(0);
@@ -100,7 +100,7 @@ impl ChangeFixture {
 
     pub fn parse_with_proc_macros(
         ra_fixture: &str,
-        mut proc_macros: Vec<(String, ProcMacro)>,
+        mut proc_macro_defs: Vec<(String, ProcMacro)>,
     ) -> ChangeFixture {
         let (mini_core, proc_macro_names, fixture) = Fixture::parse(ra_fixture);
         let mut change = Change::new();
@@ -160,7 +160,6 @@ impl ChangeFixture {
                     meta.cfg.clone(),
                     meta.cfg,
                     meta.env,
-                    Ok(Vec::new()),
                     false,
                     origin,
                     meta.target_data_layout
@@ -200,7 +199,6 @@ impl ChangeFixture {
                 default_cfg.clone(),
                 default_cfg,
                 Env::default(),
-                Ok(Vec::new()),
                 false,
                 CrateOrigin::CratesIo { repo: None, name: None },
                 default_target_data_layout
@@ -244,7 +242,6 @@ impl ChangeFixture {
                 CfgOptions::default(),
                 CfgOptions::default(),
                 Env::default(),
-                Ok(Vec::new()),
                 false,
                 CrateOrigin::Lang(LangCrateOrigin::Core),
                 target_layout.clone(),
@@ -257,12 +254,13 @@ impl ChangeFixture {
             }
         }
 
+        let mut proc_macros = ProcMacros::default();
         if !proc_macro_names.is_empty() {
             let proc_lib_file = file_id;
             file_id.0 += 1;
 
-            proc_macros.extend(default_test_proc_macros());
-            let (proc_macro, source) = filter_test_proc_macros(&proc_macro_names, proc_macros);
+            proc_macro_defs.extend(default_test_proc_macros());
+            let (proc_macro, source) = filter_test_proc_macros(&proc_macro_names, proc_macro_defs);
             let mut fs = FileSet::default();
             fs.insert(
                 proc_lib_file,
@@ -282,11 +280,11 @@ impl ChangeFixture {
                 CfgOptions::default(),
                 CfgOptions::default(),
                 Env::default(),
-                Ok(proc_macro),
                 true,
                 CrateOrigin::CratesIo { repo: None, name: None },
                 target_layout,
             );
+            proc_macros.insert(proc_macros_crate, Ok(proc_macro));
 
             for krate in all_crates {
                 crate_graph
@@ -305,6 +303,7 @@ impl ChangeFixture {
         roots.push(root);
         change.set_roots(roots);
         change.set_crate_graph(crate_graph);
+        change.set_proc_macros(proc_macros);
 
         ChangeFixture { file_position, files, change }
     }

crates/base-db/src/input.rs

@@ -6,14 +6,17 @@
 //! actual IO. See `vfs` and `project_model` in the `rust-analyzer` crate for how
 //! actual IO is done and lowered to input.
 
-use std::{fmt, ops, panic::RefUnwindSafe, str::FromStr, sync::Arc};
+use std::{fmt, mem, ops, panic::RefUnwindSafe, str::FromStr, sync::Arc};
 
 use cfg::CfgOptions;
 use rustc_hash::FxHashMap;
 use stdx::hash::{NoHashHashMap, NoHashHashSet};
 use syntax::SmolStr;
 use tt::token_id::Subtree;
-use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
+use vfs::{file_set::FileSet, AbsPathBuf, AnchoredPath, FileId, VfsPath};
+
+pub type ProcMacroPaths = FxHashMap<CrateId, Result<(Option<String>, AbsPathBuf), String>>;
+pub type ProcMacros = FxHashMap<CrateId, ProcMacroLoadResult>;
 
 /// Files are grouped into source roots. A source root is a directory on the
 /// file systems which is watched for changes. Typically it corresponds to a
@@ -269,7 +272,6 @@ pub struct CrateData {
     pub target_layout: TargetLayoutLoadResult,
     pub env: Env,
     pub dependencies: Vec<Dependency>,
-    pub proc_macro: ProcMacroLoadResult,
     pub origin: CrateOrigin,
     pub is_proc_macro: bool,
 }
@@ -322,7 +324,6 @@ impl CrateGraph {
         cfg_options: CfgOptions,
         potential_cfg_options: CfgOptions,
         env: Env,
-        proc_macro: ProcMacroLoadResult,
         is_proc_macro: bool,
         origin: CrateOrigin,
         target_layout: Result<Arc<str>, Arc<str>>,
@@ -335,7 +336,6 @@ impl CrateGraph {
             cfg_options,
             potential_cfg_options,
             env,
-            proc_macro,
             dependencies: Vec::new(),
             origin,
             target_layout,
@@ -456,11 +456,11 @@ impl CrateGraph {
     }
 
     /// Extends this crate graph by adding a complete disjoint second crate
-    /// graph.
+    /// graph and adjust the ids in the [`ProcMacroPaths`] accordingly.
     ///
     /// The ids of the crates in the `other` graph are shifted by the return
     /// amount.
-    pub fn extend(&mut self, other: CrateGraph) -> u32 {
+    pub fn extend(&mut self, other: CrateGraph, proc_macros: &mut ProcMacroPaths) -> u32 {
         let start = self.arena.len() as u32;
         self.arena.extend(other.arena.into_iter().map(|(id, mut data)| {
             let new_id = id.shift(start);
@@ -469,6 +469,11 @@ impl CrateGraph {
             }
             (new_id, data)
         }));
+
+        *proc_macros = mem::take(proc_macros)
+            .into_iter()
+            .map(|(id, macros)| (id.shift(start), macros))
+            .collect();
         start
     }
 
@@ -645,7 +650,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -658,7 +662,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -671,7 +674,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -698,7 +700,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -711,7 +712,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -735,7 +735,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -748,7 +747,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -761,7 +759,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -785,7 +782,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),
@@ -798,7 +794,6 @@ mod tests {
             CfgOptions::default(),
             CfgOptions::default(),
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("".into()),

crates/base-db/src/lib.rs

@@ -16,8 +16,8 @@ pub use crate::{
     input::{
         CrateData, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency,
         Edition, Env, LangCrateOrigin, ProcMacro, ProcMacroExpander, ProcMacroExpansionError,
-        ProcMacroId, ProcMacroKind, ProcMacroLoadResult, SourceRoot, SourceRootId,
-        TargetLayoutLoadResult,
+        ProcMacroId, ProcMacroKind, ProcMacroLoadResult, ProcMacroPaths, ProcMacros, SourceRoot,
+        SourceRootId, TargetLayoutLoadResult,
     },
 };
 pub use salsa::{self, Cancelled};
@@ -73,6 +73,10 @@ pub trait SourceDatabase: FileLoader + std::fmt::Debug {
     /// The crate graph.
     #[salsa::input]
     fn crate_graph(&self) -> Arc<CrateGraph>;
+
+    /// The crate graph.
+    #[salsa::input]
+    fn proc_macros(&self) -> Arc<ProcMacros>;
 }
 
 fn parse_query(db: &dyn SourceDatabase, file_id: FileId) -> Parse<ast::SourceFile> {

crates/hir-def/src/nameres/collector.rs

@@ -78,25 +78,35 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
     }
 
     let cfg_options = &krate.cfg_options;
-    let proc_macros = match &krate.proc_macro {
-        Ok(proc_macros) => {
-            proc_macros
-                .iter()
-                .enumerate()
-                .map(|(idx, it)| {
-                    // FIXME: a hacky way to create a Name from string.
-                    let name =
-                        tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
-                    (name.as_name(), ProcMacroExpander::new(base_db::ProcMacroId(idx as u32)))
-                })
-                .collect()
-        }
-        Err(e) => {
-            def_map.proc_macro_loading_error = Some(e.clone().into_boxed_str());
-            Vec::new()
+
+    let is_proc_macro = krate.is_proc_macro;
+    let proc_macros = if is_proc_macro {
+        match db.proc_macros().get(&def_map.krate) {
+            Some(Ok(proc_macros)) => {
+                proc_macros
+                    .iter()
+                    .enumerate()
+                    .map(|(idx, it)| {
+                        // FIXME: a hacky way to create a Name from string.
+                        let name =
+                            tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
+                        (name.as_name(), ProcMacroExpander::new(base_db::ProcMacroId(idx as u32)))
+                    })
+                    .collect()
+            }
+            Some(Err(e)) => {
+                def_map.proc_macro_loading_error = Some(e.clone().into_boxed_str());
+                Vec::new()
+            }
+            None => {
+                def_map.proc_macro_loading_error =
+                    Some("No proc-macros present for crate".to_owned().into_boxed_str());
+                Vec::new()
+            }
         }
+    } else {
+        vec![]
     };
-    let is_proc_macro = krate.is_proc_macro;
 
     let mut collector = DefCollector {
         db,

crates/hir-expand/src/proc_macro.rs

@@ -33,10 +33,10 @@ impl ProcMacroExpander {
     ) -> ExpandResult<tt::Subtree> {
         match self.proc_macro_id {
             Some(id) => {
-                let krate_graph = db.crate_graph();
-                let proc_macros = match &krate_graph[def_crate].proc_macro {
-                    Ok(proc_macros) => proc_macros,
-                    Err(_) => {
+                let proc_macros = db.proc_macros();
+                let proc_macros = match proc_macros.get(&def_crate) {
+                    Some(Ok(proc_macros)) => proc_macros,
+                    Some(Err(_)) | None => {
                         never!("Non-dummy expander even though there are no proc macros");
                         return ExpandResult::with_err(
                             tt::Subtree::empty(),
@@ -59,6 +59,7 @@ impl ProcMacroExpander {
                     }
                 };
 
+                let krate_graph = db.crate_graph();
                 // Proc macros have access to the environment variables of the invoking crate.
                 let env = &krate_graph[calling_crate].env;
                 match proc_macro.expander.expand(tt, attr_arg, env) {

crates/ide-db/src/apply_change.rs

@@ -64,6 +64,7 @@ impl RootDatabase {
             // SourceDatabase
             base_db::ParseQuery
             base_db::CrateGraphQuery
+            base_db::ProcMacrosQuery
 
             // SourceDatabaseExt
             base_db::FileTextQuery

crates/ide-db/src/lib.rs

@@ -137,6 +137,7 @@ impl RootDatabase {
     pub fn new(lru_capacity: Option<usize>) -> RootDatabase {
         let mut db = RootDatabase { storage: ManuallyDrop::new(salsa::Storage::default()) };
         db.set_crate_graph_with_durability(Default::default(), Durability::HIGH);
+        db.set_proc_macros_with_durability(Default::default(), Durability::HIGH);
         db.set_local_roots_with_durability(Default::default(), Durability::HIGH);
         db.set_library_roots_with_durability(Default::default(), Durability::HIGH);
         db.set_enable_proc_attr_macros(false);

crates/ide/src/lib.rs

@@ -235,7 +235,6 @@ impl Analysis {
             cfg_options.clone(),
             cfg_options,
             Env::default(),
-            Ok(Vec::new()),
             false,
             CrateOrigin::CratesIo { repo: None, name: None },
             Err("Analysis::from_single_file has no target layout".into()),