Skip to content

Commit b114f47

Browse files
edwin0chengedwin0cheng
committed
Async Loading outdir and proc-macro
1 parent 3cd994d commit b114f47

File tree

10 files changed

+392
-139
lines changed

10 files changed

+392
-139
lines changed

‎crates/project_model/src/build_data.rs‎

Lines changed: 166 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,13 @@ use std::{
1010
use anyhow::Result;
1111
use cargo_metadata::{BuildScript, Message, Package, PackageId};
1212
use itertools::Itertools;
13+
use la_arena::{Arena, Idx};
1314
use paths::{AbsPath, AbsPathBuf};
1415
use rustc_hash::FxHashMap;
1516
use stdx::JodChild;
1617

1718
use crate::{cfg_flag::CfgFlag, CargoConfig};
1819

19-
#[derive(Debug, Clone, Default)]
20-
pub(crate) struct BuildDataMap {
21-
data: FxHashMap<PackageId, BuildData>,
22-
}
2320
#[derive(Debug, Clone, Default, PartialEq, Eq)]
2421
pub struct BuildData {
2522
/// List of config flags defined by this package's build script
@@ -33,133 +30,187 @@ pub struct BuildData {
3330
pub out_dir: Option<AbsPathBuf>,
3431
/// Path to the proc-macro library file if this package exposes proc-macros
3532
pub proc_macro_dylib_path: Option<AbsPathBuf>,
33+
34+
/// State for build data, used for updating
35+
fetch: Option<PackageId>,
3636
}
3737

38-
impl BuildDataMap {
39-
pub(crate) fn new(
40-
cargo_toml: &AbsPath,
41-
cargo_features: &CargoConfig,
42-
packages: &Vec<Package>,
43-
progress: &dyn Fn(String),
44-
) -> Result<BuildDataMap> {
45-
let mut cmd = Command::new(toolchain::cargo());
46-
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
47-
.arg(cargo_toml.as_ref());
48-
49-
// --all-targets includes tests, benches and examples in addition to the
50-
// default lib and bins. This is an independent concept from the --targets
51-
// flag below.
52-
cmd.arg("--all-targets");
53-
54-
if let Some(target) = &cargo_features.target {
55-
cmd.args(&["--target", target]);
56-
}
38+
#[derive(Debug)]
39+
pub(crate) struct BuildDataConfig {
40+
cargo_toml: AbsPathBuf,
41+
cargo_features: CargoConfig,
42+
}
5743

58-
if cargo_features.all_features {
59-
cmd.arg("--all-features");
60-
} else {
61-
if cargo_features.no_default_features {
62-
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
63-
// https://github.com/oli-obk/cargo_metadata/issues/79
64-
cmd.arg("--no-default-features");
65-
}
66-
if !cargo_features.features.is_empty() {
67-
cmd.arg("--features");
68-
cmd.arg(cargo_features.features.join(" "));
69-
}
70-
}
44+
#[derive(Debug, Default)]
45+
pub struct BuildDataLoader {
46+
arena: Arena<BuildDataConfig>,
47+
data: FxHashMap<Idx<BuildDataConfig>, BuildDataMap>,
48+
}
7149

72-
cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());
73-
74-
let mut child = cmd.spawn().map(JodChild)?;
75-
let child_stdout = child.stdout.take().unwrap();
76-
let stdout = BufReader::new(child_stdout);
77-
78-
let mut res = BuildDataMap::default();
79-
for message in cargo_metadata::Message::parse_stream(stdout) {
80-
if let Ok(message) = message {
81-
match message {
82-
Message::BuildScriptExecuted(BuildScript {
83-
package_id,
84-
out_dir,
85-
cfgs,
86-
env,
87-
..
88-
}) => {
89-
let cfgs = {
90-
let mut acc = Vec::new();
91-
for cfg in cfgs {
92-
match cfg.parse::<CfgFlag>() {
93-
Ok(it) => acc.push(it),
94-
Err(err) => {
95-
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
96-
}
97-
};
98-
}
99-
acc
100-
};
101-
let res = res.data.entry(package_id.clone()).or_default();
102-
// cargo_metadata crate returns default (empty) path for
103-
// older cargos, which is not absolute, so work around that.
104-
if out_dir != PathBuf::default() {
105-
let out_dir = AbsPathBuf::assert(out_dir);
106-
res.out_dir = Some(out_dir);
107-
res.cfgs = cfgs;
108-
}
50+
type BuildDataMap = FxHashMap<PackageId, BuildData>;
10951

110-
res.envs = env;
111-
}
112-
Message::CompilerArtifact(message) => {
113-
progress(format!("metadata {}", message.target.name));
114-
115-
if message.target.kind.contains(&"proc-macro".to_string()) {
116-
let package_id = message.package_id;
117-
// Skip rmeta file
118-
if let Some(filename) =
119-
message.filenames.iter().find(|name| is_dylib(name))
120-
{
121-
let filename = AbsPathBuf::assert(filename.clone());
122-
let res = res.data.entry(package_id.clone()).or_default();
123-
res.proc_macro_dylib_path = Some(filename);
124-
}
125-
}
126-
}
127-
Message::CompilerMessage(message) => {
128-
progress(message.target.name.clone());
129-
}
130-
Message::Unknown => (),
131-
Message::BuildFinished(_) => {}
132-
Message::TextLine(_) => {}
52+
pub(crate) fn packages_to_build_data(packages: &Vec<Package>) -> BuildDataMap {
53+
let mut res: BuildDataMap = FxHashMap::default();
54+
for meta_pkg in packages {
55+
let mut build_data = BuildData::default();
56+
build_data.fetch = Some(meta_pkg.id.clone());
57+
inject_cargo_env(meta_pkg, &mut build_data);
58+
res.insert(meta_pkg.id.clone(), build_data);
59+
}
60+
res
61+
}
62+
63+
impl BuildData {
64+
pub(crate) fn update(&mut self, build_data_map: &BuildDataMap) {
65+
if let Some(package_id) = &self.fetch {
66+
let new_data = build_data_map.get(package_id).cloned();
67+
let BuildData { mut cfgs, mut envs, out_dir, proc_macro_dylib_path, .. } =
68+
match new_data {
69+
None => return,
70+
Some(it) => it,
71+
};
72+
73+
if let Some(out_dir) = &out_dir {
74+
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
75+
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
76+
self.envs.push(("OUT_DIR".to_string(), out_dir));
13377
}
13478
}
79+
self.cfgs.append(&mut cfgs);
80+
self.envs.append(&mut envs);
81+
self.out_dir = out_dir;
82+
self.proc_macro_dylib_path = proc_macro_dylib_path;
83+
self.fetch = None;
13584
}
136-
res.inject_cargo_env(packages);
137-
Ok(res)
13885
}
86+
}
13987

140-
pub(crate) fn with_cargo_env(packages: &Vec<Package>) -> Self {
141-
let mut res = Self::default();
142-
res.inject_cargo_env(packages);
143-
res
88+
impl BuildDataLoader {
89+
pub(crate) fn new_config(
90+
&mut self,
91+
cargo_toml: &AbsPath,
92+
cargo_features: &CargoConfig,
93+
) -> Idx<BuildDataConfig> {
94+
self.arena.alloc(BuildDataConfig {
95+
cargo_toml: cargo_toml.to_path_buf().clone(),
96+
cargo_features: cargo_features.clone(),
97+
})
14498
}
14599

146-
pub(crate) fn get(&self, id: &PackageId) -> Option<&BuildData> {
147-
self.data.get(id)
100+
pub fn load(&mut self, progress: &dyn Fn(String)) -> Result<()> {
101+
for (idx, config) in self.arena.iter() {
102+
self.data
103+
.insert(idx, load_workspace(&config.cargo_toml, &config.cargo_features, progress)?);
104+
}
105+
Ok(())
148106
}
149107

150-
fn inject_cargo_env(&mut self, packages: &Vec<Package>) {
151-
for meta_pkg in packages {
152-
let resource = self.data.entry(meta_pkg.id.clone()).or_default();
153-
inject_cargo_env(meta_pkg, &mut resource.envs);
108+
pub fn is_empty(&self) -> bool {
109+
self.arena.is_empty()
110+
}
154111

155-
if let Some(out_dir) = &resource.out_dir {
156-
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
157-
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
158-
resource.envs.push(("OUT_DIR".to_string(), out_dir));
112+
pub(crate) fn get(&self, idx: &Idx<BuildDataConfig>) -> Option<&BuildDataMap> {
113+
self.data.get(idx)
114+
}
115+
}
116+
117+
fn load_workspace(
118+
cargo_toml: &AbsPath,
119+
cargo_features: &CargoConfig,
120+
progress: &dyn Fn(String),
121+
) -> Result<BuildDataMap> {
122+
let mut cmd = Command::new(toolchain::cargo());
123+
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
124+
.arg(cargo_toml.as_ref());
125+
126+
// --all-targets includes tests, benches and examples in addition to the
127+
// default lib and bins. This is an independent concept from the --targets
128+
// flag below.
129+
cmd.arg("--all-targets");
130+
131+
if let Some(target) = &cargo_features.target {
132+
cmd.args(&["--target", target]);
133+
}
134+
135+
if cargo_features.all_features {
136+
cmd.arg("--all-features");
137+
} else {
138+
if cargo_features.no_default_features {
139+
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
140+
// https://github.com/oli-obk/cargo_metadata/issues/79
141+
cmd.arg("--no-default-features");
142+
}
143+
if !cargo_features.features.is_empty() {
144+
cmd.arg("--features");
145+
cmd.arg(cargo_features.features.join(" "));
146+
}
147+
}
148+
149+
cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());
150+
151+
let mut child = cmd.spawn().map(JodChild)?;
152+
let child_stdout = child.stdout.take().unwrap();
153+
let stdout = BufReader::new(child_stdout);
154+
155+
let mut res = FxHashMap::<PackageId, BuildData>::default();
156+
for message in cargo_metadata::Message::parse_stream(stdout) {
157+
if let Ok(message) = message {
158+
match message {
159+
Message::BuildScriptExecuted(BuildScript {
160+
package_id,
161+
out_dir,
162+
cfgs,
163+
env,
164+
..
165+
}) => {
166+
let cfgs = {
167+
let mut acc = Vec::new();
168+
for cfg in cfgs {
169+
match cfg.parse::<CfgFlag>() {
170+
Ok(it) => acc.push(it),
171+
Err(err) => {
172+
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
173+
}
174+
};
175+
}
176+
acc
177+
};
178+
let res = res.entry(package_id.clone()).or_default();
179+
// cargo_metadata crate returns default (empty) path for
180+
// older cargos, which is not absolute, so work around that.
181+
if out_dir != PathBuf::default() {
182+
let out_dir = AbsPathBuf::assert(out_dir);
183+
res.out_dir = Some(out_dir);
184+
res.cfgs = cfgs;
185+
}
186+
187+
res.envs = env;
188+
}
189+
Message::CompilerArtifact(message) => {
190+
progress(format!("metadata {}", message.target.name));
191+
192+
if message.target.kind.contains(&"proc-macro".to_string()) {
193+
let package_id = message.package_id;
194+
// Skip rmeta file
195+
if let Some(filename) = message.filenames.iter().find(|name| is_dylib(name))
196+
{
197+
let filename = AbsPathBuf::assert(filename.clone());
198+
let res = res.entry(package_id.clone()).or_default();
199+
res.proc_macro_dylib_path = Some(filename);
200+
}
201+
}
159202
}
203+
Message::CompilerMessage(message) => {
204+
progress(message.target.name.clone());
205+
}
206+
Message::Unknown => (),
207+
Message::BuildFinished(_) => {}
208+
Message::TextLine(_) => {}
160209
}
161210
}
162211
}
212+
213+
Ok(res)
163214
}
164215

165216
// FIXME: File a better way to know if it is a dylib
@@ -173,7 +224,9 @@ fn is_dylib(path: &Path) -> bool {
173224
/// Recreates the compile-time environment variables that Cargo sets.
174225
///
175226
/// Should be synced with <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates>
176-
fn inject_cargo_env(package: &cargo_metadata::Package, env: &mut Vec<(String, String)>) {
227+
fn inject_cargo_env(package: &cargo_metadata::Package, build_data: &mut BuildData) {
228+
let env = &mut build_data.envs;
229+
177230
// FIXME: Missing variables:
178231
// CARGO, CARGO_PKG_HOMEPAGE, CARGO_CRATE_NAME, CARGO_BIN_NAME, CARGO_BIN_EXE_<name>
179232

‎crates/project_model/src/cargo_workspace.rs‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use la_arena::{Arena, Idx};
99
use paths::{AbsPath, AbsPathBuf};
1010
use rustc_hash::FxHashMap;
1111

12-
use crate::build_data::{BuildData, BuildDataMap};
12+
use crate::build_data::{packages_to_build_data, BuildData, BuildDataConfig, BuildDataLoader};
1313
use crate::utf8_stdout;
1414

1515
/// `CargoWorkspace` represents the logical structure of, well, a Cargo
@@ -27,6 +27,7 @@ pub struct CargoWorkspace {
2727
packages: Arena<PackageData>,
2828
targets: Arena<TargetData>,
2929
workspace_root: AbsPathBuf,
30+
build_data_config: Option<Idx<BuildDataConfig>>,
3031
}
3132

3233
impl ops::Index<Package> for CargoWorkspace {
@@ -157,6 +158,7 @@ impl CargoWorkspace {
157158
pub fn from_cargo_metadata(
158159
cargo_toml: &AbsPath,
159160
config: &CargoConfig,
161+
build_data_loader: &mut BuildDataLoader,
160162
progress: &dyn Fn(String),
161163
) -> Result<CargoWorkspace> {
162164
let mut meta = MetadataCommand::new();
@@ -228,12 +230,13 @@ impl CargoWorkspace {
228230
)
229231
})?;
230232

231-
let resources = if config.load_out_dirs_from_check {
232-
BuildDataMap::new(cargo_toml, config, &meta.packages, progress)?
233+
let build_data_config = if config.load_out_dirs_from_check {
234+
Some(build_data_loader.new_config(cargo_toml, config))
233235
} else {
234-
BuildDataMap::with_cargo_env(&meta.packages)
236+
None
235237
};
236238

239+
let resources = packages_to_build_data(&meta.packages);
237240
let mut pkg_by_id = FxHashMap::default();
238241
let mut packages = Arena::default();
239242
let mut targets = Arena::default();
@@ -308,7 +311,7 @@ impl CargoWorkspace {
308311
}
309312

310313
let workspace_root = AbsPathBuf::assert(meta.workspace_root);
311-
Ok(CargoWorkspace { packages, targets, workspace_root: workspace_root })
314+
Ok(CargoWorkspace { packages, targets, workspace_root: workspace_root, build_data_config })
312315
}
313316

314317
pub fn packages<'a>(&'a self) -> impl Iterator<Item = Package> + ExactSizeIterator + 'a {
@@ -337,4 +340,16 @@ impl CargoWorkspace {
337340
fn is_unique(&self, name: &str) -> bool {
338341
self.packages.iter().filter(|(_, v)| v.name == name).count() == 1
339342
}
343+
344+
pub(crate) fn update_build_data(&mut self, loader: &BuildDataLoader) {
345+
let build_data_map = match self.build_data_config.and_then(|it| loader.get(&it)) {
346+
None => return,
347+
Some(it) => it,
348+
};
349+
350+
let keys: Vec<Idx<PackageData>> = self.packages.iter().map(|it| it.0).collect();
351+
for id in keys {
352+
self.packages[id].build_data.update(build_data_map);
353+
}
354+
}
340355
}

0 commit comments

Comments
 (0)