Skip to content

Async loading for outdir and proc-macro #7412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 28, 2021
Merged

Async loading for outdir and proc-macro #7412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9358eec
Async Loading outdir and proc-macro
edwin0cheng Jan 28, 2021
35310f1
Update vscode for new status
edwin0cheng Jan 28, 2021
edcafe7
Update doc
edwin0cheng Jan 28, 2021
dd47f8b
bug fix
edwin0cheng Jan 28, 2021
d069ef6
Update lsp-extension.md hash
edwin0cheng Jan 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 168 additions & 120 deletions crates/project_model/src/build_data.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,161 +5,207 @@ use std::{
io::BufReader,
path::{Path, PathBuf},
process::{Command, Stdio},
sync::Arc,
};

use anyhow::Result;
use cargo_metadata::{BuildScript, Message, Package, PackageId};
use cargo_metadata::{BuildScript, Message};
use itertools::Itertools;
use paths::{AbsPath, AbsPathBuf};
use rustc_hash::FxHashMap;
use stdx::JodChild;

use crate::{cfg_flag::CfgFlag, CargoConfig};

#[derive(Debug, Clone, Default)]
pub(crate) struct BuildDataMap {
data: FxHashMap<PackageId, BuildData>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BuildData {
pub(crate) struct BuildData {
/// List of config flags defined by this package's build script
pub cfgs: Vec<CfgFlag>,
pub(crate) cfgs: Vec<CfgFlag>,
/// List of cargo-related environment variables with their value
///
/// If the package has a build script which defines environment variables,
/// they can also be found here.
pub envs: Vec<(String, String)>,
pub(crate) envs: Vec<(String, String)>,
/// Directory where a build script might place its output
pub out_dir: Option<AbsPathBuf>,
pub(crate) out_dir: Option<AbsPathBuf>,
/// Path to the proc-macro library file if this package exposes proc-macros
pub proc_macro_dylib_path: Option<AbsPathBuf>,
pub(crate) proc_macro_dylib_path: Option<AbsPathBuf>,
}

impl BuildDataMap {
pub(crate) fn new(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
packages: &Vec<Package>,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}
#[derive(Clone, Debug)]
pub(crate) struct BuildDataConfig {
cargo_toml: AbsPathBuf,
cargo_features: CargoConfig,
packages: Arc<Vec<cargo_metadata::Package>>,
}

if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}
impl PartialEq for BuildDataConfig {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.packages, &other.packages)
}
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = BuildDataMap::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.data.entry(package_id.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}
impl Eq for BuildDataConfig {}

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) =
message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.data.entry(package_id.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
}
}
#[derive(Debug, Default)]
pub struct BuildDataCollector {
configs: FxHashMap<AbsPathBuf, BuildDataConfig>,
}

#[derive(Debug, Default, PartialEq, Eq)]
pub struct BuildDataResult {
data: FxHashMap<AbsPathBuf, BuildDataMap>,
}

pub(crate) type BuildDataMap = FxHashMap<String, BuildData>;

impl BuildDataCollector {
pub(crate) fn add_config(&mut self, workspace_root: &AbsPath, config: BuildDataConfig) {
self.configs.insert(workspace_root.to_path_buf().clone(), config);
}

pub fn collect(&mut self, progress: &dyn Fn(String)) -> Result<BuildDataResult> {
let mut res = BuildDataResult::default();
for (path, config) in self.configs.iter() {
res.data.insert(
path.clone(),
collect_from_workspace(
&config.cargo_toml,
&config.cargo_features,
&config.packages,
progress,
)?,
);
}
res.inject_cargo_env(packages);
Ok(res)
}
}

impl BuildDataResult {
pub(crate) fn get(&self, root: &AbsPath) -> Option<&BuildDataMap> {
self.data.get(&root.to_path_buf())
}
}

pub(crate) fn with_cargo_env(packages: &Vec<Package>) -> Self {
let mut res = Self::default();
res.inject_cargo_env(packages);
res
impl BuildDataConfig {
pub(crate) fn new(
cargo_toml: AbsPathBuf,
cargo_features: CargoConfig,
packages: Arc<Vec<cargo_metadata::Package>>,
) -> Self {
Self { cargo_toml, cargo_features, packages }
}
}

pub(crate) fn get(&self, id: &PackageId) -> Option<&BuildData> {
self.data.get(id)
fn collect_from_workspace(
cargo_toml: &AbsPath,
cargo_features: &CargoConfig,
packages: &Vec<cargo_metadata::Package>,
progress: &dyn Fn(String),
) -> Result<BuildDataMap> {
let mut cmd = Command::new(toolchain::cargo());
cmd.args(&["check", "--workspace", "--message-format=json", "--manifest-path"])
.arg(cargo_toml.as_ref());

// --all-targets includes tests, benches and examples in addition to the
// default lib and bins. This is an independent concept from the --targets
// flag below.
cmd.arg("--all-targets");

if let Some(target) = &cargo_features.target {
cmd.args(&["--target", target]);
}

fn inject_cargo_env(&mut self, packages: &Vec<Package>) {
for meta_pkg in packages {
let resource = self.data.entry(meta_pkg.id.clone()).or_default();
inject_cargo_env(meta_pkg, &mut resource.envs);
if cargo_features.all_features {
cmd.arg("--all-features");
} else {
if cargo_features.no_default_features {
// FIXME: `NoDefaultFeatures` is mutual exclusive with `SomeFeatures`
// https://github.com/oli-obk/cargo_metadata/issues/79
cmd.arg("--no-default-features");
}
if !cargo_features.features.is_empty() {
cmd.arg("--features");
cmd.arg(cargo_features.features.join(" "));
}
}

cmd.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null());

if let Some(out_dir) = &resource.out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
resource.envs.push(("OUT_DIR".to_string(), out_dir));
let mut child = cmd.spawn().map(JodChild)?;
let child_stdout = child.stdout.take().unwrap();
let stdout = BufReader::new(child_stdout);

let mut res = BuildDataMap::default();
for message in cargo_metadata::Message::parse_stream(stdout) {
if let Ok(message) = message {
match message {
Message::BuildScriptExecuted(BuildScript {
package_id,
out_dir,
cfgs,
env,
..
}) => {
let cfgs = {
let mut acc = Vec::new();
for cfg in cfgs {
match cfg.parse::<CfgFlag>() {
Ok(it) => acc.push(it),
Err(err) => {
anyhow::bail!("invalid cfg from cargo-metadata: {}", err)
}
};
}
acc
};
let res = res.entry(package_id.repr.clone()).or_default();
// cargo_metadata crate returns default (empty) path for
// older cargos, which is not absolute, so work around that.
if out_dir != PathBuf::default() {
let out_dir = AbsPathBuf::assert(out_dir);
res.out_dir = Some(out_dir);
res.cfgs = cfgs;
}

res.envs = env;
}
Message::CompilerArtifact(message) => {
progress(format!("metadata {}", message.target.name));

if message.target.kind.contains(&"proc-macro".to_string()) {
let package_id = message.package_id;
// Skip rmeta file
if let Some(filename) = message.filenames.iter().find(|name| is_dylib(name))
{
let filename = AbsPathBuf::assert(filename.clone());
let res = res.entry(package_id.repr.clone()).or_default();
res.proc_macro_dylib_path = Some(filename);
}
}
}
Message::CompilerMessage(message) => {
progress(message.target.name.clone());
}
Message::Unknown => (),
Message::BuildFinished(_) => {}
Message::TextLine(_) => {}
}
}
}

for package in packages {
let build_data = res.entry(package.id.repr.clone()).or_default();
inject_cargo_env(package, build_data);
if let Some(out_dir) = &build_data.out_dir {
// NOTE: cargo and rustc seem to hide non-UTF-8 strings from env! and option_env!()
if let Some(out_dir) = out_dir.to_str().map(|s| s.to_owned()) {
build_data.envs.push(("OUT_DIR".to_string(), out_dir));
}
}
}

Ok(res)
}

// FIXME: File a better way to know if it is a dylib
Expand All @@ -173,7 +219,9 @@ fn is_dylib(path: &Path) -> bool {
/// Recreates the compile-time environment variables that Cargo sets.
///
/// Should be synced with <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates>
fn inject_cargo_env(package: &cargo_metadata::Package, env: &mut Vec<(String, String)>) {
fn inject_cargo_env(package: &cargo_metadata::Package, build_data: &mut BuildData) {
let env = &mut build_data.envs;

// FIXME: Missing variables:
// CARGO_PKG_HOMEPAGE, CARGO_CRATE_NAME, CARGO_BIN_NAME, CARGO_BIN_EXE_<name>

Expand Down
Loading