From 17412bae30db7f8f72218387925b8931f42eb08a Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Fri, 7 Apr 2023 02:26:08 -0400 Subject: [PATCH 01/11] Removed use of iteration through a HashMap/HashSet in rustc_incremental and replaced with IndexMap/IndexSet --- compiler/rustc_codegen_llvm/src/lib.rs | 4 +-- compiler/rustc_codegen_ssa/src/back/write.rs | 8 +++--- .../rustc_codegen_ssa/src/traits/backend.rs | 4 +-- .../rustc_incremental/src/assert_dep_graph.rs | 28 +++++++++---------- .../src/assert_module_sources.rs | 4 +-- compiler/rustc_incremental/src/lib.rs | 1 - .../src/persist/dirty_clean.rs | 4 +-- compiler/rustc_incremental/src/persist/fs.rs | 14 +++++----- .../rustc_incremental/src/persist/load.rs | 6 ++-- .../rustc_incremental/src/persist/save.rs | 8 +++--- .../src/persist/work_product.rs | 4 +-- .../rustc_query_system/src/dep_graph/graph.rs | 10 +++---- 12 files changed, 47 insertions(+), 48 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index ff9909c720e54..24968e00cc8e5 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -34,7 +34,7 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage}; use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; @@ -356,7 +356,7 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { + ) -> Result<(CodegenResults, FxIndexMap), ErrorGuaranteed> { let (codegen_results, work_products) = ongoing_codegen .downcast::>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box") diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 10e9e5588f6cc..701d0d73ad38c 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -9,7 +9,7 @@ use crate::{ }; use jobserver::{Acquired, Client}; use rustc_ast::attr; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::profiling::TimingGuard; @@ -498,8 +498,8 @@ pub fn start_async_codegen( fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( sess: &Session, compiled_modules: &CompiledModules, -) -> FxHashMap { - let mut work_products = FxHashMap::default(); +) -> FxIndexMap { + let mut work_products = FxIndexMap::default(); if sess.opts.incremental.is_none() { return work_products; @@ -1885,7 +1885,7 @@ pub struct OngoingCodegen { } impl OngoingCodegen { - pub fn join(self, sess: &Session) -> (CodegenResults, FxHashMap) { + pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap) { let _timer = sess.timer("finish_ongoing_codegen"); self.shared_emitter_main.check(sess, true); diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index d83bfc74082f6..b3c9ecf8b938b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -6,7 +6,7 @@ use crate::back::write::TargetMachineFactoryFn; use crate::{CodegenResults, ModuleCodegen}; use rustc_ast::expand::allocator::AllocatorKind; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::sync::{DynSend, DynSync}; use rustc_errors::ErrorGuaranteed; use rustc_metadata::EncodedMetadata; @@ -101,7 +101,7 @@ pub trait CodegenBackend { ongoing_codegen: Box, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed>; + ) -> Result<(CodegenResults, FxIndexMap), ErrorGuaranteed>; /// This is called on the returned `Box` from `join_codegen` /// diff --git a/compiler/rustc_incremental/src/assert_dep_graph.rs b/compiler/rustc_incremental/src/assert_dep_graph.rs index 22bd12f2e6361..52a84b204d00b 100644 --- a/compiler/rustc_incremental/src/assert_dep_graph.rs +++ b/compiler/rustc_incremental/src/assert_dep_graph.rs @@ -35,7 +35,7 @@ use crate::errors; use rustc_ast as ast; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::FxIndexSet; use rustc_data_structures::graph::implementation::{Direction, NodeIndex, INCOMING, OUTGOING}; use rustc_graphviz as dot; use rustc_hir as hir; @@ -258,7 +258,7 @@ fn dump_graph(query: &DepGraphQuery) { } #[allow(missing_docs)] -pub struct GraphvizDepGraph(FxHashSet, Vec<(DepKind, DepKind)>); +pub struct GraphvizDepGraph(FxIndexSet, Vec<(DepKind, DepKind)>); impl<'a> dot::GraphWalk<'a> for GraphvizDepGraph { type Node = DepKind; @@ -303,7 +303,7 @@ impl<'a> dot::Labeller<'a> for GraphvizDepGraph { fn node_set<'q>( query: &'q DepGraphQuery, filter: &DepNodeFilter, -) -> Option> { +) -> Option> { debug!("node_set(filter={:?})", filter); if filter.accepts_all() { @@ -315,9 +315,9 @@ fn node_set<'q>( fn filter_nodes<'q>( query: &'q DepGraphQuery, - sources: &Option>, - targets: &Option>, -) -> FxHashSet { + sources: &Option>, + targets: &Option>, +) -> FxIndexSet { if let Some(sources) = sources { if let Some(targets) = targets { walk_between(query, sources, targets) @@ -333,10 +333,10 @@ fn filter_nodes<'q>( fn walk_nodes<'q>( query: &'q DepGraphQuery, - starts: &FxHashSet<&'q DepNode>, + starts: &FxIndexSet<&'q DepNode>, direction: Direction, -) -> FxHashSet { - let mut set = FxHashSet::default(); +) -> FxIndexSet { + let mut set = FxIndexSet::default(); for &start in starts { debug!("walk_nodes: start={:?} outgoing?={:?}", start, direction == OUTGOING); if set.insert(start.kind) { @@ -357,9 +357,9 @@ fn walk_nodes<'q>( fn walk_between<'q>( query: &'q DepGraphQuery, - sources: &FxHashSet<&'q DepNode>, - targets: &FxHashSet<&'q DepNode>, -) -> FxHashSet { + sources: &FxIndexSet<&'q DepNode>, + targets: &FxIndexSet<&'q DepNode>, +) -> FxIndexSet { // This is a bit tricky. We want to include a node only if it is: // (a) reachable from a source and (b) will reach a target. And we // have to be careful about cycles etc. Luckily efficiency is not @@ -426,8 +426,8 @@ fn walk_between<'q>( } } -fn filter_edges(query: &DepGraphQuery, nodes: &FxHashSet) -> Vec<(DepKind, DepKind)> { - let uniq: FxHashSet<_> = query +fn filter_edges(query: &DepGraphQuery, nodes: &FxIndexSet) -> Vec<(DepKind, DepKind)> { + let uniq: FxIndexSet<_> = query .edges() .into_iter() .map(|(s, t)| (s.kind, t.kind)) diff --git a/compiler/rustc_incremental/src/assert_module_sources.rs b/compiler/rustc_incremental/src/assert_module_sources.rs index c550e553bb032..82c787605d7d3 100644 --- a/compiler/rustc_incremental/src/assert_module_sources.rs +++ b/compiler/rustc_incremental/src/assert_module_sources.rs @@ -24,7 +24,7 @@ use crate::errors; use rustc_ast as ast; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::FxIndexSet; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::mir::mono::CodegenUnitNameBuilder; use rustc_middle::ty::TyCtxt; @@ -52,7 +52,7 @@ pub fn assert_module_sources(tcx: TyCtxt<'_>) { struct AssertModuleSource<'tcx> { tcx: TyCtxt<'tcx>, - available_cgus: FxHashSet, + available_cgus: FxIndexSet, } impl<'tcx> AssertModuleSource<'tcx> { diff --git a/compiler/rustc_incremental/src/lib.rs b/compiler/rustc_incremental/src/lib.rs index 11710c368cefe..b9171fad55ba3 100644 --- a/compiler/rustc_incremental/src/lib.rs +++ b/compiler/rustc_incremental/src/lib.rs @@ -4,7 +4,6 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(never_type)] #![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index 43274091cb873..848b81b9d3496 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -21,7 +21,7 @@ use crate::errors; use rustc_ast::{self as ast, Attribute, NestedMetaItem}; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; use rustc_hir::def_id::LocalDefId; use rustc_hir::intravisit; use rustc_hir::Node as HirNode; @@ -125,7 +125,7 @@ const LABELS_ADT: &[&[&str]] = &[BASE_HIR, BASE_STRUCT]; // // type_of for these. -type Labels = FxHashSet; +type Labels = FxIndexSet; /// Represents the requested configuration by rustc_clean/dirty struct Assertion { diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index e3c688b3e98cb..463481f16f371 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -104,7 +104,7 @@ //! implemented. use crate::errors; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_data_structures::svh::Svh; use rustc_data_structures::{base_n, flock}; use rustc_errors::ErrorGuaranteed; @@ -635,8 +635,8 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { // First do a pass over the crate directory, collecting lock files and // session directories - let mut session_directories = FxHashSet::default(); - let mut lock_files = FxHashSet::default(); + let mut session_directories = FxIndexSet::default(); + let mut lock_files = FxIndexSet::default(); for dir_entry in crate_directory.read_dir()? { let Ok(dir_entry) = dir_entry else { @@ -659,7 +659,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { } // Now map from lock files to session directories - let lock_file_to_session_dir: FxHashMap> = lock_files + let lock_file_to_session_dir: FxIndexMap> = lock_files .into_iter() .map(|lock_file_name| { assert!(lock_file_name.ends_with(LOCK_FILE_EXT)); @@ -705,7 +705,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { } // Filter out `None` directories - let lock_file_to_session_dir: FxHashMap = lock_file_to_session_dir + let lock_file_to_session_dir: FxIndexMap = lock_file_to_session_dir .into_iter() .filter_map(|(lock_file_name, directory_name)| directory_name.map(|n| (lock_file_name, n))) .collect(); @@ -846,7 +846,7 @@ fn delete_old(sess: &Session, path: &Path) { fn all_except_most_recent( deletion_candidates: Vec<(SystemTime, PathBuf, Option)>, -) -> FxHashMap> { +) -> FxIndexMap> { let most_recent = deletion_candidates.iter().map(|&(timestamp, ..)| timestamp).max(); if let Some(most_recent) = most_recent { @@ -856,7 +856,7 @@ fn all_except_most_recent( .map(|(_, path, lock)| (path, lock)) .collect() } else { - FxHashMap::default() + FxIndexMap::default() } } diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index a4407a93ff3ba..0f7db10912797 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -1,7 +1,7 @@ //! Code to save/load the dep-graph from files. use crate::errors; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::memmap::Mmap; use rustc_middle::dep_graph::{SerializedDepGraph, WorkProduct, WorkProductId}; use rustc_middle::query::on_disk_cache::OnDiskCache; @@ -16,7 +16,7 @@ use super::file_format; use super::fs::*; use super::work_product; -type WorkProductMap = FxHashMap; +type WorkProductMap = FxIndexMap; #[derive(Debug)] /// Represents the result of an attempt to load incremental compilation data. @@ -147,7 +147,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let report_incremental_info = sess.opts.unstable_opts.incremental_info; let expected_hash = sess.opts.dep_tracking_hash(false); - let mut prev_work_products = FxHashMap::default(); + let mut prev_work_products = FxIndexMap::default(); // If we are only building with -Zquery-dep-graph but without an actual // incr. comp. session directory, we skip this. Otherwise we'd fail diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index 7376be6be8b8a..bcffba466413c 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -1,5 +1,5 @@ use crate::errors; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::sync::join; use rustc_middle::dep_graph::{DepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -79,7 +79,7 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) { pub fn save_work_product_index( sess: &Session, dep_graph: &DepGraph, - new_work_products: FxHashMap, + new_work_products: FxIndexMap, ) { if sess.opts.incremental.is_none() { return; @@ -119,7 +119,7 @@ pub fn save_work_product_index( } fn encode_work_product_index( - work_products: &FxHashMap, + work_products: &FxIndexMap, encoder: &mut FileEncoder, ) { let serialized_products: Vec<_> = work_products @@ -146,7 +146,7 @@ fn encode_query_cache(tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult pub fn build_dep_graph( sess: &Session, prev_graph: SerializedDepGraph, - prev_work_products: FxHashMap, + prev_work_products: FxIndexMap, ) -> Option { if sess.opts.incremental.is_none() { // No incremental compilation. diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs index dc98fbeb0d166..a1dc78596a9ef 100644 --- a/compiler/rustc_incremental/src/persist/work_product.rs +++ b/compiler/rustc_incremental/src/persist/work_product.rs @@ -4,7 +4,7 @@ use crate::errors; use crate::persist::fs::*; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_fs_util::link_or_copy; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; @@ -20,7 +20,7 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( debug!(?cgu_name, ?files); sess.opts.incremental.as_ref()?; - let mut saved_files = FxHashMap::default(); + let mut saved_files = FxIndexMap::default(); for (ext, path) in files { let file_name = format!("{cgu_name}.{ext}"); let path_in_incr_dir = in_incr_comp_dir_sess(sess, &file_name); diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index c0d7386dd6adb..54eaa1d4a7f99 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -1,6 +1,6 @@ use parking_lot::Mutex; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap}; use rustc_data_structures::profiling::{EventId, QueryInvocationId, SelfProfilerRef}; use rustc_data_structures::sharded::{self, Sharded}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -93,7 +93,7 @@ pub struct DepGraphData { /// things available to us. If we find that they are not dirty, we /// load the path to the file storing those work-products here into /// this map. We can later look for and extract that data. - previous_work_products: FxHashMap, + previous_work_products: FxIndexMap, dep_node_debug: Lock, String>>, @@ -116,7 +116,7 @@ impl DepGraph { pub fn new( profiler: &SelfProfilerRef, prev_graph: SerializedDepGraph, - prev_work_products: FxHashMap, + prev_work_products: FxIndexMap, encoder: FileEncoder, record_graph: bool, record_stats: bool, @@ -688,7 +688,7 @@ impl DepGraph { /// Access the map of work-products created during the cached run. Only /// used during saving of the dep-graph. - pub fn previous_work_products(&self) -> &FxHashMap { + pub fn previous_work_products(&self) -> &FxIndexMap { &self.data.as_ref().unwrap().previous_work_products } @@ -1048,7 +1048,7 @@ pub struct WorkProduct { /// /// By convention, file extensions are currently used as identifiers, i.e. the key "o" maps to /// the object file's path, and "dwo" to the dwarf object file's path. - pub saved_files: FxHashMap, + pub saved_files: FxIndexMap, } // Index type for `DepNodeData`'s edges. From 204e2bf5a4828b044a78019a118e12d09bde772d Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Fri, 7 Apr 2023 15:56:33 -0400 Subject: [PATCH 02/11] Updated cranelift codegen to reflect modified trait signature --- compiler/rustc_codegen_cranelift/src/driver/aot.rs | 4 ++-- compiler/rustc_codegen_cranelift/src/lib.rs | 4 ++-- compiler/rustc_codegen_gcc/src/lib.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index aad9a9647f8c9..d143bcc96ef93 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -54,8 +54,8 @@ impl OngoingCodegen { self, sess: &Session, backend_config: &BackendConfig, - ) -> (CodegenResults, FxHashMap) { - let mut work_products = FxHashMap::default(); + ) -> (CodegenResults, FxIndexMap) { + let mut work_products = FxIndexMap::default(); let mut modules = vec![]; for module_codegen in self.modules { diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 9966cc2ef3c12..095fbe62c1902 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -88,7 +88,7 @@ mod prelude { }; pub(crate) use rustc_target::abi::{Abi, FieldIdx, Scalar, Size, VariantIdx, FIRST_VARIANT}; - pub(crate) use rustc_data_structures::fx::FxHashMap; + pub(crate) use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; pub(crate) use rustc_index::Idx; @@ -223,7 +223,7 @@ impl CodegenBackend for CraneliftCodegenBackend { ongoing_codegen: Box, sess: &Session, _outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { + ) -> Result<(CodegenResults, FxIndexMap), ErrorGuaranteed> { Ok(ongoing_codegen .downcast::() .unwrap() diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index 442ce0ea54209..ea013c4428cce 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -75,7 +75,7 @@ use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, ModuleConfig, use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; use rustc_codegen_ssa::target_features::supported_target_features; use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods}; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, Handler, SubdiagnosticMessage}; use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; @@ -137,7 +137,7 @@ impl CodegenBackend for GccCodegenBackend { Box::new(res) } - fn join_codegen(&self, ongoing_codegen: Box, sess: &Session, _outputs: &OutputFilenames) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { + fn join_codegen(&self, ongoing_codegen: Box, sess: &Session, _outputs: &OutputFilenames) -> Result<(CodegenResults, FxIndexMap), ErrorGuaranteed> { let (codegen_results, work_products) = ongoing_codegen .downcast::>() .expect("Expected GccCodegenBackend's OngoingCodegen, found Box") From 2a96c6e5171e5aa9e5e70639989e7cdb99dc5efc Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Fri, 7 Apr 2023 17:26:30 -0400 Subject: [PATCH 03/11] Fixed compiler error --- .../hotplug_codegen_backend/the_backend.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs b/tests/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs index 7db100a08a106..8a275751e38de 100644 --- a/tests/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs +++ b/tests/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs @@ -15,7 +15,7 @@ extern crate rustc_target; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::{CodegenResults, CrateInfo}; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_errors::ErrorGuaranteed; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; @@ -49,11 +49,11 @@ impl CodegenBackend for TheBackend { ongoing_codegen: Box, _sess: &Session, _outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { + ) -> Result<(CodegenResults, FxIndexMap), ErrorGuaranteed> { let codegen_results = ongoing_codegen .downcast::() .expect("in join_codegen: ongoing_codegen is not a CodegenResults"); - Ok((*codegen_results, FxHashMap::default())) + Ok((*codegen_results, FxIndexMap::default())) } fn link( From 1be19f710c983258282796a111354528908451c5 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Sun, 7 May 2023 19:52:19 -0400 Subject: [PATCH 04/11] Switched some uses to UnordMap --- .../src/assert_module_sources.rs | 9 +- .../src/persist/dirty_clean.rs | 22 +- compiler/rustc_incremental/src/persist/fs.rs | 206 +++++++++--------- .../rustc_incremental/src/persist/fs/tests.rs | 32 +-- .../rustc_incremental/src/persist/load.rs | 4 +- compiler/rustc_interface/src/queries.rs | 4 +- .../src/dep_graph/dep_node.rs | 9 +- 7 files changed, 150 insertions(+), 136 deletions(-) diff --git a/compiler/rustc_incremental/src/assert_module_sources.rs b/compiler/rustc_incremental/src/assert_module_sources.rs index 82c787605d7d3..3e12c9b5016be 100644 --- a/compiler/rustc_incremental/src/assert_module_sources.rs +++ b/compiler/rustc_incremental/src/assert_module_sources.rs @@ -24,7 +24,7 @@ use crate::errors; use rustc_ast as ast; -use rustc_data_structures::fx::FxIndexSet; +use rustc_data_structures::unord::UnordSet; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::mir::mono::CodegenUnitNameBuilder; use rustc_middle::ty::TyCtxt; @@ -52,7 +52,7 @@ pub fn assert_module_sources(tcx: TyCtxt<'_>) { struct AssertModuleSource<'tcx> { tcx: TyCtxt<'tcx>, - available_cgus: FxIndexSet, + available_cgus: UnordSet, } impl<'tcx> AssertModuleSource<'tcx> { @@ -118,9 +118,8 @@ impl<'tcx> AssertModuleSource<'tcx> { debug!("mapping '{}' to cgu name '{}'", self.field(attr, sym::module), cgu_name); if !self.available_cgus.contains(&cgu_name) { - let mut cgu_names: Vec<&str> = - self.available_cgus.iter().map(|cgu| cgu.as_str()).collect(); - cgu_names.sort(); + let cgu_names: Vec = + self.available_cgus.items().map(|cgu| cgu.as_str().to_owned()).into_sorted(&()); self.tcx.sess.emit_err(errors::NoModuleNamed { span: attr.span, user_path, diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index 848b81b9d3496..51bdcf8e92b52 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -21,7 +21,8 @@ use crate::errors; use rustc_ast::{self as ast, Attribute, NestedMetaItem}; -use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; +use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::unord::UnordSet; use rustc_hir::def_id::LocalDefId; use rustc_hir::intravisit; use rustc_hir::Node as HirNode; @@ -125,7 +126,7 @@ const LABELS_ADT: &[&[&str]] = &[BASE_HIR, BASE_STRUCT]; // // type_of for these. -type Labels = FxIndexSet; +type Labels = UnordSet; /// Represents the requested configuration by rustc_clean/dirty struct Assertion { @@ -197,7 +198,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { let (name, mut auto) = self.auto_labels(item_id, attr); let except = self.except(attr); let loaded_from_disk = self.loaded_from_disk(attr); - for e in except.iter() { + for e in except.to_sorted(&(), false) { if !auto.remove(e) { self.tcx.sess.emit_fatal(errors::AssertionAuto { span: attr.span, name, e }); } @@ -376,18 +377,21 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { continue; }; self.checked_attrs.insert(attr.id); - for label in assertion.clean { + assertion.clean.items().all(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_clean(item_span, dep_node); - } - for label in assertion.dirty { + true + }); + assertion.dirty.items().all(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_dirty(item_span, dep_node); - } - for label in assertion.loaded_from_disk { + true + }); + assertion.loaded_from_disk.items().all(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_loaded_from_disk(item_span, dep_node); - } + true + }); } } } diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index 463481f16f371..b44fb37fd7a93 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -104,8 +104,9 @@ //! implemented. use crate::errors; -use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; +use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; use rustc_data_structures::svh::Svh; +use rustc_data_structures::unord::{UnordMap, UnordSet}; use rustc_data_structures::{base_n, flock}; use rustc_errors::ErrorGuaranteed; use rustc_fs_util::{link_or_copy, try_canonicalize, LinkOrCopy}; @@ -636,7 +637,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { // First do a pass over the crate directory, collecting lock files and // session directories let mut session_directories = FxIndexSet::default(); - let mut lock_files = FxIndexSet::default(); + let mut lock_files = UnordSet::default(); for dir_entry in crate_directory.read_dir()? { let Ok(dir_entry) = dir_entry else { @@ -659,9 +660,8 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { } // Now map from lock files to session directories - let lock_file_to_session_dir: FxIndexMap> = lock_files - .into_iter() - .map(|lock_file_name| { + let lock_file_to_session_dir: UnordMap> = + UnordMap::from(lock_files.into_items().map(|lock_file_name| { assert!(lock_file_name.ends_with(LOCK_FILE_EXT)); let dir_prefix_end = lock_file_name.len() - LOCK_FILE_EXT.len(); let session_dir = { @@ -669,12 +669,11 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { session_directories.iter().find(|dir_name| dir_name.starts_with(dir_prefix)) }; (lock_file_name, session_dir.map(String::clone)) - }) - .collect(); + })); // Delete all lock files, that don't have an associated directory. They must // be some kind of leftover - for (lock_file_name, directory_name) in &lock_file_to_session_dir { + lock_file_to_session_dir.items().all(|(lock_file_name, directory_name)| { if directory_name.is_none() { let Ok(timestamp) = extract_timestamp_from_session_dir(lock_file_name) else { debug!( @@ -682,7 +681,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { crate_directory.join(&lock_file_name).display() ); // Ignore it - continue; + return true; }; let lock_file_path = crate_directory.join(&**lock_file_name); @@ -702,17 +701,18 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { ); } } - } + true + }); // Filter out `None` directories - let lock_file_to_session_dir: FxIndexMap = lock_file_to_session_dir - .into_iter() - .filter_map(|(lock_file_name, directory_name)| directory_name.map(|n| (lock_file_name, n))) - .collect(); + let lock_file_to_session_dir: UnordMap = + UnordMap::from(lock_file_to_session_dir.into_items().filter_map( + |(lock_file_name, directory_name)| directory_name.map(|n| (lock_file_name, n)), + )); // Delete all session directories that don't have a lock file. for directory_name in session_directories { - if !lock_file_to_session_dir.values().any(|dir| *dir == directory_name) { + if !lock_file_to_session_dir.items().any(|(_, dir)| *dir == directory_name) { let path = crate_directory.join(directory_name); if let Err(err) = safe_remove_dir_all(&path) { sess.emit_warning(errors::InvalidGcFailed { path: &path, err }); @@ -721,103 +721,103 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { } // Now garbage collect the valid session directories. - let mut deletion_candidates = vec![]; - - for (lock_file_name, directory_name) in &lock_file_to_session_dir { - debug!("garbage_collect_session_directories() - inspecting: {}", directory_name); + let deletion_candidates = + lock_file_to_session_dir.items().filter_map(|(lock_file_name, directory_name)| { + debug!("garbage_collect_session_directories() - inspecting: {}", directory_name); - let Ok(timestamp) = extract_timestamp_from_session_dir(directory_name) else { + let Ok(timestamp) = extract_timestamp_from_session_dir(directory_name) else { debug!( "found session-dir with malformed timestamp: {}", crate_directory.join(directory_name).display() ); // Ignore it - continue; + return None; }; - if is_finalized(directory_name) { - let lock_file_path = crate_directory.join(lock_file_name); - match flock::Lock::new( - &lock_file_path, - false, // don't wait - false, // don't create the lock-file - true, - ) { - // get an exclusive lock - Ok(lock) => { - debug!( - "garbage_collect_session_directories() - \ + if is_finalized(directory_name) { + let lock_file_path = crate_directory.join(lock_file_name); + match flock::Lock::new( + &lock_file_path, + false, // don't wait + false, // don't create the lock-file + true, + ) { + // get an exclusive lock + Ok(lock) => { + debug!( + "garbage_collect_session_directories() - \ successfully acquired lock" - ); - debug!( - "garbage_collect_session_directories() - adding \ + ); + debug!( + "garbage_collect_session_directories() - adding \ deletion candidate: {}", - directory_name - ); - - // Note that we are holding on to the lock - deletion_candidates.push(( - timestamp, - crate_directory.join(directory_name), - Some(lock), - )); - } - Err(_) => { - debug!( - "garbage_collect_session_directories() - \ + directory_name + ); + + // Note that we are holding on to the lock + return Some(( + (timestamp, crate_directory.join(directory_name)), + Some(lock), + )); + } + Err(_) => { + debug!( + "garbage_collect_session_directories() - \ not collecting, still in use" - ); + ); + } } - } - } else if is_old_enough_to_be_collected(timestamp) { - // When cleaning out "-working" session directories, i.e. - // session directories that might still be in use by another - // compiler instance, we only look a directories that are - // at least ten seconds old. This is supposed to reduce the - // chance of deleting a directory in the time window where - // the process has allocated the directory but has not yet - // acquired the file-lock on it. - - // Try to acquire the directory lock. If we can't, it - // means that the owning process is still alive and we - // leave this directory alone. - let lock_file_path = crate_directory.join(lock_file_name); - match flock::Lock::new( - &lock_file_path, - false, // don't wait - false, // don't create the lock-file - true, - ) { - // get an exclusive lock - Ok(lock) => { - debug!( - "garbage_collect_session_directories() - \ + } else if is_old_enough_to_be_collected(timestamp) { + // When cleaning out "-working" session directories, i.e. + // session directories that might still be in use by another + // compiler instance, we only look a directories that are + // at least ten seconds old. This is supposed to reduce the + // chance of deleting a directory in the time window where + // the process has allocated the directory but has not yet + // acquired the file-lock on it. + + // Try to acquire the directory lock. If we can't, it + // means that the owning process is still alive and we + // leave this directory alone. + let lock_file_path = crate_directory.join(lock_file_name); + match flock::Lock::new( + &lock_file_path, + false, // don't wait + false, // don't create the lock-file + true, + ) { + // get an exclusive lock + Ok(lock) => { + debug!( + "garbage_collect_session_directories() - \ successfully acquired lock" - ); + ); - delete_old(sess, &crate_directory.join(directory_name)); + delete_old(sess, &crate_directory.join(directory_name)); - // Let's make it explicit that the file lock is released at this point, - // or rather, that we held on to it until here - drop(lock); - } - Err(_) => { - debug!( - "garbage_collect_session_directories() - \ + // Let's make it explicit that the file lock is released at this point, + // or rather, that we held on to it until here + drop(lock); + } + Err(_) => { + debug!( + "garbage_collect_session_directories() - \ not collecting, still in use" - ); + ); + } } - } - } else { - debug!( - "garbage_collect_session_directories() - not finalized, not \ + } else { + debug!( + "garbage_collect_session_directories() - not finalized, not \ old enough" - ); - } - } + ); + } + None + }); + let deletion_candidates = UnordMap::from(deletion_candidates); // Delete all but the most recent of the candidates - for (path, lock) in all_except_most_recent(deletion_candidates) { + all_except_most_recent(deletion_candidates).into_items().all(|(path, lock)| { debug!("garbage_collect_session_directories() - deleting `{}`", path.display()); if let Err(err) = safe_remove_dir_all(&path) { @@ -829,7 +829,8 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { // Let's make it explicit that the file lock is released at this point, // or rather, that we held on to it until here drop(lock); - } + true + }); Ok(()) } @@ -845,18 +846,19 @@ fn delete_old(sess: &Session, path: &Path) { } fn all_except_most_recent( - deletion_candidates: Vec<(SystemTime, PathBuf, Option)>, -) -> FxIndexMap> { - let most_recent = deletion_candidates.iter().map(|&(timestamp, ..)| timestamp).max(); + deletion_candidates: UnordMap<(SystemTime, PathBuf), Option>, +) -> UnordMap> { + let most_recent = deletion_candidates.items().map(|(&(timestamp, _), _)| timestamp).max(); if let Some(most_recent) = most_recent { - deletion_candidates - .into_iter() - .filter(|&(timestamp, ..)| timestamp != most_recent) - .map(|(_, path, lock)| (path, lock)) - .collect() + UnordMap::from( + deletion_candidates + .into_items() + .filter(|&((timestamp, _), _)| timestamp != most_recent) + .map(|((_, path), lock)| (path, lock)), + ) } else { - FxIndexMap::default() + UnordMap::default() } } diff --git a/compiler/rustc_incremental/src/persist/fs/tests.rs b/compiler/rustc_incremental/src/persist/fs/tests.rs index 184796948b67d..90d8a72de07d5 100644 --- a/compiler/rustc_incremental/src/persist/fs/tests.rs +++ b/compiler/rustc_incremental/src/persist/fs/tests.rs @@ -2,26 +2,26 @@ use super::*; #[test] fn test_all_except_most_recent() { + let computed: UnordMap<_, Option> = UnordMap::from_iter([ + ((UNIX_EPOCH + Duration::new(4, 0), PathBuf::from("4")), None), + ((UNIX_EPOCH + Duration::new(1, 0), PathBuf::from("1")), None), + ((UNIX_EPOCH + Duration::new(5, 0), PathBuf::from("5")), None), + ((UNIX_EPOCH + Duration::new(3, 0), PathBuf::from("3")), None), + ((UNIX_EPOCH + Duration::new(2, 0), PathBuf::from("2")), None), + ]); + let mut paths = UnordSet::default(); + UnordSet::extend_unord(&mut paths, computed.into_items().map(|((_, path), _)| path)); assert_eq!( - all_except_most_recent(vec![ - (UNIX_EPOCH + Duration::new(4, 0), PathBuf::from("4"), None), - (UNIX_EPOCH + Duration::new(1, 0), PathBuf::from("1"), None), - (UNIX_EPOCH + Duration::new(5, 0), PathBuf::from("5"), None), - (UNIX_EPOCH + Duration::new(3, 0), PathBuf::from("3"), None), - (UNIX_EPOCH + Duration::new(2, 0), PathBuf::from("2"), None), + UnordSet::from(paths), + UnordSet::from_iter([ + PathBuf::from("1"), + PathBuf::from("2"), + PathBuf::from("3"), + PathBuf::from("4") ]) - .keys() - .cloned() - .collect::>(), - [PathBuf::from("1"), PathBuf::from("2"), PathBuf::from("3"), PathBuf::from("4"),] - .into_iter() - .collect::>() ); - assert_eq!( - all_except_most_recent(vec![]).keys().cloned().collect::>(), - FxHashSet::default() - ); + assert!(all_except_most_recent(UnordMap::default()).is_empty()); } #[test] diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index 0f7db10912797..b718bed819b2f 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -1,8 +1,8 @@ //! Code to save/load the dep-graph from files. use crate::errors; -use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::unord::UnordMap; use rustc_middle::dep_graph::{SerializedDepGraph, WorkProduct, WorkProductId}; use rustc_middle::query::on_disk_cache::OnDiskCache; use rustc_serialize::opaque::MemDecoder; @@ -16,7 +16,7 @@ use super::file_format; use super::fs::*; use super::work_product; -type WorkProductMap = FxIndexMap; +type WorkProductMap = UnordMap; #[derive(Debug)] /// Represents the result of an attempt to load incremental compilation data. diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs index c441a8ffd6f94..6975fbd917a4f 100644 --- a/compiler/rustc_interface/src/queries.rs +++ b/compiler/rustc_interface/src/queries.rs @@ -5,6 +5,7 @@ use crate::passes; use rustc_ast as ast; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; +use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::steal::Steal; use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::{AppendOnlyIndexVec, Lrc, OnceCell, RwLock, WorkerLocal}; @@ -195,7 +196,8 @@ impl<'tcx> Queries<'tcx> { .and_then(|future| { let (prev_graph, prev_work_products) = sess.time("blocked_on_dep_graph_loading", || future.open().open(sess)); - + let prev_work_products = + FxIndexMap::from_iter(prev_work_products.into_sorted(&(), false)); rustc_incremental::build_dep_graph(sess, prev_graph, prev_work_products) }) .unwrap_or_else(DepGraph::new_disabled); diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs index 9e1ca6ab515d8..e0089f2861547 100644 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs @@ -46,7 +46,7 @@ use super::{DepContext, DepKind, FingerprintStyle}; use crate::ich::StableHashingContext; use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey}; use rustc_hir::definitions::DefPathHash; use std::fmt; use std::hash::Hash; @@ -247,3 +247,10 @@ impl HashStable for WorkProductId { self.hash.hash_stable(hcx, hasher) } } +impl ToStableHashKey for WorkProductId { + type KeyType = Fingerprint; + #[inline] + fn to_stable_hash_key(&self, _: &HCX) -> Self::KeyType { + self.hash + } +} From cf7dea571695be5db843519db014785c4a0a2786 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Sun, 7 May 2023 20:15:47 -0400 Subject: [PATCH 05/11] Sorted a FxIndexSet for consistent iteration order --- compiler/rustc_incremental/src/persist/fs.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index b44fb37fd7a93..e11245549404d 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -658,6 +658,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { // This is something we don't know, leave it alone } } + session_directories.sort(); // Now map from lock files to session directories let lock_file_to_session_dir: UnordMap> = From 96b577860d4e175ccc4698b9e9a8a822b228fc19 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Sun, 7 May 2023 21:08:47 -0400 Subject: [PATCH 06/11] Fixed failing test + minor cleanup --- compiler/rustc_data_structures/src/unord.rs | 5 +++++ compiler/rustc_incremental/src/persist/dirty_clean.rs | 9 +++------ compiler/rustc_incremental/src/persist/fs.rs | 5 ++--- compiler/rustc_incremental/src/persist/fs/tests.rs | 2 +- compiler/rustc_incremental/src/persist/load.rs | 2 +- compiler/rustc_incremental/src/persist/save.rs | 4 ++-- compiler/rustc_incremental/src/persist/work_product.rs | 8 ++++---- compiler/rustc_query_system/src/dep_graph/graph.rs | 3 ++- 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_data_structures/src/unord.rs b/compiler/rustc_data_structures/src/unord.rs index 6c8d541463158..02957b38efe7b 100644 --- a/compiler/rustc_data_structures/src/unord.rs +++ b/compiler/rustc_data_structures/src/unord.rs @@ -62,6 +62,11 @@ impl> UnordItems { UnordItems(self.0.filter_map(f)) } + #[inline] + pub fn for_each ()>(self, f: F) { + self.0.for_each(|x| f(x)); + } + #[inline] pub fn max(self) -> Option where diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index 51bdcf8e92b52..b7bdffe5b06ae 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -377,20 +377,17 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { continue; }; self.checked_attrs.insert(attr.id); - assertion.clean.items().all(|label| { + assertion.clean.items().for_each(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_clean(item_span, dep_node); - true }); - assertion.dirty.items().all(|label| { + assertion.dirty.items().for_each(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_dirty(item_span, dep_node); - true }); - assertion.loaded_from_disk.items().all(|label| { + assertion.loaded_from_disk.items().for_each(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_loaded_from_disk(item_span, dep_node); - true }); } } diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index e11245549404d..c3f1abadacef2 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -674,7 +674,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { // Delete all lock files, that don't have an associated directory. They must // be some kind of leftover - lock_file_to_session_dir.items().all(|(lock_file_name, directory_name)| { + lock_file_to_session_dir.items().for_each(|(lock_file_name, directory_name)| { if directory_name.is_none() { let Ok(timestamp) = extract_timestamp_from_session_dir(lock_file_name) else { debug!( @@ -682,7 +682,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { crate_directory.join(&lock_file_name).display() ); // Ignore it - return true; + return; }; let lock_file_path = crate_directory.join(&**lock_file_name); @@ -702,7 +702,6 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { ); } } - true }); // Filter out `None` directories diff --git a/compiler/rustc_incremental/src/persist/fs/tests.rs b/compiler/rustc_incremental/src/persist/fs/tests.rs index 90d8a72de07d5..4c1cb5725dc9c 100644 --- a/compiler/rustc_incremental/src/persist/fs/tests.rs +++ b/compiler/rustc_incremental/src/persist/fs/tests.rs @@ -10,7 +10,7 @@ fn test_all_except_most_recent() { ((UNIX_EPOCH + Duration::new(2, 0), PathBuf::from("2")), None), ]); let mut paths = UnordSet::default(); - UnordSet::extend_unord(&mut paths, computed.into_items().map(|((_, path), _)| path)); + paths.extend_unord(all_except_most_recent(computed).into_items().map(|(path, _)| path)); assert_eq!( UnordSet::from(paths), UnordSet::from_iter([ diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index b718bed819b2f..0727523a22980 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -163,7 +163,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { Decodable::decode(&mut work_product_decoder); for swp in work_products { - let all_files_exist = swp.work_product.saved_files.iter().all(|(_, path)| { + let all_files_exist = swp.work_product.saved_files.items().all(|(_, path)| { let exists = in_incr_comp_dir_sess(sess, path).exists(); if !exists && sess.opts.unstable_opts.incremental_info { eprintln!("incremental: could not find file for work product: {path}",); diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index bcffba466413c..bfaa52f9c8134 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -105,7 +105,7 @@ pub fn save_work_product_index( if !new_work_products.contains_key(id) { work_product::delete_workproduct_files(sess, wp); debug_assert!( - !wp.saved_files.iter().all(|(_, path)| in_incr_comp_dir_sess(sess, path).exists()) + !wp.saved_files.items().all(|(_, path)| in_incr_comp_dir_sess(sess, path).exists()) ); } } @@ -113,7 +113,7 @@ pub fn save_work_product_index( // Check that we did not delete one of the current work-products: debug_assert!({ new_work_products.iter().all(|(_, wp)| { - wp.saved_files.iter().all(|(_, path)| in_incr_comp_dir_sess(sess, path).exists()) + wp.saved_files.items().all(|(_, path)| in_incr_comp_dir_sess(sess, path).exists()) }) }); } diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs index a1dc78596a9ef..865f90273bdec 100644 --- a/compiler/rustc_incremental/src/persist/work_product.rs +++ b/compiler/rustc_incremental/src/persist/work_product.rs @@ -4,7 +4,7 @@ use crate::errors; use crate::persist::fs::*; -use rustc_data_structures::fx::FxIndexMap; +use rustc_data_structures::unord::UnordMap; use rustc_fs_util::link_or_copy; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; @@ -20,7 +20,7 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( debug!(?cgu_name, ?files); sess.opts.incremental.as_ref()?; - let mut saved_files = FxIndexMap::default(); + let mut saved_files = UnordMap::default(); for (ext, path) in files { let file_name = format!("{cgu_name}.{ext}"); let path_in_incr_dir = in_incr_comp_dir_sess(sess, &file_name); @@ -46,10 +46,10 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( /// Removes files for a given work product. pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) { - for (_, path) in &work_product.saved_files { + work_product.saved_files.items().for_each(|(_, path)| { let path = in_incr_comp_dir_sess(sess, path); if let Err(err) = std_fs::remove_file(&path) { sess.emit_warning(errors::DeleteWorkProduct { path: &path, err }); } - } + }); } diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 54eaa1d4a7f99..c9e80a6d9bc13 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -6,6 +6,7 @@ use rustc_data_structures::sharded::{self, Sharded}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering}; +use rustc_data_structures::unord::UnordMap; use rustc_index::IndexVec; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; use smallvec::{smallvec, SmallVec}; @@ -1048,7 +1049,7 @@ pub struct WorkProduct { /// /// By convention, file extensions are currently used as identifiers, i.e. the key "o" maps to /// the object file's path, and "dwo" to the dwarf object file's path. - pub saved_files: FxIndexMap, + pub saved_files: UnordMap, } // Index type for `DepNodeData`'s edges. From 6f2d3dee1790f8066394d1198fd1864b5fae45a3 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Mon, 8 May 2023 17:26:17 -0400 Subject: [PATCH 07/11] Fixed unord mistake --- compiler/rustc_data_structures/src/unord.rs | 5 --- .../src/persist/dirty_clean.rs | 6 ++-- compiler/rustc_incremental/src/persist/fs.rs | 36 ++++++++++--------- .../src/persist/work_product.rs | 2 +- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/compiler/rustc_data_structures/src/unord.rs b/compiler/rustc_data_structures/src/unord.rs index 02957b38efe7b..6c8d541463158 100644 --- a/compiler/rustc_data_structures/src/unord.rs +++ b/compiler/rustc_data_structures/src/unord.rs @@ -62,11 +62,6 @@ impl> UnordItems { UnordItems(self.0.filter_map(f)) } - #[inline] - pub fn for_each ()>(self, f: F) { - self.0.for_each(|x| f(x)); - } - #[inline] pub fn max(self) -> Option where diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index b7bdffe5b06ae..6381a05dfdc1c 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -377,15 +377,15 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { continue; }; self.checked_attrs.insert(attr.id); - assertion.clean.items().for_each(|label| { + assertion.clean.to_sorted(&(), false).iter().for_each(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_clean(item_span, dep_node); }); - assertion.dirty.items().for_each(|label| { + assertion.dirty.to_sorted(&(), false).iter().for_each(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_dirty(item_span, dep_node); }); - assertion.loaded_from_disk.items().for_each(|label| { + assertion.loaded_from_disk.to_sorted(&(), false).iter().for_each(|label| { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_loaded_from_disk(item_span, dep_node); }); diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index c3f1abadacef2..f0e5f07230f7e 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -674,9 +674,10 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { // Delete all lock files, that don't have an associated directory. They must // be some kind of leftover - lock_file_to_session_dir.items().for_each(|(lock_file_name, directory_name)| { - if directory_name.is_none() { - let Ok(timestamp) = extract_timestamp_from_session_dir(lock_file_name) else { + lock_file_to_session_dir.to_sorted(&(), false).iter().for_each( + |(lock_file_name, directory_name)| { + if directory_name.is_none() { + let Ok(timestamp) = extract_timestamp_from_session_dir(lock_file_name) else { debug!( "found lock-file with malformed timestamp: {}", crate_directory.join(&lock_file_name).display() @@ -685,24 +686,25 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { return; }; - let lock_file_path = crate_directory.join(&**lock_file_name); + let lock_file_path = crate_directory.join(&**lock_file_name); - if is_old_enough_to_be_collected(timestamp) { - debug!( - "garbage_collect_session_directories() - deleting \ + if is_old_enough_to_be_collected(timestamp) { + debug!( + "garbage_collect_session_directories() - deleting \ garbage lock file: {}", - lock_file_path.display() - ); - delete_session_dir_lock_file(sess, &lock_file_path); - } else { - debug!( - "garbage_collect_session_directories() - lock file with \ + lock_file_path.display() + ); + delete_session_dir_lock_file(sess, &lock_file_path); + } else { + debug!( + "garbage_collect_session_directories() - lock file with \ no session dir not old enough to be collected: {}", - lock_file_path.display() - ); + lock_file_path.display() + ); + } } - } - }); + }, + ); // Filter out `None` directories let lock_file_to_session_dir: UnordMap = diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs index 865f90273bdec..b0d173ab30469 100644 --- a/compiler/rustc_incremental/src/persist/work_product.rs +++ b/compiler/rustc_incremental/src/persist/work_product.rs @@ -46,7 +46,7 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( /// Removes files for a given work product. pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) { - work_product.saved_files.items().for_each(|(_, path)| { + work_product.saved_files.to_sorted(&(), false).iter().for_each(|(_, path)| { let path = in_incr_comp_dir_sess(sess, path); if let Err(err) = std_fs::remove_file(&path) { sess.emit_warning(errors::DeleteWorkProduct { path: &path, err }); From f5f638c12434e3c277fde8f4245273c2cc3c8110 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Mon, 5 Jun 2023 00:16:20 -0400 Subject: [PATCH 08/11] Fixed to_sorted => to_sorted_stable_ord --- .../src/stable_hasher.rs | 11 ++++ .../src/assert_module_sources.rs | 4 +- .../src/persist/dirty_clean.rs | 16 +++--- compiler/rustc_incremental/src/persist/fs.rs | 55 ++++++++++--------- .../rustc_incremental/src/persist/load.rs | 2 +- .../src/persist/work_product.rs | 9 ++- 6 files changed, 60 insertions(+), 37 deletions(-) diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs index 6d57d81c56a4b..a895e28c822cb 100644 --- a/compiler/rustc_data_structures/src/stable_hasher.rs +++ b/compiler/rustc_data_structures/src/stable_hasher.rs @@ -339,6 +339,8 @@ impl, T2: HashStable, CTX> HashStable for (T1, T2) } } +unsafe impl StableOrd for (T1, T2) {} + impl HashStable for (T1, T2, T3) where T1: HashStable, @@ -353,6 +355,8 @@ where } } +unsafe impl StableOrd for (T1, T2, T3) {} + impl HashStable for (T1, T2, T3, T4) where T1: HashStable, @@ -369,6 +373,11 @@ where } } +unsafe impl StableOrd + for (T1, T2, T3, T4) +{ +} + impl, CTX> HashStable for [T] { default fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { self.len().hash_stable(ctx, hasher); @@ -459,6 +468,8 @@ impl HashStable for str { } } +unsafe impl StableOrd for &str {} + impl HashStable for String { #[inline] fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { diff --git a/compiler/rustc_incremental/src/assert_module_sources.rs b/compiler/rustc_incremental/src/assert_module_sources.rs index 3e12c9b5016be..3d9b1c2e7411f 100644 --- a/compiler/rustc_incremental/src/assert_module_sources.rs +++ b/compiler/rustc_incremental/src/assert_module_sources.rs @@ -118,8 +118,8 @@ impl<'tcx> AssertModuleSource<'tcx> { debug!("mapping '{}' to cgu name '{}'", self.field(attr, sym::module), cgu_name); if !self.available_cgus.contains(&cgu_name) { - let cgu_names: Vec = - self.available_cgus.items().map(|cgu| cgu.as_str().to_owned()).into_sorted(&()); + let cgu_names: Vec<&str> = + self.available_cgus.items().map(|cgu| cgu.as_str()).into_sorted_stable_ord(true); self.tcx.sess.emit_err(errors::NoModuleNamed { span: attr.span, user_path, diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index 6381a05dfdc1c..786a0e0d3b230 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -198,7 +198,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { let (name, mut auto) = self.auto_labels(item_id, attr); let except = self.except(attr); let loaded_from_disk = self.loaded_from_disk(attr); - for e in except.to_sorted(&(), false) { + for e in except.items().map(|x| x.as_str()).into_sorted_stable_ord(false) { if !auto.remove(e) { self.tcx.sess.emit_fatal(errors::AssertionAuto { span: attr.span, name, e }); } @@ -377,18 +377,20 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { continue; }; self.checked_attrs.insert(attr.id); - assertion.clean.to_sorted(&(), false).iter().for_each(|label| { + for label in assertion.clean.items().map(|x| x.as_str()).into_sorted_stable_ord(false) { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_clean(item_span, dep_node); - }); - assertion.dirty.to_sorted(&(), false).iter().for_each(|label| { + } + for label in assertion.dirty.items().map(|x| x.as_str()).into_sorted_stable_ord(false) { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_dirty(item_span, dep_node); - }); - assertion.loaded_from_disk.to_sorted(&(), false).iter().for_each(|label| { + } + for label in + assertion.loaded_from_disk.items().map(|x| x.as_str()).into_sorted_stable_ord(false) + { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_loaded_from_disk(item_span, dep_node); - }); + } } } } diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index f0e5f07230f7e..550772a688121 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -661,8 +661,9 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { session_directories.sort(); // Now map from lock files to session directories - let lock_file_to_session_dir: UnordMap> = - UnordMap::from(lock_files.into_items().map(|lock_file_name| { + let lock_file_to_session_dir: UnordMap> = lock_files + .into_items() + .map(|lock_file_name| { assert!(lock_file_name.ends_with(LOCK_FILE_EXT)); let dir_prefix_end = lock_file_name.len() - LOCK_FILE_EXT.len(); let session_dir = { @@ -670,41 +671,45 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { session_directories.iter().find(|dir_name| dir_name.starts_with(dir_prefix)) }; (lock_file_name, session_dir.map(String::clone)) - })); + }) + .into(); // Delete all lock files, that don't have an associated directory. They must // be some kind of leftover - lock_file_to_session_dir.to_sorted(&(), false).iter().for_each( - |(lock_file_name, directory_name)| { - if directory_name.is_none() { - let Ok(timestamp) = extract_timestamp_from_session_dir(lock_file_name) else { + let lock_file_to_session_dir_iter = lock_file_to_session_dir + .items() + .map(|(file, dir)| (file.as_str(), dir.as_ref().map(|y| y.as_str()))); + for (lock_file_name, directory_name) in + lock_file_to_session_dir_iter.into_sorted_stable_ord(false) + { + if directory_name.is_none() { + let Ok(timestamp) = extract_timestamp_from_session_dir(lock_file_name) else { debug!( "found lock-file with malformed timestamp: {}", crate_directory.join(&lock_file_name).display() ); // Ignore it - return; + continue; }; - let lock_file_path = crate_directory.join(&**lock_file_name); + let lock_file_path = crate_directory.join(&*lock_file_name); - if is_old_enough_to_be_collected(timestamp) { - debug!( - "garbage_collect_session_directories() - deleting \ - garbage lock file: {}", - lock_file_path.display() - ); - delete_session_dir_lock_file(sess, &lock_file_path); - } else { - debug!( - "garbage_collect_session_directories() - lock file with \ - no session dir not old enough to be collected: {}", - lock_file_path.display() - ); - } + if is_old_enough_to_be_collected(timestamp) { + debug!( + "garbage_collect_session_directories() - deleting \ + garbage lock file: {}", + lock_file_path.display() + ); + delete_session_dir_lock_file(sess, &lock_file_path); + } else { + debug!( + "garbage_collect_session_directories() - lock file with \ + no session dir not old enough to be collected: {}", + lock_file_path.display() + ); } - }, - ); + } + } // Filter out `None` directories let lock_file_to_session_dir: UnordMap = diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index 0727523a22980..bb479b5bdccda 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -147,7 +147,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let report_incremental_info = sess.opts.unstable_opts.incremental_info; let expected_hash = sess.opts.dep_tracking_hash(false); - let mut prev_work_products = FxIndexMap::default(); + let mut prev_work_products = UnordMap::default(); // If we are only building with -Zquery-dep-graph but without an actual // incr. comp. session directory, we skip this. Otherwise we'd fail diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs index b0d173ab30469..ae604b2ca0be7 100644 --- a/compiler/rustc_incremental/src/persist/work_product.rs +++ b/compiler/rustc_incremental/src/persist/work_product.rs @@ -46,10 +46,15 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( /// Removes files for a given work product. pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) { - work_product.saved_files.to_sorted(&(), false).iter().for_each(|(_, path)| { + for path in work_product + .saved_files + .items() + .map(|(_, path)| path.as_str()) + .into_sorted_stable_ord(false) + { let path = in_incr_comp_dir_sess(sess, path); if let Err(err) = std_fs::remove_file(&path) { sess.emit_warning(errors::DeleteWorkProduct { path: &path, err }); } - }); + } } From 54d7b327e5182b97fcdb8d90bf7853ffe54364c3 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Thu, 8 Jun 2023 00:38:50 -0400 Subject: [PATCH 09/11] Removed stable/unstable sort arg from into_sorted_stable_ord, fixed a few misc issues, added collect to UnordItems --- compiler/rustc_abi/src/lib.rs | 4 +- .../src/stable_hasher.rs | 45 +++++++++++++++---- compiler/rustc_data_structures/src/unord.rs | 8 +++- compiler/rustc_hir/src/hir_id.rs | 4 +- .../src/assert_module_sources.rs | 2 +- .../src/persist/dirty_clean.rs | 8 ++-- compiler/rustc_incremental/src/persist/fs.rs | 15 +++---- .../rustc_incremental/src/persist/fs/tests.rs | 13 ++---- .../src/persist/work_product.rs | 7 +-- compiler/rustc_interface/src/queries.rs | 11 +++-- .../src/dep_graph/dep_node.rs | 6 ++- compiler/rustc_session/src/config.rs | 4 +- .../clippy_lints/src/wildcard_imports.rs | 2 +- 13 files changed, 81 insertions(+), 48 deletions(-) diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 2ee63c286ba5c..e1b9987f57816 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -414,7 +414,9 @@ pub struct Size { // Safety: Ord is implement as just comparing numerical values and numerical values // are not changed by (de-)serialization. #[cfg(feature = "nightly")] -unsafe impl StableOrd for Size {} +unsafe impl StableOrd for Size { + const CAN_USE_UNSTABLE_SORT: bool = true; +} // This is debug-printed a lot in larger structs, don't waste too much space there impl fmt::Debug for Size { diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs index a895e28c822cb..0c1fb7518fa3b 100644 --- a/compiler/rustc_data_structures/src/stable_hasher.rs +++ b/compiler/rustc_data_structures/src/stable_hasher.rs @@ -233,7 +233,17 @@ pub trait ToStableHashKey { /// - `DefIndex`, `CrateNum`, `LocalDefId`, because their concrete /// values depend on state that might be different between /// compilation sessions. -pub unsafe trait StableOrd: Ord {} +/// +/// The associated constant `CAN_USE_UNSTABLE_SORT` denotes whether +/// unstable sorting can be used for this type. Set to true if and +/// only if `a == b` implies `a` and `b` are fully indistinguishable. +pub unsafe trait StableOrd: Ord { + const CAN_USE_UNSTABLE_SORT: bool; +} + +unsafe impl StableOrd for &T { + const CAN_USE_UNSTABLE_SORT: bool = T::CAN_USE_UNSTABLE_SORT; +} /// Implement HashStable by just calling `Hash::hash()`. Also implement `StableOrd` for the type since /// that has the same requirements. @@ -253,7 +263,9 @@ macro_rules! impl_stable_traits_for_trivial_type { } } - unsafe impl $crate::stable_hasher::StableOrd for $t {} + unsafe impl $crate::stable_hasher::StableOrd for $t { + const CAN_USE_UNSTABLE_SORT: bool = true; + } }; } @@ -339,7 +351,9 @@ impl, T2: HashStable, CTX> HashStable for (T1, T2) } } -unsafe impl StableOrd for (T1, T2) {} +unsafe impl StableOrd for (T1, T2) { + const CAN_USE_UNSTABLE_SORT: bool = T1::CAN_USE_UNSTABLE_SORT && T2::CAN_USE_UNSTABLE_SORT; +} impl HashStable for (T1, T2, T3) where @@ -355,7 +369,10 @@ where } } -unsafe impl StableOrd for (T1, T2, T3) {} +unsafe impl StableOrd for (T1, T2, T3) { + const CAN_USE_UNSTABLE_SORT: bool = + T1::CAN_USE_UNSTABLE_SORT && T2::CAN_USE_UNSTABLE_SORT && T3::CAN_USE_UNSTABLE_SORT; +} impl HashStable for (T1, T2, T3, T4) where @@ -376,6 +393,10 @@ where unsafe impl StableOrd for (T1, T2, T3, T4) { + const CAN_USE_UNSTABLE_SORT: bool = T1::CAN_USE_UNSTABLE_SORT + && T2::CAN_USE_UNSTABLE_SORT + && T3::CAN_USE_UNSTABLE_SORT + && T4::CAN_USE_UNSTABLE_SORT; } impl, CTX> HashStable for [T] { @@ -468,7 +489,9 @@ impl HashStable for str { } } -unsafe impl StableOrd for &str {} +unsafe impl StableOrd for &str { + const CAN_USE_UNSTABLE_SORT: bool = true; +} impl HashStable for String { #[inline] @@ -479,7 +502,9 @@ impl HashStable for String { // Safety: String comparison only depends on their contents and the // contents are not changed by (de-)serialization. -unsafe impl StableOrd for String {} +unsafe impl StableOrd for String { + const CAN_USE_UNSTABLE_SORT: bool = true; +} impl ToStableHashKey for String { type KeyType = String; @@ -505,7 +530,9 @@ impl HashStable for bool { } // Safety: sort order of bools is not changed by (de-)serialization. -unsafe impl StableOrd for bool {} +unsafe impl StableOrd for bool { + const CAN_USE_UNSTABLE_SORT: bool = true; +} impl HashStable for Option where @@ -523,7 +550,9 @@ where } // Safety: the Option wrapper does not add instability to comparison. -unsafe impl StableOrd for Option {} +unsafe impl StableOrd for Option { + const CAN_USE_UNSTABLE_SORT: bool = T::CAN_USE_UNSTABLE_SORT; +} impl HashStable for Result where diff --git a/compiler/rustc_data_structures/src/unord.rs b/compiler/rustc_data_structures/src/unord.rs index 6c8d541463158..e18c7b415f6cf 100644 --- a/compiler/rustc_data_structures/src/unord.rs +++ b/compiler/rustc_data_structures/src/unord.rs @@ -140,12 +140,12 @@ impl> UnordItems { } #[inline] - pub fn into_sorted_stable_ord(self, use_stable_sort: bool) -> Vec + pub fn into_sorted_stable_ord(self) -> Vec where T: Ord + StableOrd, { let mut items: Vec = self.0.collect(); - if use_stable_sort { + if !T::CAN_USE_UNSTABLE_SORT { items.sort(); } else { items.sort_unstable() @@ -161,6 +161,10 @@ impl> UnordItems { items.sort_by_cached_key(|x| x.to_stable_hash_key(hcx)); items } + + pub fn collect>>(self) -> C { + self.into() + } } /// This is a set collection type that tries very hard to not expose diff --git a/compiler/rustc_hir/src/hir_id.rs b/compiler/rustc_hir/src/hir_id.rs index d549f52f873a9..34c6157793663 100644 --- a/compiler/rustc_hir/src/hir_id.rs +++ b/compiler/rustc_hir/src/hir_id.rs @@ -166,7 +166,9 @@ impl ItemLocalId { // Safety: Ord is implement as just comparing the ItemLocalId's numerical // values and these are not changed by (de-)serialization. -unsafe impl StableOrd for ItemLocalId {} +unsafe impl StableOrd for ItemLocalId { + const CAN_USE_UNSTABLE_SORT: bool = true; +} /// The `HirId` corresponding to `CRATE_NODE_ID` and `CRATE_DEF_ID`. pub const CRATE_HIR_ID: HirId = diff --git a/compiler/rustc_incremental/src/assert_module_sources.rs b/compiler/rustc_incremental/src/assert_module_sources.rs index 3d9b1c2e7411f..0111a6d302d4a 100644 --- a/compiler/rustc_incremental/src/assert_module_sources.rs +++ b/compiler/rustc_incremental/src/assert_module_sources.rs @@ -119,7 +119,7 @@ impl<'tcx> AssertModuleSource<'tcx> { if !self.available_cgus.contains(&cgu_name) { let cgu_names: Vec<&str> = - self.available_cgus.items().map(|cgu| cgu.as_str()).into_sorted_stable_ord(true); + self.available_cgus.items().map(|cgu| cgu.as_str()).into_sorted_stable_ord(); self.tcx.sess.emit_err(errors::NoModuleNamed { span: attr.span, user_path, diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index 786a0e0d3b230..cbe77e7b16de8 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -198,7 +198,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { let (name, mut auto) = self.auto_labels(item_id, attr); let except = self.except(attr); let loaded_from_disk = self.loaded_from_disk(attr); - for e in except.items().map(|x| x.as_str()).into_sorted_stable_ord(false) { + for e in except.items().map(|x| x.as_str()).into_sorted_stable_ord() { if !auto.remove(e) { self.tcx.sess.emit_fatal(errors::AssertionAuto { span: attr.span, name, e }); } @@ -377,16 +377,16 @@ impl<'tcx> DirtyCleanVisitor<'tcx> { continue; }; self.checked_attrs.insert(attr.id); - for label in assertion.clean.items().map(|x| x.as_str()).into_sorted_stable_ord(false) { + for label in assertion.clean.items().map(|x| x.as_str()).into_sorted_stable_ord() { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_clean(item_span, dep_node); } - for label in assertion.dirty.items().map(|x| x.as_str()).into_sorted_stable_ord(false) { + for label in assertion.dirty.items().map(|x| x.as_str()).into_sorted_stable_ord() { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_dirty(item_span, dep_node); } for label in - assertion.loaded_from_disk.items().map(|x| x.as_str()).into_sorted_stable_ord(false) + assertion.loaded_from_disk.items().map(|x| x.as_str()).into_sorted_stable_ord() { let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap(); self.assert_loaded_from_disk(item_span, dep_node); diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index 550772a688121..243057b99bca2 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -676,11 +676,8 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { // Delete all lock files, that don't have an associated directory. They must // be some kind of leftover - let lock_file_to_session_dir_iter = lock_file_to_session_dir - .items() - .map(|(file, dir)| (file.as_str(), dir.as_ref().map(|y| y.as_str()))); for (lock_file_name, directory_name) in - lock_file_to_session_dir_iter.into_sorted_stable_ord(false) + lock_file_to_session_dir.items().into_sorted_stable_ord() { if directory_name.is_none() { let Ok(timestamp) = extract_timestamp_from_session_dir(lock_file_name) else { @@ -712,10 +709,10 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { } // Filter out `None` directories - let lock_file_to_session_dir: UnordMap = - UnordMap::from(lock_file_to_session_dir.into_items().filter_map( - |(lock_file_name, directory_name)| directory_name.map(|n| (lock_file_name, n)), - )); + let lock_file_to_session_dir: UnordMap = lock_file_to_session_dir + .into_items() + .filter_map(|(lock_file_name, directory_name)| directory_name.map(|n| (lock_file_name, n))) + .into(); // Delete all session directories that don't have a lock file. for directory_name in session_directories { @@ -821,7 +818,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> { } None }); - let deletion_candidates = UnordMap::from(deletion_candidates); + let deletion_candidates = deletion_candidates.into(); // Delete all but the most recent of the candidates all_except_most_recent(deletion_candidates).into_items().all(|(path, lock)| { diff --git a/compiler/rustc_incremental/src/persist/fs/tests.rs b/compiler/rustc_incremental/src/persist/fs/tests.rs index 4c1cb5725dc9c..644b8187621c9 100644 --- a/compiler/rustc_incremental/src/persist/fs/tests.rs +++ b/compiler/rustc_incremental/src/persist/fs/tests.rs @@ -2,23 +2,16 @@ use super::*; #[test] fn test_all_except_most_recent() { - let computed: UnordMap<_, Option> = UnordMap::from_iter([ + let input: UnordMap<_, Option> = UnordMap::from_iter([ ((UNIX_EPOCH + Duration::new(4, 0), PathBuf::from("4")), None), ((UNIX_EPOCH + Duration::new(1, 0), PathBuf::from("1")), None), ((UNIX_EPOCH + Duration::new(5, 0), PathBuf::from("5")), None), ((UNIX_EPOCH + Duration::new(3, 0), PathBuf::from("3")), None), ((UNIX_EPOCH + Duration::new(2, 0), PathBuf::from("2")), None), ]); - let mut paths = UnordSet::default(); - paths.extend_unord(all_except_most_recent(computed).into_items().map(|(path, _)| path)); assert_eq!( - UnordSet::from(paths), - UnordSet::from_iter([ - PathBuf::from("1"), - PathBuf::from("2"), - PathBuf::from("3"), - PathBuf::from("4") - ]) + all_except_most_recent(input).into_items().map(|(path, _)| path).into_sorted_stable_ord(), + vec![PathBuf::from("1"), PathBuf::from("2"), PathBuf::from("3"), PathBuf::from("4")] ); assert!(all_except_most_recent(UnordMap::default()).is_empty()); diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs index ae604b2ca0be7..bce5ca1e16bd1 100644 --- a/compiler/rustc_incremental/src/persist/work_product.rs +++ b/compiler/rustc_incremental/src/persist/work_product.rs @@ -46,12 +46,7 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( /// Removes files for a given work product. pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) { - for path in work_product - .saved_files - .items() - .map(|(_, path)| path.as_str()) - .into_sorted_stable_ord(false) - { + for (_, path) in work_product.saved_files.items().into_sorted_stable_ord() { let path = in_incr_comp_dir_sess(sess, path); if let Err(err) = std_fs::remove_file(&path) { sess.emit_warning(errors::DeleteWorkProduct { path: &path, err }); diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs index 6975fbd917a4f..455a8129656d4 100644 --- a/compiler/rustc_interface/src/queries.rs +++ b/compiler/rustc_interface/src/queries.rs @@ -194,10 +194,15 @@ impl<'tcx> Queries<'tcx> { let future_opt = self.dep_graph_future()?.steal(); let dep_graph = future_opt .and_then(|future| { - let (prev_graph, prev_work_products) = + let (prev_graph, mut prev_work_products) = sess.time("blocked_on_dep_graph_loading", || future.open().open(sess)); - let prev_work_products = - FxIndexMap::from_iter(prev_work_products.into_sorted(&(), false)); + // Convert from UnordMap to FxIndexMap by sorting + let prev_work_product_ids = + prev_work_products.items().map(|x| *x.0).into_sorted_stable_ord(); + let prev_work_products = prev_work_product_ids + .into_iter() + .map(|x| (x, prev_work_products.remove(&x).unwrap())) + .collect::>(); rustc_incremental::build_dep_graph(sess, prev_graph, prev_work_products) }) .unwrap_or_else(DepGraph::new_disabled); diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs index e0089f2861547..39a4cb1b179b4 100644 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs @@ -46,7 +46,7 @@ use super::{DepContext, DepKind, FingerprintStyle}; use crate::ich::StableHashingContext; use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey}; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableOrd, ToStableHashKey}; use rustc_hir::definitions::DefPathHash; use std::fmt; use std::hash::Hash; @@ -254,3 +254,7 @@ impl ToStableHashKey for WorkProductId { self.hash } } +unsafe impl StableOrd for WorkProductId { + // Fingerprint can use unstable (just a tuple of `u64`s), so WorkProductId can as well + const CAN_USE_UNSTABLE_SORT: bool = true; +} diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 0ce83e7909771..84d9f75532b08 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -311,7 +311,9 @@ pub enum OutputType { } // Safety: Trivial C-Style enums have a stable sort order across compilation sessions. -unsafe impl StableOrd for OutputType {} +unsafe impl StableOrd for OutputType { + const CAN_USE_UNSTABLE_SORT: bool = true; +} impl ToStableHashKey for OutputType { type KeyType = Self; diff --git a/src/tools/clippy/clippy_lints/src/wildcard_imports.rs b/src/tools/clippy/clippy_lints/src/wildcard_imports.rs index b6e4cd22789f9..2a3d86988bb04 100644 --- a/src/tools/clippy/clippy_lints/src/wildcard_imports.rs +++ b/src/tools/clippy/clippy_lints/src/wildcard_imports.rs @@ -160,7 +160,7 @@ impl LateLintPass<'_> for WildcardImports { ) }; - let mut imports = used_imports.items().map(ToString::to_string).into_sorted_stable_ord(false); + let mut imports = used_imports.items().map(ToString::to_string).into_sorted_stable_ord(); let imports_string = if imports.len() == 1 { imports.pop().unwrap() } else if braced_glob { From 4ae250bf4e285cbd0a474207e8e3fc8d2c5ed9ee Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Thu, 8 Jun 2023 00:40:29 -0400 Subject: [PATCH 10/11] fixup! Removed stable/unstable sort arg from into_sorted_stable_ord, fixed a few misc issues, added collect to UnordItems --- src/tools/clippy/clippy_lints/src/wildcard_imports.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/clippy/clippy_lints/src/wildcard_imports.rs b/src/tools/clippy/clippy_lints/src/wildcard_imports.rs index 2a3d86988bb04..b6e4cd22789f9 100644 --- a/src/tools/clippy/clippy_lints/src/wildcard_imports.rs +++ b/src/tools/clippy/clippy_lints/src/wildcard_imports.rs @@ -160,7 +160,7 @@ impl LateLintPass<'_> for WildcardImports { ) }; - let mut imports = used_imports.items().map(ToString::to_string).into_sorted_stable_ord(); + let mut imports = used_imports.items().map(ToString::to_string).into_sorted_stable_ord(false); let imports_string = if imports.len() == 1 { imports.pop().unwrap() } else if braced_glob { From 3f324a8b7d62f44f97ba76eeb76b67d6ff0bf744 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Thu, 8 Jun 2023 01:05:38 -0400 Subject: [PATCH 11/11] Whoops, submodule change was actually valid - undoing fixup --- src/tools/clippy/clippy_lints/src/wildcard_imports.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/clippy/clippy_lints/src/wildcard_imports.rs b/src/tools/clippy/clippy_lints/src/wildcard_imports.rs index b6e4cd22789f9..2a3d86988bb04 100644 --- a/src/tools/clippy/clippy_lints/src/wildcard_imports.rs +++ b/src/tools/clippy/clippy_lints/src/wildcard_imports.rs @@ -160,7 +160,7 @@ impl LateLintPass<'_> for WildcardImports { ) }; - let mut imports = used_imports.items().map(ToString::to_string).into_sorted_stable_ord(false); + let mut imports = used_imports.items().map(ToString::to_string).into_sorted_stable_ord(); let imports_string = if imports.len() == 1 { imports.pop().unwrap() } else if braced_glob {