Use real atomic instructions instead of a global lock

Author: bjorn3

scripts/config.sh

@@ -26,7 +26,7 @@ export RUSTC=$dir"/bin/cg_clif"
 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
 '-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
 
-# FIXME remove once the atomic shim is gone
+# FIXME fix `#[linkage = "extern_weak"]` without this
 if [[ "$unamestr" == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi

src/atomic_shim.rs

@@ -8,7 +8,7 @@ use rustc_session::Session;
 use cranelift_module::FuncId;
 
 use object::write::*;
-use object::{RelocationEncoding, RelocationKind, SectionKind, SymbolFlags};
+use object::{RelocationEncoding, SectionKind, SymbolFlags};
 
 use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
 
@@ -118,49 +118,6 @@ impl WriteDebugInfo for ObjectProduct {
     }
 }
 
-// FIXME remove once atomic instructions are implemented in Cranelift.
-pub(crate) trait AddConstructor {
-    fn add_constructor(&mut self, func_id: FuncId);
-}
-
-impl AddConstructor for ObjectProduct {
-    fn add_constructor(&mut self, func_id: FuncId) {
-        let symbol = self.function_symbol(func_id);
-        let segment = self
-            .object
-            .segment_name(object::write::StandardSegment::Data);
-        let init_array_section =
-            self.object
-                .add_section(segment.to_vec(), b".init_array".to_vec(), SectionKind::Data);
-        let address_size = self
-            .object
-            .architecture()
-            .address_size()
-            .expect("address_size must be known")
-            .bytes();
-        self.object.append_section_data(
-            init_array_section,
-            &std::iter::repeat(0)
-                .take(address_size.into())
-                .collect::<Vec<u8>>(),
-            8,
-        );
-        self.object
-            .add_relocation(
-                init_array_section,
-                object::write::Relocation {
-                    offset: 0,
-                    size: address_size * 8,
-                    kind: RelocationKind::Absolute,
-                    encoding: RelocationEncoding::Generic,
-                    symbol,
-                    addend: 0,
-                },
-            )
-            .unwrap();
-    }
-}
-
 pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
     let triple = crate::build_isa(sess).triple().clone();
 

src/backend.rs

@@ -12,12 +12,10 @@ use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{DebugInfo, OutputType};
 
-use cranelift_object::{ObjectModule, ObjectProduct};
+use cranelift_object::ObjectModule;
 
 use crate::{prelude::*, BackendConfig};
 
-use crate::backend::AddConstructor;
-
 fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule {
     let module = crate::backend::make_module(tcx.sess, name);
     assert_eq!(pointer_ty(tcx), module.target_config().pointer_type());
@@ -39,7 +37,6 @@ fn emit_module(
     module: ObjectModule,
     debug: Option<DebugContext<'_>>,
     unwind_context: UnwindContext<'_>,
-    map_product: impl FnOnce(ObjectProduct) -> ObjectProduct,
 ) -> ModuleCodegenResult {
     let mut product = module.finish();
 
@@ -49,8 +46,6 @@ fn emit_module(
 
     unwind_context.emit(&mut product);
 
-    let product = map_product(product);
-
     let tmp_file = tcx
         .output_filenames(LOCAL_CRATE)
         .temp_path(OutputType::Object, Some(&name));
@@ -124,30 +119,7 @@ fn module_codegen(
     let cgu = tcx.codegen_unit(cgu_name);
     let mono_items = cgu.items_in_deterministic_order(tcx);
 
-    let mut module = new_module(tcx, cgu_name.as_str().to_string());
-
-    // Initialize the global atomic mutex using a constructor for proc-macros.
-    // FIXME implement atomic instructions in Cranelift.
-    let mut init_atomics_mutex_from_constructor = None;
-    if tcx
-        .sess
-        .crate_types()
-        .contains(&rustc_session::config::CrateType::ProcMacro)
-    {
-        if mono_items.iter().any(|(mono_item, _)| match mono_item {
-            rustc_middle::mir::mono::MonoItem::Static(def_id) => tcx
-                .symbol_name(Instance::mono(tcx, *def_id))
-                .name
-                .contains("__rustc_proc_macro_decls_"),
-            _ => false,
-        }) {
-            init_atomics_mutex_from_constructor =
-                Some(crate::atomic_shim::init_global_lock_constructor(
-                    &mut module,
-                    &format!("{}_init_atomics_mutex", cgu_name.as_str()),
-                ));
-        }
-    }
+    let module = new_module(tcx, cgu_name.as_str().to_string());
 
     let mut cx = crate::CodegenCx::new(
         tcx,
@@ -180,7 +152,7 @@ fn module_codegen(
     }
     let (mut module, global_asm, debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context, false);
+    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context);
 
     let codegen_result = emit_module(
         tcx,
@@ -189,13 +161,6 @@ fn module_codegen(
         module,
         debug,
         unwind_context,
-        |mut product| {
-            if let Some(func_id) = init_atomics_mutex_from_constructor {
-                product.add_constructor(func_id);
-            }
-
-            product
-        },
     );
 
     codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm);
@@ -275,7 +240,6 @@ pub(super) fn run_aot(
             allocator_module,
             None,
             allocator_unwind_context,
-            |product| product,
         );
         if let Some((id, product)) = work_product {
             work_products.insert(id, product);

src/driver/aot.rs

@@ -23,24 +23,6 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
     }
 
-    #[cfg(unix)]
-    unsafe {
-        // When not using our custom driver rustc will open us without the RTLD_GLOBAL flag, so
-        // __cg_clif_global_atomic_mutex will not be exported. We fix this by opening ourself again
-        // as global.
-        // FIXME remove once atomic_shim is gone
-
-        let mut dl_info: libc::Dl_info = std::mem::zeroed();
-        assert_ne!(
-            libc::dladdr(run_jit as *const libc::c_void, &mut dl_info),
-            0
-        );
-        assert_ne!(
-            libc::dlopen(dl_info.dli_fname, libc::RTLD_NOW | libc::RTLD_GLOBAL),
-            std::ptr::null_mut(),
-        );
-    }
-
     let imported_symbols = load_imported_symbols_for_jit(tcx);
 
     let mut jit_builder = JITBuilder::with_isa(
@@ -111,7 +93,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
 
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
+    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
     tcx.sess.abort_if_errors();