Add support to cdylib, dylib and staticlib (rust-lang#2039)

Refactor how we were creating the rlib files in the link stage. We cannot invoke rustc's linker directly for the native types since they will call the native linker that will fail. Instead, we just manually create an rlib file that includes only a metadata file.

I cleaned up our archive builder since we no longer need to conform to rustc's interface.

Author: celinval

kani-compiler/src/codegen_cprover_gotoc/archive.rs

@@ -2,114 +2,45 @@
 //
 // Modifications Copyright Kani Contributors
 // See GitHub history for details.
-// This file is a lightly modified version of upstream rustc:
+// This file is a heavily modified version of upstream rustc:
 //     compiler/rustc_codegen_cranelift/src/archive.rs
 
-// Along with lifting the deps:
-//  object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
-//  ar = "0.8.0"
-
-// Also: I removed all mentions of 'ranlib' which caused issues on macos.
-// We don't actually need these to be passed to a real linker anyway.
-// 'ranlib' is about building a global symbol table out of all the object
-// files in the archive, and we just don't put object files in our archives.
-
 //! Creation of ar archives like for the lib and staticlib crate type
+//! We now call the ArchiveBuilder directly, so we don't bother trying to fit into the rustc's
+//! `ArchiveBuilder`.
+//! Note that once we update to a version newer than 2022-12-04 we should be able to remove the
+//! logic here and use the compiler ArArchiveBuilder introduced here:
+//! <https://github.com/rust-lang/rust/pull/97485>
 
+use rustc_session::Session;
 use std::fs::File;
-use std::io::{self, Read, Seek};
 use std::path::{Path, PathBuf};
 
-use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
-use rustc_session::Session;
-
-use object::read::archive::ArchiveFile;
-use object::ReadCache;
-
-#[derive(Debug)]
-enum ArchiveEntry {
-    FromArchive { archive_index: usize, file_range: (u64, u64) },
-    File(PathBuf),
-}
-
-pub(crate) struct ArArchiveBuilder<'a> {
+pub(crate) struct ArchiveBuilder<'a> {
     sess: &'a Session,
     use_gnu_style_archive: bool,
 
-    src_archives: Vec<File>,
     // Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at
     // the end of an archive for linkers to not get confused.
-    entries: Vec<(Vec<u8>, ArchiveEntry)>,
+    entries: Vec<(Vec<u8>, PathBuf)>,
 }
 
-impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
-    fn add_file(&mut self, file: &Path) {
+impl<'a> ArchiveBuilder<'a> {
+    pub fn add_file(&mut self, file: &Path) {
         self.entries.push((
             file.file_name().unwrap().to_str().unwrap().to_string().into_bytes(),
-            ArchiveEntry::File(file.to_owned()),
+            file.to_owned(),
         ));
     }
 
-    fn add_archive(
-        &mut self,
-        archive_path: &Path,
-        mut skip: Box<dyn FnMut(&str) -> bool + 'static>,
-    ) -> std::io::Result<()> {
-        let read_cache = ReadCache::new(std::fs::File::open(&archive_path)?);
-        let archive = ArchiveFile::parse(&read_cache).unwrap();
-        let archive_index = self.src_archives.len();
-
-        for entry in archive.members() {
-            let entry = entry.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
-            let file_name = String::from_utf8(entry.name().to_vec())
-                .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
-            if !skip(&file_name) {
-                self.entries.push((
-                    file_name.into_bytes(),
-                    ArchiveEntry::FromArchive { archive_index, file_range: entry.file_range() },
-                ));
-            }
-        }
-
-        self.src_archives.push(read_cache.into_inner());
-        Ok(())
-    }
-
-    fn build(mut self: Box<Self>, output: &Path) -> bool {
+    pub fn build(&self, output: &Path) -> bool {
         enum BuilderKind {
             Bsd(ar::Builder<File>),
             Gnu(ar::GnuBuilder<File>),
         }
 
         let sess = self.sess;
 
-        let mut entries = Vec::new();
-
-        for (entry_name, entry) in self.entries {
-            // FIXME only read the symbol table of the object files to avoid having to keep all
-            // object files in memory at once, or read them twice.
-            let data = match entry {
-                ArchiveEntry::FromArchive { archive_index, file_range } => {
-                    // FIXME read symbols from symtab
-                    let src_read_cache = &mut self.src_archives[archive_index];
-
-                    src_read_cache.seek(io::SeekFrom::Start(file_range.0)).unwrap();
-                    let mut data = std::vec::from_elem(0, usize::try_from(file_range.1).unwrap());
-                    src_read_cache.read_exact(&mut data).unwrap();
-
-                    data
-                }
-                ArchiveEntry::File(file) => std::fs::read(file).unwrap_or_else(|err| {
-                    sess.fatal(&format!(
-                        "error while reading object file during archive building: {}",
-                        err
-                    ));
-                }),
-            };
-
-            entries.push((entry_name, data));
-        }
-
         let mut builder = if self.use_gnu_style_archive {
             BuilderKind::Gnu(ar::GnuBuilder::new(
                 File::create(&output).unwrap_or_else(|err| {
@@ -118,50 +49,45 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                         err
                     ));
                 }),
-                entries.iter().map(|(name, _)| name.clone()).collect(),
+                self.entries.iter().map(|(name, _)| name.clone()).collect(),
             ))
         } else {
             BuilderKind::Bsd(ar::Builder::new(File::create(&output).unwrap_or_else(|err| {
                 sess.fatal(&format!("error opening destination during archive building: {err}"));
             })))
         };
 
+        let entries = self.entries.iter().map(|(entry_name, file)| {
+            let data = std::fs::read(file).unwrap_or_else(|err| {
+                sess.fatal(&format!(
+                    "error while reading object file during archive building: {}",
+                    err
+                ));
+            });
+            (entry_name, data)
+        });
+
         // Add all files
-        let any_members = !entries.is_empty();
-        for (entry_name, data) in entries.into_iter() {
-            let header = ar::Header::new(entry_name, data.len() as u64);
+        let mut any_members = false;
+        for (entry_name, data) in entries {
+            let header = ar::Header::new(entry_name.clone(), data.len() as u64);
             match builder {
                 BuilderKind::Bsd(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
                 BuilderKind::Gnu(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
             }
+            any_members = true;
         }
 
         // Finalize archive
         std::mem::drop(builder);
         any_members
     }
-}
 
-pub(crate) struct ArArchiveBuilderBuilder;
-
-impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
-    fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> {
-        Box::new(ArArchiveBuilder {
+    pub fn new(sess: &'a Session) -> Self {
+        ArchiveBuilder {
             sess,
             use_gnu_style_archive: sess.target.archive_format == "gnu",
-            src_archives: vec![],
             entries: vec![],
-        })
-    }
-
-    fn create_dll_import_lib(
-        &self,
-        _sess: &Session,
-        _lib_name: &str,
-        _dll_imports: &[rustc_session::cstore::DllImport],
-        _tmpdir: &Path,
-        _is_direct_dependency: bool,
-    ) -> PathBuf {
-        unimplemented!("injecting dll imports is not supported");
+        }
     }
 }

kani-compiler/src/codegen_cprover_gotoc/compiler_interface.rs

@@ -3,6 +3,7 @@
 
 //! This file contains the code necessary to interface with the compiler backend
 
+use crate::codegen_cprover_gotoc::archive::ArchiveBuilder;
 use crate::codegen_cprover_gotoc::GotocCtx;
 use crate::kani_middle::provide;
 use crate::kani_middle::reachability::{
@@ -13,20 +14,24 @@ use cbmc::goto_program::Location;
 use cbmc::{InternedString, MachineModel};
 use kani_metadata::{ArtifactType, HarnessMetadata, KaniMetadata};
 use kani_queries::{QueryDb, ReachabilityType, UserInput};
+use rustc_codegen_ssa::back::metadata::create_wrapper_file;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::{CodegenResults, CrateInfo};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LOCAL_CRATE;
+use rustc_metadata::fs::{emit_wrapper_file, METADATA_FILENAME};
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_middle::mir::write_mir_pretty;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, InstanceDef, TyCtxt};
-use rustc_session::config::{OutputFilenames, OutputType};
+use rustc_session::config::{CrateType, OutputFilenames, OutputType};
 use rustc_session::cstore::MetadataLoaderDyn;
+use rustc_session::output::out_filename;
 use rustc_session::Session;
 use rustc_span::def_id::DefId;
 use rustc_target::abi::Endian;
@@ -42,6 +47,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::rc::Rc;
 use std::time::Instant;
+use tempfile::Builder as TempFileBuilder;
 use tracing::{debug, error, info, warn};
 
 #[derive(Clone)]
@@ -72,15 +78,15 @@ impl CodegenBackend for GotocCodegenBackend {
         &self,
         tcx: TyCtxt,
         rustc_metadata: EncodedMetadata,
-        need_metadata_module: bool,
+        _need_metadata_module: bool,
     ) -> Box<dyn Any> {
         super::utils::init();
 
         // Follow rustc naming convention (cx is abbrev for context).
         // https://rustc-dev-guide.rust-lang.org/conventions.html#naming-conventions
         let mut gcx = GotocCtx::new(tcx, self.queries.clone());
         check_target(tcx.sess);
-        check_options(tcx.sess, need_metadata_module);
+        check_options(tcx.sess);
         check_crate_items(&gcx);
 
         let items = with_timer(|| collect_codegen_items(&gcx), "codegen reachability analysis");
@@ -188,38 +194,45 @@ impl CodegenBackend for GotocCodegenBackend {
             .unwrap())
     }
 
+    /// Emit `rlib` files during the link stage if it was requested.
+    ///
+    /// We need to emit `rlib` files normally if requested. Cargo expects these in some
+    /// circumstances and sends them to subsequent builds with `-L`.
+    ///
+    /// We CAN NOT invoke the native linker, because that will fail. We don't have real objects.
+    /// What determines whether the native linker is invoked or not is the set of `crate_types`.
+    /// Types such as `bin`, `cdylib`, `dylib` will trigger the native linker.
+    ///
+    /// Thus, we manually build the rlib file including only the `rmeta` file.
     fn link(
         &self,
         sess: &Session,
         codegen_results: CodegenResults,
         outputs: &OutputFilenames,
     ) -> Result<(), ErrorGuaranteed> {
-        // In `link`, we need to do two things:
-        // 1. We need to emit `rlib` files normally. Cargo expects these in some circumstances and sends
-        //    them to subsequent builds with `-L`.
-        // 2. We MUST NOT try to invoke the native linker, because that will fail. We don't have real objects.
-        // This is normally not a problem: usually we only get one requested `crate-type`.
-        // But let's be careful and fail loudly if we get conflicting requests:
         let requested_crate_types = sess.crate_types();
-        // Quit successfully if we don't need an `rlib`:
-        if !requested_crate_types.contains(&rustc_session::config::CrateType::Rlib) {
+        if !requested_crate_types.contains(&CrateType::Rlib) {
+            // Quit successfully if we don't need an `rlib`:
             return Ok(());
         }
-        // Fail loudly if we need an `rlib` (above!) and *also* an executable, which
-        // we can't produce, and can't easily suppress in `link_binary`:
-        if requested_crate_types.contains(&rustc_session::config::CrateType::Executable) {
-            sess.err("Build crate-type requested both rlib and executable, and Kani cannot handle this situation");
-            sess.abort_if_errors();
-        }
 
-        // All this ultimately boils down to is emitting an `rlib` that contains just one file: `lib.rmeta`
-        use rustc_codegen_ssa::back::link::link_binary;
-        link_binary(
+        // Emit the `rlib` that contains just one file: `<crate>.rmeta`
+        let mut builder = ArchiveBuilder::new(sess);
+        let tmp_dir = TempFileBuilder::new().prefix("kani").tempdir().unwrap();
+        let path = MaybeTempDir::new(tmp_dir, sess.opts.cg.save_temps);
+        let (metadata, _metadata_position) =
+            create_wrapper_file(sess, b".rmeta".to_vec(), codegen_results.metadata.raw_data());
+        let metadata = emit_wrapper_file(sess, &metadata, &path, METADATA_FILENAME);
+        builder.add_file(&metadata);
+
+        let rlib = out_filename(
             sess,
-            &crate::codegen_cprover_gotoc::archive::ArArchiveBuilderBuilder,
-            &codegen_results,
+            CrateType::Rlib,
             outputs,
-        )
+            codegen_results.crate_info.local_crate_name.as_str(),
+        );
+        builder.build(&rlib);
+        Ok(())
     }
 }
 
@@ -246,7 +259,7 @@ fn check_target(session: &Session) {
     session.abort_if_errors();
 }
 
-fn check_options(session: &Session, need_metadata_module: bool) {
+fn check_options(session: &Session) {
     // The requirements for `min_global_align` and `endian` are needed to build
     // a valid CBMC machine model in function `machine_model_from_session` from
     // src/kani-compiler/src/codegen_cprover_gotoc/context/goto_ctx.rs
@@ -277,10 +290,6 @@ fn check_options(session: &Session, need_metadata_module: bool) {
         );
     }
 
-    if need_metadata_module {
-        session.err("Kani cannot generate metadata module.");
-    }
-
     session.abort_if_errors();
 }
 

kani-compiler/src/main.rs

@@ -26,6 +26,8 @@ extern crate rustc_middle;
 extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_target;
+// We can't add this directly as a dependency because we need the version to match rustc
+extern crate tempfile;
 
 #[cfg(feature = "cprover")]
 mod codegen_cprover_gotoc;