Run all proof harnesses by default (rust-lang#962)

* Introduce --harness and parse kani-metadata.

* run all proof harnesses

* enable function-less 'expected' tests for cargo-kani in compiletest

* Only mention --harness not --function in docs

* use --harness instead of --function in tests, mostly.

* Run all harnesses, even if one fails. Also generate all reports in separate directories

* Output reports (and harness) with - not :: in file names

* Print all failed harnesses in a final summary

* Make harness search robust against possible future name collisions

Author: tedinski

docs/src/kani-single-file.md

@@ -22,9 +22,9 @@ kani filenames.rs --visualize --cbmc-args --object-bits 11 --unwind 15
 **`--visualize`** will generate a report in the local directory accessible through `report/html/index.html`.
 This report will shows coverage information, as well as give traces for each failure Kani finds.
 
-**`--function <name>`** Kani defaults to assuming the starting function is called `main`.
-You can change it to a different function with this argument.
-Note that to "find" the function given, it needs to be given the `#[kani::proof]` annotation.
+**`--harness <name>`** Kani defaults to checking all proof harnesses.
+You can switch to checking just one harness using this flag.
+Proof harnesses are functions that have been given the `#[kani::proof]` annotation.
 
 **`--gen-c`** will generate a C file that roughly corresponds to the input Rust file.
 This can sometimes be helpful when trying to debug a problem with Kani.

docs/src/tutorial-first-steps.md

@@ -185,7 +185,7 @@ Fortunately, Kani is able to report a coverage metric for each proof harness.
 Try running:
 
 ```
-kani --visualize src/final_form.rs --function verify_success
+kani --visualize src/final_form.rs --harness verify_success
 open report/html/index.html
 ```
 

docs/src/tutorial-kinds-of-failure.md

@@ -43,7 +43,7 @@ But we're able to check this unsafe code with Kani:
 ```
 
 ```
-# kani src/bounds_check.rs --function bound_check
+# kani src/bounds_check.rs --harness bound_check
 [...]
 ** 15 of 448 failed
 [...]
@@ -56,7 +56,7 @@ Kani is inserting a lot more checks than appear as asserts in our code, so the o
 Let's narrow that output down a bit:
 
 ```
-# kani src/bounds_check.rs --function bound_check | grep Failed
+# kani src/bounds_check.rs --harness bound_check | grep Failed
 Failed Checks: attempt to compute offset which would overflow
 Failed Checks: attempt to calculate the remainder with a divisor of zero
 Failed Checks: attempt to add with overflow
@@ -92,7 +92,7 @@ Consider trying a few more small exercises with this example:
 Having switched back to the safe indexing operation, Kani reports two failures:
 
 ```
-# kani src/bounds_check.rs --function bound_check | grep Failed
+# kani src/bounds_check.rs --harness bound_check | grep Failed
 Failed Checks: index out of bounds: the length is less than or equal to the given index
 Failed Checks: dereference failure: pointer outside object bounds
 ```
@@ -176,7 +176,7 @@ Kani will find these failures as well.
 Here's the output from Kani:
 
 ```
-# kani src/overflow.rs --function add_overflow
+# kani src/overflow.rs --harness add_overflow
 [...]
 RESULTS:
 Check 1: simple_addition.assertion.1

docs/src/tutorial-nondeterministic-variables.md

@@ -32,7 +32,7 @@ You can try it out by running the example under
 [arbitrary-variables directory](https://github.com/model-checking/kani/tree/main/docs/src/tutorial/arbitrary-variables/):
 
 ```
-cargo kani --function safe_update
+cargo kani --harness safe_update
 ```
 
 ## Unsafe nondeterministic variables
@@ -57,7 +57,7 @@ The compiler is able to represent `Option<NonZeroU32>` using `32` bits by using
 You can try it out by running the example under [arbitrary-variables directory](https://github.com/model-checking/kani/tree/main/docs/src/tutorial/arbitrary-variables/):
 
 ```
-cargo kani --function unsafe_update
+cargo kani --harness unsafe_update
 ```
 
 ## Safe nondeterministic variables for custom types
@@ -88,5 +88,5 @@ You can try it out by running the example under
 [`docs/src/tutorial/arbitrary-variables`](https://github.com/model-checking/kani/tree/main/docs/src/tutorial/arbitrary-variables/):
 
 ```
-cargo kani --function check_rating
+cargo kani --harness check_rating
 ```

src/cargo-kani/src/args.rs

@@ -72,9 +72,14 @@ pub struct KaniArgs {
     #[structopt(flatten)]
     pub checks: CheckArgs,
 
-    /// Entry point for verification
-    #[structopt(long, default_value = "main")]
-    pub function: String,
+    /// Entry point for verification (symbol name)
+    #[structopt(long, hidden = true)]
+    pub function: Option<String>,
+    /// Entry point for verification (proof harness)
+    // In a dry-run, we don't have kani-metadata.json to read, so we can't use this flag
+    #[structopt(long, conflicts_with = "function", conflicts_with = "dry-run")]
+    pub harness: Option<String>,
+
     /// Link external C files referenced by Rust code.
     /// This is an experimental feature.
     #[structopt(long, parse(from_os_str), hidden = true)]
@@ -286,6 +291,14 @@ impl KaniArgs {
             )
             .exit();
         }
+
+        if self.cbmc_args.contains(&OsString::from("--function")) {
+            Error::with_description(
+                "Invalid flag: --function should be provided to Kani directly, not via --cbmc-args.",
+                ErrorKind::ArgumentConflict,
+            )
+            .exit();
+        }
     }
 }
 
@@ -317,4 +330,13 @@ mod tests {
             assert_eq!(t, format!("{}", AbstractionType::from_str(t).unwrap()));
         }
     }
+
+    #[test]
+    fn check_dry_run_harness_conflicts() {
+        // harness needs metadata which we don't have with dry-run
+        let args = vec!["kani", "file.rs", "--dry-run", "--harness", "foo"];
+        let app = StandaloneArgs::clap();
+        let err = app.get_matches_from_safe(args).unwrap_err();
+        assert!(err.kind == ErrorKind::ArgumentConflict);
+    }
 }

src/cargo-kani/src/call_cargo.rs

@@ -3,7 +3,7 @@
 
 use anyhow::{Context, Result};
 use std::ffi::OsString;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use crate::session::KaniSession;
@@ -17,6 +17,8 @@ pub struct CargoOutputs {
     pub symtabs: Vec<PathBuf>,
     /// The location of vtable restrictions files (a directory of *.restrictions.json)
     pub restrictions: Option<PathBuf>,
+    /// The kani-metadata.json files written by kani-compiler.
+    pub metadata: Vec<PathBuf>,
 }
 
 impl KaniSession {
@@ -44,7 +46,7 @@ impl KaniSession {
         args.push(build_target.into());
 
         args.push("--target-dir".into());
-        args.push(target_dir.clone().into());
+        args.push(target_dir.into());
 
         if self.args.verbose {
             args.push("-v".into());
@@ -59,26 +61,26 @@ impl KaniSession {
         self.run_terminal(cmd)?;
 
         if self.args.dry_run {
-            // mock an answer
+            // mock an answer: mostly the same except we don't/can't expand the globs
             return Ok(CargoOutputs {
                 outdir: outdir.clone(),
-                symtabs: vec![outdir.join("dry-run.symtab.json")],
+                symtabs: vec![outdir.join("*.symtab.json")],
+                metadata: vec![outdir.join("*.kani-metadata.json")],
                 restrictions: self.args.restrict_vtable().then(|| outdir),
             });
         }
 
-        let symtabs = glob(&outdir.join("*.symtab.json"));
-
         Ok(CargoOutputs {
             outdir: outdir.clone(),
-            symtabs: symtabs?,
+            symtabs: glob(&outdir.join("*.symtab.json"))?,
+            metadata: glob(&outdir.join("*.kani-metadata.json"))?,
             restrictions: self.args.restrict_vtable().then(|| outdir),
         })
     }
 }
 
 /// Given a `path` with glob characters in it (e.g. `*.json`), return a vector of matching files
-fn glob(path: &PathBuf) -> Result<Vec<PathBuf>> {
+fn glob(path: &Path) -> Result<Vec<PathBuf>> {
     let results = glob::glob(path.to_str().context("Non-UTF-8 path enountered")?)?;
     // the logic to turn "Iter<Result<T, E>>" into "Result<Vec<T>, E>" doesn't play well
     // with anyhow, so a type annotation is required

src/cargo-kani/src/call_cbmc_viewer.rs

@@ -12,7 +12,7 @@ use crate::util::alter_extension;
 impl KaniSession {
     /// Run CBMC appropriately to produce 3 output XML files, then run cbmc-viewer on them to produce a report.
     /// Viewer doesn't give different error codes depending on verification failure, so as long as it works, we report success.
-    pub fn run_visualize(&self, file: &Path, default_reportdir: &str) -> Result<()> {
+    pub fn run_visualize(&self, file: &Path, report_dir: &Path) -> Result<()> {
         let results_filename = alter_extension(file, "results.xml");
         let coverage_filename = alter_extension(file, "coverage.xml");
         let property_filename = alter_extension(file, "property.xml");
@@ -28,12 +28,6 @@ impl KaniSession {
         self.cbmc_variant(file, &["--xml-ui", "--cover", "location"], &coverage_filename)?;
         self.cbmc_variant(file, &["--xml-ui", "--show-properties"], &property_filename)?;
 
-        let reportdir = if let Some(pb) = &self.args.target_dir {
-            pb.join("report").into_os_string()
-        } else {
-            default_reportdir.into()
-        };
-
         let args: Vec<OsString> = vec![
             "--result".into(),
             results_filename.into(),
@@ -48,7 +42,7 @@ impl KaniSession {
             "--goto".into(),
             file.into(),
             "--reportdir".into(),
-            reportdir.clone(),
+            report_dir.into(),
         ];
 
         // TODO get cbmc-viewer path from self
@@ -59,7 +53,7 @@ impl KaniSession {
 
         // Let the user know
         if !self.args.quiet {
-            println!("Report written to: {}/html/index.html", reportdir.to_string_lossy());
+            println!("Report written to: {}/html/index.html", report_dir.to_string_lossy());
         }
 
         Ok(())

src/cargo-kani/src/call_goto_instrument.rs

@@ -6,14 +6,10 @@ use std::ffi::OsString;
 use std::path::Path;
 use std::process::Command;
 
+use crate::metadata::collect_and_link_function_pointer_restrictions;
 use crate::session::KaniSession;
 use crate::util::alter_extension;
 
-use kani_metadata::{InternedString, TraitDefinedMethod, VtableCtxResults};
-use std::collections::HashMap;
-use std::fs::File;
-use std::io::{BufReader, BufWriter};
-
 impl KaniSession {
     /// Postprocess a goto binary (before cbmc, after linking) in-place by calling goto-instrument
     pub fn run_goto_instrument(&self, input: &Path, output: &Path, function: &str) -> Result<()> {
@@ -28,6 +24,9 @@ impl KaniSession {
         }
 
         if self.args.gen_c {
+            if !self.args.quiet {
+                println!("Generated C code written to {}", output.to_string_lossy());
+            }
             self.gen_c(output)?;
         }
 
@@ -124,77 +123,3 @@ impl KaniSession {
         self.run_suppress(cmd)
     }
 }
-
-/// Collect all vtable restriction metadata together, and write one combined output in CBMC's format
-fn link_function_pointer_restrictions(
-    data_per_crate: Vec<VtableCtxResults>,
-    output_filename: &Path,
-) -> Result<()> {
-    // Combine all method possibilities into one global mapping
-    let mut combined_possible_methods: HashMap<TraitDefinedMethod, Vec<InternedString>> =
-        HashMap::new();
-    for crate_data in &data_per_crate {
-        for entry in &crate_data.possible_methods {
-            combined_possible_methods
-                .insert(entry.trait_method.clone(), entry.possibilities.clone());
-        }
-    }
-
-    // Emit a restriction for every call site
-    let mut output = HashMap::new();
-    for crate_data in data_per_crate {
-        for call_site in crate_data.call_sites {
-            // CBMC Now supports referencing callsites by label:
-            // https://github.com/diffblue/cbmc/pull/6508
-            let cbmc_call_site_name = format!("{}.{}", call_site.function_name, call_site.label);
-            let trait_def = call_site.trait_method;
-
-            // Look up all possibilities, defaulting to the empty set
-            let possibilities =
-                combined_possible_methods.get(&trait_def).unwrap_or(&vec![]).clone();
-            output.insert(cbmc_call_site_name, possibilities);
-        }
-    }
-
-    let f = File::create(output_filename)?;
-    let f = BufWriter::new(f);
-    serde_json::to_writer(f, &output)?;
-    Ok(())
-}
-
-/// From either a file or a path with multiple files, output the CBMC restrictions file we should use.
-fn collect_and_link_function_pointer_restrictions(
-    path: &Path,
-    output_filename: &Path,
-) -> Result<()> {
-    let md = std::fs::metadata(path)?;
-
-    // Fill with data from all files in that path with the expected suffix
-    let mut per_crate_restrictions = Vec::new();
-
-    if md.is_dir() {
-        for element in path.read_dir()? {
-            let path = element?.path();
-            if path.as_os_str().to_str().unwrap().ends_with(".restrictions.json") {
-                let restrictions = read_restrictions(&path)?;
-                per_crate_restrictions.push(restrictions);
-            }
-        }
-    } else if md.is_file() {
-        assert!(path.as_os_str().to_str().unwrap().ends_with(".restrictions.json"));
-        let restrictions = read_restrictions(path)?;
-        per_crate_restrictions.push(restrictions);
-    } else {
-        unreachable!("Path must be restrcitions file or directory containing restrictions files")
-    }
-
-    link_function_pointer_restrictions(per_crate_restrictions, output_filename)
-}
-
-/// Deserialize a *.restrictions.json file
-fn read_restrictions(path: &Path) -> Result<VtableCtxResults> {
-    let file = File::open(path)?;
-    let reader = BufReader::new(file);
-    let restrictions = serde_json::from_reader(reader)?;
-    Ok(restrictions)
-}

src/cargo-kani/src/call_single_file.rs

@@ -18,6 +18,8 @@ pub struct SingleOutputs {
     pub symtab: PathBuf,
     /// The vtable restrictions files, if any.
     pub restrictions: Option<PathBuf>,
+    /// The kani-metadata.json file written by kani-compiler.
+    pub metadata: PathBuf,
 }
 
 impl KaniSession {
@@ -34,7 +36,7 @@ impl KaniSession {
             let mut temps = self.temporaries.borrow_mut();
             temps.push(output_filename.clone());
             temps.push(typemap_filename);
-            temps.push(metadata_filename);
+            temps.push(metadata_filename.clone());
             if self.args.restrict_vtable() {
                 temps.push(restrictions_filename.clone());
             }
@@ -55,11 +57,11 @@ impl KaniSession {
                 args.push(t);
             }
         } else {
-            if self.args.function != "main" {
-                // Unless entry function for proof harness is main, compile code as lib.
-                // This ensures that rustc won't prune functions that are not reachable
-                // from main as well as enable compilation of crates that don't have a main
-                // function.
+            // If we specifically request "--function main" then don't override crate type
+            if Some("main".to_string()) != self.args.function {
+                // We only run against proof harnesses normally, and this change
+                // 1. Means we do not require a `fn main` to exist
+                // 2. Don't forget it also changes visibility rules.
                 args.push("--crate-type".into());
                 args.push("lib".into());
             }
@@ -80,6 +82,7 @@ impl KaniSession {
         Ok(SingleOutputs {
             outdir,
             symtab: output_filename,
+            metadata: metadata_filename,
             restrictions: if self.args.restrict_vtable() {
                 Some(restrictions_filename)
             } else {