Source-code based coverage is failing to identify some executable lines #83985

wpbrown · 2021-04-07T21:52:47Z

I tried to generate test code coverage with this source file using -Zinstrument-coverage and the LLVM tools.

Source File:
https://github.com/wpbrown/rust-codecov/blob/main/codecovsample/src/main.rs

I expected to see this happen: 76% line coverage

Instead, this happened: 83% line coverage.

cargo-kcov, -Zprofile+grcov, and tarpaulin agree on 76%.

In the output below lines 65-67, 116-118, 163-165 were not executable but should have been executable+uncovered. Line 169-170 should be executable+covered.

coverage output

#![allow(dead_code)]#![allow(dead_code)]
    2|       |
    3|       |use async_trait::async_trait;
    4|       |
    5|      1|fn main() {
    6|      1|    println!("codecovsample::main");
    7|      1|}
    8|       |
    9|       |enum Covered {
   10|       |    Variant1,
   11|       |    Variant2,
   12|       |}
   13|       |enum Uncovered {
   14|       |    Variant1,
   15|       |    Variant2,
   16|       |}
   17|       |enum PartiallyCovered {
   18|       |    Variant1,
   19|       |    Variant2,
   20|       |}
   21|       |
   22|      2|fn fn_covered_enum(input: Covered) {
   23|      2|    match input {
   24|      2|        Covered::Variant1 => { println!("Variant1"); }
                                           ^1
   25|      1|        Covered::Variant2 => { println!("Variant2"); }
   26|       |    }
   27|      2|}
   28|       |
   29|      0|fn fn_uncovered_enum(input: Uncovered) {
   30|      0|    match input {
   31|      0|        Uncovered::Variant1 => { println!("Variant1"); }
   32|      0|        Uncovered::Variant2 => { println!("Variant2"); }
   33|       |    }
   34|      0|}
   35|       |
   36|      1|fn fn_partially_covered_enum(input: PartiallyCovered) {
   37|      1|    match input {
   38|      1|        PartiallyCovered::Variant1 => { println!("Variant1"); }
   39|      0|        PartiallyCovered::Variant2 => { println!("Variant2"); }
   40|       |    }
   41|      1|}
   42|       |
   43|       |trait ATrait {
   44|       |    fn covered(&self);
   45|       |    fn uncovered(&self);
   46|       |    fn func_covered();    
   47|       |    fn func_uncovered();
   48|       |
   49|      2|    fn default_covered(&self) {
   50|      2|        println!("default_covered");
   51|      2|    }
  ------------------
  | <codecovsample::ATraitImplDirect as codecovsample::ATrait>::default_covered:
  |   49|      1|    fn default_covered(&self) {
  |   50|      1|        println!("default_covered");
  |   51|      1|    }
  ------------------
  | <codecovsample::ATraitImplGeneric as codecovsample::ATrait>::default_covered:
  |   49|      1|    fn default_covered(&self) {
  |   50|      1|        println!("default_covered");
  |   51|      1|    }
  ------------------
   52|       |    
   53|      0|    fn default_uncovered(&self) {
   54|      0|        println!("default_uncovered");
   55|      0|    }
   56|       |}
   57|       |trait BTrait {
   58|       |    fn covered(&self);
   59|       |    fn uncovered(&self);
   60|       |
   61|      1|    fn default_covered(&self) {
   62|      1|        println!("default_covered");
   63|      1|    }
   64|       |    
   65|       |    fn default_uncovered(&self) {
   66|       |        println!("default_uncovered");
   67|       |    }
   68|       |}
   69|       |
   70|       |struct ATraitImplDirect;
   71|       |
   72|       |impl ATrait for ATraitImplDirect {
   73|      1|    fn covered(&self) {
   74|      1|        println!("covered")
   75|      1|    }
   76|       |
   77|      0|    fn uncovered(&self) {
   78|      0|        println!("uncovered");
   79|      0|    }
   80|       |
   81|      1|    fn func_covered() {
   82|      1|        println!("func_covered");
   83|      1|    }
   84|       |
   85|      0|    fn func_uncovered() {
   86|      0|        println!("func_covered");
   87|      0|    }
   88|       |}
   89|       |
   90|       |struct ATraitImplGeneric;
   91|       |
   92|       |impl ATrait for ATraitImplGeneric {
   93|      1|    fn covered(&self) {
   94|      1|        println!("covered")
   95|      1|    }
   96|       |
   97|      0|    fn uncovered(&self) {
   98|      0|        println!("uncovered");
   99|      0|    }
  100|       |
  101|      1|    fn func_covered() {
  102|      1|        println!("func_covered");
  103|      1|    }
  104|       |
  105|      0|    fn func_uncovered() {
  106|      0|        println!("func_covered");
  107|      0|    }
  108|       |}
  109|       |struct BTraitImplBoxed;
  110|       |
  111|       |impl BTrait for BTraitImplBoxed {
  112|      1|    fn covered(&self) {
  113|      1|        println!("covered")
  114|      1|    }
  115|       |
  116|       |    fn uncovered(&self) {
  117|       |        println!("uncovered");
  118|       |    }
  119|       |}
  120|       |
  121|       |macro_rules! simple_rule {
  122|       |    () => {
  123|       |        println!("simple rule");
  124|       |    };
  125|       |}
  126|       |
  127|      1|fn call_simple_rule() {
  128|      1|    simple_rule!();
  129|      1|}
  130|       |
  131|      1|fn call_generic_atrait<T: ATrait>(input: T) {
  132|      1|    input.covered();
  133|      1|    input.default_covered();
  134|      1|    T::func_covered();
  135|      1|}
  136|       |
  137|      1|async fn async_func() {
  138|       |    println!("async_func");
  139|       |}
  140|       |
  141|      1|async fn async_func_anon() {
  142|      1|    let x = async {
  143|      1|        println!("async_func");
  144|      1|    };
  145|      1|    x.await;
  146|      1|}
  147|       |
  148|       |#[async_trait]
  149|       |trait AsyncTrait {
  150|       |    async fn covered(&self);
  151|       |    async fn uncovered(&self);
  152|       |}
  153|       |
  154|       |struct AsyncTraitImpl;
  155|       |
  156|       |#[async_trait]
  157|       |impl AsyncTrait for AsyncTraitImpl {
  158|      1|    async fn covered(&self) {
  159|      1|        println!("covered");
  160|      1|        async_func_from_trait_covered().await;
  161|      1|    }
  162|       |
  163|       |    async fn uncovered(&self) {
  164|       |        println!("uncovered");
  165|       |    }
  166|       |}
  167|       |
  168|      1|async fn async_func_from_trait_covered() {
  169|       |    println!("covered async func from trait");
  170|       |}
  171|       |
  172|       |#[cfg(test)]
  173|       |mod tests {
  174|       |    use futures::executor::block_on;
  175|       |
  176|       |    use super::*;
  177|       |
  178|       |    #[test]
  179|      1|    fn test_main() {
  ------------------
  | codecovsample::tests::test_main::{closure#0}:
  |  179|      1|    fn test_main() {
  ------------------
  180|      1|        main();
  181|      1|    }
  ------------------
  | codecovsample::tests::test_main:
  |  179|      1|    fn test_main() {
  |  180|      1|        main();
  |  181|      1|    }
  ------------------
  182|       |
  183|       |    #[test]
  184|      1|    fn cover_enum() {
  ------------------
  | codecovsample::tests::cover_enum::{closure#0}:
  |  184|      1|    fn cover_enum() {
  ------------------
  185|      1|        fn_covered_enum(Covered::Variant1);
  186|      1|        fn_covered_enum(Covered::Variant2);
  187|      1|    }
  ------------------
  | codecovsample::tests::cover_enum:
  |  184|      1|    fn cover_enum() {
  |  185|      1|        fn_covered_enum(Covered::Variant1);
  |  186|      1|        fn_covered_enum(Covered::Variant2);
  |  187|      1|    }
  ------------------
  188|       |
  189|       |    #[test]
  190|      1|    fn partially_cover_enum() {
  ------------------
  | codecovsample::tests::partially_cover_enum::{closure#0}:
  |  190|      1|    fn partially_cover_enum() {
  ------------------
  191|      1|        fn_partially_covered_enum(PartiallyCovered::Variant1);
  192|      1|    }
  ------------------
  | codecovsample::tests::partially_cover_enum:
  |  190|      1|    fn partially_cover_enum() {
  |  191|      1|        fn_partially_covered_enum(PartiallyCovered::Variant1);
  |  192|      1|    }
  ------------------
  193|       |
  194|       |    #[test]
  195|      1|    fn cover_atrait_direct() {
  ------------------
  | codecovsample::tests::cover_atrait_direct::{closure#0}:
  |  195|      1|    fn cover_atrait_direct() {
  ------------------
  196|      1|        let x = ATraitImplDirect;
  197|      1|        x.covered();
  198|      1|        x.default_covered();
  199|      1|        <ATraitImplDirect as ATrait>::func_covered();
  200|      1|    }
  ------------------
  | codecovsample::tests::cover_atrait_direct:
  |  195|      1|    fn cover_atrait_direct() {
  |  196|      1|        let x = ATraitImplDirect;
  |  197|      1|        x.covered();
  |  198|      1|        x.default_covered();
  |  199|      1|        <ATraitImplDirect as ATrait>::func_covered();
  |  200|      1|    }
  ------------------
  201|       |    #[test]
  202|      1|    fn cover_atrait_boxed() {
  ------------------
  | codecovsample::tests::cover_atrait_boxed::{closure#0}:
  |  202|      1|    fn cover_atrait_boxed() {
  ------------------
  203|      1|        let x: Box<dyn BTrait> = Box::new(BTraitImplBoxed);
  204|      1|        x.covered();
  205|      1|        x.default_covered();
  206|      1|    }
  ------------------
  | codecovsample::tests::cover_atrait_boxed:
  |  202|      1|    fn cover_atrait_boxed() {
  |  203|      1|        let x: Box<dyn BTrait> = Box::new(BTraitImplBoxed);
  |  204|      1|        x.covered();
  |  205|      1|        x.default_covered();
  |  206|      1|    }
  ------------------
  207|       |
  208|       |    #[test]
  209|      1|    fn cover_simple_rule() {
  ------------------
  | codecovsample::tests::cover_simple_rule::{closure#0}:
  |  209|      1|    fn cover_simple_rule() {
  ------------------
  210|      1|        call_simple_rule();
  211|      1|    }
  ------------------
  | codecovsample::tests::cover_simple_rule:
  |  209|      1|    fn cover_simple_rule() {
  |  210|      1|        call_simple_rule();
  |  211|      1|    }
  ------------------
  212|       |
  213|       |    #[test]
  214|      1|    fn cover_generic_atrait() {
  ------------------
  | codecovsample::tests::cover_generic_atrait::{closure#0}:
  |  214|      1|    fn cover_generic_atrait() {
  ------------------
  215|      1|        let x = ATraitImplGeneric;
  216|      1|        call_generic_atrait(x);
  217|      1|    }
  ------------------
  | codecovsample::tests::cover_generic_atrait:
  |  214|      1|    fn cover_generic_atrait() {
  |  215|      1|        let x = ATraitImplGeneric;
  |  216|      1|        call_generic_atrait(x);
  |  217|      1|    }
  ------------------
  218|       |
  219|       |    #[test]
  220|      1|    fn cover_async_funcs() {
  ------------------
  | codecovsample::tests::cover_async_funcs::{closure#0}:
  |  220|      1|    fn cover_async_funcs() {
  ------------------
  221|      1|        block_on(async {
  222|      1|            async_func().await;
  223|      1|            async_func_anon().await;
  224|      1|        });
  225|      1|    }
  ------------------
  | codecovsample::tests::cover_async_funcs:
  |  220|      1|    fn cover_async_funcs() {
  |  221|      1|        block_on(async {
  |  222|       |            async_func().await;
  |  223|       |            async_func_anon().await;
  |  224|      1|        });
  |  225|      1|    }
  ------------------
  226|       |
  227|       |    #[test]
  228|      1|    fn cover_async_trait() {
  ------------------
  | codecovsample::tests::cover_async_trait::{closure#0}:
  |  228|      1|    fn cover_async_trait() {
  ------------------
  229|      1|        block_on(async {
  230|      1|            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  231|      1|            x.covered().await;
  232|      1|        });
  233|      1|    }
  ------------------
  | codecovsample::tests::cover_async_trait:
  |  228|      1|    fn cover_async_trait() {
  |  229|      1|        block_on(async {
  |  230|       |            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  |  231|       |            x.covered().await;
  |  232|      1|        });
  |  233|      1|    }
  ------------------
  234|       |}

Meta

rustc --version --verbose:

rustc 1.53.0-nightly (07e0e2ec2 2021-03-24)
binary: rustc
commit-hash: 07e0e2ec268c140e607e1ac7f49f145612d0f597
commit-date: 2021-03-24
host: x86_64-unknown-linux-gnu
release: 1.53.0-nightly
LLVM version: 12.0.0

The text was updated successfully, but these errors were encountered:

eggyal · 2021-04-07T21:55:04Z

@rustbot label: +A-code-coverage
cc @richkadel

richkadel · 2021-04-07T23:40:25Z

Thanks for the bug report @wpbrown !

There was a recent bug fix (PR #83774) that addresses most of the missing coverage in your example (except for the last one).

I just tested with rustc 1.53.0-nightly (c755ee4ce 2021-04-04) and I see the expected coverage now for the first three ranges you reported seeing no coverage. These three are now showing coverage (with 0 executions) as expected:

In the output below lines 65-67, 116-118, 163-165 were not executable but should have been executable+uncovered.

I was surprised to see no coverage for line 169, and I do see the println!() did print, so that does seem to be a bug.

So as of today, this issue is still a bug that I will need to look into further:

Line 169-170 should be executable+covered.

richkadel · 2021-04-08T00:19:48Z

cc: @tmandry @wesleywiser

Note, here is a snippet showing the missing coverage and code relevant to its codepath.

I haven't thoroughly investigated this. I should be able to reproduce this in a simplified test (without the async_trait and executor dependencies) that I can add to our unit tests.

I don't have a theory yet. It could either be the closure's MIR is not included in coverage (seems unlikely to me), or llvm-cov is silently dropping coverage because it doesn't like something. (This is what was happening for the other missing coverage reported in this issue, that has since been resolved.)

The test (shown at the bottom) calls and awaits the implementation of covered() below:

  148|       |#[async_trait]
  149|       |trait AsyncTrait {
  150|       |    async fn covered(&self);
  151|       |    async fn uncovered(&self);
  152|       |}
  153|       |
  154|       |struct AsyncTraitImpl;
  155|       |
  156|       |#[async_trait]
  157|       |impl AsyncTrait for AsyncTraitImpl {
  158|      1|    async fn covered(&self) {
  159|      1|        println!("covered");
  160|      1|        async_func_from_trait_covered().await;
  161|      1|    }
  162|       |
  163|      0|    async fn uncovered(&self) {
  164|      0|        println!("uncovered");
  165|      0|    }
  166|       |}
  167|       |

I do see the test output, including the println!() from covered() above, and from the awaited call to async_func_from_trait_covered(), below:

covered
covered async func from trait

The async closure seems to be missing coverage, as shown here. Note the wrapper function is covered and called (showing the coverage of 1 on line 168), but there should also be coverage for the awaited implicit closure (lines 168-170, I believe).

Since the program output proves this is called, the coverage should be showing 1 execution.

  168|      1|async fn async_func_from_trait_covered() {
  169|       |    println!("covered async func from trait");
  170|       |}

And the test that invokes it is covered:

  226|       |
  227|       |    #[test]
  228|      1|    fn cover_async_trait() {
  ------------------
  | codecovsample::tests::cover_async_trait::{closure#0}:
  |  228|      1|    fn cover_async_trait() {
  ------------------
  229|      1|        block_on(async {
  230|      1|            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  231|      1|            x.covered().await;
  232|      1|        });
  233|      1|    }
  ------------------
  | codecovsample::tests::cover_async_trait:
  |  228|      1|    fn cover_async_trait() {
  |  229|      1|        block_on(async {
  |  230|       |            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  |  231|       |            x.covered().await;
  |  232|      1|        });
  |  233|      1|    }
  ------------------
  234|       |}

wpbrown · 2021-04-13T02:48:00Z

Thank you. I see improvement in the latest nightly. I minified the test case to the async issues for anyone looking at this (but it still uses executor and async_trait.)

https://github.com/wpbrown/rust-codecov/blob/async_reduce/codecovsample/src/main.rs

Since this function is missing coverage too the issue isn't specific to async_trait thought.

    8|       |
    9|      1|async fn async_func() {
   10|       |    println!("async_func");
   11|       |}
   12|       |

richkadel · 2021-04-18T22:13:38Z

So it turns out that the internal (auto-generated) closure actually does get coverage, but for some reason the code region for the println!() macro, by itself, is not added or not counted. I've created a draft PR with a new test and current results, demonstrating this issue. Once I figure out how to fix it, I'll submit the PR with the updated results.

rust/src/test/run-make-fulldeps/coverage-reports/expected_show_coverage.async2.txt

Lines 9 to 34 in 3ece606

    
            9|      1|fn non_async_func() { 
        
           10|      1|    println!("non_async_func was covered"); 
        
           11|      1|    let b = true; 
        
           12|      1|    if b { 
        
           13|      1|        println!("non_async_func println in block"); 
        
           14|      1|    } 
        
           15|      1|} 
        
           16|       | 
        
           17|       |// FIXME(#83985): The auto-generated closure in an async function is failing to include 
        
           18|       |// the println!() and `let` assignment lines in the coverage code region(s), as it does in the 
        
           19|       |// non-async function above, unless the `println!()` is inside a covered block. 
        
           20|      1|async fn async_func() { 
        
           21|       |    println!("async_func was covered"); 
        
           22|       |    let b = true; 
        
           23|      1|    if b { 
        
           24|      1|        println!("async_func println in block"); 
        
           25|      1|    } 
        
                           ^0 
        
           26|      1|} 
        
           27|       | 
        
           28|       |// FIXME(#83985): As above, this async function only has the `println!()` macro call, which is not 
        
           29|       |// showing coverage, so the entire async closure _appears_ uncovered; but this is not exactly true. 
        
           30|       |// It's only certain kinds of lines and/or their context that results in missing coverage. 
        
           31|      1|async fn async_func_just_println() { 
        
           32|       |    println!("async_func_just_println was covered"); 
        
           33|       |}

richkadel · 2021-04-19T16:46:04Z

Update: #84323 fixes the issue.

coverage of async function bodies should match non-async This fixes some missing coverage within async function bodies. Commit 1 demonstrates the problem in the fixed issue, and commit 2 corrects it. Fixes: rust-lang#83985

wpbrown added the C-bug Category: This is a bug. label Apr 7, 2021

rustbot added the A-code-coverage Area: Source-based code coverage (-Cinstrument-coverage) label Apr 7, 2021

This was referenced Apr 8, 2021

Some Rust functions still missing coverage (-Z instrument-coverage) #79617

Closed

Tracking Issue for LLVM InstrProf code coverage (current flag: -Zinstrument-coverage) #79121

Closed

richkadel mentioned this issue Apr 18, 2021

coverage of async function bodies should match non-async #84323

Merged

bors closed this as completed in 3ece606 Apr 20, 2021

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Source-code based coverage is failing to identify some executable lines #83985

Source-code based coverage is failing to identify some executable lines #83985

wpbrown commented Apr 7, 2021

eggyal commented Apr 7, 2021

richkadel commented Apr 7, 2021 •

edited

Loading

richkadel commented Apr 8, 2021

wpbrown commented Apr 13, 2021

richkadel commented Apr 18, 2021

richkadel commented Apr 19, 2021

Source-code based coverage is failing to identify some executable lines #83985

Source-code based coverage is failing to identify some executable lines #83985

Comments

wpbrown commented Apr 7, 2021

Meta

eggyal commented Apr 7, 2021

richkadel commented Apr 7, 2021 • edited Loading

richkadel commented Apr 8, 2021

wpbrown commented Apr 13, 2021

richkadel commented Apr 18, 2021

richkadel commented Apr 19, 2021

richkadel commented Apr 7, 2021 •

edited

Loading