more efficient bit counting

Author: alamb

datafusion/core/src/physical_plan/file_format/row_filter.rs

@@ -118,9 +118,7 @@ impl ArrowPredicate for DatafusionArrowPredicate {
             Ok(array) => {
                 if let Some(mask) = array.as_any().downcast_ref::<BooleanArray>() {
                     let bool_arr = BooleanArray::from(mask.data().clone());
-                    // TODO is there a more efficient way to count the rows that are filtered?
-                    let num_filtered =
-                        bool_arr.iter().filter(|p| !matches!(p, Some(true))).count();
+                    let num_filtered = bool_arr.len() - true_count(&bool_arr);
                     self.rows_filtered.add(num_filtered);
                     timer.stop();
                     Ok(bool_arr)
@@ -138,6 +136,27 @@ impl ArrowPredicate for DatafusionArrowPredicate {
     }
 }
 
+/// Return the number of non null true vaulues in an array
+// TODO remove when https://github.com/apache/arrow-rs/issues/2963 is released
+fn true_count(arr: &BooleanArray) -> usize {
+    match arr.data().null_buffer() {
+        Some(nulls) => {
+            let null_chunks = nulls.bit_chunks(arr.offset(), arr.len());
+            let value_chunks = arr.values().bit_chunks(arr.offset(), arr.len());
+            null_chunks
+                .iter()
+                .zip(value_chunks.iter())
+                .chain(std::iter::once((
+                    null_chunks.remainder_bits(),
+                    value_chunks.remainder_bits(),
+                )))
+                .map(|(a, b)| (a & b).count_ones() as usize)
+                .sum()
+        }
+        None => arr.values().count_set_bits_offset(arr.offset(), arr.len()),
+    }
+}
+
 /// A candidate expression for creating a `RowFilter` contains the
 /// expression as well as data to estimate the cost of evaluating
 /// the resulting expression.