Provide a list of tokens

If we cannot reproduce the original whitespace anyway, then we should
let the callee decide what to do with the tokens, rather than smashing
them together in a "safe" way.

Author: kulp

bindgen-integration/build.rs

@@ -65,7 +65,7 @@ impl ParseCallbacks for MacroCallback {
         }
     }
 
-    fn func_macro(&self, name: &str, value: &[u8]) {
+    fn func_macro(&self, name: &str, value: &[&[u8]]) {
         match name {
             "TESTMACRO_NONFUNCTIONAL" => {
                 panic!("func_macro was called for a non-functional macro");
@@ -76,19 +76,29 @@ impl ParseCallbacks for MacroCallback {
                 // change in the future, but it is safe by the definition of a
                 // token in C, whereas leaving the spaces out could change
                 // tokenization.
-                assert_eq!(value, b"- TESTMACRO_INTEGER");
+                assert_eq!(
+                    value,
+                    &["-".as_bytes(), "TESTMACRO_INTEGER".as_bytes()]
+                );
                 *self.seen_funcs.lock().unwrap() += 1;
             }
             "TESTMACRO_FUNCTIONAL_EMPTY(TESTMACRO_INTEGER)" => {
-                assert_eq!(value, b"");
+                assert_eq!(value, &[] as &[&[u8]]);
                 *self.seen_funcs.lock().unwrap() += 1;
             }
             "TESTMACRO_FUNCTIONAL_TOKENIZED(a,b,c,d,e)" => {
-                assert_eq!(value, b"a / b c d ## e");
+                assert_eq!(
+                    value,
+                    ["a", "/", "b", "c", "d", "##", "e"]
+                        .iter()
+                        .map(|s| s.as_bytes())
+                        .collect::<Vec<_>>()
+                        .as_slice()
+                );
                 *self.seen_funcs.lock().unwrap() += 1;
             }
             "TESTMACRO_FUNCTIONAL_SPLIT(a,b)" => {
-                assert_eq!(value, b"b , a");
+                assert_eq!(value, &[b"b", b",", b"a"]);
                 *self.seen_funcs.lock().unwrap() += 1;
             }
             _ => {

src/callbacks.rs

@@ -45,10 +45,8 @@ pub trait ParseCallbacks: fmt::Debug + UnwindSafe {
     ///
     /// The first parameter represents the name and argument list (including the
     /// parentheses) of the function-like macro. The second parameter represents
-    /// the expansion of the macro. It is not guaranteed that the whitespace of
-    /// the original is preserved, but it is guaranteed that tokenization will
-    /// not be changed.
-    fn func_macro(&self, _name: &str, _value: &[u8]) {}
+    /// the expansion of the macro as a sequence of tokens.
+    fn func_macro(&self, _name: &str, _value: &[&[u8]]) {}
 
     /// This function should return whether, given an enum variant
     /// name, and value, this enum variant will forcibly be a constant.

src/ir/var.rs

@@ -137,7 +137,7 @@ fn default_macro_constant_type(value: i64) -> IntKind {
 /// be infallible), then `Err(ParseError::Continue)` is returned as well.
 fn handle_function_macro(
     cursor: &clang::Cursor,
-    func_macro: impl FnOnce(&str, &[u8]),
+    func_macro: impl FnOnce(&str, &[&[u8]]),
 ) -> Result<(), ParseError> {
     // cexpr explicitly does not handle function-like macros, except for a
     // degenerate case in which it behaves in a non-conformant way. We prevent
@@ -161,7 +161,7 @@ fn handle_function_macro(
             let (left, right) = joined.split_at(boundary);
             let left = String::from_utf8(left.concat())
                 .map_err(|_| ParseError::Continue)?;
-            let right = right.join(&b' ');
+            let right: Vec<_> = right.iter().map(Vec::as_slice).collect();
             func_macro(&left, &right);
 
             // We handled the macro, skip future macro processing.