Generate func_macro callbacks

Author: kulp

src/callbacks.rs

@@ -45,10 +45,8 @@ pub trait ParseCallbacks: fmt::Debug + UnwindSafe {
     ///
     /// The first parameter represents the name and argument list (including the
     /// parentheses) of the function-like macro. The second parameter represents
-    /// the expansion of the macro. It is not guaranteed that the whitespace of
-    /// the original is preserved, but it is guaranteed that tokenization will
-    /// not be changed.
-    fn func_macro(&self, _name: &str, _value: &str) {}
+    /// the expansion of the macro as a sequence of tokens.
+    fn func_macro(&self, _name: &str, _value: &[&[u8]]) {}
 
     /// This function should return whether, given an enum variant
     /// name, and value, this enum variant will forcibly be a constant.

src/clang.rs

@@ -709,30 +709,9 @@ impl Cursor {
 
     /// Gets the tokens that correspond to that cursor as  `cexpr` tokens.
     pub fn cexpr_tokens(self) -> Vec<cexpr::token::Token> {
-        use cexpr::token;
-
         self.tokens()
             .iter()
-            .filter_map(|token| {
-                let kind = match token.kind {
-                    CXToken_Punctuation => token::Kind::Punctuation,
-                    CXToken_Literal => token::Kind::Literal,
-                    CXToken_Identifier => token::Kind::Identifier,
-                    CXToken_Keyword => token::Kind::Keyword,
-                    // NB: cexpr is not too happy about comments inside
-                    // expressions, so we strip them down here.
-                    CXToken_Comment => return None,
-                    _ => {
-                        error!("Found unexpected token kind: {:?}", token);
-                        return None;
-                    }
-                };
-
-                Some(token::Token {
-                    kind,
-                    raw: token.spelling().to_vec().into_boxed_slice(),
-                })
-            })
+            .filter_map(|token| token.as_cexpr_token())
             .collect()
     }
 
@@ -826,6 +805,30 @@ impl ClangToken {
         };
         c_str.to_bytes()
     }
+
+    /// Converts a ClangToken to a `cexpr` token if possible.
+    pub fn as_cexpr_token(&self) -> Option<cexpr::token::Token> {
+        use cexpr::token;
+
+        let kind = match self.kind {
+            CXToken_Punctuation => token::Kind::Punctuation,
+            CXToken_Literal => token::Kind::Literal,
+            CXToken_Identifier => token::Kind::Identifier,
+            CXToken_Keyword => token::Kind::Keyword,
+            // NB: cexpr is not too happy about comments inside
+            // expressions, so we strip them down here.
+            CXToken_Comment => return None,
+            _ => {
+                error!("Found unexpected token kind: {:?}", self);
+                return None;
+            }
+        };
+
+        Some(token::Token {
+            kind,
+            raw: self.spelling().to_vec().into_boxed_slice(),
+        })
+    }
 }
 
 impl Drop for ClangToken {

src/ir/var.rs

@@ -8,6 +8,7 @@ use super::item::Item;
 use super::ty::{FloatKind, TypeKind};
 use crate::callbacks::MacroParsingBehavior;
 use crate::clang;
+use crate::clang::ClangToken;
 use crate::parse::{
     ClangItemParser, ClangSubItemParser, ParseError, ParseResult,
 };
@@ -130,6 +131,75 @@ fn default_macro_constant_type(value: i64) -> IntKind {
     }
 }
 
+/// Determines whether a set of tokens from a CXCursor_MacroDefinition
+/// represent a function-like macro. If so, calls the func_macro callback
+/// and returns `Err(ParseError::Continue)` to signal to skip further
+/// processing. If conversion to UTF-8 fails (it is performed only where it
+/// should be infallible), then `Err(ParseError::Continue)` is returned as well.
+fn handle_function_macro(
+    cursor: &clang::Cursor,
+    tokens: &[ClangToken],
+    callbacks: &dyn crate::callbacks::ParseCallbacks,
+) -> Result<(), ParseError> {
+    fn is_abutting(a: &ClangToken, b: &ClangToken) -> bool {
+        unsafe {
+            clang_sys::clang_equalLocations(
+                clang_sys::clang_getRangeEnd(a.extent),
+                clang_sys::clang_getRangeStart(b.extent),
+            ) != 0
+        }
+    }
+
+    let is_functional_macro =
+        // If we have libclang >= 3.9, we can use `is_macro_function_like()` and
+        // avoid checking for abutting tokens ourselves.
+        cursor.is_macro_function_like().unwrap_or_else(|| {
+            // If we cannot get a definitive answer from clang, we instead check
+            // for a parenthesis token immediately adjacent to (that is,
+            // abutting) the first token in the macro definition.
+            // TODO: Once we don't need the fallback check here, we can hoist
+            // the `is_macro_function_like` check into this function's caller,
+            // and thus avoid allocating the `tokens` vector for non-functional
+            // macros.
+            match tokens.get(0..2) {
+                Some([a, b]) => is_abutting(&a, &b) && b.spelling() == b"(",
+                _ => false,
+            }
+        });
+
+    if !is_functional_macro {
+        return Ok(());
+    }
+
+    let is_closing_paren = |t: &ClangToken| {
+        // Test cheap token kind before comparing exact spellings.
+        t.kind == clang_sys::CXToken_Punctuation && t.spelling() == b")"
+    };
+    let boundary = tokens.iter().position(is_closing_paren);
+
+    let mut spelled = tokens.iter().map(ClangToken::spelling);
+    // Add 1, to convert index to length.
+    let left = spelled
+        .by_ref()
+        .take(boundary.ok_or(ParseError::Continue)? + 1);
+    let left = left.collect::<Vec<_>>().concat();
+    let left = String::from_utf8(left).map_err(|_| ParseError::Continue)?;
+    let right = spelled;
+    // Drop last token with LLVM < 4.0, due to an LLVM bug.
+    //
+    // See:
+    //   https://bugs.llvm.org//show_bug.cgi?id=9069
+    let len = match (right.len(), crate::clang_version().parsed) {
+        (len, Some((v, _))) if len > 0 && v < 4 => len - 1,
+        (len, _) => len,
+    };
+    let right: Vec<_> = right.take(len).collect();
+    callbacks.func_macro(&left, &right);
+
+    // We handled the macro, skip future macro processing.
+    Err(ParseError::Continue)
+}
+
 impl ClangSubItemParser for Var {
     fn parse(
         cursor: clang::Cursor,
@@ -140,16 +210,20 @@ impl ClangSubItemParser for Var {
         use clang_sys::*;
         match cursor.kind() {
             CXCursor_MacroDefinition => {
+                let tokens: Vec<_> = cursor.tokens().iter().collect();
+
                 if let Some(callbacks) = ctx.parse_callbacks() {
                     match callbacks.will_parse_macro(&cursor.spelling()) {
                         MacroParsingBehavior::Ignore => {
                             return Err(ParseError::Continue);
                         }
                         MacroParsingBehavior::Default => {}
                     }
+
+                    handle_function_macro(&cursor, &tokens, callbacks)?;
                 }
 
-                let value = parse_macro(ctx, &cursor);
+                let value = parse_macro(ctx, &tokens);
 
                 let (id, value) = match value {
                     Some(v) => v,
@@ -316,11 +390,14 @@ impl ClangSubItemParser for Var {
 /// Try and parse a macro using all the macros parsed until now.
 fn parse_macro(
     ctx: &BindgenContext,
-    cursor: &clang::Cursor,
+    tokens: &[ClangToken],
 ) -> Option<(Vec<u8>, cexpr::expr::EvalResult)> {
     use cexpr::expr;
 
-    let mut cexpr_tokens = cursor.cexpr_tokens();
+    let mut cexpr_tokens: Vec<_> = tokens
+        .iter()
+        .filter_map(ClangToken::as_cexpr_token)
+        .collect();
 
     let parser = expr::IdentifierParser::new(ctx.parsed_macros());