diff --git a/bindgen-cli/options.rs b/bindgen-cli/options.rs index 66bb3f1439..815a28f42a 100644 --- a/bindgen-cli/options.rs +++ b/bindgen-cli/options.rs @@ -400,6 +400,12 @@ struct BindgenCommand { /// Wrap unsafe operations in unsafe blocks. #[arg(long)] wrap_unsafe_ops: bool, + /// Enable fallback for clang macro parsing. + #[arg(long)] + clang_macro_fallback: bool, + /// Set path for temporary files generated by fallback for clang macro parsing. + #[arg(long)] + clang_macro_fallback_build_dir: Option, /// Derive custom traits on any kind of type. The CUSTOM value must be of the shape REGEX=DERIVE where DERIVE is a coma-separated list of derive macros. #[arg(long, value_name = "CUSTOM", value_parser = parse_custom_derive)] with_derive_custom: Vec<(Vec, String)>, @@ -554,6 +560,8 @@ where merge_extern_blocks, override_abi, wrap_unsafe_ops, + clang_macro_fallback, + clang_macro_fallback_build_dir, with_derive_custom, with_derive_custom_struct, with_derive_custom_enum, @@ -1023,6 +1031,14 @@ where builder = builder.wrap_unsafe_ops(true); } + if clang_macro_fallback { + builder = builder.clang_macro_fallback(); + } + + if let Some(path) = clang_macro_fallback_build_dir { + builder = builder.clang_macro_fallback_build_dir(path); + } + #[derive(Debug)] struct CustomDeriveCallback { derives: Vec, diff --git a/bindgen-tests/tests/expectations/tests/issue-753.rs b/bindgen-tests/tests/expectations/tests/issue-753.rs new file mode 100644 index 0000000000..3119ec569e --- /dev/null +++ b/bindgen-tests/tests/expectations/tests/issue-753.rs @@ -0,0 +1,4 @@ +#![allow(dead_code, non_snake_case, non_camel_case_types, non_upper_case_globals)] +pub const CONST: u32 = 5; +pub const OTHER_CONST: u32 = 6; +pub const LARGE_CONST: u32 = 1536; diff --git a/bindgen-tests/tests/expectations/tests/libclang-9/issue-753.rs b/bindgen-tests/tests/expectations/tests/libclang-9/issue-753.rs new file mode 100644 index 0000000000..fe64295a68 --- /dev/null +++ b/bindgen-tests/tests/expectations/tests/libclang-9/issue-753.rs @@ -0,0 +1 @@ +#![allow(dead_code, non_snake_case, non_camel_case_types, non_upper_case_globals)] \ No newline at end of file diff --git a/bindgen-tests/tests/headers/issue-753.h b/bindgen-tests/tests/headers/issue-753.h new file mode 100644 index 0000000000..94bb8e1c31 --- /dev/null +++ b/bindgen-tests/tests/headers/issue-753.h @@ -0,0 +1,7 @@ +// bindgen-flags: --clang-macro-fallback + +#define UINT32_C(c) c ## U + +#define CONST UINT32_C(5) +#define OTHER_CONST UINT32_C(6) +#define LARGE_CONST UINT32_C(6 << 8) diff --git a/bindgen/clang.rs b/bindgen/clang.rs index a9a0ec814f..04465b6be9 100644 --- a/bindgen/clang.rs +++ b/bindgen/clang.rs @@ -10,6 +10,7 @@ use std::cmp; use std::ffi::{CStr, CString}; use std::fmt; +use std::fs::OpenOptions; use std::hash::Hash; use std::hash::Hasher; use std::os::raw::{c_char, c_int, c_longlong, c_uint, c_ulong, c_ulonglong}; @@ -1868,6 +1869,27 @@ impl TranslationUnit { } } + /// Save a translation unit to the given file. + pub(crate) fn save(&mut self, file: &str) -> Result<(), CXSaveError> { + let file = if let Ok(cstring) = CString::new(file) { + cstring + } else { + return Err(CXSaveError_Unknown); + }; + let ret = unsafe { + clang_saveTranslationUnit( + self.x, + file.as_ptr(), + clang_defaultSaveOptions(self.x), + ) + }; + if ret != 0 { + Err(ret) + } else { + Ok(()) + } + } + /// Is this the null translation unit? pub(crate) fn is_null(&self) -> bool { self.x.is_null() @@ -1882,6 +1904,89 @@ impl Drop for TranslationUnit { } } +/// Translation unit used for macro fallback parsing +pub(crate) struct FallbackTranslationUnit { + file_path: String, + pch_paths: Vec, + idx: Box, + tu: TranslationUnit, +} + +impl fmt::Debug for FallbackTranslationUnit { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "FallbackTranslationUnit {{ }}") + } +} + +impl FallbackTranslationUnit { + /// Create a new fallback translation unit + pub(crate) fn new( + file: String, + pch_paths: Vec, + c_args: &[Box], + ) -> Option { + // Create empty file + OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&file) + .ok()?; + + let f_index = Box::new(Index::new(true, false)); + let f_translation_unit = TranslationUnit::parse( + &f_index, + &file, + c_args, + &[], + CXTranslationUnit_None, + )?; + Some(FallbackTranslationUnit { + file_path: file, + pch_paths, + tu: f_translation_unit, + idx: f_index, + }) + } + + /// Get reference to underlying translation unit. + pub(crate) fn translation_unit(&self) -> &TranslationUnit { + &self.tu + } + + /// Reparse a translation unit. + pub(crate) fn reparse( + &mut self, + unsaved_contents: &str, + ) -> Result<(), CXErrorCode> { + let unsaved = &[UnsavedFile::new(&self.file_path, unsaved_contents)]; + let mut c_unsaved: Vec = + unsaved.iter().map(|f| f.x).collect(); + let ret = unsafe { + clang_reparseTranslationUnit( + self.tu.x, + unsaved.len() as c_uint, + c_unsaved.as_mut_ptr(), + clang_defaultReparseOptions(self.tu.x), + ) + }; + if ret != 0 { + Err(ret) + } else { + Ok(()) + } + } +} + +impl Drop for FallbackTranslationUnit { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.file_path); + for pch in self.pch_paths.iter() { + let _ = std::fs::remove_file(pch); + } + } +} + /// A diagnostic message generated while parsing a translation unit. pub(crate) struct Diagnostic { x: CXDiagnostic, diff --git a/bindgen/ir/context.rs b/bindgen/ir/context.rs index 26247cdcc5..c4465df9ec 100644 --- a/bindgen/ir/context.rs +++ b/bindgen/ir/context.rs @@ -30,6 +30,7 @@ use std::borrow::Cow; use std::cell::{Cell, RefCell}; use std::collections::{BTreeSet, HashMap as StdHashMap}; use std::mem; +use std::path::Path; /// An identifier for some kind of IR item. #[derive(Debug, Copy, Clone, Eq, PartialOrd, Ord, Hash)] @@ -376,6 +377,9 @@ pub(crate) struct BindgenContext { /// The translation unit for parsing. translation_unit: clang::TranslationUnit, + /// The translation unit for macro fallback parsing. + fallback_tu: Option, + /// Target information that can be useful for some stuff. target_info: clang::TargetInfo, @@ -584,6 +588,7 @@ If you encounter an error missing from this list, please file an issue or a PR!" collected_typerefs: false, in_codegen: false, translation_unit, + fallback_tu: None, target_info, options, generated_bindgen_complex: Cell::new(false), @@ -2060,6 +2065,77 @@ If you encounter an error missing from this list, please file an issue or a PR!" &self.translation_unit } + /// Initialize fallback translation unit if it does not exist and + /// then return a mutable reference to the fallback translation unit. + pub(crate) fn try_ensure_fallback_translation_unit( + &mut self, + ) -> Option<&mut clang::FallbackTranslationUnit> { + if self.fallback_tu.is_none() { + let file = format!( + "{}/.macro_eval.c", + match self.options().clang_macro_fallback_build_dir { + Some(ref path) => path.as_os_str().to_str()?, + None => ".", + } + ); + + let index = clang::Index::new(false, false); + + let mut c_args = Vec::new(); + let mut pch_paths = Vec::new(); + for input_header in self.options().input_headers.iter() { + let path = Path::new(input_header.as_ref()); + let header_name = path + .file_name() + .and_then(|hn| hn.to_str()) + .map(|s| s.to_owned()); + let header_path = path + .parent() + .and_then(|hp| hp.to_str()) + .map(|s| s.to_owned()); + + let (header, pch) = if let (Some(ref hp), Some(hn)) = + (header_path, header_name) + { + let header_path = if hp.is_empty() { "." } else { hp }; + let header = format!("{header_path}/{hn}"); + let pch_path = if let Some(ref path) = + self.options().clang_macro_fallback_build_dir + { + path.as_os_str().to_str()? + } else { + header_path + }; + (header, format!("{pch_path}/{hn}.pch")) + } else { + return None; + }; + + let mut tu = clang::TranslationUnit::parse( + &index, + &header, + &[ + "-x".to_owned().into_boxed_str(), + "c-header".to_owned().into_boxed_str(), + ], + &[], + clang_sys::CXTranslationUnit_ForSerialization, + )?; + tu.save(&pch).ok()?; + + c_args.push("-include-pch".to_string().into_boxed_str()); + c_args.push(pch.clone().into_boxed_str()); + pch_paths.push(pch); + } + + self.fallback_tu = Some(clang::FallbackTranslationUnit::new( + file, pch_paths, &c_args, + )?); + } + + self.fallback_tu.as_mut() + } + /// Have we parsed the macro named `macro_name` already? pub(crate) fn parsed_macro(&self, macro_name: &[u8]) -> bool { self.parsed_macros.contains_key(macro_name) diff --git a/bindgen/ir/var.rs b/bindgen/ir/var.rs index a548ec881b..a970cf1588 100644 --- a/bindgen/ir/var.rs +++ b/bindgen/ir/var.rs @@ -389,9 +389,51 @@ impl ClangSubItemParser for Var { } } +/// This function uses a [`FallbackTranslationUnit`][clang::FallbackTranslationUnit] to parse each +/// macro that cannot be parsed by the normal bindgen process for `#define`s. +/// +/// To construct the [`FallbackTranslationUnit`][clang::FallbackTranslationUnit], first precompiled +/// headers are generated for all input headers. An empty temporary `.c` file is generated to pass +/// to the translation unit. On the evaluation of each macro, a [`String`] is generated with the +/// new contents of the empty file and passed in for reparsing. The precompiled headers and +/// preservation of the [`FallbackTranslationUnit`][clang::FallbackTranslationUnit] across macro +/// evaluations are both optimizations that have significantly improved the performance. +fn parse_macro_clang_fallback( + ctx: &mut BindgenContext, + cursor: &clang::Cursor, +) -> Option<(Vec, cexpr::expr::EvalResult)> { + if !ctx.options().clang_macro_fallback { + return None; + } + + let ftu = ctx.try_ensure_fallback_translation_unit()?; + let contents = format!("int main() {{ {}; }}", cursor.spelling(),); + ftu.reparse(&contents).ok()?; + // Children of root node of AST + let root_children = ftu.translation_unit().cursor().collect_children(); + // Last child in root is function declaration + // Should be FunctionDecl + let main_func = root_children.last()?; + // Children should all be statements in function declaration + let all_stmts = main_func.collect_children(); + // First child in all_stmts should be the statement containing the macro to evaluate + // Should be CompoundStmt + let macro_stmt = all_stmts.first()?; + // Children should all be expressions from the compound statement + let paren_exprs = macro_stmt.collect_children(); + // First child in all_exprs is the expression utilizing the given macro to be evaluated + // Should be ParenExpr + let paren = paren_exprs.first()?; + + Some(( + cursor.spelling().into_bytes(), + cexpr::expr::EvalResult::Int(Wrapping(paren.evaluate()?.as_int()?)), + )) +} + /// Try and parse a macro using all the macros parsed until now. fn parse_macro( - ctx: &BindgenContext, + ctx: &mut BindgenContext, cursor: &clang::Cursor, ) -> Option<(Vec, cexpr::expr::EvalResult)> { use cexpr::expr; @@ -402,7 +444,7 @@ fn parse_macro( match parser.macro_definition(&cexpr_tokens) { Ok((_, (id, val))) => Some((id.into(), val)), - _ => None, + _ => parse_macro_clang_fallback(ctx, cursor), } } diff --git a/bindgen/options/mod.rs b/bindgen/options/mod.rs index 3b3224caf0..dc453b13aa 100644 --- a/bindgen/options/mod.rs +++ b/bindgen/options/mod.rs @@ -21,9 +21,7 @@ use crate::HashMap; use crate::DEFAULT_ANON_FIELDS_PREFIX; use std::env; -#[cfg(feature = "experimental")] -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::rc::Rc; use as_args::AsArgs; @@ -2107,5 +2105,38 @@ options! { } }, as_args: "--emit-diagnostics", + }, + /// Whether to use Clang evaluation on temporary files as a fallback for macros that fail to + /// parse. + clang_macro_fallback: bool { + methods: { + /// Use Clang as a fallback for macros that fail to parse using `CExpr`. + /// + /// This uses a workaround to evaluate each macro in a temporary file. Because this + /// results in slower compilation, this option is opt-in. + pub fn clang_macro_fallback(mut self) -> Self { + self.options.clang_macro_fallback = true; + self + } + }, + as_args: "--clang-macro-fallback", + } + /// Path to use for temporary files created by clang macro fallback code like precompiled + /// headers. + clang_macro_fallback_build_dir: Option { + methods: { + /// Set a path to a directory to which `.c` and `.h.pch` files should be written for the + /// purpose of using clang to evaluate macros that can't be easily parsed. + /// + /// The default location for `.h.pch` files is the directory that the corresponding + /// `.h` file is located in. The default for the temporary `.c` file used for clang + /// parsing is the current working directory. Both of these defaults are overridden + /// by this option. + pub fn clang_macro_fallback_build_dir>(mut self, path: P) -> Self { + self.options.clang_macro_fallback_build_dir = Some(path.as_ref().to_owned()); + self + } + }, + as_args: "--clang-macro-fallback-build-dir", } }