From e1f314eb44b09268c6d05ff152f3ddd61e635135 Mon Sep 17 00:00:00 2001 From: Christian Poveda Date: Fri, 23 Sep 2022 15:20:56 -0500 Subject: [PATCH 1/8] move codegen postprocessing to its own module --- src/codegen/mod.rs | 5 +- src/codegen/postprocessing.rs | 148 ++++++++++++++++++++++++++++++++++ src/lib.rs | 113 +------------------------- 3 files changed, 152 insertions(+), 114 deletions(-) create mode 100644 src/codegen/postprocessing.rs diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index 843d5111a1..bc2c9fe21b 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -3,6 +3,7 @@ mod error; mod helpers; mod impl_debug; mod impl_partialeq; +mod postprocessing; pub mod struct_layout; #[cfg(test)] @@ -4414,7 +4415,7 @@ impl CodeGenerator for ObjCInterface { pub(crate) fn codegen( context: BindgenContext, -) -> (Vec, BindgenOptions, Vec) { +) -> (proc_macro2::TokenStream, BindgenOptions, Vec) { context.gen(|context| { let _t = context.timer("codegen"); let counter = Cell::new(0); @@ -4464,7 +4465,7 @@ pub(crate) fn codegen( result.push(dynamic_items_tokens); } - result.items + postprocessing::postprocessing(result.items, context.options()) }) } diff --git a/src/codegen/postprocessing.rs b/src/codegen/postprocessing.rs new file mode 100644 index 0000000000..728322980f --- /dev/null +++ b/src/codegen/postprocessing.rs @@ -0,0 +1,148 @@ +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::Item; + +use crate::BindgenOptions; + +macro_rules! decl_postprocessing { + ($($ty:ty),*) => { + pub(crate) fn postprocessing( + items: Vec, + options: &BindgenOptions, + ) -> TokenStream { + // Whether any of the enabled options requires `syn`. + let require_syn = $(<$ty as PostProcessing>::should_run(options))||*; + + if !require_syn { + return items.into_iter().collect(); + } + + let module_wrapped_tokens = + quote!(mod wrapper_for_sorting_hack { #( #items )* }); + + // This syn business is a hack, for now. This means that we are re-parsing already + // generated code using `syn` (as opposed to `quote`) because `syn` provides us more + // control over the elements. + // One caveat is that some of the items coming from `quote`d output might have + // multiple items within them. Hence, we have to wrap the incoming in a `mod`. + // The two `unwrap`s here are deliberate because + // The first one won't panic because we build the `mod` and know it is there + // The second one won't panic because we know original output has something in + // it already. + let mut items = + syn::parse2::(module_wrapped_tokens) + .unwrap() + .content + .unwrap() + .1; + + $(if <$ty as PostProcessing>::should_run(options) { + <$ty as PostProcessing>::run(&mut items); + })* + + let synful_items = items + .into_iter() + .map(|item| item.into_token_stream()); + + quote! { #( #synful_items )* } + } + }; +} + +decl_postprocessing! { + MergeExternBlocks, + SortSemantically +} + +trait PostProcessing { + fn should_run(options: &BindgenOptions) -> bool; + + fn run(items: &mut Vec); +} + +struct SortSemantically; + +impl PostProcessing for SortSemantically { + #[inline] + fn should_run(options: &BindgenOptions) -> bool { + options.sort_semantically + } + + fn run(items: &mut Vec) { + items.sort_by_key(|item| match item { + Item::Type(_) => 0, + Item::Struct(_) => 1, + Item::Const(_) => 2, + Item::Fn(_) => 3, + Item::Enum(_) => 4, + Item::Union(_) => 5, + Item::Static(_) => 6, + Item::Trait(_) => 7, + Item::TraitAlias(_) => 8, + Item::Impl(_) => 9, + Item::Mod(_) => 10, + Item::Use(_) => 11, + Item::Verbatim(_) => 12, + Item::ExternCrate(_) => 13, + Item::ForeignMod(_) => 14, + Item::Macro(_) => 15, + Item::Macro2(_) => 16, + _ => 18, + }); + } +} + +struct MergeExternBlocks; + +impl PostProcessing for MergeExternBlocks { + #[inline] + fn should_run(options: &BindgenOptions) -> bool { + options.merge_extern_blocks + } + + fn run(items: &mut Vec) { + // Keep all the extern blocks in a different `Vec` for faster search. + let mut foreign_mods = Vec::::new(); + + for item in std::mem::take(items) { + match item { + Item::ForeignMod(syn::ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }) => { + let mut exists = false; + for foreign_mod in &mut foreign_mods { + // Check if there is a extern block with the same ABI and + // attributes. + if foreign_mod.attrs == attrs && foreign_mod.abi == abi + { + // Merge the items of the two blocks. + foreign_mod.items.extend_from_slice(&foreign_items); + exists = true; + break; + } + } + // If no existing extern block had the same ABI and attributes, store + // it. + if !exists { + foreign_mods.push(syn::ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }); + } + } + // If the item is not an extern block, we don't have to do anything. + _ => items.push(item), + } + } + + // Move all the extern blocks alongiside the rest of the items. + for foreign_mod in foreign_mods { + items.push(Item::ForeignMod(foreign_mod)); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 3c89368f11..d0be8604d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -88,7 +88,6 @@ use std::{env, iter}; // Some convenient typedefs for a fast hash map and hash set. type HashMap = ::rustc_hash::FxHashMap; type HashSet = ::rustc_hash::FxHashSet; -use quote::ToTokens; pub(crate) use std::collections::hash_map::Entry; /// Default prefix for the anon fields. @@ -2118,11 +2117,6 @@ struct BindgenOptions { impl ::std::panic::UnwindSafe for BindgenOptions {} impl BindgenOptions { - /// Whether any of the enabled options requires `syn`. - fn require_syn(&self) -> bool { - self.sort_semantically || self.merge_extern_blocks - } - fn build(&mut self) { let mut regex_sets = [ &mut self.allowlisted_vars, @@ -2555,112 +2549,7 @@ impl Bindings { parse(&mut context)?; } - let (items, options, warnings) = codegen::codegen(context); - - let module = if options.require_syn() { - let module_wrapped_tokens = - quote!(mod wrapper_for_sorting_hack { #( #items )* }); - - // This syn business is a hack, for now. This means that we are re-parsing already - // generated code using `syn` (as opposed to `quote`) because `syn` provides us more - // control over the elements. - // One caveat is that some of the items coming from `quote`d output might have - // multiple items within them. Hence, we have to wrap the incoming in a `mod`. - // The two `unwrap`s here are deliberate because - // The first one won't panic because we build the `mod` and know it is there - // The second one won't panic because we know original output has something in - // it already. - let mut syn_parsed_items = - syn::parse2::(module_wrapped_tokens) - .unwrap() - .content - .unwrap() - .1; - - if options.merge_extern_blocks { - // Here we will store all the items after deduplication. - let mut items = Vec::new(); - - // Keep all the extern blocks in a different `Vec` for faster search. - let mut foreign_mods = Vec::::new(); - for item in syn_parsed_items { - match item { - syn::Item::ForeignMod(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }) => { - let mut exists = false; - for foreign_mod in &mut foreign_mods { - // Check if there is a extern block with the same ABI and - // attributes. - if foreign_mod.attrs == attrs && - foreign_mod.abi == abi - { - // Merge the items of the two blocks. - foreign_mod - .items - .extend_from_slice(&foreign_items); - exists = true; - break; - } - } - // If no existing extern block had the same ABI and attributes, store - // it. - if !exists { - foreign_mods.push(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }); - } - } - // If the item is not an extern block, we don't have to do anything. - _ => items.push(item), - } - } - - // Move all the extern blocks alongiside the rest of the items. - for foreign_mod in foreign_mods { - items.push(syn::Item::ForeignMod(foreign_mod)); - } - - syn_parsed_items = items; - } - - if options.sort_semantically { - syn_parsed_items.sort_by_key(|item| match item { - syn::Item::Type(_) => 0, - syn::Item::Struct(_) => 1, - syn::Item::Const(_) => 2, - syn::Item::Fn(_) => 3, - syn::Item::Enum(_) => 4, - syn::Item::Union(_) => 5, - syn::Item::Static(_) => 6, - syn::Item::Trait(_) => 7, - syn::Item::TraitAlias(_) => 8, - syn::Item::Impl(_) => 9, - syn::Item::Mod(_) => 10, - syn::Item::Use(_) => 11, - syn::Item::Verbatim(_) => 12, - syn::Item::ExternCrate(_) => 13, - syn::Item::ForeignMod(_) => 14, - syn::Item::Macro(_) => 15, - syn::Item::Macro2(_) => 16, - _ => 18, - }); - } - - let synful_items = syn_parsed_items - .into_iter() - .map(|item| item.into_token_stream()); - - quote! { #( #synful_items )* } - } else { - quote! { #( #items )* } - }; + let (module, options, warnings) = codegen::codegen(context); Ok(Bindings { options, From 0e4c1ae92c8ab89bf2285cf3b4540821fb480850 Mon Sep 17 00:00:00 2001 From: Christian Poveda Date: Fri, 23 Sep 2022 15:24:19 -0500 Subject: [PATCH 2/8] update `CONTRIBUTING.md` section about `syn` --- CONTRIBUTING.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2974ba4254..3d757ccdff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -323,14 +323,12 @@ generated Rust code are implemented using the [`syn`](https://docs.rs/syn) crate ### Implementing new options using `syn` -Here is a list of recommendations to be followed if a new option can be -implemented using the `syn` crate: +If a new option can be implemented using the `syn` crate it should be added to +the `codegen::postprocessing` module by following these steps: -- The `BindgenOptions::require_syn` method must be updated to reflect that this - new option requires parsing the generated Rust code with `syn`. - -- The implementation of the new option should be added at the end of - `Bindings::generate`, inside the `if options.require_syn() { ... }` block. +- Introduce a new `struct` with no fields. +- Implement the `PostProcessing` trait for the `struct`. +- Add the `struct` to the `decl_postprocessing` macro invocation. ## Pull Requests and Code Reviews From a372499b938c817719ccc1958bbdcf71f21e8447 Mon Sep 17 00:00:00 2001 From: Christian Poveda Date: Fri, 23 Sep 2022 15:20:56 -0500 Subject: [PATCH 3/8] move codegen postprocessing to its own module --- src/codegen/mod.rs | 5 +- src/codegen/postprocessing.rs | 148 ++++++++++++++++++++++++++++++++++ src/lib.rs | 113 +------------------------- 3 files changed, 152 insertions(+), 114 deletions(-) create mode 100644 src/codegen/postprocessing.rs diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index 8eb7b0134c..5660b1262f 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -3,6 +3,7 @@ mod error; mod helpers; mod impl_debug; mod impl_partialeq; +mod postprocessing; pub mod struct_layout; #[cfg(test)] @@ -4439,7 +4440,7 @@ impl CodeGenerator for ObjCInterface { pub(crate) fn codegen( context: BindgenContext, -) -> (Vec, BindgenOptions, Vec) { +) -> (proc_macro2::TokenStream, BindgenOptions, Vec) { context.gen(|context| { let _t = context.timer("codegen"); let counter = Cell::new(0); @@ -4489,7 +4490,7 @@ pub(crate) fn codegen( result.push(dynamic_items_tokens); } - result.items + postprocessing::postprocessing(result.items, context.options()) }) } diff --git a/src/codegen/postprocessing.rs b/src/codegen/postprocessing.rs new file mode 100644 index 0000000000..728322980f --- /dev/null +++ b/src/codegen/postprocessing.rs @@ -0,0 +1,148 @@ +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::Item; + +use crate::BindgenOptions; + +macro_rules! decl_postprocessing { + ($($ty:ty),*) => { + pub(crate) fn postprocessing( + items: Vec, + options: &BindgenOptions, + ) -> TokenStream { + // Whether any of the enabled options requires `syn`. + let require_syn = $(<$ty as PostProcessing>::should_run(options))||*; + + if !require_syn { + return items.into_iter().collect(); + } + + let module_wrapped_tokens = + quote!(mod wrapper_for_sorting_hack { #( #items )* }); + + // This syn business is a hack, for now. This means that we are re-parsing already + // generated code using `syn` (as opposed to `quote`) because `syn` provides us more + // control over the elements. + // One caveat is that some of the items coming from `quote`d output might have + // multiple items within them. Hence, we have to wrap the incoming in a `mod`. + // The two `unwrap`s here are deliberate because + // The first one won't panic because we build the `mod` and know it is there + // The second one won't panic because we know original output has something in + // it already. + let mut items = + syn::parse2::(module_wrapped_tokens) + .unwrap() + .content + .unwrap() + .1; + + $(if <$ty as PostProcessing>::should_run(options) { + <$ty as PostProcessing>::run(&mut items); + })* + + let synful_items = items + .into_iter() + .map(|item| item.into_token_stream()); + + quote! { #( #synful_items )* } + } + }; +} + +decl_postprocessing! { + MergeExternBlocks, + SortSemantically +} + +trait PostProcessing { + fn should_run(options: &BindgenOptions) -> bool; + + fn run(items: &mut Vec); +} + +struct SortSemantically; + +impl PostProcessing for SortSemantically { + #[inline] + fn should_run(options: &BindgenOptions) -> bool { + options.sort_semantically + } + + fn run(items: &mut Vec) { + items.sort_by_key(|item| match item { + Item::Type(_) => 0, + Item::Struct(_) => 1, + Item::Const(_) => 2, + Item::Fn(_) => 3, + Item::Enum(_) => 4, + Item::Union(_) => 5, + Item::Static(_) => 6, + Item::Trait(_) => 7, + Item::TraitAlias(_) => 8, + Item::Impl(_) => 9, + Item::Mod(_) => 10, + Item::Use(_) => 11, + Item::Verbatim(_) => 12, + Item::ExternCrate(_) => 13, + Item::ForeignMod(_) => 14, + Item::Macro(_) => 15, + Item::Macro2(_) => 16, + _ => 18, + }); + } +} + +struct MergeExternBlocks; + +impl PostProcessing for MergeExternBlocks { + #[inline] + fn should_run(options: &BindgenOptions) -> bool { + options.merge_extern_blocks + } + + fn run(items: &mut Vec) { + // Keep all the extern blocks in a different `Vec` for faster search. + let mut foreign_mods = Vec::::new(); + + for item in std::mem::take(items) { + match item { + Item::ForeignMod(syn::ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }) => { + let mut exists = false; + for foreign_mod in &mut foreign_mods { + // Check if there is a extern block with the same ABI and + // attributes. + if foreign_mod.attrs == attrs && foreign_mod.abi == abi + { + // Merge the items of the two blocks. + foreign_mod.items.extend_from_slice(&foreign_items); + exists = true; + break; + } + } + // If no existing extern block had the same ABI and attributes, store + // it. + if !exists { + foreign_mods.push(syn::ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }); + } + } + // If the item is not an extern block, we don't have to do anything. + _ => items.push(item), + } + } + + // Move all the extern blocks alongiside the rest of the items. + for foreign_mod in foreign_mods { + items.push(Item::ForeignMod(foreign_mod)); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 9a5887df8e..3ee46f7540 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -88,7 +88,6 @@ use std::{env, iter}; // Some convenient typedefs for a fast hash map and hash set. type HashMap = ::rustc_hash::FxHashMap; type HashSet = ::rustc_hash::FxHashSet; -use quote::ToTokens; pub(crate) use std::collections::hash_map::Entry; /// Default prefix for the anon fields. @@ -2118,11 +2117,6 @@ struct BindgenOptions { impl ::std::panic::UnwindSafe for BindgenOptions {} impl BindgenOptions { - /// Whether any of the enabled options requires `syn`. - fn require_syn(&self) -> bool { - self.sort_semantically || self.merge_extern_blocks - } - fn build(&mut self) { let mut regex_sets = [ &mut self.allowlisted_vars, @@ -2555,112 +2549,7 @@ impl Bindings { parse(&mut context)?; } - let (items, options, warnings) = codegen::codegen(context); - - let module = if options.require_syn() { - let module_wrapped_tokens = - quote!(mod wrapper_for_sorting_hack { #( #items )* }); - - // This syn business is a hack, for now. This means that we are re-parsing already - // generated code using `syn` (as opposed to `quote`) because `syn` provides us more - // control over the elements. - // One caveat is that some of the items coming from `quote`d output might have - // multiple items within them. Hence, we have to wrap the incoming in a `mod`. - // The two `unwrap`s here are deliberate because - // The first one won't panic because we build the `mod` and know it is there - // The second one won't panic because we know original output has something in - // it already. - let mut syn_parsed_items = - syn::parse2::(module_wrapped_tokens) - .unwrap() - .content - .unwrap() - .1; - - if options.merge_extern_blocks { - // Here we will store all the items after deduplication. - let mut items = Vec::new(); - - // Keep all the extern blocks in a different `Vec` for faster search. - let mut foreign_mods = Vec::::new(); - for item in syn_parsed_items { - match item { - syn::Item::ForeignMod(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }) => { - let mut exists = false; - for foreign_mod in &mut foreign_mods { - // Check if there is a extern block with the same ABI and - // attributes. - if foreign_mod.attrs == attrs && - foreign_mod.abi == abi - { - // Merge the items of the two blocks. - foreign_mod - .items - .extend_from_slice(&foreign_items); - exists = true; - break; - } - } - // If no existing extern block had the same ABI and attributes, store - // it. - if !exists { - foreign_mods.push(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }); - } - } - // If the item is not an extern block, we don't have to do anything. - _ => items.push(item), - } - } - - // Move all the extern blocks alongiside the rest of the items. - for foreign_mod in foreign_mods { - items.push(syn::Item::ForeignMod(foreign_mod)); - } - - syn_parsed_items = items; - } - - if options.sort_semantically { - syn_parsed_items.sort_by_key(|item| match item { - syn::Item::Type(_) => 0, - syn::Item::Struct(_) => 1, - syn::Item::Const(_) => 2, - syn::Item::Fn(_) => 3, - syn::Item::Enum(_) => 4, - syn::Item::Union(_) => 5, - syn::Item::Static(_) => 6, - syn::Item::Trait(_) => 7, - syn::Item::TraitAlias(_) => 8, - syn::Item::Impl(_) => 9, - syn::Item::Mod(_) => 10, - syn::Item::Use(_) => 11, - syn::Item::Verbatim(_) => 12, - syn::Item::ExternCrate(_) => 13, - syn::Item::ForeignMod(_) => 14, - syn::Item::Macro(_) => 15, - syn::Item::Macro2(_) => 16, - _ => 18, - }); - } - - let synful_items = syn_parsed_items - .into_iter() - .map(|item| item.into_token_stream()); - - quote! { #( #synful_items )* } - } else { - quote! { #( #items )* } - }; + let (module, options, warnings) = codegen::codegen(context); Ok(Bindings { options, From 3dfbc602ae004f142811ad5ef17435dd04bc16ed Mon Sep 17 00:00:00 2001 From: Christian Poveda Date: Fri, 23 Sep 2022 15:24:19 -0500 Subject: [PATCH 4/8] update `CONTRIBUTING.md` section about `syn` --- CONTRIBUTING.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2974ba4254..3d757ccdff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -323,14 +323,12 @@ generated Rust code are implemented using the [`syn`](https://docs.rs/syn) crate ### Implementing new options using `syn` -Here is a list of recommendations to be followed if a new option can be -implemented using the `syn` crate: +If a new option can be implemented using the `syn` crate it should be added to +the `codegen::postprocessing` module by following these steps: -- The `BindgenOptions::require_syn` method must be updated to reflect that this - new option requires parsing the generated Rust code with `syn`. - -- The implementation of the new option should be added at the end of - `Bindings::generate`, inside the `if options.require_syn() { ... }` block. +- Introduce a new `struct` with no fields. +- Implement the `PostProcessing` trait for the `struct`. +- Add the `struct` to the `decl_postprocessing` macro invocation. ## Pull Requests and Code Reviews From 0798bdaccfd352c9b2394b1741508000b99ba1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= Date: Fri, 23 Sep 2022 18:10:38 -1000 Subject: [PATCH 5/8] Clean-up postprocessing to use less macro magic. --- src/codegen/postprocessing.rs | 227 ++++++++++++++++------------------ 1 file changed, 104 insertions(+), 123 deletions(-) diff --git a/src/codegen/postprocessing.rs b/src/codegen/postprocessing.rs index 728322980f..6d935a6149 100644 --- a/src/codegen/postprocessing.rs +++ b/src/codegen/postprocessing.rs @@ -4,145 +4,126 @@ use syn::Item; use crate::BindgenOptions; -macro_rules! decl_postprocessing { - ($($ty:ty),*) => { - pub(crate) fn postprocessing( - items: Vec, - options: &BindgenOptions, - ) -> TokenStream { - // Whether any of the enabled options requires `syn`. - let require_syn = $(<$ty as PostProcessing>::should_run(options))||*; - - if !require_syn { - return items.into_iter().collect(); - } - - let module_wrapped_tokens = - quote!(mod wrapper_for_sorting_hack { #( #items )* }); - - // This syn business is a hack, for now. This means that we are re-parsing already - // generated code using `syn` (as opposed to `quote`) because `syn` provides us more - // control over the elements. - // One caveat is that some of the items coming from `quote`d output might have - // multiple items within them. Hence, we have to wrap the incoming in a `mod`. - // The two `unwrap`s here are deliberate because - // The first one won't panic because we build the `mod` and know it is there - // The second one won't panic because we know original output has something in - // it already. - let mut items = - syn::parse2::(module_wrapped_tokens) - .unwrap() - .content - .unwrap() - .1; - - $(if <$ty as PostProcessing>::should_run(options) { - <$ty as PostProcessing>::run(&mut items); - })* - - let synful_items = items - .into_iter() - .map(|item| item.into_token_stream()); +struct PostProcessingPass { + should_run: fn(&BindgenOptions) -> bool, + run: fn(&mut Vec), +} - quote! { #( #synful_items )* } +// TODO: This can be a const fn when mutable references are allowed in const +// context. +macro_rules! pass { + ($pass:ident) => { + PostProcessingPass { + should_run: |options| options.$pass, + run: $pass, } }; } -decl_postprocessing! { - MergeExternBlocks, - SortSemantically -} - -trait PostProcessing { - fn should_run(options: &BindgenOptions) -> bool; - - fn run(items: &mut Vec); -} - -struct SortSemantically; +static PASSES: [PostProcessingPass; 2] = + [pass!(merge_extern_blocks), pass!(sort_semantically)]; -impl PostProcessing for SortSemantically { - #[inline] - fn should_run(options: &BindgenOptions) -> bool { - options.sort_semantically +pub(crate) fn postprocessing( + items: Vec, + options: &BindgenOptions, +) -> TokenStream { + let require_syn = PASSES.iter().any(|pass| (pass.should_run)(options)); + if !require_syn { + return items.into_iter().collect(); } - - fn run(items: &mut Vec) { - items.sort_by_key(|item| match item { - Item::Type(_) => 0, - Item::Struct(_) => 1, - Item::Const(_) => 2, - Item::Fn(_) => 3, - Item::Enum(_) => 4, - Item::Union(_) => 5, - Item::Static(_) => 6, - Item::Trait(_) => 7, - Item::TraitAlias(_) => 8, - Item::Impl(_) => 9, - Item::Mod(_) => 10, - Item::Use(_) => 11, - Item::Verbatim(_) => 12, - Item::ExternCrate(_) => 13, - Item::ForeignMod(_) => 14, - Item::Macro(_) => 15, - Item::Macro2(_) => 16, - _ => 18, - }); + let module_wrapped_tokens = + quote!(mod wrapper_for_sorting_hack { #( #items )* }); + + // This syn business is a hack, for now. This means that we are re-parsing already + // generated code using `syn` (as opposed to `quote`) because `syn` provides us more + // control over the elements. + // One caveat is that some of the items coming from `quote`d output might have + // multiple items within them. Hence, we have to wrap the incoming in a `mod`. + // The two `unwrap`s here are deliberate because + // The first one won't panic because we build the `mod` and know it is there + // The second one won't panic because we know original output has something in + // it already. + let mut items = syn::parse2::(module_wrapped_tokens) + .unwrap() + .content + .unwrap() + .1; + + for pass in PASSES.iter() { + if (pass.should_run)(options) { + (pass.run)(&mut items); + } } -} -struct MergeExternBlocks; + let synful_items = items.into_iter().map(|item| item.into_token_stream()); -impl PostProcessing for MergeExternBlocks { - #[inline] - fn should_run(options: &BindgenOptions) -> bool { - options.merge_extern_blocks - } + quote! { #( #synful_items )* } +} - fn run(items: &mut Vec) { - // Keep all the extern blocks in a different `Vec` for faster search. - let mut foreign_mods = Vec::::new(); +fn sort_semantically(items: &mut Vec) { + items.sort_by_key(|item| match item { + Item::Type(_) => 0, + Item::Struct(_) => 1, + Item::Const(_) => 2, + Item::Fn(_) => 3, + Item::Enum(_) => 4, + Item::Union(_) => 5, + Item::Static(_) => 6, + Item::Trait(_) => 7, + Item::TraitAlias(_) => 8, + Item::Impl(_) => 9, + Item::Mod(_) => 10, + Item::Use(_) => 11, + Item::Verbatim(_) => 12, + Item::ExternCrate(_) => 13, + Item::ForeignMod(_) => 14, + Item::Macro(_) => 15, + Item::Macro2(_) => 16, + _ => 18, + }); +} - for item in std::mem::take(items) { - match item { - Item::ForeignMod(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }) => { - let mut exists = false; - for foreign_mod in &mut foreign_mods { - // Check if there is a extern block with the same ABI and - // attributes. - if foreign_mod.attrs == attrs && foreign_mod.abi == abi - { - // Merge the items of the two blocks. - foreign_mod.items.extend_from_slice(&foreign_items); - exists = true; - break; - } - } - // If no existing extern block had the same ABI and attributes, store - // it. - if !exists { - foreign_mods.push(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }); +fn merge_extern_blocks(items: &mut Vec) { + // Keep all the extern blocks in a different `Vec` for faster search. + let mut foreign_mods = Vec::::new(); + + for item in std::mem::take(items) { + match item { + Item::ForeignMod(syn::ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }) => { + let mut exists = false; + for foreign_mod in &mut foreign_mods { + // Check if there is a extern block with the same ABI and + // attributes. + if foreign_mod.attrs == attrs && foreign_mod.abi == abi { + // Merge the items of the two blocks. + foreign_mod.items.extend_from_slice(&foreign_items); + exists = true; + break; } } - // If the item is not an extern block, we don't have to do anything. - _ => items.push(item), + // If no existing extern block had the same ABI and attributes, store + // it. + if !exists { + foreign_mods.push(syn::ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }); + } } + // If the item is not an extern block, we don't have to do anything. + _ => items.push(item), } + } - // Move all the extern blocks alongiside the rest of the items. - for foreign_mod in foreign_mods { - items.push(Item::ForeignMod(foreign_mod)); - } + // Move all the extern blocks alongside the rest of the items. + for foreign_mod in foreign_mods { + items.push(Item::ForeignMod(foreign_mod)); } } From 4312df317a2fdf26ec889cb2f8d3ffe0b12ea4d3 Mon Sep 17 00:00:00 2001 From: Christian Poveda Date: Mon, 26 Sep 2022 10:55:05 -0500 Subject: [PATCH 6/8] update CONTRIBUTING.md --- CONTRIBUTING.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d757ccdff..48e4a4025e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -326,9 +326,11 @@ generated Rust code are implemented using the [`syn`](https://docs.rs/syn) crate If a new option can be implemented using the `syn` crate it should be added to the `codegen::postprocessing` module by following these steps: -- Introduce a new `struct` with no fields. -- Implement the `PostProcessing` trait for the `struct`. -- Add the `struct` to the `decl_postprocessing` macro invocation. +- Introduce a new field to `BindgenOptions` for the option. +- Write a free function inside `codegen::postprocessing` implementing the + option. This function with the same name of the `BindgenOptions` field. +- Add a new value to the `codegen::postprocessing::PASSES` for the option using + the `pass!` macro. ## Pull Requests and Code Reviews From 06a6479e9e6fe9b556a54b3d7f28c23b1268b048 Mon Sep 17 00:00:00 2001 From: Christian Poveda Date: Mon, 26 Sep 2022 11:02:42 -0500 Subject: [PATCH 7/8] s/static/const --- src/codegen/postprocessing.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/codegen/postprocessing.rs b/src/codegen/postprocessing.rs index 6d935a6149..6550ca57db 100644 --- a/src/codegen/postprocessing.rs +++ b/src/codegen/postprocessing.rs @@ -20,8 +20,8 @@ macro_rules! pass { }; } -static PASSES: [PostProcessingPass; 2] = - [pass!(merge_extern_blocks), pass!(sort_semantically)]; +const PASSES: &'static [PostProcessingPass] = + &[pass!(merge_extern_blocks), pass!(sort_semantically)]; pub(crate) fn postprocessing( items: Vec, @@ -43,13 +43,12 @@ pub(crate) fn postprocessing( // The first one won't panic because we build the `mod` and know it is there // The second one won't panic because we know original output has something in // it already. - let mut items = syn::parse2::(module_wrapped_tokens) + let (_, mut items) = syn::parse2::(module_wrapped_tokens) .unwrap() .content - .unwrap() - .1; + .unwrap(); - for pass in PASSES.iter() { + for pass in PASSES { if (pass.should_run)(options) { (pass.run)(&mut items); } From ddb319ce3579b687cdd06d934ac40c253e96221f Mon Sep 17 00:00:00 2001 From: Christian Poveda Date: Tue, 27 Sep 2022 13:43:33 -0500 Subject: [PATCH 8/8] split `processing` module --- src/codegen/postprocessing.rs | 128 ------------------ .../postprocessing/merge_extern_blocks.rs | 46 +++++++ src/codegen/postprocessing/mod.rs | 66 +++++++++ .../postprocessing/sort_semantically.rs | 24 ++++ 4 files changed, 136 insertions(+), 128 deletions(-) delete mode 100644 src/codegen/postprocessing.rs create mode 100644 src/codegen/postprocessing/merge_extern_blocks.rs create mode 100644 src/codegen/postprocessing/mod.rs create mode 100644 src/codegen/postprocessing/sort_semantically.rs diff --git a/src/codegen/postprocessing.rs b/src/codegen/postprocessing.rs deleted file mode 100644 index 6550ca57db..0000000000 --- a/src/codegen/postprocessing.rs +++ /dev/null @@ -1,128 +0,0 @@ -use proc_macro2::TokenStream; -use quote::ToTokens; -use syn::Item; - -use crate::BindgenOptions; - -struct PostProcessingPass { - should_run: fn(&BindgenOptions) -> bool, - run: fn(&mut Vec), -} - -// TODO: This can be a const fn when mutable references are allowed in const -// context. -macro_rules! pass { - ($pass:ident) => { - PostProcessingPass { - should_run: |options| options.$pass, - run: $pass, - } - }; -} - -const PASSES: &'static [PostProcessingPass] = - &[pass!(merge_extern_blocks), pass!(sort_semantically)]; - -pub(crate) fn postprocessing( - items: Vec, - options: &BindgenOptions, -) -> TokenStream { - let require_syn = PASSES.iter().any(|pass| (pass.should_run)(options)); - if !require_syn { - return items.into_iter().collect(); - } - let module_wrapped_tokens = - quote!(mod wrapper_for_sorting_hack { #( #items )* }); - - // This syn business is a hack, for now. This means that we are re-parsing already - // generated code using `syn` (as opposed to `quote`) because `syn` provides us more - // control over the elements. - // One caveat is that some of the items coming from `quote`d output might have - // multiple items within them. Hence, we have to wrap the incoming in a `mod`. - // The two `unwrap`s here are deliberate because - // The first one won't panic because we build the `mod` and know it is there - // The second one won't panic because we know original output has something in - // it already. - let (_, mut items) = syn::parse2::(module_wrapped_tokens) - .unwrap() - .content - .unwrap(); - - for pass in PASSES { - if (pass.should_run)(options) { - (pass.run)(&mut items); - } - } - - let synful_items = items.into_iter().map(|item| item.into_token_stream()); - - quote! { #( #synful_items )* } -} - -fn sort_semantically(items: &mut Vec) { - items.sort_by_key(|item| match item { - Item::Type(_) => 0, - Item::Struct(_) => 1, - Item::Const(_) => 2, - Item::Fn(_) => 3, - Item::Enum(_) => 4, - Item::Union(_) => 5, - Item::Static(_) => 6, - Item::Trait(_) => 7, - Item::TraitAlias(_) => 8, - Item::Impl(_) => 9, - Item::Mod(_) => 10, - Item::Use(_) => 11, - Item::Verbatim(_) => 12, - Item::ExternCrate(_) => 13, - Item::ForeignMod(_) => 14, - Item::Macro(_) => 15, - Item::Macro2(_) => 16, - _ => 18, - }); -} - -fn merge_extern_blocks(items: &mut Vec) { - // Keep all the extern blocks in a different `Vec` for faster search. - let mut foreign_mods = Vec::::new(); - - for item in std::mem::take(items) { - match item { - Item::ForeignMod(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }) => { - let mut exists = false; - for foreign_mod in &mut foreign_mods { - // Check if there is a extern block with the same ABI and - // attributes. - if foreign_mod.attrs == attrs && foreign_mod.abi == abi { - // Merge the items of the two blocks. - foreign_mod.items.extend_from_slice(&foreign_items); - exists = true; - break; - } - } - // If no existing extern block had the same ABI and attributes, store - // it. - if !exists { - foreign_mods.push(syn::ItemForeignMod { - attrs, - abi, - brace_token, - items: foreign_items, - }); - } - } - // If the item is not an extern block, we don't have to do anything. - _ => items.push(item), - } - } - - // Move all the extern blocks alongside the rest of the items. - for foreign_mod in foreign_mods { - items.push(Item::ForeignMod(foreign_mod)); - } -} diff --git a/src/codegen/postprocessing/merge_extern_blocks.rs b/src/codegen/postprocessing/merge_extern_blocks.rs new file mode 100644 index 0000000000..2b7614941e --- /dev/null +++ b/src/codegen/postprocessing/merge_extern_blocks.rs @@ -0,0 +1,46 @@ +use syn::{Item, ItemForeignMod}; + +pub(super) fn merge_extern_blocks(items: &mut Vec) { + // Keep all the extern blocks in a different `Vec` for faster search. + let mut foreign_mods = Vec::::new(); + + for item in std::mem::take(items) { + match item { + Item::ForeignMod(ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }) => { + let mut exists = false; + for foreign_mod in &mut foreign_mods { + // Check if there is a extern block with the same ABI and + // attributes. + if foreign_mod.attrs == attrs && foreign_mod.abi == abi { + // Merge the items of the two blocks. + foreign_mod.items.extend_from_slice(&foreign_items); + exists = true; + break; + } + } + // If no existing extern block had the same ABI and attributes, store + // it. + if !exists { + foreign_mods.push(ItemForeignMod { + attrs, + abi, + brace_token, + items: foreign_items, + }); + } + } + // If the item is not an extern block, we don't have to do anything. + _ => items.push(item), + } + } + + // Move all the extern blocks alongside the rest of the items. + for foreign_mod in foreign_mods { + items.push(Item::ForeignMod(foreign_mod)); + } +} diff --git a/src/codegen/postprocessing/mod.rs b/src/codegen/postprocessing/mod.rs new file mode 100644 index 0000000000..c6612f2b91 --- /dev/null +++ b/src/codegen/postprocessing/mod.rs @@ -0,0 +1,66 @@ +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::Item; + +use crate::BindgenOptions; + +mod merge_extern_blocks; +mod sort_semantically; + +use merge_extern_blocks::merge_extern_blocks; +use sort_semantically::sort_semantically; + +struct PostProcessingPass { + should_run: fn(&BindgenOptions) -> bool, + run: fn(&mut Vec), +} + +// TODO: This can be a const fn when mutable references are allowed in const +// context. +macro_rules! pass { + ($pass:ident) => { + PostProcessingPass { + should_run: |options| options.$pass, + run: |items| $pass(items), + } + }; +} + +const PASSES: &[PostProcessingPass] = + &[pass!(merge_extern_blocks), pass!(sort_semantically)]; + +pub(crate) fn postprocessing( + items: Vec, + options: &BindgenOptions, +) -> TokenStream { + let require_syn = PASSES.iter().any(|pass| (pass.should_run)(options)); + if !require_syn { + return items.into_iter().collect(); + } + let module_wrapped_tokens = + quote!(mod wrapper_for_sorting_hack { #( #items )* }); + + // This syn business is a hack, for now. This means that we are re-parsing already + // generated code using `syn` (as opposed to `quote`) because `syn` provides us more + // control over the elements. + // One caveat is that some of the items coming from `quote`d output might have + // multiple items within them. Hence, we have to wrap the incoming in a `mod`. + // The two `unwrap`s here are deliberate because + // The first one won't panic because we build the `mod` and know it is there + // The second one won't panic because we know original output has something in + // it already. + let (_, mut items) = syn::parse2::(module_wrapped_tokens) + .unwrap() + .content + .unwrap(); + + for pass in PASSES { + if (pass.should_run)(options) { + (pass.run)(&mut items); + } + } + + let synful_items = items.into_iter().map(|item| item.into_token_stream()); + + quote! { #( #synful_items )* } +} diff --git a/src/codegen/postprocessing/sort_semantically.rs b/src/codegen/postprocessing/sort_semantically.rs new file mode 100644 index 0000000000..96596cb01e --- /dev/null +++ b/src/codegen/postprocessing/sort_semantically.rs @@ -0,0 +1,24 @@ +use syn::Item; + +pub(super) fn sort_semantically(items: &mut [Item]) { + items.sort_by_key(|item| match item { + Item::Type(_) => 0, + Item::Struct(_) => 1, + Item::Const(_) => 2, + Item::Fn(_) => 3, + Item::Enum(_) => 4, + Item::Union(_) => 5, + Item::Static(_) => 6, + Item::Trait(_) => 7, + Item::TraitAlias(_) => 8, + Item::Impl(_) => 9, + Item::Mod(_) => 10, + Item::Use(_) => 11, + Item::Verbatim(_) => 12, + Item::ExternCrate(_) => 13, + Item::ForeignMod(_) => 14, + Item::Macro(_) => 15, + Item::Macro2(_) => 16, + _ => 18, + }); +}