proc-macro process stack overflow

[umanwizard]

While browsing this file in Materialize: https://github.com/MaterializeInc/materialize/blob/c5b9619ba/src/storage/src/source/persist_source.rs

Stack trace (snipped, the end repeats for hundreds of frames)

#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=140252985230912) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=6, threadid=140252985230912) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=140252985230912, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#3  0x00007f8f44ecb476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4  0x00007f8f44eb17f3 in __GI_abort () at ./stdlib/abort.c:79
#5  0x0000556a681ca307 in std::sys::unix::abort_internal () at library/std/src/sys/unix/mod.rs:259
#6  0x0000556a681c9735 in std::sys::unix::stack_overflow::imp::signal_handler () at library/std/src/sys/unix/stack_overflow.rs:93
#7  <signal handler called>
#8  0x0000556a67226f5b in <proc_macro_srv::abis::abi_1_58::proc_macro::bridge::TokenTree<G,P,I,L> as proc_macro_srv::abis::abi_1_58::proc_macro::bridge::rpc::DecodeMut<S>>::decode ()
#9  0x0000556a672676f2 in std::panicking::try ()
#10 0x0000556a672144de in <proc_macro_srv::abis::abi_1_58::proc_macro::bridge::server::Dispatcher<proc_macro_srv::abis::abi_1_58::proc_macro::bridge::server::MarkedTypes<S>> as proc_macro_srv::abis::abi_1_58::proc_macro::bridge::server::DispatcherTrait>::dispatch ()
#11 0x0000556a6726bc70 in _ZN142_$LT$proc_macro_srv..abis..abi_1_58..proc_macro..bridge..closure..Closure$LT$A$C$R$GT$$u20$as$u20$core..convert..From$LT$$RF$mut$u20$F$GT$$GT$4from4call17h895f5d143678e9c3E.llvm.3698797192250470128 ()
#12 0x00007f8f35f90315 in proc_macro::bridge::closure::Closure::call<proc_macro::bridge::buffer::Buffer<u8>, proc_macro::bridge::buffer::Buffer<u8>> () at library/proc_macro/src/bridge/closure.rs:26
#13 proc_macro::bridge::client::{impl#130}::from_token_tree::{closure#0} () at library/proc_macro/src/bridge/client.rs:259
#14 proc_macro::bridge::client::{impl#9}::with::{closure#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}> ()
    at library/proc_macro/src/bridge/client.rs:351
#15 proc_macro::bridge::client::{impl#8}::with::{closure#0}::{closure#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#9}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}>> () at library/proc_macro/src/bridge/client.rs:308
#16 proc_macro::bridge::scoped_cell::ScopedCell::replace<proc_macro::bridge::client::BridgeStateL, proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#8}::with::{closure#0}::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#9}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}>>> () at library/proc_macro/src/bridge/scoped_cell.rs:74
#17 proc_macro::bridge::client::{impl#8}::with::{closure#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#9}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}>> () at library/proc_macro/src/bridge/client.rs:306
#18 std::thread::local::LocalKey::try_with<proc_macro::bridge::scoped_cell::ScopedCell<proc_macro::bridge::client::BridgeStateL>, proc_macro::bridge::client::{impl#8}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#9}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}>>, proc_macro::bridge::client::TokenStream> () at /rustc/a8314ef7d0ec7b75c336af2c9857bfaf43002bfc/library/std/src/thread/local.rs:445
#19 std::thread::local::LocalKey::with<proc_macro::bridge::scoped_cell::ScopedCell<proc_macro::bridge::client::BridgeStateL>, proc_macro::bridge::client::{impl#8}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#9}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}>>, proc_macro::bridge::client::TokenStream> () at /rustc/a8314ef7d0ec7b75c336af2c9857bfaf43002bfc/library/std/src/thread/local.rs:421
#20 proc_macro::bridge::client::BridgeState::with<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#9}::with::{closure_env#0}<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}>> () at library/proc_macro/src/bridge/client.rs:305
#21 proc_macro::bridge::Bridge::with<proc_macro::bridge::client::TokenStream, proc_macro::bridge::client::{impl#130}::from_token_tree::{closure_env#0}> () at library/proc_macro/src/bridge/client.rs:344
#22 proc_macro::bridge::client::TokenStream::from_token_tree () at library/proc_macro/src/bridge/client.rs:252
#23 proc_macro::{impl#16}::from () at library/proc_macro/src/lib.rs:216
#24 0x00007f8f35fa83a6 in proc_macro::{impl#59}::to_string () at library/proc_macro/src/lib.rs:988
#25 0x00007f8f35dbdc3e in proc_macro2::imp::{impl#32}::eq<&&str> (self=0x7f8f313d14d0, other=0x7f8f313d1458)
    at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/proc-macro2-1.0.40/src/wrapper.rs:762
#26 0x00007f8f35de5e69 in proc_macro2::{impl#35}::eq<&str> (self=0x7f8f313d14d0, other=0x7f8f313d1478) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/proc-macro2-1.0.40/src/lib.rs:991
#27 0x00007f8f35dbe336 in syn::token::parsing::peek_keyword (cursor=..., token=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/token.rs:909
#28 0x00007f8f35eee6c0 in syn::token::{impl#484}::peek (cursor=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/token.rs:329
#29 0x00007f8f35dd7349 in syn::parse::ParseBuffer::peek<fn(syn::lookahead::TokenMarker) -> syn::token::Super> (self=0x7f8f313da490, token=0x0)
    at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/parse.rs:585
#30 0x00007f8f35dcdd6b in syn::path::PathSegment::parse_helper (input=0x7f8f313da490, expr_style=true) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/path.rs:407
#31 0x00007f8f35dcf8c6 in syn::path::Path::parse_helper (input=0x7f8f313da490, expr_style=true) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/path.rs:609
#32 0x00007f8f35ee3e46 in syn::path::parsing::qpath (input=0x7f8f313da490, expr_style=true) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/path.rs:680
#33 0x00007f8f35da701b in syn::expr::parsing::{impl#39}::parse (input=0x7f8f313da490) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:2815
#34 0x00007f8f35dd5547 in syn::parse::ParseBuffer::parse<syn::expr::ExprPath> (self=0x7f8f313da490) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/parse.rs:468
#35 0x00007f8f35e1adaf in syn::expr::parsing::path_or_macro_or_struct (input=0x7f8f313da490, allow_struct=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1830
#36 0x00007f8f35e19d5f in syn::expr::parsing::atom_expr (input=0x7f8f313da490, allow_struct=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1744
#37 0x00007f8f35e163aa in syn::expr::parsing::trailer_expr (begin=..., attrs=..., input=0x7f8f313da490, allow_struct=...)
    at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1560
#38 0x00007f8f35e16240 in syn::expr::parsing::unary_expr (input=0x7f8f313da490, allow_struct=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1530
#39 0x00007f8f35e14c6b in syn::expr::parsing::ambiguous_expr (input=0x7f8f313da490, allow_struct=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1465
#40 0x00007f8f35d9d483 in syn::expr::parsing::{impl#1}::parse (input=0x7f8f313da490) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1162
#41 0x00007f8f35ea9bb1 in syn::punctuated::Punctuated<syn::expr::Expr, syn::token::Comma>::parse_terminated_with<syn::expr::Expr, syn::token::Comma> (input=0x7f8f313da490, 
    parser=0x7f8f35d9d460 <syn::expr::parsing::{impl#1}::parse>) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/punctuated.rs:301
#42 0x00007f8f35dd855b in syn::parse::ParseBuffer::parse_terminated<syn::expr::Expr, syn::token::Comma> (self=0x7f8f313da490, parser=0x7f8f35d9d460 <syn::expr::parsing::{impl#1}::parse>)
    at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/parse.rs:703
#43 0x00007f8f35e16c97 in syn::expr::parsing::trailer_helper (input=0x7f8f313df5a0, e=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1583
#44 0x00007f8f35e16493 in syn::expr::parsing::trailer_expr (begin=..., attrs=..., input=0x7f8f313df5a0, allow_struct=...)
    at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1561
#45 0x00007f8f35e16240 in syn::expr::parsing::unary_expr (input=0x7f8f313df5a0, allow_struct=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1530
#46 0x00007f8f35e14c6b in syn::expr::parsing::ambiguous_expr (input=0x7f8f313df5a0, allow_struct=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1465
#47 0x00007f8f35d9d483 in syn::expr::parsing::{impl#1}::parse (input=0x7f8f313df5a0) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1162
#48 0x00007f8f35ea9bb1 in syn::punctuated::Punctuated<syn::expr::Expr, syn::token::Comma>::parse_terminated_with<syn::expr::Expr, syn::token::Comma> (input=0x7f8f313df5a0, parser=0x7f8f35d9d460 <syn::expr::parsing::{impl#1}::parse>) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/punctuated.rs:301
#49 0x00007f8f35dd855b in syn::parse::ParseBuffer::parse_terminated<syn::expr::Expr, syn::token::Comma> (self=0x7f8f313df5a0, parser=0x7f8f35d9d460 <syn::expr::parsing::{impl#1}::parse>) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/parse.rs:703
#50 0x00007f8f35e16c97 in syn::expr::parsing::trailer_helper (input=0x7f8f313e46b0, e=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1583
#51 0x00007f8f35e16493 in syn::expr::parsing::trailer_expr (begin=..., attrs=..., input=0x7f8f313e46b0, allow_struct=...) at /home/brennan/.cargo/registry/src/git.colasdn.top-1ecc6299db9ec823/syn-1.0.98/src/expr.rs:1561

rust-analyzer version: rust-analyzer 0.0.0 (c2eebd7 2022-07-26)

rustc version: rustc 1.62.0 (a8314ef7d 2022-06-27)

relevant settings: using emacs lsp-mode, with lsp-rust-analyzer-proc-macro-enable set to t

[bjorn3]

I did guess this proc macro relies on rustc using a stack size larger than the OS default to avoid a stack overflow. It would likely also crash with rustc if you started to nest your code deeper. Rustc itself uses stacker at various places where there can be deep recursion to grow the stack when it would almost overflow. Syn however doesn't do this and is thus susceptible to stack overflow.

[jonas-schievink]

We're already using quite a large stack (8 MiB):

rust-analyzer/crates/rust-analyzer/src/bin/main.rs

Line 86 in 84a6fac

with_extra_thread("MacroExpander", || proc_macro_srv::cli::run().map_err(Into::into))?;

[bjorn3]

The actual proc macro expansion now happens on a new thread without the extra stack:

rust-analyzer/crates/proc-macro-srv/src/lib.rs

Line 70 in 7e2b983

.spawn(|_| {