Skip to content

Less TokenTree cloning #114115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jul 28, 2023
Merged

Less TokenTree cloning #114115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
53379a7
Simplify the `ttdelim_span` test.
nnethercote Jul 26, 2023
a9d8459
Replace `into_trees` with `trees` in a test.
nnethercote Jul 27, 2023
103bd4a
Use `TokenStream::trees` instead of `into_trees` for attributes.
nnethercote Jul 27, 2023
55a7324
Make `TokenTree::uninterpolate` take `&self` and return a `Cow`.
nnethercote Jul 27, 2023
853f453
Avoid some token cloning in `filter_tokens_from_list`.
nnethercote Jul 27, 2023
d2f7f67
Avoid some token tree cloning in decl macro parsing.
nnethercote Jul 27, 2023
ee6ed60
Avoid `into_trees` usage in rustfmt.
nnethercote Jul 27, 2023
4ebf2be
Remove `Iterator` impl for `TokenTreeCursor`.
nnethercote Jul 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 18 additions & 19 deletions compiler/rustc_ast/src/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,17 +285,17 @@ impl MetaItem {
self.kind.value_str()
}

fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
where
I: Iterator<Item = TokenTree>,
I: Iterator<Item = &'a TokenTree>,
{
// FIXME: Share code with `parse_path`.
let path = match tokens.next().map(TokenTree::uninterpolate) {
Some(TokenTree::Token(
Token { kind: kind @ (token::Ident(..) | token::ModSep), span },
let path = match tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref() {
Some(&TokenTree::Token(
Token { kind: ref kind @ (token::Ident(..) | token::ModSep), span },
_,
)) => 'arm: {
let mut segments = if let token::Ident(name, _) = kind {
let mut segments = if let &token::Ident(name, _) = kind {
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
tokens.peek()
{
Expand All @@ -308,8 +308,8 @@ impl MetaItem {
thin_vec![PathSegment::path_root(span)]
};
loop {
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
tokens.next().map(TokenTree::uninterpolate)
if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
{
segments.push(PathSegment::from_ident(Ident::new(name, span)));
} else {
Expand All @@ -326,7 +326,7 @@ impl MetaItem {
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
Path { span, segments, tokens: None }
}
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &*nt {
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &**nt {
token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
token::Nonterminal::NtPath(path) => (**path).clone(),
_ => return None,
Expand Down Expand Up @@ -354,7 +354,7 @@ impl MetaItemKind {
}

fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<NestedMetaItem>> {
let mut tokens = tokens.into_trees().peekable();
let mut tokens = tokens.trees().peekable();
let mut result = ThinVec::new();
while tokens.peek().is_some() {
let item = NestedMetaItem::from_tokens(&mut tokens)?;
Expand All @@ -367,12 +367,12 @@ impl MetaItemKind {
Some(result)
}

fn name_value_from_tokens(
tokens: &mut impl Iterator<Item = TokenTree>,
fn name_value_from_tokens<'a>(
tokens: &mut impl Iterator<Item = &'a TokenTree>,
) -> Option<MetaItemKind> {
match tokens.next() {
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
}
Some(TokenTree::Token(token, _)) => {
MetaItemLit::from_token(&token).map(MetaItemKind::NameValue)
Expand All @@ -381,8 +381,8 @@ impl MetaItemKind {
}
}

fn from_tokens(
tokens: &mut iter::Peekable<impl Iterator<Item = TokenTree>>,
fn from_tokens<'a>(
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
) -> Option<MetaItemKind> {
match tokens.peek() {
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
Expand Down Expand Up @@ -501,9 +501,9 @@ impl NestedMetaItem {
self.meta_item().is_some()
}

fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
where
I: Iterator<Item = TokenTree>,
I: Iterator<Item = &'a TokenTree>,
{
match tokens.peek() {
Some(TokenTree::Token(token, _))
Expand All @@ -513,9 +513,8 @@ impl NestedMetaItem {
return Some(NestedMetaItem::Lit(lit));
}
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
let inner_tokens = inner_tokens.clone();
tokens.next();
return NestedMetaItem::from_tokens(&mut inner_tokens.into_trees().peekable());
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
}
_ => {}
}
Expand Down
33 changes: 15 additions & 18 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
use rustc_span::{Span, DUMMY_SP};
use smallvec::{smallvec, SmallVec};

use std::borrow::Cow;
use std::{fmt, iter, mem};

/// When the main Rust parser encounters a syntax-extension invocation, it
Expand Down Expand Up @@ -98,12 +99,13 @@ impl TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
}

pub fn uninterpolate(self) -> TokenTree {
pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
match self {
TokenTree::Token(token, spacing) => {
TokenTree::Token(token.uninterpolate().into_owned(), spacing)
}
tt => tt,
TokenTree::Token(token, spacing) => match token.uninterpolate() {
Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)),
Cow::Borrowed(_) => Cow::Borrowed(self),
},
_ => Cow::Borrowed(self),
}
}
}
Expand Down Expand Up @@ -595,26 +597,21 @@ impl<'t> Iterator for RefTokenTreeCursor<'t> {
}
}

/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree`
/// Owning by-value iterator over a [`TokenStream`], that produces `&TokenTree`
/// items.
// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
///
/// Doesn't impl `Iterator` because Rust doesn't permit an owning iterator to
/// return `&T` from `next`; the need for an explicit lifetime in the `Item`
/// associated type gets in the way. Instead, use `next_ref` (which doesn't
/// involve associated types) for getting individual elements, or
/// `RefTokenTreeCursor` if you really want an `Iterator`, e.g. in a `for`
/// loop.
#[derive(Clone)]
pub struct TokenTreeCursor {
pub stream: TokenStream,
index: usize,
}

impl Iterator for TokenTreeCursor {
type Item = TokenTree;

fn next(&mut self) -> Option<TokenTree> {
self.stream.0.get(self.index).map(|tree| {
self.index += 1;
tree.clone()
})
}
}

impl TokenTreeCursor {
fn new(stream: TokenStream) -> Self {
TokenTreeCursor { stream, index: 0 }
Expand Down
6 changes: 3 additions & 3 deletions compiler/rustc_expand/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,9 +365,9 @@ impl<'a> StripUnconfigured<'a> {

// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
// for `attr` when we expand it to `#[attr]`
let mut orig_trees = orig_tokens.into_trees();
let mut orig_trees = orig_tokens.trees();
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) =
orig_trees.next().unwrap()
orig_trees.next().unwrap().clone()
else {
panic!("Bad tokens for attribute {:?}", attr);
};
Expand All @@ -377,7 +377,7 @@ impl<'a> StripUnconfigured<'a> {
if attr.style == AttrStyle::Inner {
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) =
orig_trees.next().unwrap()
orig_trees.next().unwrap().clone()
else {
panic!("Bad tokens for attribute {:?}", attr);
};
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_expand/src/mbe/macro_rules.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ pub fn compile_declarative_macro(
.map(|m| {
if let MatchedTokenTree(tt) = m {
let tt = mbe::quoted::parse(
TokenStream::new(vec![tt.clone()]),
&TokenStream::new(vec![tt.clone()]),
true,
&sess.parse_sess,
def.id,
Expand All @@ -524,7 +524,7 @@ pub fn compile_declarative_macro(
.map(|m| {
if let MatchedTokenTree(tt) = m {
return mbe::quoted::parse(
TokenStream::new(vec![tt.clone()]),
&TokenStream::new(vec![tt.clone()]),
false,
&sess.parse_sess,
def.id,
Expand Down
42 changes: 21 additions & 21 deletions compiler/rustc_expand/src/mbe/quoted.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \
///
/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
pub(super) fn parse(
input: tokenstream::TokenStream,
input: &tokenstream::TokenStream,
parsing_patterns: bool,
sess: &ParseSess,
node_id: NodeId,
Expand All @@ -48,15 +48,15 @@ pub(super) fn parse(

// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
// additional trees if need be.
let mut trees = input.into_trees();
let mut trees = input.trees();
while let Some(tree) = trees.next() {
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
let tree = parse_tree(tree, &mut trees, parsing_patterns, sess, node_id, features, edition);
match tree {
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
let span = match trees.next() {
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
Some(&tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
match trees.next() {
Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
Some((frag, _)) => {
Expand Down Expand Up @@ -96,10 +96,10 @@ pub(super) fn parse(
}
_ => token.span,
},
tree => tree.as_ref().map_or(span, tokenstream::TokenTree::span),
tree => tree.map_or(span, tokenstream::TokenTree::span),
}
}
tree => tree.as_ref().map_or(start_sp, tokenstream::TokenTree::span),
tree => tree.map_or(start_sp, tokenstream::TokenTree::span),
};

result.push(TokenTree::MetaVarDecl(span, ident, None));
Expand Down Expand Up @@ -134,9 +134,9 @@ fn maybe_emit_macro_metavar_expr_feature(features: &Features, sess: &ParseSess,
/// - `parsing_patterns`: same as [parse].
/// - `sess`: the parsing session. Any errors will be emitted to this session.
/// - `features`: language features so we can do feature gating.
fn parse_tree(
tree: tokenstream::TokenTree,
outer_trees: &mut impl Iterator<Item = tokenstream::TokenTree>,
fn parse_tree<'a>(
tree: &'a tokenstream::TokenTree,
outer_trees: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
parsing_patterns: bool,
sess: &ParseSess,
node_id: NodeId,
Expand All @@ -146,21 +146,21 @@ fn parse_tree(
// Depending on what `tree` is, we could be parsing different parts of a macro
match tree {
// `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing.
let mut next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>;
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
trees = Box::new(tts.into_trees());
trees = Box::new(tts.trees());
next = trees.next();
} else {
trees = Box::new(outer_trees);
}

match next {
// `tree` is followed by a delimited set of token trees.
Some(tokenstream::TokenTree::Delimited(delim_span, delim, tts)) => {
Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts)) => {
if parsing_patterns {
if delim != Delimiter::Parenthesis {
span_dollar_dollar_or_metavar_in_the_lhs_err(
Expand Down Expand Up @@ -228,7 +228,7 @@ fn parse_tree(
}

// `tree` is followed by another `$`. This is an escaped `$`.
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
Some(&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
if parsing_patterns {
span_dollar_dollar_or_metavar_in_the_lhs_err(
sess,
Expand Down Expand Up @@ -256,11 +256,11 @@ fn parse_tree(
}

// `tree` is an arbitrary token. Keep it.
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token),
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token.clone()),

// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it.
tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
&tokenstream::TokenTree::Delimited(span, delim, ref tts) => TokenTree::Delimited(
span,
Delimited {
delim,
Expand All @@ -286,16 +286,16 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp
/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
/// - Err(span) if the next token tree is not a token
fn parse_kleene_op(
input: &mut impl Iterator<Item = tokenstream::TokenTree>,
fn parse_kleene_op<'a>(
input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
span: Span,
) -> Result<Result<(KleeneOp, Span), Token>, Span> {
match input.next() {
Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) {
Some(op) => Ok(Ok((op, token.span))),
None => Ok(Err(token)),
None => Ok(Err(token.clone())),
},
tree => Err(tree.as_ref().map_or(span, tokenstream::TokenTree::span)),
tree => Err(tree.map_or(span, tokenstream::TokenTree::span)),
}
}

Expand All @@ -311,8 +311,8 @@ fn parse_kleene_op(
/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
/// error with the appropriate span is emitted to `sess` and a dummy value is returned.
fn parse_sep_and_kleene_op(
input: &mut impl Iterator<Item = tokenstream::TokenTree>,
fn parse_sep_and_kleene_op<'a>(
input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
span: Span,
sess: &ParseSess,
) -> (Option<Token>, KleeneToken) {
Expand Down
9 changes: 3 additions & 6 deletions compiler/rustc_expand/src/parse/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,8 @@ fn bad_path_expr_1() {
#[test]
fn string_to_tts_macro() {
create_default_session_globals_then(|| {
let tts: Vec<_> =
string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).into_trees().collect();
let tts: &[TokenTree] = &tts[..];
let stream = string_to_stream("macro_rules! zip (($a)=>($a))".to_string());
let tts = &stream.trees().collect::<Vec<_>>()[..];

match tts {
[
Expand Down Expand Up @@ -294,9 +293,7 @@ fn ttdelim_span() {
.unwrap();

let ast::ExprKind::MacCall(mac) = &expr.kind else { panic!("not a macro") };
let tts: Vec<_> = mac.args.tokens.clone().into_trees().collect();

let span = tts.iter().rev().next().unwrap().span();
let span = mac.args.tokens.trees().last().unwrap().span();

match sess.source_map().span_to_snippet(span) {
Ok(s) => assert_eq!(&s[..], "{ body }"),
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_expand/src/proc_macro_server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
// Estimate the capacity as `stream.len()` rounded up to the next power
// of two to limit the number of required reallocations.
let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
let mut cursor = stream.into_trees();
let mut cursor = stream.trees();

while let Some(tree) = cursor.next() {
let (Token { kind, span }, joint) = match tree {
let (Token { kind, span }, joint) = match tree.clone() {
tokenstream::TokenTree::Delimited(span, delim, tts) => {
let delimiter = pm::Delimiter::from_internal(delim);
trees.push(TokenTree::Group(Group {
Expand Down
Loading