From bc18868a9b471f6603a2c08d8c8e2f576b424ddc Mon Sep 17 00:00:00 2001 From: Eduard-Mihai Burtescu Date: Sat, 20 Oct 2018 13:47:01 +0300 Subject: [PATCH] Go back to external iteration (for `all`) without generators. --- src/generate/rust.rs | 122 ++++++------------- src/runtime.rs | 276 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 279 insertions(+), 119 deletions(-) diff --git a/src/generate/rust.rs b/src/generate/rust.rs index 22f19f6..48b39a5 100644 --- a/src/generate/rust.rs +++ b/src/generate/rust.rs @@ -221,8 +221,6 @@ impl fmt::Display for CodeLabel { #[derive(Default)] pub struct Options { - /// Enable the use of unstable features, such as generators. - pub unstable: bool, /// Disable generating macros (e.g. `P!(...)` sugar for `_P::...`). pub no_macros: bool, @@ -247,34 +245,13 @@ impl Grammar { let mut code_labels = OrderMap::new(); put!(" -use gll::runtime::{Call, Continuation, ParseNodeKind, CodeLabel, ParseNodeShape, ParseNode, Range, traverse}; +use gll::runtime::{Call, Continuation, ParseNodeKind, CodeLabel, ParseNodeShape, ParseNode, Range, traverse, nd::Arrow}; use std::any; use std::fmt; use std::marker::PhantomData;"); - if options.unstable { - put!(" -use std::ops::{GeneratorState, Generator}; -"); - } // HACK(eddyb) see `out += out_body` at the end let out_imports = mem::replace(&mut out, String::new()); - if options.unstable { - put!(" -struct GenIter(G); - -impl> Iterator for GenIter { - type Item = G::Yield; - - fn next(&mut self) -> Option { - match unsafe { self.0.resume() } { - GeneratorState::Complete(..) => None, - GeneratorState::Yielded(v) => Some(v), - } - } -} -"); - } put!(" #[derive(Debug)] pub enum ParseError { @@ -621,13 +598,14 @@ impl<'a, 'i, I: ::gll::runtime::Input> fmt::Debug for Handle<'a, 'i, I, ", name, fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, \"{:?} => \", self.source_info())?; let mut first = true; - self.try_for_each(|x| { + for x in self.all() { if !first { write!(f, \" | \")?; } first = false; - fmt::Debug::fmt(&x, f) - }) + fmt::Debug::fmt(&x, f)?; + } + Ok(()) } } @@ -761,76 +739,51 @@ impl<'a, 'i, I: ::gll::runtime::Input> Handle<'a, 'i, I, ", name, "<'a, 'i, I>> } put!(" }))().map_err(|::gll::runtime::MoreThanOne| Ambiguity(self)) - }"); - if options.unstable { - put!(" + } pub fn all(self) -> impl Iterator> { let sppf = &self.parser.sppf; - GenIter(move || { - let node = self.node.unpack_alias();"); + let node = self.node.unpack_alias();"); if let Some(variants) = &variants { put!(" - for node in sppf.all_choices(node) { - match node.kind {"); - for (rule, variant, _) in variants { + #[derive(Clone)] + enum Iter<"); for i in 0..variants.len() { put!("_", i, ","); } put!("> { + "); for i in 0..variants.len() { put!("_", i, "(_", i, "),"); } put!(" + } + impl,"); } put!("> Iterator + for Iter<"); for i in 0..variants.len() { put!("_", i, ","); } put!("> + { + type Item = T; + fn next(&mut self) -> Option { + match self {"); + for i in 0..variants.len() { put!(" - ", rule.parse_node_kind(&parse_nodes), " => { - traverse!(all(sppf, node) ", rule.generate_traverse_shape(false, &parse_nodes), ", - r => yield ", name, "::", variant, "_from_sppf(self.parser, node, r)); - }"); + Iter::_", i, "(iter) => iter.next(),"); } put!(" - _ => unreachable!(), - } - }"); - } else { - put!(" - traverse!(all(sppf, node) ", rule.rule.generate_traverse_shape(false, &parse_nodes), ", - r => yield ", name, "::from_sppf(self.parser, node, r));"); } - put!(" - }) - }"); } - put!(" - pub fn for_each(self, mut f: impl FnMut(", name, "<'a, 'i, I>)) { - let sppf = &self.parser.sppf; - let node = self.node.unpack_alias();"); - if let Some(variants) = &variants { - put!(" - for node in sppf.all_choices(node) { + } + sppf.all_choices(node).flat_map(move |node| { match node.kind {"); - for (rule, variant, _) in variants { + for (i, (rule, variant, _)) in variants.iter().enumerate() { + put!(" + ", rule.parse_node_kind(&parse_nodes), " => Iter::_", i, "( + traverse!(all(sppf) ", rule.generate_traverse_shape(false, &parse_nodes), ") + .apply(node) + .map(move |r| ", name, "::", variant, "_from_sppf(self.parser, node, r)) + ),"); + } put!(" - ", rule.parse_node_kind(&parse_nodes), " => { - traverse!(all(sppf, node) ", rule.generate_traverse_shape(false, &parse_nodes), ", - r => f(", name, "::", variant, "_from_sppf(self.parser, node, r))); - }"); - } - put!(" _ => unreachable!(), } - }"); - } else { + })"); + } else { + put!(" + traverse!(all(sppf) ", rule.rule.generate_traverse_shape(false, &parse_nodes), ") + .apply(node) + .map(move |r| ", name, "::from_sppf(self.parser, node, r))"); + } put!(" - traverse!(all(sppf, node) ", rule.rule.generate_traverse_shape(false, &parse_nodes), ", - r => f(", name, "::from_sppf(self.parser, node, r)));"); - } - put!(" - } - pub fn try_for_each( - self, - mut f: impl FnMut(", name, "<'a, 'i, I>) -> Result<(), E>, - ) -> Result<(), E> { - let mut r = Ok(()); - self.for_each(|x| { - // FIXME(eddyb) this should stop the iteration early - if r.is_err() { - return; - } - r = f(x); - }); - r } }"); } @@ -1523,7 +1476,8 @@ impl Rule { for (i, rule) in rules.iter().enumerate() { write!( s, - "{}: {} => {},", + "{} _{}: {} => {},", + i, i, rule.parse_node_kind(parse_nodes), rule.generate_traverse_shape(true, parse_nodes) diff --git a/src/runtime.rs b/src/runtime.rs index 752fb00..8a9c3ed 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -501,7 +501,7 @@ impl<'i, P: ParseNodeKind> ParseForest<'i, P> { pub fn all_choices<'a>( &'a self, node: ParseNode<'i, P>, - ) -> impl Iterator> + 'a { + ) -> impl Iterator> + Clone + 'a { match node.kind.shape() { ParseNodeShape::Choice => self .possibilities @@ -547,7 +547,7 @@ impl<'i, P: ParseNodeKind> ParseForest<'i, P> { pub fn all_splits<'a>( &'a self, node: ParseNode<'i, P>, - ) -> impl Iterator, ParseNode<'i, P>)> + 'a { + ) -> impl Iterator, ParseNode<'i, P>)> + Clone + 'a { match node.kind.shape() { ParseNodeShape::Split(left_kind, right_kind) => self .possibilities @@ -695,6 +695,205 @@ pub trait CodeLabel: fmt::Debug + Ord + Hash + Copy + 'static { fn enclosing_fn(self) -> Self; } +// FIXME(rust-lang/rust#54175) work around iterator adapter compile-time +// blowup issues by using a makeshift "non-determinism arrow toolkit". +pub mod nd { + use std::iter; + use std::marker::PhantomData; + + pub trait Arrow: Copy { + type Input; + type Output; + type Iter: Iterator + Clone; + fn apply(&self, x: Self::Input) -> Self::Iter; + + fn map R, R>(self, f: F) -> Map { + Map(self, f) + } + fn then>(self, b: B) -> Then { + Then(self, b) + } + fn pairs(self, b: B) -> Pairs + where + Self::Output: Copy, + B::Input: Copy, + { + Pairs(self, b) + } + } + + macro_rules! derive_copy { + ($name:ident<$($param:ident $(: $bound:ident)*),*>) => { + impl<$($param $(: $bound)*),*> Copy for $name<$($param),*> {} + impl<$($param $(: $bound)*),*> Clone for $name<$($param),*> { + fn clone(&self) -> Self { + *self + } + } + } + } + + pub struct Id(PhantomData); + derive_copy!(Id); + impl Id { + pub fn new() -> Self { + Id(PhantomData) + } + } + impl Arrow for Id { + type Input = T; + type Output = T; + type Iter = iter::Once; + fn apply(&self, x: T) -> Self::Iter { + iter::once(x) + } + } + + pub struct FromIter(F, PhantomData); + derive_copy!(FromIter); + impl FromIter { + pub fn new(f: F) -> Self { + FromIter(f, PhantomData) + } + } + impl I, I: Iterator + Clone> Arrow for FromIter { + type Input = T; + type Output = I::Item; + type Iter = I; + fn apply(&self, x: T) -> I { + self.0(x) + } + } + + pub struct FromIterK(K, F, PhantomData); + derive_copy!(FromIterK); + impl FromIterK { + pub fn new(k: K, f: F) -> Self { + FromIterK(k, f, PhantomData) + } + } + impl I, I: Iterator + Clone> Arrow for FromIterK { + type Input = T; + type Output = I::Item; + type Iter = I; + fn apply(&self, x: T) -> I { + self.1(self.0, x) + } + } + + #[derive(Copy, Clone)] + pub struct Map(A, F); + impl R, R> Arrow for Map { + type Input = A::Input; + type Output = R; + type Iter = iter::Map; + fn apply(&self, x: Self::Input) -> Self::Iter { + self.0.apply(x).map(self.1) + } + } + + #[derive(Clone)] + pub struct ThenIter> { + a_iter: A::Iter, + b_arrow: B, + b_iter: Option, + // HACK(eddyb) this field is useless (never set to `Some`) + // (see `match self.b_iter_backwards` below for more details). + b_iter_backwards: Option, + } + impl> Iterator for ThenIter { + type Item = B::Output; + fn next(&mut self) -> Option { + loop { + if let Some(ref mut b_iter) = self.b_iter { + if let x @ Some(_) = b_iter.next() { + return x; + } + } + match self.a_iter.next() { + // HACK(eddyb) this never does anything, but without a *second* + // call to `B::Iter::next`, LLVM spends more time optimizing. + None => { + return match self.b_iter_backwards { + Some(ref mut b_iter) => b_iter.next(), + None => None, + } + } + Some(x) => self.b_iter = Some(self.b_arrow.apply(x)), + } + } + } + } + + #[derive(Copy, Clone)] + pub struct Then(A, B); + impl> Arrow for Then { + type Input = A::Input; + type Output = B::Output; + type Iter = ThenIter; + fn apply(&self, x: Self::Input) -> Self::Iter { + ThenIter { + a_iter: self.0.apply(x), + b_arrow: self.1, + b_iter: None, + b_iter_backwards: None, + } + } + } + + #[derive(Clone)] + pub struct PairsIter + where + A::Output: Copy, + B::Input: Copy, + { + a_iter: A::Iter, + b_iter0: B::Iter, + a_output_b_iter: Option<(A::Output, B::Iter)>, + } + impl Iterator for PairsIter + where + A::Output: Copy, + B::Input: Copy, + { + type Item = (A::Output, B::Output); + fn next(&mut self) -> Option { + loop { + if let Some((x, ref mut b_iter)) = self.a_output_b_iter { + if let Some(y) = b_iter.next() { + return Some((x, y)); + } + } + match self.a_iter.next() { + None => return None, + Some(x) => { + self.a_output_b_iter = Some((x, self.b_iter0.clone())); + } + } + } + } + } + + #[derive(Copy, Clone)] + pub struct Pairs(A, B); + impl Arrow for Pairs + where + A::Output: Copy, + B::Input: Copy, + { + type Input = (A::Input, B::Input); + type Output = (A::Output, B::Output); + type Iter = PairsIter; + fn apply(&self, (x, y): Self::Input) -> Self::Iter { + PairsIter { + a_iter: self.0.apply(x), + b_iter0: self.1.apply(y), + a_output_b_iter: None, + } + } + } +} + // HACK(eddyb) work around `macro_rules` not being `use`-able. pub use crate::traverse; @@ -703,7 +902,7 @@ macro_rules! traverse { (typeof($leaf:ty) _) => { $leaf }; (typeof($leaf:ty) ?) => { Option }; (typeof($leaf:ty) ($l_shape:tt, $r_shape:tt)) => { (traverse!(typeof($leaf) $l_shape), traverse!(typeof($leaf) $r_shape)) }; - (typeof($leaf:ty) { $($i:tt: $kind:pat => $shape:tt,)* }) => { ($(traverse!(typeof($leaf) $shape),)*) }; + (typeof($leaf:ty) { $($i:tt $_i:ident: $kind:pat => $shape:tt,)* }) => { ($(traverse!(typeof($leaf) $shape),)*) }; (typeof($leaf:ty) [$shape:tt]) => { (traverse!(typeof($leaf) $shape),) }; (one($sppf:ident, $node:ident) _) => { @@ -721,7 +920,7 @@ macro_rules! traverse { ) } }; - (one($sppf:ident, $node:ident) { $($i:tt: $kind:pat => $shape:tt,)* }) => { + (one($sppf:ident, $node:ident) { $($i:tt $_i:ident: $kind:pat => $shape:tt,)* }) => { { let node = $sppf.one_choice($node)?; let mut r = <($(traverse!(typeof(_) $shape),)*)>::default(); @@ -742,46 +941,53 @@ macro_rules! traverse { } }; - (all($sppf:ident, $node:ident) _, $result:pat => $cont:expr) => { - match $node { $result => $cont } + (all($sppf:ident) _) => { + $crate::runtime::nd::Id::new() }; - (all($sppf:ident, $node:ident) ?, $result:pat => $cont:expr) => { - match Some($node) { $result => $cont } + (all($sppf:ident) ?) => { + $crate::runtime::nd::Id::new().map(Some) }; - (all($sppf:ident, $node:ident) ($l_shape:tt, $r_shape:tt), $result:pat => $cont:expr) => { - for (left, right) in $sppf.all_splits($node) { - traverse!(all($sppf, left) $l_shape, left => - traverse!(all($sppf, right) $r_shape, right => match (left, right) { $result => $cont })) - } + (all($sppf:ident) ($l_shape:tt, $r_shape:tt)) => { + $crate::runtime::nd::FromIterK::new($sppf, $crate::runtime::ParseForest::all_splits) + .then(traverse!(all($sppf) $l_shape).pairs(traverse!(all($sppf) $r_shape))) }; - (all($sppf:ident, $node:ident) { $($i:tt: $kind:pat => $shape:tt,)* }, $result:pat => $cont:expr) => { - for node in $sppf.all_choices($node) { - let tuple_template = <($(traverse!(typeof(_) $shape),)*)>::default(); - match node.kind { - $($kind => traverse!(all($sppf, node) $shape, x => { - let mut r = tuple_template; - r.$i = x; - match r { $result => $cont } - }),)* - _ => unreachable!(), + (all($sppf:ident) { $($i:tt $_i:ident: $kind:pat => $shape:tt,)* }) => { + $crate::runtime::nd::FromIter::new(move |node| { + #[derive(Clone)] + enum Iter<$($_i),*> { + $($_i($_i)),* } - } + impl<$($_i: Iterator),*> Iterator for Iter<$($_i),*> + where $($_i::Item: Default),* + { + type Item = ($($_i::Item),*); + fn next(&mut self) -> Option { + let mut r = Self::Item::default(); + match self { + $(Iter::$_i(iter) => r.$i = iter.next()?),* + } + Some(r) + } + } + $sppf.all_choices(node).flat_map(move |node| { + match node.kind { + $($kind => Iter::$_i(traverse!(all($sppf) $shape).apply(node)),)* + _ => unreachable!(), + } + }) + }) }; - (all($sppf:ident, $node:ident) [$shape:tt], $result:pat => $cont:expr) => { - { - let tuple_template = <(traverse!(typeof(_) $shape),)>::default(); - match $node.unpack_opt() { + (all($sppf:ident) [$shape:tt]) => { + $crate::runtime::nd::FromIter::new(move |node| { + match $crate::runtime::ParseNode::unpack_opt(node) { Some(node) => { - traverse!(all($sppf, node) $shape, x => { - let mut r = tuple_template; - r.0 = x; - match r { $result => $cont } - }) + Some(traverse!(all($sppf) $shape).apply(node).map(|x| (x,))) + .into_iter().flatten().chain(None) } None => { - match tuple_template { $result => $cont } + None.into_iter().flatten().chain(Some(<_>::default())) } } - } + }) } }