fix: improve nested parsing and cleanup

Author: psteinroe

crates/parser/src/parser.rs

@@ -7,13 +7,14 @@ use crate::syntax_error::SyntaxError;
 use crate::syntax_kind::{SyntaxKind, SyntaxKindType};
 use crate::syntax_node::SyntaxNode;
 
-#[derive(Default, Debug)]
+#[derive(Debug)]
 pub struct Parser {
     inner: GreenNodeBuilder<'static, 'static, SyntaxKind>,
     token_buffer: Vec<(SyntaxKind, String)>,
     curr_depth: i32,
     errors: Vec<SyntaxError>,
     stmts: Vec<RawStmt>,
+    checkpoint: Option<i32>,
     is_parsing_erronous_node: bool,
 }
 
@@ -26,34 +27,64 @@ pub struct Parse {
 
 /// Main parser that controls the cst building process, and collects errors and statements
 impl Parser {
+    pub fn new() -> Self {
+        Self {
+            curr_depth: -1,
+            inner: GreenNodeBuilder::new(),
+            token_buffer: Vec::new(),
+            errors: Vec::new(),
+            stmts: Vec::new(),
+            checkpoint: None,
+            is_parsing_erronous_node: false,
+        }
+    }
+
     pub fn close_until_depth(&mut self, depth: i32) {
-        while self.curr_depth >= depth && depth > 0 {
+        while self.curr_depth >= depth {
             self.finish_node();
             self.curr_depth -= 1;
         }
     }
 
+    pub fn set_checkpoint(&mut self, is_parsing_erronous_node: bool) {
+        assert!(self.checkpoint.is_none());
+        assert!(self.token_buffer.is_empty());
+        println!("set_checkpoint at {}", self.curr_depth);
+        self.checkpoint = Some(self.curr_depth);
+        self.is_parsing_erronous_node = is_parsing_erronous_node;
+    }
+
+    pub fn close_checkpoint(&mut self) {
+        self.consume_token_buffer();
+        if self.checkpoint.is_some() {
+            self.close_until_depth(self.checkpoint.unwrap());
+        }
+        self.checkpoint = None;
+        self.is_parsing_erronous_node = false;
+    }
+
+    pub fn start_node(&mut self, kind: SyntaxKind) {
+        self.inner.start_node(kind);
+    }
+
     /// start a new node of `SyntaxKind` at `depth`
     /// handles closing previous nodes if necessary
     /// and consumes token buffer before starting new node
     ///
     /// if `SyntaxKind` is `SyntaxKind::AnyStatement`, sets `is_parsing_erronous_node` to true
-    pub fn start_node(&mut self, kind: SyntaxKind, depth: i32) {
+    pub fn start_node_at(&mut self, kind: SyntaxKind, depth: Option<i32>) {
+        let depth = depth.unwrap_or(self.curr_depth + 1);
         // close until target depth
         self.close_until_depth(depth);
 
         self.consume_token_buffer();
 
         self.curr_depth = depth;
-        self.inner.start_node(kind);
-        if kind == SyntaxKind::AnyStatement {
-            self.is_parsing_erronous_node = true;
-        }
+        self.start_node(kind);
     }
 
     pub fn finish_node(&mut self) {
         self.inner.finish_node();
-        self.is_parsing_erronous_node = false;
     }
 
     /// Drains the token buffer and applies all tokens

crates/parser/src/source_file.rs

@@ -26,7 +26,7 @@ impl Parser {
     pub fn parse_source_file(&mut self, text: &str) {
         let mut lexer = SourceFileToken::lexer(text);
 
-        self.start_node(SyntaxKind::SourceFile, 0);
+        self.start_node_at(SyntaxKind::SourceFile, Some(0));
         while let Some(token) = lexer.next() {
             match token {
                 Ok(token) => {
@@ -92,7 +92,7 @@ select 1;
 
 ";
 
-        let mut parser = Parser::default();
+        let mut parser = Parser::new();
         println!("input {:?}", input);
         parser.parse_source_file(input);
         let parsed = parser.finish();

crates/parser/src/statement.rs

@@ -103,7 +103,6 @@ impl StatementToken {
 
 impl Parser {
     pub fn parse_statement(&mut self, text: &str, at_offset: Option<u32>) {
-        println!("#### parse_statement: {}", text);
         let offset = at_offset.unwrap_or(0);
         let range = TextRange::new(
             TextSize::from(offset),
@@ -117,7 +116,6 @@ impl Parser {
                 Vec::new().into_iter().peekable()
             }
         };
-        println!("pg_query_tokens: {:?}", pg_query_tokens);
 
         let parsed = pg_query::parse(text);
         let proto;
@@ -139,18 +137,19 @@ impl Parser {
                 Vec::new().into_iter().peekable()
             }
         };
-        println!("pg_query_nodes: {:?}", pg_query_nodes);
 
         let mut lexer = StatementToken::lexer(&text);
 
         // parse root node if no syntax errors
         if pg_query_nodes.peek().is_some() {
             let (node, depth, _) = pg_query_nodes.next().unwrap();
             self.stmt(node.to_enum(), range);
-            self.start_node(SyntaxKind::from_pg_query_node(&node), depth);
+            self.start_node_at(SyntaxKind::from_pg_query_node(&node), Some(depth));
+            self.set_checkpoint(false);
         } else {
             // fallback to generic node as root
-            self.start_node(SyntaxKind::AnyStatement, 1);
+            self.start_node_at(SyntaxKind::Stmt, None);
+            self.set_checkpoint(true);
         }
 
         while let Some(token) = lexer.next() {
@@ -166,7 +165,7 @@ impl Parser {
                         } else {
                             // node is within span
                             let (node, depth, _) = pg_query_nodes.next().unwrap();
-                            self.start_node(SyntaxKind::from_pg_query_node(&node), depth);
+                            self.start_node_at(SyntaxKind::from_pg_query_node(&node), Some(depth));
                         }
                     }
 
@@ -192,8 +191,7 @@ impl Parser {
         }
 
         // close up nodes
-        self.consume_token_buffer();
-        self.close_until_depth(1);
+        self.close_checkpoint();
     }
 }
 
@@ -251,7 +249,7 @@ mod tests {
     fn test_statement_parser() {
         let input = "select *,some_col from contact where id = '123 4 5';";
 
-        let mut parser = Parser::default();
+        let mut parser = Parser::new();
         parser.parse_statement(input, None);
         let parsed = parser.finish();
 
@@ -264,7 +262,7 @@ mod tests {
     fn test_invalid_statement() {
         let input = "select select;";
 
-        let mut parser = Parser::default();
+        let mut parser = Parser::new();
         parser.parse_statement(input, None);
         let parsed = parser.finish();
 

crates/parser/src/syntax_kind.rs

@@ -12,8 +12,8 @@ pub enum SyntaxKind {
     Whitespace,
     Newline,
     Tab,
-    Word,         // common value for all keywords (select, from, ...)
-    AnyStatement, // node for unknown statements (e.g. when parsing fails)
+    Word, // common value for all keywords (select, from, ...)
+    Stmt, // node for unknown statements (e.g. when parsing fails)
     // from here copyied from NodeEnum
     Alias,
     RangeVar,