Merge pull request #157 from zevaverbach/main

fix: small bug in whitespace_tokens, added a few tests for such bugs.

Author: psteinroe

crates/pg_lexer/src/lib.rs

@@ -89,7 +89,7 @@ fn whitespace_tokens(input: &str) -> VecDeque<Token> {
         } else if let Some(tab) = cap.name("tab") {
             tokens.push_back(Token {
                 token_type: TokenType::Whitespace,
-                kind: SyntaxKind::Newline,
+                kind: SyntaxKind::Tab,
                 text: tab.as_str().to_string(),
                 span: TextRange::new(
                     TextSize::from(u32::try_from(tab.start()).unwrap()),
@@ -177,9 +177,30 @@ pub fn lex(text: &str) -> Vec<Token> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_tab_tokens() {
+        let input = "select\t1";
+        let tokens = lex(input);
+        assert_eq!(tokens[1].kind, SyntaxKind::Tab);
+    }
+
+    #[test]
+    fn test_newline_tokens() {
+        let input = "select\n1";
+        let tokens = lex(input);
+        assert_eq!(tokens[1].kind, SyntaxKind::Newline);
+    }
+
+    #[test]
+    fn test_whitespace_tokens() {
+        let input = "select 1";
+        let tokens = lex(input);
+        assert_eq!(tokens[1].kind, SyntaxKind::Whitespace);
+    }
+
     #[test]
     fn test_lexer() {
-        let input = "select 1; \n -- some comment \n select 2";
+        let input = "select 1; \n -- some comment \n select 2\t";
 
         let tokens = lex(input);
         let mut tokens_iter = tokens.iter();
@@ -227,5 +248,8 @@ mod tests {
         let token = tokens_iter.next().unwrap();
         assert_eq!(token.kind, SyntaxKind::Iconst);
         assert_eq!(token.text, "2");
+
+        let token = tokens_iter.next().unwrap();
+        assert_eq!(token.kind, SyntaxKind::Tab);
     }
 }